From a04cc8a00fe7cbe48fbcc1719d2f4603ac9f92db Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Mon, 26 Oct 2026 17:16:09 -0700 Subject: [PATCH 1/2] virtiofs: factor the integration test harness for input-gate tests The integration test harness built its own VirtioFs device, started its one queue only from scratch, and waited for the reply to the FUSE_INIT that it posted before it returned. A test of a device whose host input is quiesced, as a restore leaves a device behind its post-restore input gate, needs more than that: the fixed microVM device, the only one that honors an input quiesce; a request that is posted while the device must not serve it, and whose reply is awaited later; and a queue that starts again from the state that its stop saved. Factor the harness along those lines, without changing what the existing tests do: - with_device builds the harness around a given device and the temporary directory that backs it. new passes it the same VirtioFs device as before. - start_queue starts queue 0, which carries every request of the harness, from an optional saved state, as a restore starts it. enable passes no state. - post_fuse_init posts FUSE_INIT and returns its unique ID and response buffer, and wait_for_init awaits and checks its reply. fuse_init calls one after the other. See microsoft/nvx#904. Co-authored-by: Enrique Saurez Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../virtio/virtiofs/src/integration_tests.rs | 44 +++++++++++++++---- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/vm/devices/virtio/virtiofs/src/integration_tests.rs b/vm/devices/virtio/virtiofs/src/integration_tests.rs index 831b8df387c..0abf1ea0cd5 100644 --- a/vm/devices/virtio/virtiofs/src/integration_tests.rs +++ b/vm/devices/virtio/virtiofs/src/integration_tests.rs @@ -18,6 +18,7 @@ use test_with_tracing::test; use virtio::QueueResources; use virtio::VirtioDevice; use virtio::queue::QueueParams; +use virtio::queue::QueueState; use virtio::spec::VirtioDeviceFeatures; use virtio::spec::queue::DescriptorFlags; use virtio::test_helpers::init_avail_ring; @@ -64,15 +65,23 @@ struct TestHarness { impl TestHarness { fn new(driver: &DefaultDriver) -> Self { let tmpdir = tempfile::tempdir().unwrap(); + let fs = VirtioFs::new(tmpdir.path(), None).unwrap(); + let driver_source = VmTaskDriverSource::new(SingleDriverBackend::new(driver.clone())); + let device = VirtioFsDevice::new(&driver_source, "testfs", fs, 0, None); + Self::with_device(driver, device, tmpdir) + } + /// Creates a harness that drives `device`, whose filesystem is backed by + /// `tmpdir`. + fn with_device( + driver: &DefaultDriver, + device: VirtioFsDevice, + tmpdir: tempfile::TempDir, + ) -> Self { let mem = GuestMemory::allocate(TOTAL_MEM_SIZE); init_avail_ring(&mem, AVAIL_ADDR); init_used_ring(&mem, USED_ADDR); - let fs = VirtioFs::new(tmpdir.path(), None).unwrap(); - let driver_source = VmTaskDriverSource::new(SingleDriverBackend::new(driver.clone())); - let device = VirtioFsDevice::new(&driver_source, "testfs", fs, 0, None); - let queue_event = Event::new(); let interrupt_event = Event::new(); @@ -91,6 +100,12 @@ impl TestHarness { } async fn enable(&mut self) { + self.start_queue(None).await; + } + + /// Starts queue 0, which carries every request of the harness, from + /// `initial_state` if there is one, as a restore starts it. + async fn start_queue(&mut self, initial_state: Option) { let interrupt = Interrupt::from_event(self.interrupt_event.clone()); self.device .start_queue( @@ -108,7 +123,7 @@ impl TestHarness { guest_memory: self.mem.clone(), }, &VirtioDeviceFeatures::new(), - None, + initial_state, ) .await .unwrap(); @@ -277,8 +292,9 @@ impl TestHarness { T::read_from_bytes(&buf).unwrap() } - /// Send FUSE_INIT and wait for the response. Panics on failure. - async fn fuse_init(&mut self, head_desc: u16) { + /// Post FUSE_INIT without waiting for the response. Returns + /// `(unique, resp_gpa)`, as [`Self::post_fuse_request`] does. + fn post_fuse_init(&mut self, head_desc: u16) -> (u64, u64) { let init_args = fuse_init_in { major: FUSE_KERNEL_VERSION, minor: FUSE_KERNEL_MINOR_VERSION, @@ -289,9 +305,13 @@ impl TestHarness { }; let resp_size = OUT_HEADER_SIZE + size_of::() as u32; - let (unique, resp_gpa) = - self.post_fuse_request(head_desc, FUSE_INIT, 0, init_args.as_bytes(), resp_size); + self.post_fuse_request(head_desc, FUSE_INIT, 0, init_args.as_bytes(), resp_size) + } + /// Wait for the response to the FUSE_INIT that [`Self::post_fuse_init`] + /// posted as `unique`, with its response buffer at `resp_gpa`. Panics on + /// failure. + async fn wait_for_init(&mut self, unique: u64, resp_gpa: u64) { let (_used_id, used_len) = self.wait_for_used().await; assert!(used_len > 0, "FUSE_INIT response should not be empty"); @@ -303,6 +323,12 @@ impl TestHarness { assert_eq!(init_out.major, FUSE_KERNEL_VERSION); } + /// Send FUSE_INIT and wait for the response. Panics on failure. + async fn fuse_init(&mut self, head_desc: u16) { + let (unique, resp_gpa) = self.post_fuse_init(head_desc); + self.wait_for_init(unique, resp_gpa).await; + } + /// Return the path to the temp directory backing the filesystem. fn tmpdir_path(&self) -> &std::path::Path { self._tmpdir.path() From e68871cc6f14fceb6d38a899504ae8105f89e310 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Mon, 26 Oct 2026 17:23:17 -0700 Subject: [PATCH 2/2] virtiofs: wait for request admission instead of ending a queue worker A virtio-fs queue worker ended as soon as it found request admission closed. It returned Ok(()), so task_control marked it done, and only a new queue start, which inserts a new worker, would serve the queue again. An input quiesce closes admission on the fixed microVM device, and the VM keeps or starts queues behind a closed admission on three paths: - A restore under the post-restore input gate, which every tiered snapshot requires, quiesces device input before the VM's state units start, so its queues start with admission closed and their workers end at once. When the guest acknowledged the restore with its snapshot boundary and input resumed, no worker was left, and every request of the guest, from its FUSE_INIT on, waited forever. A filesystem attached at the restore, such as a clone's directory export bound to a dormant slot, could be mounted, since the guest's mount does not wait for FUSE_INIT, but never used. - The rollback of a failed save starts the state units again while input is still quiesced, and only then releases the snapshot boundary, with the same result. - A worker that was serving a request when a snapshot boundary closed admission ended once it completed that request, and nothing started it again if the boundary was then released without a save. Make the worker wait for admission to reopen instead. RequestAdmission now notifies an event_listener::Event when resume() reopens admission, and accept() waits for it. accept() registers its listener before it releases the admission lock, so no resume between its check and its wait goes unnoticed, and dropping the wait unregisters the listener, so a queue that stops while it waits, however often, leaves nothing registered. The worker awaits admission through StopTask::until_stopped, so a stop, such as a save's, still cancels the wait at once. Admission still precedes queue.next(), so a closed gate leaves descriptors on the virtqueue, and a waiting worker holds no request, so a save still finds none in flight. A device whose input is never quiesced, such as a virtio-fs device outside the microVM profile, is admitted at once, as before. virtiofs now depends on event-listener, which the workspace already uses. Unit tests check that closed admission admits nothing, that resume() admits a waiting request, and that a dropped wait leaves no listener. Integration tests start a microVM queue while input is quiesced, check that a posted FUSE_INIT stays on the virtqueue, resume input, and wait for the reply; and stop such a queue while its worker waits, check that it took no request and that the device saves, start it again from the saved state, and wait for the reply once input resumes. Without the change, both integration tests time out waiting for the reply. The snapshot guide and the virtio-fs reference say that a tiered restore gates filesystem input too: the guest's virtio-fs requests wait on their virtqueues until OpenVMM releases device input, so the guest's post-restore repair must not wait for a virtio-fs reply. See microsoft/nvx#904. Co-authored-by: Enrique Saurez Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Cargo.lock | 1 + .../src/reference/devices/virtio/virtio-fs.md | 6 ++ Guide/src/user_guide/openvmm/snapshots.md | 12 +-- vm/devices/virtio/virtiofs/Cargo.toml | 1 + .../virtio/virtiofs/src/integration_tests.rs | 81 +++++++++++++++++++ vm/devices/virtio/virtiofs/src/virtio.rs | 67 +++++++++++---- 6 files changed, 149 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ef55d05df26..8820ea7548e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10390,6 +10390,7 @@ version = "0.0.0" dependencies = [ "anyhow", "chipset_device", + "event-listener", "fuse", "futures", "guestmem", diff --git a/Guide/src/reference/devices/virtio/virtio-fs.md b/Guide/src/reference/devices/virtio/virtio-fs.md index 6c9ad64ecf2..576ee12e2bb 100644 --- a/Guide/src/reference/devices/virtio/virtio-fs.md +++ b/Guide/src/reference/devices/virtio/virtio-fs.md @@ -363,6 +363,12 @@ mkdir -p /mnt/share mount -t virtiofs microvm /mnt/share ``` +A tiered restore starts the device behind the post-restore input gate that +[Snapshots](../../../user_guide/openvmm/snapshots.md) describes. Requests that +the guest sends before it acknowledges the restore stay on their virtqueues, +and OpenVMM serves them once it releases device input, so post-restore repair +must not wait for a virtio-fs reply. + Snapshots created before the dormant-slot capability cannot add a restore-time attachment and fail with a compatibility error. Snapshots that an earlier OpenVMM captured with a second virtio-fs slot, `fs:microvm1`, cannot be diff --git a/Guide/src/user_guide/openvmm/snapshots.md b/Guide/src/user_guide/openvmm/snapshots.md index c93975ec24f..84a7021f015 100644 --- a/Guide/src/user_guide/openvmm/snapshots.md +++ b/Guide/src/user_guide/openvmm/snapshots.md @@ -174,11 +174,13 @@ releasing gated input. The peer must accept and read while resume is in progress; on Windows, flush completion waits until the named-pipe peer consumes the complete frame. -For a tiered microVM restore, OpenVMM starts device workers with network and -control input gated. The guest performs post-restore repair and writes the -existing snapshot port (`0x605`) to acknowledge completion. OpenVMM stops at -that exact post-write boundary, completes the deferred write while the vCPU is -stopped, and then releases device input before resuming the vCPU. The acknowledgement is bounded by +For a tiered microVM restore, OpenVMM starts device workers with network, +control, and filesystem input gated. The guest performs post-restore repair +and writes the existing snapshot port (`0x605`) to acknowledge completion. +OpenVMM stops at that exact post-write boundary, completes the deferred write +while the vCPU is stopped, and then releases device input before resuming the +vCPU. Until then, the guest's virtio-fs requests wait on their virtqueues, so +repair must not wait for a virtio-fs reply. The acknowledgement is bounded by `--restore-gate-timeout-ms` (60000 milliseconds by default). A control-console host can connect while input is gated, but OpenVMM reads its authentication only after the release, so its `--microvm-control-auth-timeout-ms` deadline diff --git a/vm/devices/virtio/virtiofs/Cargo.toml b/vm/devices/virtio/virtiofs/Cargo.toml index b07c564aa85..31002e4e7d0 100644 --- a/vm/devices/virtio/virtiofs/Cargo.toml +++ b/vm/devices/virtio/virtiofs/Cargo.toml @@ -22,6 +22,7 @@ mesh.workspace = true task_control.workspace = true anyhow.workspace = true +event-listener.workspace = true futures.workspace = true pal_async.workspace = true parking_lot.workspace = true diff --git a/vm/devices/virtio/virtiofs/src/integration_tests.rs b/vm/devices/virtio/virtiofs/src/integration_tests.rs index 0abf1ea0cd5..39f3fd83aa0 100644 --- a/vm/devices/virtio/virtiofs/src/integration_tests.rs +++ b/vm/devices/virtio/virtiofs/src/integration_tests.rs @@ -8,12 +8,17 @@ //! as a guest kernel would. use crate::VirtioFs; +use crate::profile::MICROVM_ATTACHMENT_ID; +use crate::profile::MicroVmOwnerMode; +use crate::profile::microvm_root_identity; use crate::virtio::VirtioFsDevice; use fuse::protocol::*; use guestmem::GuestMemory; use pal_async::DefaultDriver; use pal_async::async_test; +use pal_async::timer::PolledTimer; use pal_event::Event; +use std::time::Duration; use test_with_tracing::test; use virtio::QueueResources; use virtio::VirtioDevice; @@ -24,6 +29,7 @@ use virtio::spec::queue::DescriptorFlags; use virtio::test_helpers::init_avail_ring; use virtio::test_helpers::init_used_ring; use virtio::test_helpers::make_available; +use virtio::test_helpers::read_used_idx; use virtio::test_helpers::wait_for_used; use virtio::test_helpers::write_descriptor; use vmcore::interrupt::Interrupt; @@ -71,6 +77,28 @@ impl TestHarness { Self::with_device(driver, device, tmpdir) } + /// Creates a harness for the fixed microVM device of a writable host + /// directory, whose requests run as the VMM. Unlike the device of + /// [`Self::new`], it honors an input quiesce. + fn new_microvm(driver: &DefaultDriver) -> Self { + let tmpdir = tempfile::tempdir().unwrap(); + let driver_source = VmTaskDriverSource::new(SingleDriverBackend::new(driver.clone())); + let device = VirtioFsDevice::new_microvm_hostfs( + &driver_source, + MICROVM_ATTACHMENT_ID.to_owned(), + microvm_root_identity(tmpdir.path()).unwrap(), + false, + Vec::new(), + Vec::new(), + Vec::new(), + MicroVmOwnerMode::Vmm, + tmpdir.path(), + None, + ) + .unwrap(); + Self::with_device(driver, device, tmpdir) + } + /// Creates a harness that drives `device`, whose filesystem is backed by /// `tmpdir`. fn with_device( @@ -277,6 +305,19 @@ impl TestHarness { .await } + /// Gives the device time to serve the posted requests, then checks that + /// it completed none of them. + async fn assert_no_request_completes(&self) { + PolledTimer::new(&self.driver) + .sleep(Duration::from_millis(100)) + .await; + assert_eq!( + read_used_idx(&self.mem, USED_ADDR), + self.used_idx, + "the device completed a request that it must leave on the virtqueue" + ); + } + /// Read the FUSE out_header from a response buffer GPA. fn read_out_header(&self, resp_gpa: u64) -> fuse_out_header { let mut buf = [0u8; size_of::()]; @@ -345,6 +386,46 @@ async fn fuse_init_succeeds(driver: DefaultDriver) { harness.fuse_init(0).await; } +/// A microVM queue that starts while host input is quiesced, as a restore +/// starts its queues behind the post-restore input gate, leaves its requests +/// on the virtqueue until input resumes, and then serves them. +#[async_test] +async fn microvm_queue_started_while_input_is_quiesced_serves_once_input_resumes( + driver: DefaultDriver, +) { + let mut harness = TestHarness::new_microvm(&driver); + harness.device.quiesce_input().await.unwrap(); + harness.enable().await; + let (unique, resp_gpa) = harness.post_fuse_init(0); + harness.assert_no_request_completes().await; + + harness.device.resume_input().await.unwrap(); + harness.wait_for_init(unique, resp_gpa).await; +} + +/// A microVM queue that stops while it waits for host input to resume, as a +/// capture under the post-restore input gate stops it, leaves its requests on +/// the virtqueue, and the device saves with no request in flight. The queue +/// that starts again from the saved state, as after a failed save, serves +/// them once input resumes. +#[async_test] +async fn microvm_queue_stopped_while_input_is_quiesced_keeps_its_requests(driver: DefaultDriver) { + let mut harness = TestHarness::new_microvm(&driver); + harness.device.quiesce_input().await.unwrap(); + harness.enable().await; + let (unique, resp_gpa) = harness.post_fuse_init(0); + harness.assert_no_request_completes().await; + + let state = harness.device.stop_queue(0).await.unwrap(); + assert_eq!(state.avail_index, 0, "the stopped queue took a request"); + harness.device.save_device().unwrap(); + + harness.start_queue(Some(state)).await; + harness.assert_no_request_completes().await; + harness.device.resume_input().await.unwrap(); + harness.wait_for_init(unique, resp_gpa).await; +} + /// GETATTR on the root inode returns a directory after INIT. #[async_test] async fn getattr_root_returns_directory(driver: DefaultDriver) { diff --git a/vm/devices/virtio/virtiofs/src/virtio.rs b/vm/devices/virtio/virtiofs/src/virtio.rs index 31960ac9989..e7697271e1d 100644 --- a/vm/devices/virtio/virtiofs/src/virtio.rs +++ b/vm/devices/virtio/virtiofs/src/virtio.rs @@ -103,6 +103,8 @@ struct AdmissionState { pub(crate) struct RequestAdmission { state: Mutex, + /// Notified when admission reopens. + reopened: event_listener::Event, } struct RequestGuard { @@ -116,18 +118,31 @@ impl RequestAdmission { accepting: true, in_flight: 0, }), + reopened: event_listener::Event::new(), } } - fn accept(self: &Arc) -> Option { - let mut state = self.state.lock(); - if !state.accepting { - return None; + /// Admits one request, waiting while admission is closed for + /// [`Self::resume`] to reopen it. Returns `None` if the in-flight count + /// would overflow. + /// + /// Dropping the future cancels the wait and leaves nothing registered. + async fn accept(self: &Arc) -> Option { + loop { + let reopened = { + let mut state = self.state.lock(); + if state.accepting { + state.in_flight = state.in_flight.checked_add(1)?; + return Some(RequestGuard { + admission: Arc::clone(self), + }); + } + // Listen before releasing the lock, so that no resume between + // the check and the wait goes unnoticed. + self.reopened.listen() + }; + reopened.await; } - state.in_flight = state.in_flight.checked_add(1)?; - Some(RequestGuard { - admission: Arc::clone(self), - }) } pub(crate) fn quiesce(&self) { @@ -135,9 +150,10 @@ impl RequestAdmission { state.accepting = false; } + /// Reopens admission and wakes the requests that wait for it. pub(crate) fn resume(&self) { - let mut state = self.state.lock(); - state.accepting = true; + self.state.lock().accepting = true; + self.reopened.notify(usize::MAX); } pub(crate) fn verify_drained(&self) -> anyhow::Result<()> { @@ -413,7 +429,11 @@ impl AsyncRun for VirtioFsWorker { // Admission must precede queue.next(): after save closes the // gate, an unowned descriptor must remain on the virtqueue rather // than being dequeued and falsely completed with an empty reply. - let Some(request_guard) = self.admission.accept() else { + // While the gate is closed, wait for it to reopen rather than + // end: a restore starts the queues behind its post-restore input + // gate, and a worker that ended would not run again when input + // resumes. + let Some(request_guard) = stop.until_stopped(self.admission.accept()).await? else { break; }; let work = stop.until_stopped(state.queue.next()).await?; @@ -569,8 +589,10 @@ impl fuse::Mapper for VirtioMapper<'_> { mod tests { use super::*; use crate::VirtioFs; + use futures::FutureExt; use pal_async::DefaultDriver; use pal_async::async_test; + use std::pin::pin; use test_with_tracing::test; use vmcore::vm_task::SingleDriverBackend; @@ -630,14 +652,31 @@ mod tests { } #[test] - fn closed_admission_rejects_new_work_until_all_requests_drain() { + fn closed_admission_admits_nothing_and_drains_once_requests_complete() { let admission = Arc::new(RequestAdmission::new()); - let request = admission.accept().unwrap(); + let request = admission.accept().now_or_never().flatten().unwrap(); admission.quiesce(); - assert!(admission.accept().is_none()); + assert!(admission.accept().now_or_never().is_none()); assert!(admission.verify_drained().is_err()); drop(request); admission.verify_drained().unwrap(); } + + #[test] + fn resume_admits_a_waiting_request_and_a_dropped_wait_stays_unregistered() { + let admission = Arc::new(RequestAdmission::new()); + admission.quiesce(); + + // A queue that stops while it waits drops its wait. + assert!(admission.accept().now_or_never().is_none()); + assert_eq!(admission.reopened.total_listeners(), 0); + + let mut waiting = pin!(admission.accept()); + assert!(waiting.as_mut().now_or_never().is_none()); + assert_eq!(admission.reopened.total_listeners(), 1); + admission.resume(); + assert!(waiting.now_or_never().flatten().is_some()); + assert_eq!(admission.reopened.total_listeners(), 0); + } }