From 1d1ac833d8a09b3f6eadb42385da4231b9337e6b Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Thu, 5 Nov 2026 17:14:14 -0800 Subject: [PATCH 1/3] virtio_net: fail the queue start when the endpoint cannot create queues The coordinator created the endpoint queues only after the device task had started it. When the endpoint failed to create them, the coordinator logged "failed to restart queues" and started workers that wait forever for an endpoint queue: the guest's rings stalled, the transport never marked the device as needing reset, and a resume after a snapshot restore reported success. Create the endpoint queues in `start_queue` when it completes the first queue pair, before the coordinator starts. A failure now fails the queue start like the device's other start failures: on activation the transport refuses DRIVER_OK and resets the device, and on resume it marks the device DEVICE_NEEDS_RESET. The coordinator still re-creates the queues when the endpoint requests a restart, and still re-establishes a latched input gate after creating them. The new test fails the endpoint's first queue creation: the start fails and leaves no queue started, and after a reset the pair starts again and transmits. It fails at the base revision, whose start succeeds. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- vm/devices/virtio/virtio_net/src/lib.rs | 28 ++- vm/devices/virtio/virtio_net/src/tests.rs | 1 + .../virtio/virtio_net/src/tests/activation.rs | 162 ++++++++++++++++++ 3 files changed, 190 insertions(+), 1 deletion(-) create mode 100644 vm/devices/virtio/virtio_net/src/tests/activation.rs diff --git a/vm/devices/virtio/virtio_net/src/lib.rs b/vm/devices/virtio/virtio_net/src/lib.rs index 82aeca71adc..0ea1dd8069b 100644 --- a/vm/devices/virtio/virtio_net/src/lib.rs +++ b/vm/devices/virtio/virtio_net/src/lib.rs @@ -466,6 +466,11 @@ impl VirtioDevice for Device { ); if first_pair { + if let Err(err) = self.create_endpoint_queues().await { + self.coordinator.remove(); + self.pairs[pair_idx] = QueuePairState::Empty; + return Err(err); + } self.coordinator.start(); } } @@ -748,11 +753,27 @@ impl Device { num_queues, enabled_rx_offloads: negotiated_rx_offload_support(negotiated_features), restart: true, + endpoint_queues_created: false, input_quiesced: self.lifecycle.input_quiesced, }, ); } + /// Creates the endpoint queues of the coordinator that `start_queue` has + /// just inserted, before the coordinator starts, so that an endpoint that + /// cannot provide them fails the queue start instead of leaving the device + /// without endpoint queues. + async fn create_endpoint_queues(&mut self) -> anyhow::Result<()> { + let (state, coordinator) = self.coordinator.get_mut(); + let coordinator = coordinator.context("virtio-net coordinator state is unavailable")?; + coordinator + .restart_queues(state) + .await + .context("failed to create the network endpoint queues")?; + coordinator.endpoint_queues_created = true; + Ok(()) + } + /// Allocates and inserts a worker. /// /// The coordinator must be stopped. @@ -789,6 +810,9 @@ struct Coordinator { num_queues: u16, enabled_rx_offloads: RxOffloadSupport, restart: bool, + /// Whether `start_queue` has created the endpoint queues that the first + /// restart would otherwise create. + endpoint_queues_created: bool, input_quiesced: bool, } @@ -844,7 +868,9 @@ impl Coordinator { stop.until_stopped(self.stop_workers()).await?; // The queue restart operation is not restartable, so do not // poll on `stop` here. - if let Err(err) = self.restart_queues(state).await { + if !std::mem::take(&mut self.endpoint_queues_created) + && let Err(err) = self.restart_queues(state).await + { tracing::error!( error = &err as &dyn std::error::Error, "failed to restart queues" diff --git a/vm/devices/virtio/virtio_net/src/tests.rs b/vm/devices/virtio/virtio_net/src/tests.rs index d107600c4d6..1fbb436fd39 100644 --- a/vm/devices/virtio/virtio_net/src/tests.rs +++ b/vm/devices/virtio/virtio_net/src/tests.rs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +mod activation; mod egress; mod harness; mod quiesce; diff --git a/vm/devices/virtio/virtio_net/src/tests/activation.rs b/vm/devices/virtio/virtio_net/src/tests/activation.rs new file mode 100644 index 00000000000..df2df4cbf9d --- /dev/null +++ b/vm/devices/virtio/virtio_net/src/tests/activation.rs @@ -0,0 +1,162 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Activation tests: a queue pair whose endpoint cannot create its queues +//! fails its start, instead of leaving the device without endpoint queues. + +use super::MockEndpoint; +use super::QUEUE_SIZE; +use super::TX_AVAIL_ADDR; +use super::TX_DESC_ADDR; +use super::TX_USED_ADDR; +use super::TestHarness; +use crate::Device; +use async_trait::async_trait; +use inspect::InspectMut; +use net_backend::Endpoint; +use net_backend::EndpointAction; +use net_backend::MultiQueueSupport; +use net_backend::QueueConfig; +use net_backend::RssConfig; +use net_backend::TxOffloadSupport; +use net_backend_resources::mac_address::MacAddress; +use pal_async::DefaultDriver; +use pal_async::async_test; +use test_with_tracing::test; +use virtio::QueueResources; +use virtio::VirtioDevice; +use virtio::queue::QueueParams; +use virtio::spec::VirtioDeviceFeatures; +use vmcore::interrupt::Interrupt; +use vmcore::vm_task::SingleDriverBackend; +use vmcore::vm_task::VmTaskDriverSource; + +/// A [`MockEndpoint`] that fails to create queues the first `failures` times. +struct FailingEndpoint { + endpoint: MockEndpoint, + failures: usize, +} + +impl InspectMut for FailingEndpoint { + fn inspect_mut(&mut self, req: inspect::Request<'_>) { + self.endpoint.inspect_mut(req); + } +} + +#[async_trait] +impl Endpoint for FailingEndpoint { + fn endpoint_type(&self) -> &'static str { + self.endpoint.endpoint_type() + } + + async fn get_queues( + &mut self, + config: Vec, + rss: Option<&RssConfig<'_>>, + queues: &mut Vec>, + ) -> anyhow::Result<()> { + if self.failures > 0 { + self.failures -= 1; + anyhow::bail!("injected endpoint queue failure"); + } + self.endpoint.get_queues(config, rss, queues).await + } + + async fn stop(&mut self) { + self.endpoint.stop().await + } + + fn is_ordered(&self) -> bool { + self.endpoint.is_ordered() + } + + fn tx_offload_support(&self) -> TxOffloadSupport { + self.endpoint.tx_offload_support() + } + + fn multiqueue_support(&self) -> MultiQueueSupport { + self.endpoint.multiqueue_support() + } + + fn tx_fast_completions(&self) -> bool { + self.endpoint.tx_fast_completions() + } + + async fn wait_for_endpoint_action(&mut self) -> EndpointAction { + self.endpoint.wait_for_endpoint_action().await + } +} + +impl TestHarness { + /// Replaces the harness device with one whose endpoint fails to create + /// queues the first `failures` times. + fn new_with_endpoint_failures(driver: &DefaultDriver, failures: usize) -> Self { + let mut harness = Self::new(driver); + let (queue_tx, queue_handle_rx) = mesh::channel(); + let (queue_config_tx, queue_config_rx) = mesh::channel(); + let (endpoint_action_tx, endpoint_action_rx) = mesh::channel(); + let endpoint = FailingEndpoint { + endpoint: MockEndpoint { + queue_tx, + queue_config_tx, + endpoint_action_rx, + is_ordered: true, + }, + failures, + }; + let driver_source = VmTaskDriverSource::new(SingleDriverBackend::new(driver.clone())); + let mac = MacAddress::new([0x00, 0x15, 0x5d, 0xaa, 0xbb, 0xcc]); + harness.device = Device::builder() + .build(&driver_source, Box::new(endpoint), mac) + .unwrap(); + harness.queue_handle_rx = queue_handle_rx; + harness.queue_config_rx = queue_config_rx; + harness.endpoint_action_tx = endpoint_action_tx; + harness + } + + fn tx_queue_resources(&self) -> QueueResources { + QueueResources { + params: QueueParams { + size: QUEUE_SIZE, + enable: true, + desc_addr: TX_DESC_ADDR, + avail_addr: TX_AVAIL_ADDR, + used_addr: TX_USED_ADDR, + }, + notify: Interrupt::from_event(self.tx_interrupt_event.clone()), + event: self.tx_event.clone(), + guest_memory: self.mem.clone(), + } + } +} + +#[async_test] +async fn endpoint_queue_failure_fails_the_queue_start(driver: DefaultDriver) { + let mut harness = TestHarness::new_with_endpoint_failures(&driver, 1); + let features = VirtioDeviceFeatures::new(); + + // The TX queue completes the pair, whose endpoint queues fail. + harness.start_rx_queue(&features).await; + let error = harness + .device + .start_queue(1, harness.tx_queue_resources(), &features, None) + .await + .unwrap_err(); + assert!( + format!("{error:#}").contains("injected endpoint queue failure"), + "{error:#}" + ); + + // No queue of the failed pair remains started. + assert!(harness.device.stop_queue(0).await.is_none()); + assert!(harness.device.stop_queue(1).await.is_none()); + + // After the reset that follows a failed start, the pair starts again and + // transmits. + harness.device.reset().await; + let _handle = harness.enable_and_get_handle().await; + harness.post_tx_and_signal(0, 64); + let (used_id, _) = harness.wait_for_used().await; + assert_eq!(used_id, 0); +} From f3fbffb63ac97a16882a60cd4ae20ab80f5cd1db Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Thu, 5 Nov 2026 17:17:33 -0800 Subject: [PATCH 2/3] virtio: test removal before start and stop/start queue progress A restored PCI device that the VM removes before its first start must release its device, as a test already checked for the MMIO transport. And an ordinary pause and resume, which stop and start a transport without restoring it, must resume each queue from the progress that the stop captured, without restoring or resetting the device; no test checked that on either transport. Add a PCI counterpart of the MMIO removal test, which shares the wait for the device task to drop the device, and a stop/start test for both transports: each of two starts resumes both queues from the progress of the stop before it, each stop leaves no queue running, and the device keeps its private state without a restore or a reset. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../virtio/virtio/src/tests/saved_state.rs | 87 +++++++++++++++++-- 1 file changed, 79 insertions(+), 8 deletions(-) diff --git a/vm/devices/virtio/virtio/src/tests/saved_state.rs b/vm/devices/virtio/virtio/src/tests/saved_state.rs index c238da9cac9..3118de02056 100644 --- a/vm/devices/virtio/virtio/src/tests/saved_state.rs +++ b/vm/devices/virtio/virtio/src/tests/saved_state.rs @@ -3,7 +3,8 @@ //! Tests for device-private saved state and restore: private-state round trips, //! restore validation, the restore of private state into the stopped device, -//! and the start, reset, and removal that follow it. +//! and the start, reset, and removal that follow it. Also tests that ordinary +//! stops and starts, which restore nothing, keep queue progress. use super::*; use std::sync::atomic::AtomicBool; @@ -782,6 +783,17 @@ async fn pci_reset_discards_restored_queue_progress(driver: DefaultDriver) { assert_eq!(probe.initial_used.load(Ordering::Relaxed), usize::MAX); } +/// Waits for the device task of a dropped transport to drop its device. +async fn assert_device_dropped(probe: &PrivateStateProbe) { + for _ in 0..10 { + if probe.dropped.load(Ordering::Relaxed) { + break; + } + yield_now().await; + } + assert!(probe.dropped.load(Ordering::Relaxed)); +} + #[async_test] async fn mmio_removal_before_first_start_releases_device(driver: DefaultDriver) { use vmcore::save_restore::AsyncSaveRestore; @@ -794,13 +806,24 @@ async fn mmio_removal_before_first_start_releases_device(driver: DefaultDriver) assert!(!probe.dropped.load(Ordering::Relaxed)); drop(destination); - for _ in 0..10 { - if probe.dropped.load(Ordering::Relaxed) { - break; - } - yield_now().await; - } - assert!(probe.dropped.load(Ordering::Relaxed)); + assert_device_dropped(&probe).await; + assert_eq!(probe.start_count.load(Ordering::Relaxed), 0); +} + +#[async_test] +async fn pci_removal_before_first_start_releases_device(driver: DefaultDriver) { + use vmcore::save_restore::AsyncSaveRestore; + + let test_mem = VirtioTestMemoryAccess::new(); + let saved = save_active_private_pci(&driver, &test_mem).await; + let (device, probe) = PrivateStateTestDevice::new(0, 1, false); + let mut destination = + VirtioPciTestDevice::new_with_device(&driver, &test_mem, Box::new(device)); + destination.pci_device.restore(saved).await.unwrap(); + assert!(!probe.dropped.load(Ordering::Relaxed)); + + drop(destination); + assert_device_dropped(&probe).await; assert_eq!(probe.start_count.load(Ordering::Relaxed), 0); } @@ -915,6 +938,54 @@ async fn stop_applies_failed_start_pci(driver: DefaultDriver) { verify_stop_applies_failed_start(&mut transport, &probe).await; } +/// Stops and starts an active device twice, as ordinary pauses and resumes +/// do: each start resumes both queues from the progress of the stop before +/// it, and nothing restores or resets the device. +async fn verify_stop_start_keeps_queue_progress( + transport: &mut impl TestTransport, + probe: &PrivateStateProbe, +) { + transport.write_driver_ok(); + yield_and_poll(transport).await; + assert_eq!(probe.running_count.load(Ordering::Relaxed), 2); + probe.value.store(42, Ordering::Relaxed); + transport.stop().await; + assert_eq!(probe.running_count.load(Ordering::Relaxed), 0); + + for starts in [4, 6] { + probe.initial_avail.store(usize::MAX, Ordering::Relaxed); + probe.initial_used.store(usize::MAX, Ordering::Relaxed); + transport.start(); + // The stop waits for the start, then stops the queues again. + transport.stop().await; + assert_eq!(probe.start_count.load(Ordering::Relaxed), starts); + assert_eq!(probe.running_count.load(Ordering::Relaxed), 0); + assert_eq!(probe.initial_avail.load(Ordering::Relaxed), 3); + assert_eq!(probe.initial_used.load(Ordering::Relaxed), 1); + } + + assert_eq!(probe.restore_count.load(Ordering::Relaxed), 0); + assert_eq!(probe.reset_count.load(Ordering::Relaxed), 0); + assert_eq!(probe.value.load(Ordering::Relaxed), 42); + let status = transport.read_status(); + assert_ne!(status & VIRTIO_DRIVER_OK, 0); + assert_eq!(status & VIRTIO_DEVICE_NEEDS_RESET, 0); +} + +#[async_test] +async fn stop_start_keeps_queue_progress_mmio(driver: DefaultDriver) { + let (device, probe) = PrivateStateTestDevice::new(0, 2, false); + let mut transport = MmioTestTransport::new(Box::new(device), &driver, 2); + verify_stop_start_keeps_queue_progress(&mut transport, &probe).await; +} + +#[async_test] +async fn stop_start_keeps_queue_progress_pci(driver: DefaultDriver) { + let (device, probe) = PrivateStateTestDevice::new(0, 2, false); + let mut transport = PciTestTransport::new(Box::new(device), &driver, 2); + verify_stop_start_keeps_queue_progress(&mut transport, &probe).await; +} + #[async_test] async fn mmio_restore_skips_disabled_zero_address_ring(driver: DefaultDriver) { use vmcore::save_restore::AsyncSaveRestore; From bdea9ab194297a31895979aaf5f78dfba902e0b5 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Thu, 5 Nov 2026 17:20:16 -0800 Subject: [PATCH 3/3] virtio: document how a queue that fails to start after restore is reported A restore completes the restoration of every device before any vCPU runs, and the Guide says that invalid device state or an unreopenable virtio-fs object fails the restore rather than a later resume. It did not say what a device whose queues fail to start reports: resuming the VM starts the state units without waiting for the devices to start their queues, so the resume and the restore readiness event succeed, and the transport marks the device as needing reset. State that in the snapshot guide's restore section, and refer to it from the restore descriptions of the command line and of the management API. Document on `VirtioDevice::start_queue` how the transports report its failure, and that host resources which restored state needs belong in `restore_device`, where a missing one fails the restore. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Guide/src/reference/openvmm/management/cli.md | 6 +++++- Guide/src/reference/openvmm/management/grpc.md | 6 +++++- Guide/src/user_guide/openvmm/snapshots.md | 11 +++++++++++ vm/devices/virtio/virtio/src/device.rs | 7 +++++++ 4 files changed, 28 insertions(+), 2 deletions(-) diff --git a/Guide/src/reference/openvmm/management/cli.md b/Guide/src/reference/openvmm/management/cli.md index c3c06064cc2..3acb7eb7014 100644 --- a/Guide/src/reference/openvmm/management/cli.md +++ b/Guide/src/reference/openvmm/management/cli.md @@ -482,7 +482,11 @@ describes the source definitions. Restore completes the restoration of every device while the VM is stopped, before any vCPU starts, even when the VM stays paused: invalid device-private state, or a saved virtio-fs object that OpenVMM cannot - reopen, fails the restore itself rather than a later resume. + reopen, fails the restore itself rather than a later resume. The resume does + not wait for the devices to start their queues: a queue that fails to start + sets `DEVICE_NEEDS_RESET` on its device instead of failing the resume, as + [Snapshots](../../../user_guide/openvmm/snapshots.md#restoring-a-snapshot) + describes. When the snapshot contains virtio-net, restore also requires `--network-profile portable`; the snapshot's profile and canonical egress diff --git a/Guide/src/reference/openvmm/management/grpc.md b/Guide/src/reference/openvmm/management/grpc.md index 210b2c8e9d0..0ac7d1e285f 100644 --- a/Guide/src/reference/openvmm/management/grpc.md +++ b/Guide/src/reference/openvmm/management/grpc.md @@ -335,7 +335,11 @@ state, or a saved virtio-fs object that OpenVMM cannot reopen, fails the restore rather than `ResumeVM`. The readiness endpoint is a process-local orchestration attachment and is not part of saved state. Each successful restore publishes one event; validation, -attachment, or worker-start failure publishes none. +attachment, or worker-start failure publishes none. Neither `ResumeVM` nor the +event waits for the devices to start their queues: a queue that fails to start +sets `DEVICE_NEEDS_RESET` on its device instead of failing either, as +[Snapshots](../../../user_guide/openvmm/snapshots.md#restoring-a-snapshot) +describes. `VMConfig.MICROVM` is numeric value 1 and accepts exactly 1, 2, 4, or 8 processors. RPC construction is currently blockless; role-bearing sandbox diff --git a/Guide/src/user_guide/openvmm/snapshots.md b/Guide/src/user_guide/openvmm/snapshots.md index 4b2b3587ed5..eb92b1c6dda 100644 --- a/Guide/src/user_guide/openvmm/snapshots.md +++ b/Guide/src/user_guide/openvmm/snapshots.md @@ -173,6 +173,17 @@ releasing gated input. The peer must accept and read while resume is in progress; on Windows, flush completion waits until the named-pipe peer consumes the complete frame. +Restore completes the restoration of every device before any vCPU runs, even +when the VM stays paused: invalid device state, or a host resource that a +device's state refers to and that OpenVMM cannot reacquire, fails the restore +itself. Restoring starts no virtqueue. Each device starts its queues when the +VM first runs, and neither the resume nor the readiness event waits for them. +A queue that fails to start then, such as one whose network backend cannot +create its queues, fails neither: its transport sets `DEVICE_NEEDS_RESET` +instead, with a configuration-change notification if the driver has set +`DRIVER_OK`, and OpenVMM logs the failure. Once the guest resets the device, +setting `DRIVER_OK` starts its queues again. + For a tiered microVM restore, and for a restore that replaces the network policy of a snapshot without a tier, OpenVMM starts device workers with network, control, and filesystem input gated. The guest performs post-restore diff --git a/vm/devices/virtio/virtio/src/device.rs b/vm/devices/virtio/virtio/src/device.rs index d05eb0a2b24..b19cbed2835 100644 --- a/vm/devices/virtio/virtio/src/device.rs +++ b/vm/devices/virtio/virtio/src/device.rs @@ -88,6 +88,13 @@ pub trait VirtioDevice: InspectMut + Send { /// `initial_state` provides restored queue indices for save/restore /// or vhost-user `SET_VRING_BASE`. If `None`, the queue starts fresh /// (indices at 0). + /// + /// An error fails the start. When the guest sets DRIVER_OK, the + /// transport fails DRIVER_OK; when the VM resumes, which does not wait + /// for queues to start, the transport marks the device as needing reset. + /// Acquire the host resources that restored state refers to in + /// [`restore_device`](Self::restore_device) instead, so that a missing + /// one fails the restore. fn start_queue( &mut self, idx: u16,