diff --git a/Guide/src/reference/openvmm/management/cli.md b/Guide/src/reference/openvmm/management/cli.md index c3c06064cc2..3acb7eb7014 100644 --- a/Guide/src/reference/openvmm/management/cli.md +++ b/Guide/src/reference/openvmm/management/cli.md @@ -482,7 +482,11 @@ describes the source definitions. Restore completes the restoration of every device while the VM is stopped, before any vCPU starts, even when the VM stays paused: invalid device-private state, or a saved virtio-fs object that OpenVMM cannot - reopen, fails the restore itself rather than a later resume. + reopen, fails the restore itself rather than a later resume. The resume does + not wait for the devices to start their queues: a queue that fails to start + sets `DEVICE_NEEDS_RESET` on its device instead of failing the resume, as + [Snapshots](../../../user_guide/openvmm/snapshots.md#restoring-a-snapshot) + describes. When the snapshot contains virtio-net, restore also requires `--network-profile portable`; the snapshot's profile and canonical egress diff --git a/Guide/src/reference/openvmm/management/grpc.md b/Guide/src/reference/openvmm/management/grpc.md index 210b2c8e9d0..0ac7d1e285f 100644 --- a/Guide/src/reference/openvmm/management/grpc.md +++ b/Guide/src/reference/openvmm/management/grpc.md @@ -335,7 +335,11 @@ state, or a saved virtio-fs object that OpenVMM cannot reopen, fails the restore rather than `ResumeVM`. The readiness endpoint is a process-local orchestration attachment and is not part of saved state. Each successful restore publishes one event; validation, -attachment, or worker-start failure publishes none. +attachment, or worker-start failure publishes none. Neither `ResumeVM` nor the +event waits for the devices to start their queues: a queue that fails to start +sets `DEVICE_NEEDS_RESET` on its device instead of failing either, as +[Snapshots](../../../user_guide/openvmm/snapshots.md#restoring-a-snapshot) +describes. `VMConfig.MICROVM` is numeric value 1 and accepts exactly 1, 2, 4, or 8 processors. RPC construction is currently blockless; role-bearing sandbox diff --git a/Guide/src/user_guide/openvmm/snapshots.md b/Guide/src/user_guide/openvmm/snapshots.md index 4b2b3587ed5..eb92b1c6dda 100644 --- a/Guide/src/user_guide/openvmm/snapshots.md +++ b/Guide/src/user_guide/openvmm/snapshots.md @@ -173,6 +173,17 @@ releasing gated input. The peer must accept and read while resume is in progress; on Windows, flush completion waits until the named-pipe peer consumes the complete frame. +Restore completes the restoration of every device before any vCPU runs, even +when the VM stays paused: invalid device state, or a host resource that a +device's state refers to and that OpenVMM cannot reacquire, fails the restore +itself. Restoring starts no virtqueue. Each device starts its queues when the +VM first runs, and neither the resume nor the readiness event waits for them. +A queue that fails to start then, such as one whose network backend cannot +create its queues, fails neither: its transport sets `DEVICE_NEEDS_RESET` +instead, with a configuration-change notification if the driver has set +`DRIVER_OK`, and OpenVMM logs the failure. Once the guest resets the device, +setting `DRIVER_OK` starts its queues again. + For a tiered microVM restore, and for a restore that replaces the network policy of a snapshot without a tier, OpenVMM starts device workers with network, control, and filesystem input gated. The guest performs post-restore diff --git a/vm/devices/virtio/virtio/src/device.rs b/vm/devices/virtio/virtio/src/device.rs index d05eb0a2b24..b19cbed2835 100644 --- a/vm/devices/virtio/virtio/src/device.rs +++ b/vm/devices/virtio/virtio/src/device.rs @@ -88,6 +88,13 @@ pub trait VirtioDevice: InspectMut + Send { /// `initial_state` provides restored queue indices for save/restore /// or vhost-user `SET_VRING_BASE`. If `None`, the queue starts fresh /// (indices at 0). + /// + /// An error fails the start. When the guest sets DRIVER_OK, the + /// transport fails DRIVER_OK; when the VM resumes, which does not wait + /// for queues to start, the transport marks the device as needing reset. + /// Acquire the host resources that restored state refers to in + /// [`restore_device`](Self::restore_device) instead, so that a missing + /// one fails the restore. fn start_queue( &mut self, idx: u16, diff --git a/vm/devices/virtio/virtio/src/tests/saved_state.rs b/vm/devices/virtio/virtio/src/tests/saved_state.rs index c238da9cac9..3118de02056 100644 --- a/vm/devices/virtio/virtio/src/tests/saved_state.rs +++ b/vm/devices/virtio/virtio/src/tests/saved_state.rs @@ -3,7 +3,8 @@ //! Tests for device-private saved state and restore: private-state round trips, //! restore validation, the restore of private state into the stopped device, -//! and the start, reset, and removal that follow it. +//! and the start, reset, and removal that follow it. Also tests that ordinary +//! stops and starts, which restore nothing, keep queue progress. use super::*; use std::sync::atomic::AtomicBool; @@ -782,6 +783,17 @@ async fn pci_reset_discards_restored_queue_progress(driver: DefaultDriver) { assert_eq!(probe.initial_used.load(Ordering::Relaxed), usize::MAX); } +/// Waits for the device task of a dropped transport to drop its device. +async fn assert_device_dropped(probe: &PrivateStateProbe) { + for _ in 0..10 { + if probe.dropped.load(Ordering::Relaxed) { + break; + } + yield_now().await; + } + assert!(probe.dropped.load(Ordering::Relaxed)); +} + #[async_test] async fn mmio_removal_before_first_start_releases_device(driver: DefaultDriver) { use vmcore::save_restore::AsyncSaveRestore; @@ -794,13 +806,24 @@ async fn mmio_removal_before_first_start_releases_device(driver: DefaultDriver) assert!(!probe.dropped.load(Ordering::Relaxed)); drop(destination); - for _ in 0..10 { - if probe.dropped.load(Ordering::Relaxed) { - break; - } - yield_now().await; - } - assert!(probe.dropped.load(Ordering::Relaxed)); + assert_device_dropped(&probe).await; + assert_eq!(probe.start_count.load(Ordering::Relaxed), 0); +} + +#[async_test] +async fn pci_removal_before_first_start_releases_device(driver: DefaultDriver) { + use vmcore::save_restore::AsyncSaveRestore; + + let test_mem = VirtioTestMemoryAccess::new(); + let saved = save_active_private_pci(&driver, &test_mem).await; + let (device, probe) = PrivateStateTestDevice::new(0, 1, false); + let mut destination = + VirtioPciTestDevice::new_with_device(&driver, &test_mem, Box::new(device)); + destination.pci_device.restore(saved).await.unwrap(); + assert!(!probe.dropped.load(Ordering::Relaxed)); + + drop(destination); + assert_device_dropped(&probe).await; assert_eq!(probe.start_count.load(Ordering::Relaxed), 0); } @@ -915,6 +938,54 @@ async fn stop_applies_failed_start_pci(driver: DefaultDriver) { verify_stop_applies_failed_start(&mut transport, &probe).await; } +/// Stops and starts an active device twice, as ordinary pauses and resumes +/// do: each start resumes both queues from the progress of the stop before +/// it, and nothing restores or resets the device. +async fn verify_stop_start_keeps_queue_progress( + transport: &mut impl TestTransport, + probe: &PrivateStateProbe, +) { + transport.write_driver_ok(); + yield_and_poll(transport).await; + assert_eq!(probe.running_count.load(Ordering::Relaxed), 2); + probe.value.store(42, Ordering::Relaxed); + transport.stop().await; + assert_eq!(probe.running_count.load(Ordering::Relaxed), 0); + + for starts in [4, 6] { + probe.initial_avail.store(usize::MAX, Ordering::Relaxed); + probe.initial_used.store(usize::MAX, Ordering::Relaxed); + transport.start(); + // The stop waits for the start, then stops the queues again. + transport.stop().await; + assert_eq!(probe.start_count.load(Ordering::Relaxed), starts); + assert_eq!(probe.running_count.load(Ordering::Relaxed), 0); + assert_eq!(probe.initial_avail.load(Ordering::Relaxed), 3); + assert_eq!(probe.initial_used.load(Ordering::Relaxed), 1); + } + + assert_eq!(probe.restore_count.load(Ordering::Relaxed), 0); + assert_eq!(probe.reset_count.load(Ordering::Relaxed), 0); + assert_eq!(probe.value.load(Ordering::Relaxed), 42); + let status = transport.read_status(); + assert_ne!(status & VIRTIO_DRIVER_OK, 0); + assert_eq!(status & VIRTIO_DEVICE_NEEDS_RESET, 0); +} + +#[async_test] +async fn stop_start_keeps_queue_progress_mmio(driver: DefaultDriver) { + let (device, probe) = PrivateStateTestDevice::new(0, 2, false); + let mut transport = MmioTestTransport::new(Box::new(device), &driver, 2); + verify_stop_start_keeps_queue_progress(&mut transport, &probe).await; +} + +#[async_test] +async fn stop_start_keeps_queue_progress_pci(driver: DefaultDriver) { + let (device, probe) = PrivateStateTestDevice::new(0, 2, false); + let mut transport = PciTestTransport::new(Box::new(device), &driver, 2); + verify_stop_start_keeps_queue_progress(&mut transport, &probe).await; +} + #[async_test] async fn mmio_restore_skips_disabled_zero_address_ring(driver: DefaultDriver) { use vmcore::save_restore::AsyncSaveRestore; diff --git a/vm/devices/virtio/virtio_net/src/lib.rs b/vm/devices/virtio/virtio_net/src/lib.rs index 82aeca71adc..0ea1dd8069b 100644 --- a/vm/devices/virtio/virtio_net/src/lib.rs +++ b/vm/devices/virtio/virtio_net/src/lib.rs @@ -466,6 +466,11 @@ impl VirtioDevice for Device { ); if first_pair { + if let Err(err) = self.create_endpoint_queues().await { + self.coordinator.remove(); + self.pairs[pair_idx] = QueuePairState::Empty; + return Err(err); + } self.coordinator.start(); } } @@ -748,11 +753,27 @@ impl Device { num_queues, enabled_rx_offloads: negotiated_rx_offload_support(negotiated_features), restart: true, + endpoint_queues_created: false, input_quiesced: self.lifecycle.input_quiesced, }, ); } + /// Creates the endpoint queues of the coordinator that `start_queue` has + /// just inserted, before the coordinator starts, so that an endpoint that + /// cannot provide them fails the queue start instead of leaving the device + /// without endpoint queues. + async fn create_endpoint_queues(&mut self) -> anyhow::Result<()> { + let (state, coordinator) = self.coordinator.get_mut(); + let coordinator = coordinator.context("virtio-net coordinator state is unavailable")?; + coordinator + .restart_queues(state) + .await + .context("failed to create the network endpoint queues")?; + coordinator.endpoint_queues_created = true; + Ok(()) + } + /// Allocates and inserts a worker. /// /// The coordinator must be stopped. @@ -789,6 +810,9 @@ struct Coordinator { num_queues: u16, enabled_rx_offloads: RxOffloadSupport, restart: bool, + /// Whether `start_queue` has created the endpoint queues that the first + /// restart would otherwise create. + endpoint_queues_created: bool, input_quiesced: bool, } @@ -844,7 +868,9 @@ impl Coordinator { stop.until_stopped(self.stop_workers()).await?; // The queue restart operation is not restartable, so do not // poll on `stop` here. - if let Err(err) = self.restart_queues(state).await { + if !std::mem::take(&mut self.endpoint_queues_created) + && let Err(err) = self.restart_queues(state).await + { tracing::error!( error = &err as &dyn std::error::Error, "failed to restart queues" diff --git a/vm/devices/virtio/virtio_net/src/tests.rs b/vm/devices/virtio/virtio_net/src/tests.rs index d107600c4d6..1fbb436fd39 100644 --- a/vm/devices/virtio/virtio_net/src/tests.rs +++ b/vm/devices/virtio/virtio_net/src/tests.rs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +mod activation; mod egress; mod harness; mod quiesce; diff --git a/vm/devices/virtio/virtio_net/src/tests/activation.rs b/vm/devices/virtio/virtio_net/src/tests/activation.rs new file mode 100644 index 00000000000..df2df4cbf9d --- /dev/null +++ b/vm/devices/virtio/virtio_net/src/tests/activation.rs @@ -0,0 +1,162 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Activation tests: a queue pair whose endpoint cannot create its queues +//! fails its start, instead of leaving the device without endpoint queues. + +use super::MockEndpoint; +use super::QUEUE_SIZE; +use super::TX_AVAIL_ADDR; +use super::TX_DESC_ADDR; +use super::TX_USED_ADDR; +use super::TestHarness; +use crate::Device; +use async_trait::async_trait; +use inspect::InspectMut; +use net_backend::Endpoint; +use net_backend::EndpointAction; +use net_backend::MultiQueueSupport; +use net_backend::QueueConfig; +use net_backend::RssConfig; +use net_backend::TxOffloadSupport; +use net_backend_resources::mac_address::MacAddress; +use pal_async::DefaultDriver; +use pal_async::async_test; +use test_with_tracing::test; +use virtio::QueueResources; +use virtio::VirtioDevice; +use virtio::queue::QueueParams; +use virtio::spec::VirtioDeviceFeatures; +use vmcore::interrupt::Interrupt; +use vmcore::vm_task::SingleDriverBackend; +use vmcore::vm_task::VmTaskDriverSource; + +/// A [`MockEndpoint`] that fails to create queues the first `failures` times. +struct FailingEndpoint { + endpoint: MockEndpoint, + failures: usize, +} + +impl InspectMut for FailingEndpoint { + fn inspect_mut(&mut self, req: inspect::Request<'_>) { + self.endpoint.inspect_mut(req); + } +} + +#[async_trait] +impl Endpoint for FailingEndpoint { + fn endpoint_type(&self) -> &'static str { + self.endpoint.endpoint_type() + } + + async fn get_queues( + &mut self, + config: Vec, + rss: Option<&RssConfig<'_>>, + queues: &mut Vec>, + ) -> anyhow::Result<()> { + if self.failures > 0 { + self.failures -= 1; + anyhow::bail!("injected endpoint queue failure"); + } + self.endpoint.get_queues(config, rss, queues).await + } + + async fn stop(&mut self) { + self.endpoint.stop().await + } + + fn is_ordered(&self) -> bool { + self.endpoint.is_ordered() + } + + fn tx_offload_support(&self) -> TxOffloadSupport { + self.endpoint.tx_offload_support() + } + + fn multiqueue_support(&self) -> MultiQueueSupport { + self.endpoint.multiqueue_support() + } + + fn tx_fast_completions(&self) -> bool { + self.endpoint.tx_fast_completions() + } + + async fn wait_for_endpoint_action(&mut self) -> EndpointAction { + self.endpoint.wait_for_endpoint_action().await + } +} + +impl TestHarness { + /// Replaces the harness device with one whose endpoint fails to create + /// queues the first `failures` times. + fn new_with_endpoint_failures(driver: &DefaultDriver, failures: usize) -> Self { + let mut harness = Self::new(driver); + let (queue_tx, queue_handle_rx) = mesh::channel(); + let (queue_config_tx, queue_config_rx) = mesh::channel(); + let (endpoint_action_tx, endpoint_action_rx) = mesh::channel(); + let endpoint = FailingEndpoint { + endpoint: MockEndpoint { + queue_tx, + queue_config_tx, + endpoint_action_rx, + is_ordered: true, + }, + failures, + }; + let driver_source = VmTaskDriverSource::new(SingleDriverBackend::new(driver.clone())); + let mac = MacAddress::new([0x00, 0x15, 0x5d, 0xaa, 0xbb, 0xcc]); + harness.device = Device::builder() + .build(&driver_source, Box::new(endpoint), mac) + .unwrap(); + harness.queue_handle_rx = queue_handle_rx; + harness.queue_config_rx = queue_config_rx; + harness.endpoint_action_tx = endpoint_action_tx; + harness + } + + fn tx_queue_resources(&self) -> QueueResources { + QueueResources { + params: QueueParams { + size: QUEUE_SIZE, + enable: true, + desc_addr: TX_DESC_ADDR, + avail_addr: TX_AVAIL_ADDR, + used_addr: TX_USED_ADDR, + }, + notify: Interrupt::from_event(self.tx_interrupt_event.clone()), + event: self.tx_event.clone(), + guest_memory: self.mem.clone(), + } + } +} + +#[async_test] +async fn endpoint_queue_failure_fails_the_queue_start(driver: DefaultDriver) { + let mut harness = TestHarness::new_with_endpoint_failures(&driver, 1); + let features = VirtioDeviceFeatures::new(); + + // The TX queue completes the pair, whose endpoint queues fail. + harness.start_rx_queue(&features).await; + let error = harness + .device + .start_queue(1, harness.tx_queue_resources(), &features, None) + .await + .unwrap_err(); + assert!( + format!("{error:#}").contains("injected endpoint queue failure"), + "{error:#}" + ); + + // No queue of the failed pair remains started. + assert!(harness.device.stop_queue(0).await.is_none()); + assert!(harness.device.stop_queue(1).await.is_none()); + + // After the reset that follows a failed start, the pair starts again and + // transmits. + harness.device.reset().await; + let _handle = harness.enable_and_get_handle().await; + harness.post_tx_and_signal(0, 64); + let (used_id, _) = harness.wait_for_used().await; + assert_eq!(used_id, 0); +}