diff --git a/Guide/src/reference/openvmm/management/cli.md b/Guide/src/reference/openvmm/management/cli.md index 1dac32f8366..aa0ae46c22d 100644 --- a/Guide/src/reference/openvmm/management/cli.md +++ b/Guide/src/reference/openvmm/management/cli.md @@ -482,9 +482,9 @@ describes the source definitions. Restore completes the restoration of every device while the VM is stopped, before any vCPU starts, even when the VM stays paused: invalid device-private state, or a saved virtio-fs object that OpenVMM cannot - reopen, fails the restore itself rather than a later resume. The resume does - not wait for the devices to start their queues: a queue that fails to start - sets `DEVICE_NEEDS_RESET` on its device instead of failing the resume, as + reopen, fails the restore itself rather than a later resume. The resume + waits for the devices to start their queues before any vCPU runs: a queue + that fails to start fails the resume, as [Snapshots](../../../user_guide/openvmm/snapshots.md#restoring-a-snapshot) describes. diff --git a/Guide/src/reference/openvmm/management/grpc.md b/Guide/src/reference/openvmm/management/grpc.md index 0ac7d1e285f..16e994823e2 100644 --- a/Guide/src/reference/openvmm/management/grpc.md +++ b/Guide/src/reference/openvmm/management/grpc.md @@ -335,9 +335,9 @@ state, or a saved virtio-fs object that OpenVMM cannot reopen, fails the restore rather than `ResumeVM`. The readiness endpoint is a process-local orchestration attachment and is not part of saved state. Each successful restore publishes one event; validation, -attachment, or worker-start failure publishes none. Neither `ResumeVM` nor the -event waits for the devices to start their queues: a queue that fails to start -sets `DEVICE_NEEDS_RESET` on its device instead of failing either, as +attachment, or worker-start failure publishes none. `ResumeVM` waits for the +devices to start their queues before any vCPU runs: a queue that fails to start +fails it, and the restore then publishes no event, as [Snapshots](../../../user_guide/openvmm/snapshots.md#restoring-a-snapshot) describes. diff --git a/Guide/src/user_guide/openvmm/snapshots.md b/Guide/src/user_guide/openvmm/snapshots.md index 731cbd54d3a..17ae8f42f21 100644 --- a/Guide/src/user_guide/openvmm/snapshots.md +++ b/Guide/src/user_guide/openvmm/snapshots.md @@ -179,12 +179,12 @@ Restore completes the restoration of every device before any vCPU runs, even when the VM stays paused: invalid device state, or a host resource that a device's state refers to and that OpenVMM cannot reacquire, fails the restore itself. Restoring starts no virtqueue. Each device starts its queues when the -VM first runs, and neither the resume nor the readiness event waits for them. -A queue that fails to start then, such as one whose network backend cannot -create its queues, fails neither: its transport sets `DEVICE_NEEDS_RESET` -instead, with a configuration-change notification if the driver has set -`DRIVER_OK`, and OpenVMM logs the failure. Once the guest resets the device, -setting `DRIVER_OK` starts its queues again. +VM first runs, and the resume waits for them before any vCPU runs. A queue that +fails to start then, such as one whose network backend cannot create its +queues, fails the resume: OpenVMM stops the devices that started again and +logs the failure, no vCPU runs, and the restore publishes no readiness event. +The devices keep their restored state, so a later resume starts their queues +again. For a tiered microVM restore, and for a restore that replaces the network policy of a snapshot without a tier, OpenVMM starts device workers with diff --git a/vm/chipset_device_resources/src/lib.rs b/vm/chipset_device_resources/src/lib.rs index 8b93378f358..e3dab1affbd 100644 --- a/vm/chipset_device_resources/src/lib.rs +++ b/vm/chipset_device_resources/src/lib.rs @@ -92,6 +92,7 @@ pub trait ConfigureChipsetDevice: Send { #[async_trait] trait DynChipsetDevice: ChipsetDevice + AsyncProtobufSaveRestore + InspectMut { fn start(&mut self); + async fn confirm_start(&mut self) -> anyhow::Result<()>; async fn quiesce_input(&mut self) -> anyhow::Result<()>; async fn resume_input(&mut self) -> anyhow::Result<()>; async fn stop(&mut self); @@ -108,6 +109,9 @@ impl anyhow::Result<()> { + ChangeDeviceState::confirm_start(self).await + } async fn quiesce_input(&mut self) -> anyhow::Result<()> { ChangeDeviceState::quiesce_input(self).await } @@ -152,6 +156,10 @@ impl ChangeDeviceState for ErasedChipsetDevice { self.0.start() } + async fn confirm_start(&mut self) -> anyhow::Result<()> { + self.0.confirm_start().await + } + async fn quiesce_input(&mut self) -> anyhow::Result<()> { self.0.quiesce_input().await } diff --git a/vm/devices/virtio/virtio/src/device.rs b/vm/devices/virtio/virtio/src/device.rs index b19cbed2835..fe684411e54 100644 --- a/vm/devices/virtio/virtio/src/device.rs +++ b/vm/devices/virtio/virtio/src/device.rs @@ -90,8 +90,9 @@ pub trait VirtioDevice: InspectMut + Send { /// (indices at 0). /// /// An error fails the start. When the guest sets DRIVER_OK, the - /// transport fails DRIVER_OK; when the VM resumes, which does not wait - /// for queues to start, the transport marks the device as needing reset. + /// transport fails DRIVER_OK; when the VM starts, the transport fails the + /// confirmation of the start, and the VM stops the device again before + /// any vCPU runs, which keeps the progress of the queues that did start. /// Acquire the host resources that restored state refers to in /// [`restore_device`](Self::restore_device) instead, so that a missing /// one fails the restore. diff --git a/vm/devices/virtio/virtio/src/tests.rs b/vm/devices/virtio/virtio/src/tests.rs index 1c2f0bbe844..a9a9292b9b4 100644 --- a/vm/devices/virtio/virtio/src/tests.rs +++ b/vm/devices/virtio/virtio/src/tests.rs @@ -9,6 +9,7 @@ mod dma; mod interrupt; mod saved_state; +mod start; use crate::DeviceTraits; use crate::DynVirtioDevice; @@ -4334,6 +4335,8 @@ trait TestTransport { fn stop(&mut self) -> impl Future; /// Drive ChangeDeviceState::start(). fn start(&mut self); + /// Drive ChangeDeviceState::confirm_start(). + fn confirm_start(&mut self) -> impl Future>; fn teardown(self) -> impl Future; } @@ -4409,6 +4412,9 @@ impl TestTransport for MmioTestTransport { fn start(&mut self) { ChangeDeviceState::start(&mut self.dev); } + async fn confirm_start(&mut self) -> anyhow::Result<()> { + ChangeDeviceState::confirm_start(&mut self.dev).await + } async fn teardown(self) { self.dev.teardown().await; } @@ -4550,6 +4556,9 @@ impl TestTransport for PciTestTransport { fn start(&mut self) { ChangeDeviceState::start(&mut self.dev); } + async fn confirm_start(&mut self) -> anyhow::Result<()> { + ChangeDeviceState::confirm_start(&mut self.dev).await + } async fn teardown(self) { self.dev.teardown().await; } diff --git a/vm/devices/virtio/virtio/src/tests/saved_state.rs b/vm/devices/virtio/virtio/src/tests/saved_state.rs index 3118de02056..0767a77e0e1 100644 --- a/vm/devices/virtio/virtio/src/tests/saved_state.rs +++ b/vm/devices/virtio/virtio/src/tests/saved_state.rs @@ -877,29 +877,36 @@ async fn fail_resume(transport: &mut impl TestTransport, probe: &PrivateStatePro transport.start(); } -async fn verify_failed_resume_marks_device_needs_reset( +/// A resume whose second queue fails to start fails its confirmation and +/// leaves the device as it was: the queue that started is stopped again, and +/// the device is not marked as needing reset, since the guest does not run +/// until a start succeeds. The VM's next start then starts both queues. +async fn verify_failed_resume_fails_its_confirmation( transport: &mut impl TestTransport, probe: &PrivateStateProbe, ) { fail_resume(transport, probe).await; - yield_and_poll(transport).await; - let status = transport.read_status(); - assert_ne!( - status & VIRTIO_DEVICE_NEEDS_RESET, - 0, - "a failed start must mark the device as needing reset" + let error = transport.confirm_start().await.unwrap_err(); + assert!( + format!("{error:#}").contains("intentional queue start failure"), + "{error:#}" ); + let status = transport.read_status(); + assert_eq!(status & VIRTIO_DEVICE_NEEDS_RESET, 0); assert_ne!(status & VIRTIO_DRIVER_OK, 0); - // The queue that started before the failure is stopped again. assert_eq!(probe.running_count.load(Ordering::Relaxed), 0); - // The guest's reset recovers the device. + // The VM stops the device again, and its next start succeeds. + transport.stop().await; + assert_eq!(transport.read_status() & VIRTIO_DEVICE_NEEDS_RESET, 0); probe.fail_start_at.store(usize::MAX, Ordering::Relaxed); - transport.write_status_zero(); - yield_and_poll(transport).await; - assert_eq!(transport.read_status(), 0); + transport.start(); + transport.confirm_start().await.unwrap(); + assert_eq!(probe.running_count.load(Ordering::Relaxed), 2); } +/// A resume that nothing confirms leaves the failure to the next stop, which +/// marks the device as needing reset. async fn verify_stop_applies_failed_start( transport: &mut impl TestTransport, probe: &PrivateStateProbe, @@ -911,17 +918,17 @@ async fn verify_stop_applies_failed_start( } #[async_test] -async fn failed_resume_marks_device_needs_reset_mmio(driver: DefaultDriver) { +async fn failed_resume_fails_its_confirmation_mmio(driver: DefaultDriver) { let (device, probe) = PrivateStateTestDevice::new(0, 2, false); let mut transport = MmioTestTransport::new(Box::new(device), &driver, 2); - verify_failed_resume_marks_device_needs_reset(&mut transport, &probe).await; + verify_failed_resume_fails_its_confirmation(&mut transport, &probe).await; } #[async_test] -async fn failed_resume_marks_device_needs_reset_pci(driver: DefaultDriver) { +async fn failed_resume_fails_its_confirmation_pci(driver: DefaultDriver) { let (device, probe) = PrivateStateTestDevice::new(0, 2, false); let mut transport = PciTestTransport::new(Box::new(device), &driver, 2); - verify_failed_resume_marks_device_needs_reset(&mut transport, &probe).await; + verify_failed_resume_fails_its_confirmation(&mut transport, &probe).await; } #[async_test] diff --git a/vm/devices/virtio/virtio/src/tests/start.rs b/vm/devices/virtio/virtio/src/tests/start.rs new file mode 100644 index 00000000000..4dd3cd5adb4 --- /dev/null +++ b/vm/devices/virtio/virtio/src/tests/start.rs @@ -0,0 +1,288 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Tests for restarting the queues of an active device when the VM starts: +//! the confirmation of the start, a queue that fails to start, and the +//! progress of the queues that started before it. + +use super::*; +use test_with_tracing::test; + +/// The state that a [`ProgressDevice`] queue reports when it stops, having +/// completed one request since it started at `state`. +fn advanced(state: QueueState) -> QueueState { + QueueState { + avail_index: state.avail_index.wrapping_add(1), + used_index: state.used_index.wrapping_add(1), + } +} + +/// A two-queue device whose queues each complete one request while they run, +/// and whose queue start fails for the queue that the probe names. +#[derive(InspectMut)] +#[inspect(skip)] +struct ProgressDevice { + probe: ProgressProbe, + running: [Option; 2], +} + +#[derive(Clone, Default)] +struct ProgressProbe { + /// Per queue, the initial state of its last start. + last_start: Arc; 2]>>, + /// The queue whose start fails. + fail_at: Arc>>, + /// The queue whose stop reports no progress. + forget_at: Arc>>, +} + +impl ProgressProbe { + fn last_start(&self, idx: usize) -> Option { + self.last_start.lock()[idx] + } + + fn fail_at(&self, idx: Option) { + *self.fail_at.lock() = idx; + } + + fn forget_at(&self, idx: Option) { + *self.forget_at.lock() = idx; + } +} + +impl ProgressDevice { + fn new() -> (Self, ProgressProbe) { + let probe = ProgressProbe::default(); + ( + Self { + probe: probe.clone(), + running: [None; 2], + }, + probe, + ) + } +} + +impl VirtioDevice for ProgressDevice { + fn traits(&self) -> DeviceTraits { + DeviceTraits { + device_id: VirtioDeviceType::CONSOLE, + device_features: VirtioDeviceFeatures::new(), + max_queues: 2, + device_register_length: 0, + ..Default::default() + } + } + + async fn read_registers_u32(&mut self, _offset: u16) -> u32 { + 0 + } + + async fn write_registers_u32(&mut self, _offset: u16, _val: u32) {} + + async fn start_queue( + &mut self, + idx: u16, + _resources: QueueResources, + _features: &VirtioDeviceFeatures, + initial_state: Option, + ) -> anyhow::Result<()> { + anyhow::ensure!( + *self.probe.fail_at.lock() != Some(idx), + "intentional queue start failure" + ); + self.probe.last_start.lock()[usize::from(idx)] = initial_state; + self.running[usize::from(idx)] = Some(initial_state.unwrap_or_default()); + Ok(()) + } + + async fn stop_queue(&mut self, idx: u16) -> Option { + let state = self.running[usize::from(idx)].take().map(advanced); + // A device may report no progress, as virtio-net does for the queue + // pair that a failed start drops. + if *self.probe.forget_at.lock() == Some(idx) { + None + } else { + state + } + } +} + +/// Starts the device's queues, then resumes it twice, first with the second +/// queue failing to start, as `verify_failed_start_keeps_queue_progress` +/// does, but with the first queue reporting no progress when the failed start +/// stops it: the next start resumes that queue from the progress that the +/// transport saved before the failed start. +async fn verify_failed_start_keeps_saved_progress( + transport: &mut impl TestTransport, + probe: &ProgressProbe, +) { + transport.write_driver_ok(); + yield_and_poll(transport).await; + transport.stop().await; + + probe.fail_at(Some(1)); + probe.forget_at(Some(0)); + transport.start(); + transport.stop().await; + + probe.fail_at(None); + probe.forget_at(None); + transport.start(); + transport.stop().await; + assert_eq!( + probe.last_start(0), + Some(QueueState { + avail_index: 1, + used_index: 1 + }) + ); +} + +#[async_test] +async fn failed_start_keeps_saved_progress_mmio(driver: DefaultDriver) { + let (device, probe) = ProgressDevice::new(); + let mut transport = MmioTestTransport::new(Box::new(device), &driver, 2); + verify_failed_start_keeps_saved_progress(&mut transport, &probe).await; +} + +#[async_test] +async fn failed_start_keeps_saved_progress_pci(driver: DefaultDriver) { + let (device, probe) = ProgressDevice::new(); + let mut transport = PciTestTransport::new(Box::new(device), &driver, 2); + verify_failed_start_keeps_saved_progress(&mut transport, &probe).await; +} + +/// Starts the device's queues, then resumes it twice: first with the second +/// queue failing to start after the first queue has completed a request, +/// then successfully. +async fn verify_failed_start_keeps_queue_progress( + transport: &mut impl TestTransport, + probe: &ProgressProbe, +) { + let progress = |n| QueueState { + avail_index: n, + used_index: n, + }; + + transport.write_driver_ok(); + yield_and_poll(transport).await; + assert_ne!(transport.read_status() & VIRTIO_DRIVER_OK, 0); + transport.stop().await; + + probe.fail_at(Some(1)); + transport.start(); + transport.stop().await; + assert_eq!(probe.last_start(0), Some(progress(1))); + + // The next start resumes the first queue after the request that it + // completed before the failed start stopped it, rather than repeating + // that request. + probe.fail_at(None); + transport.start(); + transport.stop().await; + assert_eq!(probe.last_start(0), Some(progress(2))); + assert_eq!(probe.last_start(1), Some(progress(1))); +} + +#[async_test] +async fn failed_start_keeps_queue_progress_mmio(driver: DefaultDriver) { + let (device, probe) = ProgressDevice::new(); + let mut transport = MmioTestTransport::new(Box::new(device), &driver, 2); + verify_failed_start_keeps_queue_progress(&mut transport, &probe).await; +} + +#[async_test] +async fn failed_start_keeps_queue_progress_pci(driver: DefaultDriver) { + let (device, probe) = ProgressDevice::new(); + let mut transport = PciTestTransport::new(Box::new(device), &driver, 2); + verify_failed_start_keeps_queue_progress(&mut transport, &probe).await; +} + +/// Confirms the starts of an inactive device, which starts nothing, and of +/// the same device once its queues are active. +async fn verify_start_confirmation(transport: &mut impl TestTransport, probe: &ProgressProbe) { + transport.start(); + transport.confirm_start().await.unwrap(); + transport.stop().await; + assert_eq!(probe.last_start(0), None); + + transport.write_driver_ok(); + yield_and_poll(transport).await; + transport.stop().await; + transport.start(); + transport.confirm_start().await.unwrap(); + assert_eq!( + probe.last_start(0), + Some(QueueState { + avail_index: 1, + used_index: 1 + }) + ); + // A confirmed start leaves no answer for the stop to apply. + transport.stop().await; + assert_eq!(transport.read_status() & VIRTIO_DEVICE_NEEDS_RESET, 0); +} + +#[async_test] +async fn start_confirmation_mmio(driver: DefaultDriver) { + let (device, probe) = ProgressDevice::new(); + let mut transport = MmioTestTransport::new(Box::new(device), &driver, 2); + verify_start_confirmation(&mut transport, &probe).await; +} + +#[async_test] +async fn start_confirmation_pci(driver: DefaultDriver) { + let (device, probe) = ProgressDevice::new(); + let mut transport = PciTestTransport::new(Box::new(device), &driver, 2); + verify_start_confirmation(&mut transport, &probe).await; +} + +/// Fails the confirmation of a start whose second queue fails to start, then +/// stops and starts the device again, as the VM does when it rolls the start +/// back and later resumes: the device is never marked as needing reset, and +/// the first queue resumes after the request that it completed. +async fn verify_failed_confirmation_keeps_queue_progress( + transport: &mut impl TestTransport, + probe: &ProgressProbe, +) { + transport.write_driver_ok(); + yield_and_poll(transport).await; + transport.stop().await; + + probe.fail_at(Some(1)); + transport.start(); + let error = transport.confirm_start().await.unwrap_err(); + assert!( + format!("{error:#}").contains("intentional queue start failure"), + "{error:#}" + ); + transport.stop().await; + + probe.fail_at(None); + transport.start(); + transport.confirm_start().await.unwrap(); + assert_eq!( + probe.last_start(0), + Some(QueueState { + avail_index: 2, + used_index: 2 + }) + ); + transport.stop().await; + assert_eq!(transport.read_status() & VIRTIO_DEVICE_NEEDS_RESET, 0); +} + +#[async_test] +async fn failed_confirmation_keeps_queue_progress_mmio(driver: DefaultDriver) { + let (device, probe) = ProgressDevice::new(); + let mut transport = MmioTestTransport::new(Box::new(device), &driver, 2); + verify_failed_confirmation_keeps_queue_progress(&mut transport, &probe).await; +} + +#[async_test] +async fn failed_confirmation_keeps_queue_progress_pci(driver: DefaultDriver) { + let (device, probe) = ProgressDevice::new(); + let mut transport = PciTestTransport::new(Box::new(device), &driver, 2); + verify_failed_confirmation_keeps_queue_progress(&mut transport, &probe).await; +} diff --git a/vm/devices/virtio/virtio/src/transport/core.rs b/vm/devices/virtio/virtio/src/transport/core.rs index 0cc72019c1e..76f529cf327 100644 --- a/vm/devices/virtio/virtio/src/transport/core.rs +++ b/vm/devices/virtio/virtio/src/transport/core.rs @@ -18,8 +18,8 @@ use crate::queue::QueueParams; use crate::queue::QueueState; use crate::spec::VirtioDeviceFeatures; use crate::spec::VirtioDeviceStatus; +use anyhow::Context as _; use chipset_device::io::deferred::DeferredWrite; -use futures::FutureExt; use futures::task::AtomicWaker; use guestmem::DoorbellRegistration; use guestmem::GuestMemory; @@ -141,7 +141,7 @@ pub(crate) struct VirtioTransportCore { pub pending_status_deferred: Option, #[inspect(with = "Vec::len")] pub stalled_io: Vec, - /// The device task's answer to the last start, until `poll_device` or + /// The device task's answer to the last start, until `confirm_start()` or /// `stop()` takes it. #[inspect(skip)] pending_start: Option>, @@ -489,35 +489,43 @@ impl VirtioTransportCore { self.mark_needs_reset(ops); } } - - if let Some(start) = &mut self.pending_start { - if let Poll::Ready(result) = start.poll_unpin(cx) { - self.pending_start = None; - self.complete_start(ops, result); - } - } } /// `ChangeDeviceState::start()` implementation. /// - /// `start()` cannot wait for the device task, so `poll_device` or the next - /// `stop()` takes its answer. + /// `start()` cannot wait for the device task, so `confirm_start()`, or the + /// next `stop()` when nothing confirms the start, takes its answer. pub fn start(&mut self, ops: &mut dyn TransportOps) { if let Some(params) = self.start_params(ops) { self.pending_start = Some( self.device_sender .call_failable(DeviceCommand::Start, params), ); - if let Some(waker) = self.poll_waker.take() { - waker.wake(); - } } } - /// Applies the device task's answer to a start: a device whose queues - /// did not all start processes nothing, so the transport marks it as - /// needing reset, as virtio 1.2 section 2.1.2 requires of a device in an - /// error state, rather than let the guest wait on it. + /// `ChangeDeviceState::confirm_start()` implementation. + /// + /// Waits for the device task's answer to the last start and fails if a + /// queue did not start. The device task has then stopped the queues that + /// did and kept their progress, so the caller's stop leaves the device as + /// it was before the start. The transport does not mark the device as + /// needing reset, since the caller stops it again rather than let the + /// guest use it. + pub async fn confirm_start(&mut self) -> anyhow::Result<()> { + if let Some(start) = self.pending_start.take() { + start + .await + .context("virtio device failed to start its queues")?; + } + Ok(()) + } + + /// Applies the device task's answer to a start that nothing confirmed: a + /// device whose queues did not all start processes nothing, so the + /// transport marks it as needing reset, as virtio 1.2 section 2.1.2 + /// requires of a device in an error state, rather than let the guest wait + /// on it. fn complete_start( &mut self, ops: &mut dyn TransportOps, diff --git a/vm/devices/virtio/virtio/src/transport/mmio.rs b/vm/devices/virtio/virtio/src/transport/mmio.rs index 314006ec500..f716a31c2d3 100644 --- a/vm/devices/virtio/virtio/src/transport/mmio.rs +++ b/vm/devices/virtio/virtio/src/transport/mmio.rs @@ -418,6 +418,10 @@ impl ChangeDeviceState for VirtioMmioDevice { self.core.start(&mut self.mmio); } + async fn confirm_start(&mut self) -> anyhow::Result<()> { + self.core.confirm_start().await + } + async fn quiesce_input(&mut self) -> anyhow::Result<()> { self.core.quiesce_input().await } diff --git a/vm/devices/virtio/virtio/src/transport/pci.rs b/vm/devices/virtio/virtio/src/transport/pci.rs index f265a0427a7..fcbccd55337 100644 --- a/vm/devices/virtio/virtio/src/transport/pci.rs +++ b/vm/devices/virtio/virtio/src/transport/pci.rs @@ -831,6 +831,10 @@ impl ChangeDeviceState for VirtioPciDevice { self.core.start(&mut self.pci); } + async fn confirm_start(&mut self) -> anyhow::Result<()> { + self.core.confirm_start().await + } + async fn quiesce_input(&mut self) -> anyhow::Result<()> { self.core.quiesce_input().await } diff --git a/vm/devices/virtio/virtio/src/transport/task.rs b/vm/devices/virtio/virtio/src/transport/task.rs index 4a2acf1dfcd..d54e7842caf 100644 --- a/vm/devices/virtio/virtio/src/transport/task.rs +++ b/vm/devices/virtio/virtio/src/transport/task.rs @@ -200,8 +200,12 @@ struct DeviceTask { max_queues: u16, /// The device type reported by restore lifecycle traces. device_type: u16, - /// Per queue, whether it is started. + /// Per queue, whether it started since the last stop, and so has progress + /// for the next stop to report. started: Vec, + /// Per queue, the progress of a started queue that a failed start stopped + /// again, which the next stop reports instead of stopping the queue. + stopped_progress: Vec>, } impl DeviceTask { @@ -262,14 +266,18 @@ impl DeviceTask { async fn stop(&mut self) -> restore::StopResult { let mut states = vec![None; self.max_queues as usize]; for idx in 0..self.max_queues { - states[idx as usize] = self.device.stop_queue(idx).await; + states[idx as usize] = match self.stopped_progress[idx as usize].take() { + Some(progress) => Some(progress), + None => self.device.stop_queue(idx).await, + }; } self.stop_result(states) } /// Restarts the queues of a running device. If one fails to start, stops - /// the queues that did, so that the device processes nothing until the - /// guest resets it. + /// the queues that did, so that the device processes nothing, and keeps + /// their progress for the next stop, so that a later start resumes them + /// where they stopped rather than repeating their work. async fn start(&mut self, params: StartParams) -> anyhow::Result<()> { for (idx, resources, initial_state) in params.queues { if let Err(err) = self @@ -287,7 +295,18 @@ impl DeviceTask { idx, "virtio device start_queue failed on resume" ); - self.stop_all_queues().await; + for queue in 0..self.max_queues { + if self.started[queue as usize] { + match self.device.stop_queue(queue).await { + Some(progress) => { + self.stopped_progress[queue as usize] = Some(progress); + } + // A queue without progress to report keeps the + // progress that the transport saved. + None => self.started[queue as usize] = false, + } + } + } return Err(err); } } @@ -303,6 +322,7 @@ impl DeviceTask { for idx in 0..self.max_queues { self.device.stop_queue(idx).await; self.started[idx as usize] = false; + self.stopped_progress[idx as usize] = None; } } } @@ -317,6 +337,7 @@ pub async fn run_device_task( max_queues: traits.max_queues, device_type: traits.device_id.0, started: vec![false; traits.max_queues as usize], + stopped_progress: vec![None; traits.max_queues as usize], device, }; diff --git a/vm/vmcore/src/device_state.rs b/vm/vmcore/src/device_state.rs index c1d9578615c..6eaa01c28e3 100644 --- a/vm/vmcore/src/device_state.rs +++ b/vm/vmcore/src/device_state.rs @@ -19,15 +19,29 @@ pub trait ChangeDeviceState { /// this method. /// /// This is a synchronous method instead of an asynchronous one because it - /// is a notification only--callers do not need to wait for the device to - /// finish starting to consider the VM started. Devices should kick off any - /// tasks that need to run any return, without waiting. + /// is a notification only: devices should kick off any tasks that need to + /// run and return, without waiting. A device whose start can still fail + /// once it returns reports that failure from [`Self::confirm_start`], + /// which callers await before they consider the device started. /// // FUTURE: make this asynchronous if any device has a good reason to need // it. This may also require changing state units to wait for device start // to finish, which may require a bunch of other changes. fn start(&mut self); + /// Waits for the start that [`Self::start`] began to finish, and fails if + /// the device did not start. + /// + /// Callers call this after each [`Self::start`], before any other state + /// change, and stop the device again if it fails. A device that fails + /// keeps the state that it had before the start, so that stopping it + /// leaves it as it was and a later start can try again. + /// + /// The default reports success, for devices whose start cannot fail. + fn confirm_start(&mut self) -> impl Send + Future> { + async { Ok(()) } + } + /// Stops accepting new host input before a snapshot vCPU boundary. fn quiesce_input(&mut self) -> impl Send + Future> { async { Ok(()) } diff --git a/vmm_core/state_unit/src/lib.rs b/vmm_core/state_unit/src/lib.rs index 7cef7bd4f84..df8d2ffaff9 100644 --- a/vmm_core/state_unit/src/lib.rs +++ b/vmm_core/state_unit/src/lib.rs @@ -78,6 +78,9 @@ pub enum StateRequest { /// Start asynchronous operations. Start(Rpc<(), ()>), + /// Confirm that the start of a just-started unit succeeded. + ConfirmStart(FailableRpc<(), ()>), + /// Stop asynchronous operations. Stop(Rpc<(), ()>), @@ -113,6 +116,14 @@ pub trait StateUnit: InspectMut { /// Start asynchronous processing. async fn start(&mut self); + /// Waits for the start that [`Self::start`] began to finish, and fails if + /// the unit did not start. [`StateUnits::start`] sends this request right + /// after each start, starts the unit's dependents only once it succeeds, + /// and stops the started units again when it fails. + async fn confirm_start(&mut self) -> anyhow::Result<()> { + Ok(()) + } + /// Stop asynchronous processing. async fn stop(&mut self); @@ -197,6 +208,7 @@ impl StateRequest { } StateRequest::Start(_) + | StateRequest::ConfirmStart(_) | StateRequest::Stop(_) | StateRequest::QuiesceInput(_) | StateRequest::ResumeInput(_) @@ -228,6 +240,10 @@ impl StateRequest { pub async fn apply(self, unit: &mut impl StateUnit) { match self { StateRequest::Start(rpc) => rpc.handle(async |()| unit.start().await).await, + StateRequest::ConfirmStart(rpc) => { + rpc.handle_failable(async |()| unit.confirm_start().await) + .await + } StateRequest::Stop(rpc) => rpc.handle(async |()| unit.stop().await).await, StateRequest::QuiesceInput(rpc) => { rpc.handle_failable(async |()| unit.quiesce_input().await) diff --git a/vmm_core/state_unit/src/start.rs b/vmm_core/state_unit/src/start.rs index 468d0525d48..dedb26ec200 100644 --- a/vmm_core/state_unit/src/start.rs +++ b/vmm_core/state_unit/src/start.rs @@ -1,9 +1,10 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -//! Start with rollback. A unit's start cannot fail, but its task can stop -//! answering: then its dependents do not start, and the units that did start -//! are stopped again, rather than the whole process panicking. +//! Start with rollback. A unit's start cannot fail, but its confirmation can, +//! and its task can stop answering: then its dependents do not start, and the +//! units that did start are stopped again, rather than the whole process +//! panicking. use super::State; use super::StateRequest; @@ -15,14 +16,21 @@ use std::time::Instant; impl StateUnits { /// Starts all stopped units in dependency order. /// - /// Fails without starting any unit if one is in a terminal state. A unit - /// does not start if one of its dependencies - /// did not start. If a unit's task does not answer its start request, the - /// units that did start are stopped again, and the unit is left in a - /// terminal state that fails later starts. + /// Fails without starting any unit if one is in a terminal state. Each + /// unit confirms its start, with [`StateUnit::confirm_start`], before its + /// dependents start. A unit does not start if one of its dependencies did + /// not start or did not confirm its start. If a unit fails to confirm its + /// start, or its task does not answer its start or confirmation request, + /// the units that did start, the failed unit among them, are stopped + /// again. A unit whose task does not answer is left in a terminal state + /// that fails later starts; a unit that failed to confirm its start can + /// start again. + /// + /// [`StateUnit::confirm_start`]: crate::StateUnit::confirm_start pub(super) async fn start_with_rollback(&mut self) -> anyhow::Result<()> { enum StartResult { Started, + NotConfirmed(anyhow::Error), DependencyFailed, Uncertain(anyhow::Error), } @@ -59,6 +67,8 @@ impl StateUnits { let dependencies = unit.dependencies.iter().copied().collect::>(); let ready_set = ready_set.clone(); let start = state_change(name.clone(), unit, StateRequest::Start, Some(())); + let confirm = + state_change(name.clone(), unit, StateRequest::ConfirmStart, Some(())); operations.push(async move { if !ready_set.wait("start", id, &dependencies).await { ready_set.done(id, false); @@ -66,7 +76,14 @@ impl StateUnits { } let result = match start.await { - Ok(Some(())) => StartResult::Started, + Ok(Some(())) => match confirm.await { + Ok(Some(Ok(()))) => StartResult::Started, + Ok(Some(Err(error))) => StartResult::NotConfirmed(error.into()), + Ok(None) => StartResult::Uncertain(anyhow::anyhow!( + "confirmation request was not sent" + )), + Err(error) => StartResult::Uncertain(error.into()), + }, Ok(None) => { StartResult::Uncertain(anyhow::anyhow!("start request was not sent")) } @@ -96,6 +113,11 @@ impl StateUnits { unit.state = State::Running; started_ids.push(id); } + StartResult::NotConfirmed(error) => { + unit.state = State::Running; + started_ids.push(id); + failures.push(format!("{name}: {error:#}")); + } StartResult::DependencyFailed => { unit.state = State::Stopped; failures.push(format!("{name}: a dependency did not start")); diff --git a/vmm_core/state_unit/src/tests/start.rs b/vmm_core/state_unit/src/tests/start.rs index 7cd277b288a..d41d23f3bc9 100644 --- a/vmm_core/state_unit/src/tests/start.rs +++ b/vmm_core/state_unit/src/tests/start.rs @@ -11,9 +11,11 @@ use futures::StreamExt; use inspect::InspectMut; use pal_async::DefaultDriver; use pal_async::async_test; +use parking_lot::Mutex; use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; +use std::time::Duration; use test_with_tracing::test; use vmcore::save_restore::RestoreError; use vmcore::save_restore::SaveError; @@ -105,3 +107,219 @@ async fn an_unanswered_start_rolls_back_and_blocks_dependents(driver: DefaultDri let retry_error = units.start().await.unwrap_err(); assert!(retry_error.to_string().contains("terminal state")); } + +/// The record of the starts, confirmations, and stops of [`ConfirmUnit`]s. +#[derive(Clone, Default)] +struct Events(Arc>>); + +impl Events { + fn push(&self, unit: &str, event: &str) { + self.0.lock().push(format!("{unit} {event}")); + } + + fn take(&self) -> Vec { + std::mem::take(&mut *self.0.lock()) + } +} + +/// A unit whose start confirmation takes `delay`, then fails while `fail` +/// is set, as one whose device failed to start would. +struct ConfirmUnit { + name: &'static str, + driver: DefaultDriver, + delay: Duration, + fail: Arc, + events: Events, +} + +impl StateUnit for ConfirmUnit { + async fn start(&mut self) { + self.events.push(self.name, "started"); + } + + async fn confirm_start(&mut self) -> anyhow::Result<()> { + pal_async::timer::PolledTimer::new(&self.driver) + .sleep(self.delay) + .await; + anyhow::ensure!( + !self.fail.load(Ordering::Relaxed), + "the unit's device did not start" + ); + self.events.push(self.name, "confirmed"); + Ok(()) + } + + async fn stop(&mut self) { + self.events.push(self.name, "stopped"); + } + + async fn reset(&mut self) -> anyhow::Result<()> { + Ok(()) + } + + async fn save(&mut self) -> Result, SaveError> { + Ok(None) + } + + async fn restore(&mut self, _state: SavedStateBlob) -> Result<(), RestoreError> { + Err(RestoreError::SavedStateNotSupported) + } +} + +impl InspectMut for ConfirmUnit { + fn inspect_mut(&mut self, req: inspect::Request<'_>) { + req.respond(); + } +} + +/// Adds a [`ConfirmUnit`] named `name`, which depends on `dependency` if +/// one is given. +fn add_confirm_unit( + units: &StateUnits, + driver: &DefaultDriver, + name: &'static str, + dependency: Option<&crate::UnitHandle>, + delay: Duration, + fail: &Arc, + events: &Events, +) -> crate::SpawnedUnit { + let mut builder = units.add(name); + if let Some(dependency) = dependency { + builder = builder.depends_on(dependency); + } + let unit = ConfirmUnit { + name, + driver: driver.clone(), + delay, + fail: fail.clone(), + events: events.clone(), + }; + builder.spawn(driver, |recv| run_unit(unit, recv)).unwrap() +} + +#[async_test] +async fn a_failed_confirmation_rolls_back_and_blocks_dependents(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let events = Events::default(); + let no_failure = Arc::new(AtomicBool::new(false)); + let fail = Arc::new(AtomicBool::new(true)); + let _independent = add_confirm_unit( + &units, + &driver, + "independent", + None, + Duration::ZERO, + &no_failure, + &events, + ); + let failing = add_confirm_unit( + &units, + &driver, + "failing", + None, + Duration::ZERO, + &fail, + &events, + ); + let _dependent = add_confirm_unit( + &units, + &driver, + "dependent", + Some(failing.handle()), + Duration::ZERO, + &no_failure, + &events, + ); + + let error = format!("{:#}", units.start().await.unwrap_err()); + assert!( + error.contains("failing: the unit's device did not start"), + "{error}" + ); + assert!( + error.contains("dependent: a dependency did not start"), + "{error}" + ); + assert!(!units.is_running()); + // The unit whose confirmation failed did start, so it stops again with + // the others, and its dependent never starts. + let first = events.take(); + for event in ["independent stopped", "failing started", "failing stopped"] { + assert!(first.iter().any(|e| e == event), "{first:?}"); + } + assert!( + !first.iter().any(|e| e.starts_with("dependent")), + "{first:?}" + ); + + // Unlike a unit that does not answer, the unit can start again. + fail.store(false, Ordering::Relaxed); + units.start().await.unwrap(); + assert!(units.is_running()); + let second = events.take(); + assert!( + second.iter().any(|e| e == "dependent confirmed"), + "{second:?}" + ); +} + +#[async_test] +async fn dependents_start_once_their_dependencies_confirm(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let events = Events::default(); + let no_failure = Arc::new(AtomicBool::new(false)); + let dependency = add_confirm_unit( + &units, + &driver, + "dependency", + None, + Duration::from_millis(50), + &no_failure, + &events, + ); + let _dependent = add_confirm_unit( + &units, + &driver, + "dependent", + Some(dependency.handle()), + Duration::ZERO, + &no_failure, + &events, + ); + + units.start().await.unwrap(); + assert_eq!( + events.take(), + [ + "dependency started", + "dependency confirmed", + "dependent started", + "dependent confirmed" + ] + ); +} + +#[async_test] +async fn an_unanswered_confirmation_leaves_a_terminal_unit(driver: DefaultDriver) { + let mut units = StateUnits::new(); + // A unit whose task starts, then drops its confirmation request without + // answering, as a task that has failed would. + let _unanswered = units + .add("unanswered") + .spawn(&driver, |mut recv| async move { + while let Some(request) = recv.next().await { + match request { + StateRequest::Start(rpc) => rpc.complete(()), + StateRequest::ConfirmStart(_) => break, + _ => {} + } + } + }) + .unwrap(); + + let error = units.start().await.unwrap_err(); + assert!(error.to_string().contains("unanswered"), "{error:#}"); + assert!(!units.is_running()); + let retry_error = units.start().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); +} diff --git a/vmm_core/vmotherboard/src/chipset/backing/arc_mutex/state_unit.rs b/vmm_core/vmotherboard/src/chipset/backing/arc_mutex/state_unit.rs index 6a9b382f67a..0a69e7c4773 100644 --- a/vmm_core/vmotherboard/src/chipset/backing/arc_mutex/state_unit.rs +++ b/vmm_core/vmotherboard/src/chipset/backing/arc_mutex/state_unit.rs @@ -96,6 +96,7 @@ impl InspectMut for ArcMutexChipsetDeviceUnit { #[async_trait] trait DynDevice: InspectMut + Send { fn start(&mut self); + async fn confirm_start(&mut self) -> anyhow::Result<()>; async fn stop(&mut self); async fn quiesce_input(&mut self) -> anyhow::Result<()>; async fn resume_input(&mut self) -> anyhow::Result<()>; @@ -112,6 +113,10 @@ impl DynDevice for T { self.start() } + async fn confirm_start(&mut self) -> anyhow::Result<()> { + vmcore::device_state::ChangeDeviceState::confirm_start(self).await + } + async fn stop(&mut self) { self.stop().await } @@ -233,6 +238,10 @@ impl StateUnit for ArcMutexChipsetDeviceUnit { device.poll_device(&mut Context::from_waker(&waker_ref(&self.poll_event))); } + async fn confirm_start(&mut self) -> anyhow::Result<()> { + self.device.clone().close().confirm_start().await + } + async fn stop(&mut self) { self.device.clone().close().stop().await; self.running = false; @@ -415,6 +424,57 @@ mod tests { assert!(!device.lock().restored); } + /// A device whose start fails once `gate` opens. + #[derive(InspectMut)] + #[inspect(skip)] + struct StartFailingDevice { + gate: Option>, + } + + impl ChipsetDevice for StartFailingDevice {} + + impl ChangeDeviceState for StartFailingDevice { + fn start(&mut self) {} + async fn confirm_start(&mut self) -> anyhow::Result<()> { + if let Some(gate) = self.gate.take() { + gate.await?; + } + anyhow::bail!("the device did not start") + } + async fn stop(&mut self) {} + async fn reset(&mut self) {} + } + + impl AsyncSaveRestore for StartFailingDevice { + type SavedState = NoSavedState; + + async fn save(&mut self) -> Result { + Ok(NoSavedState) + } + + async fn restore(&mut self, _: NoSavedState) -> Result<(), RestoreError> { + Ok(()) + } + } + + #[async_test] + async fn start_confirmation_waits_with_exclusive_device_access() { + let (send, recv) = mesh::oneshot(); + let device = Arc::new(CloseableMutex::new(StartFailingDevice { gate: Some(recv) })); + let mut unit = ArcMutexChipsetDeviceUnit::new(device.clone(), false); + unit.start().await; + { + let mut confirm = std::pin::pin!(unit.confirm_start()); + assert!(poll!(&mut confirm).is_pending()); + assert!(device.lock_if_open().is_none()); + send.send(()); + let error = confirm.await.unwrap_err(); + assert!(error.to_string().contains("did not start"), "{error:#}"); + } + assert!(device.lock_if_open().is_some()); + unit.stop().await; + } + struct DropMarker(Arc); impl Drop for DropMarker { diff --git a/workers/chipset_device_worker/src/protocol.rs b/workers/chipset_device_worker/src/protocol.rs index 3a479f9dc5b..31152264f11 100644 --- a/workers/chipset_device_worker/src/protocol.rs +++ b/workers/chipset_device_worker/src/protocol.rs @@ -61,6 +61,8 @@ pub(crate) enum DeviceRequest { PciConfigWrite(WriteRequest, PciConfigByteEnable), /// Start the device Start, + /// Confirm that the start of the device succeeded. + ConfirmStart(mesh::rpc::FailableRpc<(), ()>), /// Stop accepting new host input before snapshot. QuiesceInput(mesh::rpc::FailableRpc<(), ()>), /// Resume host input after a failed snapshot. diff --git a/workers/chipset_device_worker/src/proxy.rs b/workers/chipset_device_worker/src/proxy.rs index f750268646f..dbf7854ed19 100644 --- a/workers/chipset_device_worker/src/proxy.rs +++ b/workers/chipset_device_worker/src/proxy.rs @@ -275,6 +275,13 @@ impl ChangeDeviceState for ChipsetDeviceProxy { self.req_send.send(DeviceRequest::Start) } + async fn confirm_start(&mut self) -> anyhow::Result<()> { + self.req_send + .call_failable(DeviceRequest::ConfirmStart, ()) + .await?; + Ok(()) + } + async fn quiesce_input(&mut self) -> anyhow::Result<()> { self.req_send .call_failable(DeviceRequest::QuiesceInput, ()) @@ -402,6 +409,10 @@ mod tests { impl ChangeDeviceState for TestDevice { fn start(&mut self) {} + async fn confirm_start(&mut self) -> anyhow::Result<()> { + anyhow::ensure!(self.value != 0xbad, "the device did not start"); + Ok(()) + } async fn stop(&mut self) { self.access_memory(); } @@ -604,4 +615,23 @@ mod tests { ); stop_worker(&mut proxy).await; } + + #[async_test] + async fn remote_start_confirmations_are_reported(driver: DefaultDriver) { + let vmtime = VmTimeKeeper::new(&driver, VmTime::from_100ns(0)); + let mut proxy = new_proxy(&vmtime, &driver).await; + proxy.start(); + proxy.confirm_start().await.unwrap(); + proxy.stop().await; + + proxy.restore(state(0xbad)).await.unwrap(); + proxy.start(); + let error = proxy.confirm_start().await.unwrap_err(); + assert!( + format!("{error:#}").contains("the device did not start"), + "{error:#}" + ); + proxy.stop().await; + stop_worker(&mut proxy).await; + } } diff --git a/workers/chipset_device_worker/src/worker.rs b/workers/chipset_device_worker/src/worker.rs index eb62d7ecb1f..3fb8ecde75f 100644 --- a/workers/chipset_device_worker/src/worker.rs +++ b/workers/chipset_device_worker/src/worker.rs @@ -265,6 +265,10 @@ impl Worker for RemoteChipsetDeviceWorker { self.device.start(); self.running = true; } + DeviceRequest::ConfirmStart(rpc) => { + rpc.handle_failable(async |()| self.device.confirm_start().await) + .await + } DeviceRequest::Teardown(rpc) => { assert!( !self.running,