From d8e8f271c4081ad9aecd83549c6a2293c3669429 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Fri, 6 Nov 2026 13:26:49 -0800 Subject: [PATCH 1/2] virtio: keep the progress of queues that a failed start stops When the VM starts an active virtio device, its device task starts each queue from the progress that the last stop saved. If a queue fails to start, the task stops the queues that had started, so that the device processes nothing, but it discarded the progress that they made in the meantime: the next stop reported them as never started, the transport kept their earlier progress, and a later start resumed them from it, repeating the requests that they had completed, such as a transmit or a file system operation. Keep that progress for the next stop to report, as it reports the progress of a queue that it stops itself, so that the next start resumes each queue where it stopped. A queue that the failed start stopped still counts as started until that stop, and a reset or a guest reset still discards it. A queue that reports no progress when the failed start stops it, as virtio-net's does for the queue pair that its failed start drops, no longer counts as started, so it keeps the progress that the transport saved, as before. This prepares the VM's start to stop a device whose start fails, and to start it again later, for microsoft/nvx#1232. Tests: on both transports, a device whose queues each complete a request while they run starts, stops, fails to start its second queue, stops, and starts again; the first queue must resume after the request that it completed before the failed start stopped it. Without the change, the tests fail: the queue resumes from its earlier progress. When the first queue reports no progress as the failed start stops it, it must resume from the progress that the transport saved; counting it as started would lose that progress, which the second pair of tests checks. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- vm/devices/virtio/virtio/src/tests.rs | 1 + vm/devices/virtio/virtio/src/tests/start.rs | 200 ++++++++++++++++++ .../virtio/virtio/src/transport/task.rs | 31 ++- 3 files changed, 227 insertions(+), 5 deletions(-) create mode 100644 vm/devices/virtio/virtio/src/tests/start.rs diff --git a/vm/devices/virtio/virtio/src/tests.rs b/vm/devices/virtio/virtio/src/tests.rs index 1c2f0bbe844..0ca85c3b71d 100644 --- a/vm/devices/virtio/virtio/src/tests.rs +++ b/vm/devices/virtio/virtio/src/tests.rs @@ -9,6 +9,7 @@ mod dma; mod interrupt; mod saved_state; +mod start; use crate::DeviceTraits; use crate::DynVirtioDevice; diff --git a/vm/devices/virtio/virtio/src/tests/start.rs b/vm/devices/virtio/virtio/src/tests/start.rs new file mode 100644 index 00000000000..912a5d09c75 --- /dev/null +++ b/vm/devices/virtio/virtio/src/tests/start.rs @@ -0,0 +1,200 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Tests for restarting the queues of an active device when the VM starts: +//! a queue that fails to start, and the progress of the queues that started +//! before it. + +use super::*; +use test_with_tracing::test; + +/// The state that a [`ProgressDevice`] queue reports when it stops, having +/// completed one request since it started at `state`. +fn advanced(state: QueueState) -> QueueState { + QueueState { + avail_index: state.avail_index.wrapping_add(1), + used_index: state.used_index.wrapping_add(1), + } +} + +/// A two-queue device whose queues each complete one request while they run, +/// and whose queue start fails for the queue that the probe names. +#[derive(InspectMut)] +#[inspect(skip)] +struct ProgressDevice { + probe: ProgressProbe, + running: [Option; 2], +} + +#[derive(Clone, Default)] +struct ProgressProbe { + /// Per queue, the initial state of its last start. + last_start: Arc; 2]>>, + /// The queue whose start fails. + fail_at: Arc>>, + /// The queue whose stop reports no progress. + forget_at: Arc>>, +} + +impl ProgressProbe { + fn last_start(&self, idx: usize) -> Option { + self.last_start.lock()[idx] + } + + fn fail_at(&self, idx: Option) { + *self.fail_at.lock() = idx; + } + + fn forget_at(&self, idx: Option) { + *self.forget_at.lock() = idx; + } +} + +impl ProgressDevice { + fn new() -> (Self, ProgressProbe) { + let probe = ProgressProbe::default(); + ( + Self { + probe: probe.clone(), + running: [None; 2], + }, + probe, + ) + } +} + +impl VirtioDevice for ProgressDevice { + fn traits(&self) -> DeviceTraits { + DeviceTraits { + device_id: VirtioDeviceType::CONSOLE, + device_features: VirtioDeviceFeatures::new(), + max_queues: 2, + device_register_length: 0, + ..Default::default() + } + } + + async fn read_registers_u32(&mut self, _offset: u16) -> u32 { + 0 + } + + async fn write_registers_u32(&mut self, _offset: u16, _val: u32) {} + + async fn start_queue( + &mut self, + idx: u16, + _resources: QueueResources, + _features: &VirtioDeviceFeatures, + initial_state: Option, + ) -> anyhow::Result<()> { + anyhow::ensure!( + *self.probe.fail_at.lock() != Some(idx), + "intentional queue start failure" + ); + self.probe.last_start.lock()[usize::from(idx)] = initial_state; + self.running[usize::from(idx)] = Some(initial_state.unwrap_or_default()); + Ok(()) + } + + async fn stop_queue(&mut self, idx: u16) -> Option { + let state = self.running[usize::from(idx)].take().map(advanced); + // A device may report no progress, as virtio-net does for the queue + // pair that a failed start drops. + if *self.probe.forget_at.lock() == Some(idx) { + None + } else { + state + } + } +} + +/// Starts the device's queues, then resumes it twice, first with the second +/// queue failing to start, as `verify_failed_start_keeps_queue_progress` +/// does, but with the first queue reporting no progress when the failed start +/// stops it: the next start resumes that queue from the progress that the +/// transport saved before the failed start. +async fn verify_failed_start_keeps_saved_progress( + transport: &mut impl TestTransport, + probe: &ProgressProbe, +) { + transport.write_driver_ok(); + yield_and_poll(transport).await; + transport.stop().await; + + probe.fail_at(Some(1)); + probe.forget_at(Some(0)); + transport.start(); + transport.stop().await; + + probe.fail_at(None); + probe.forget_at(None); + transport.start(); + transport.stop().await; + assert_eq!( + probe.last_start(0), + Some(QueueState { + avail_index: 1, + used_index: 1 + }) + ); +} + +#[async_test] +async fn failed_start_keeps_saved_progress_mmio(driver: DefaultDriver) { + let (device, probe) = ProgressDevice::new(); + let mut transport = MmioTestTransport::new(Box::new(device), &driver, 2); + verify_failed_start_keeps_saved_progress(&mut transport, &probe).await; +} + +#[async_test] +async fn failed_start_keeps_saved_progress_pci(driver: DefaultDriver) { + let (device, probe) = ProgressDevice::new(); + let mut transport = PciTestTransport::new(Box::new(device), &driver, 2); + verify_failed_start_keeps_saved_progress(&mut transport, &probe).await; +} + +/// Starts the device's queues, then resumes it twice: first with the second +/// queue failing to start after the first queue has completed a request, +/// then successfully. +async fn verify_failed_start_keeps_queue_progress( + transport: &mut impl TestTransport, + probe: &ProgressProbe, +) { + let progress = |n| QueueState { + avail_index: n, + used_index: n, + }; + + transport.write_driver_ok(); + yield_and_poll(transport).await; + assert_ne!(transport.read_status() & VIRTIO_DRIVER_OK, 0); + transport.stop().await; + + probe.fail_at(Some(1)); + transport.start(); + transport.stop().await; + assert_eq!(probe.last_start(0), Some(progress(1))); + + // The next start resumes the first queue after the request that it + // completed before the failed start stopped it, rather than repeating + // that request. + probe.fail_at(None); + transport.start(); + transport.stop().await; + assert_eq!(probe.last_start(0), Some(progress(2))); + assert_eq!(probe.last_start(1), Some(progress(1))); +} + +#[async_test] +async fn failed_start_keeps_queue_progress_mmio(driver: DefaultDriver) { + let (device, probe) = ProgressDevice::new(); + let mut transport = MmioTestTransport::new(Box::new(device), &driver, 2); + verify_failed_start_keeps_queue_progress(&mut transport, &probe).await; +} + +#[async_test] +async fn failed_start_keeps_queue_progress_pci(driver: DefaultDriver) { + let (device, probe) = ProgressDevice::new(); + let mut transport = PciTestTransport::new(Box::new(device), &driver, 2); + verify_failed_start_keeps_queue_progress(&mut transport, &probe).await; +} diff --git a/vm/devices/virtio/virtio/src/transport/task.rs b/vm/devices/virtio/virtio/src/transport/task.rs index 4a2acf1dfcd..d54e7842caf 100644 --- a/vm/devices/virtio/virtio/src/transport/task.rs +++ b/vm/devices/virtio/virtio/src/transport/task.rs @@ -200,8 +200,12 @@ struct DeviceTask { max_queues: u16, /// The device type reported by restore lifecycle traces. device_type: u16, - /// Per queue, whether it is started. + /// Per queue, whether it started since the last stop, and so has progress + /// for the next stop to report. started: Vec, + /// Per queue, the progress of a started queue that a failed start stopped + /// again, which the next stop reports instead of stopping the queue. + stopped_progress: Vec>, } impl DeviceTask { @@ -262,14 +266,18 @@ impl DeviceTask { async fn stop(&mut self) -> restore::StopResult { let mut states = vec![None; self.max_queues as usize]; for idx in 0..self.max_queues { - states[idx as usize] = self.device.stop_queue(idx).await; + states[idx as usize] = match self.stopped_progress[idx as usize].take() { + Some(progress) => Some(progress), + None => self.device.stop_queue(idx).await, + }; } self.stop_result(states) } /// Restarts the queues of a running device. If one fails to start, stops - /// the queues that did, so that the device processes nothing until the - /// guest resets it. + /// the queues that did, so that the device processes nothing, and keeps + /// their progress for the next stop, so that a later start resumes them + /// where they stopped rather than repeating their work. async fn start(&mut self, params: StartParams) -> anyhow::Result<()> { for (idx, resources, initial_state) in params.queues { if let Err(err) = self @@ -287,7 +295,18 @@ impl DeviceTask { idx, "virtio device start_queue failed on resume" ); - self.stop_all_queues().await; + for queue in 0..self.max_queues { + if self.started[queue as usize] { + match self.device.stop_queue(queue).await { + Some(progress) => { + self.stopped_progress[queue as usize] = Some(progress); + } + // A queue without progress to report keeps the + // progress that the transport saved. + None => self.started[queue as usize] = false, + } + } + } return Err(err); } } @@ -303,6 +322,7 @@ impl DeviceTask { for idx in 0..self.max_queues { self.device.stop_queue(idx).await; self.started[idx as usize] = false; + self.stopped_progress[idx as usize] = None; } } } @@ -317,6 +337,7 @@ pub async fn run_device_task( max_queues: traits.max_queues, device_type: traits.device_id.0, started: vec![false; traits.max_queues as usize], + stopped_progress: vec![None; traits.max_queues as usize], device, }; From db42fde0444ca6e1a05cfd7c1124e995ffe78638 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Fri, 6 Nov 2026 13:27:35 -0800 Subject: [PATCH 2/2] vmcore: confirm device starts before the VM runs A device's start is a notification that nothing awaits: the state units start the devices, then the partition starts the vCPUs, and a resume, or a restore's readiness event, reports success. A virtio device starts its queues on its device task after the start returns, so a queue that failed to start, such as one whose network backend could not create its queues, only marked the device DEVICE_NEEDS_RESET and logged the failure, while the resume and the readiness event succeeded and the guest ran with a device that processes nothing. microsoft/nvx#1232 requires that a partially activated VM never report success. Add ChangeDeviceState::confirm_start, and StateUnit::confirm_start with the ConfirmStart state request, which wait for the start that start began and fail if the device did not start. Both report success by default. StateUnits::start confirms each unit's start before it starts the unit's dependents. A unit that fails to confirm fails the start as a unit whose task does not answer does: its dependents do not start, and the units that did start, the failed unit among them, stop again. Unlike an unanswered unit, it is not left in a terminal state, so a later start can try again. The chipset device units forward the confirmation to their devices, and the remote chipset device workers carry it in a ConfirmStart device request. The virtio transports confirm with the device task's answer to the start. poll_device no longer takes that answer, so the confirmation always does. It fails without marking the device as needing reset: the device task has stopped the queues that started and kept their progress, so the stop that follows leaves the device as it was. A stop that finds a start that nothing confirmed still marks the device. The chipset devices are dependencies of the chipset unit, on which the partition depends, so no vCPU starts until every device has confirmed its start. A device that fails to start fails the VM's resume, with the device's error, before any vCPU runs, and a restore then publishes no readiness event. The snapshot, CLI, and management API guides say so, and VirtioDevice::start_queue says how its failure is reported. Tests: state units start their dependents only after confirming; a failed confirmation stops the started units, blocks the dependents, and leaves the unit able to start again; an unanswered one leaves it in a terminal state. A chipset device unit confirms with exclusive access to its device, and a remote device's confirmation crosses its worker. On both virtio transports, the confirmation waits for the device task, succeeds for an inactive device, and fails for a resume whose queue fails to start, which leaves the device unmarked, keeps the queue progress across the stop, and starts again. That test replaces one that expected the failed resume to mark the device as needing reset. Without the changes, the new tests fail. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Guide/src/reference/openvmm/management/cli.md | 6 +- .../src/reference/openvmm/management/grpc.md | 6 +- Guide/src/user_guide/openvmm/snapshots.md | 12 +- vm/chipset_device_resources/src/lib.rs | 8 + vm/devices/virtio/virtio/src/device.rs | 5 +- vm/devices/virtio/virtio/src/tests.rs | 8 + .../virtio/virtio/src/tests/saved_state.rs | 39 ++-- vm/devices/virtio/virtio/src/tests/start.rs | 92 +++++++- .../virtio/virtio/src/transport/core.rs | 44 ++-- .../virtio/virtio/src/transport/mmio.rs | 4 + vm/devices/virtio/virtio/src/transport/pci.rs | 4 + vm/vmcore/src/device_state.rs | 20 +- vmm_core/state_unit/src/lib.rs | 16 ++ vmm_core/state_unit/src/start.rs | 40 +++- vmm_core/state_unit/src/tests/start.rs | 218 ++++++++++++++++++ .../chipset/backing/arc_mutex/state_unit.rs | 60 +++++ workers/chipset_device_worker/src/protocol.rs | 2 + workers/chipset_device_worker/src/proxy.rs | 30 +++ workers/chipset_device_worker/src/worker.rs | 4 + 19 files changed, 556 insertions(+), 62 deletions(-) diff --git a/Guide/src/reference/openvmm/management/cli.md b/Guide/src/reference/openvmm/management/cli.md index 1dac32f8366..aa0ae46c22d 100644 --- a/Guide/src/reference/openvmm/management/cli.md +++ b/Guide/src/reference/openvmm/management/cli.md @@ -482,9 +482,9 @@ describes the source definitions. Restore completes the restoration of every device while the VM is stopped, before any vCPU starts, even when the VM stays paused: invalid device-private state, or a saved virtio-fs object that OpenVMM cannot - reopen, fails the restore itself rather than a later resume. The resume does - not wait for the devices to start their queues: a queue that fails to start - sets `DEVICE_NEEDS_RESET` on its device instead of failing the resume, as + reopen, fails the restore itself rather than a later resume. The resume + waits for the devices to start their queues before any vCPU runs: a queue + that fails to start fails the resume, as [Snapshots](../../../user_guide/openvmm/snapshots.md#restoring-a-snapshot) describes. diff --git a/Guide/src/reference/openvmm/management/grpc.md b/Guide/src/reference/openvmm/management/grpc.md index 0ac7d1e285f..16e994823e2 100644 --- a/Guide/src/reference/openvmm/management/grpc.md +++ b/Guide/src/reference/openvmm/management/grpc.md @@ -335,9 +335,9 @@ state, or a saved virtio-fs object that OpenVMM cannot reopen, fails the restore rather than `ResumeVM`. The readiness endpoint is a process-local orchestration attachment and is not part of saved state. Each successful restore publishes one event; validation, -attachment, or worker-start failure publishes none. Neither `ResumeVM` nor the -event waits for the devices to start their queues: a queue that fails to start -sets `DEVICE_NEEDS_RESET` on its device instead of failing either, as +attachment, or worker-start failure publishes none. `ResumeVM` waits for the +devices to start their queues before any vCPU runs: a queue that fails to start +fails it, and the restore then publishes no event, as [Snapshots](../../../user_guide/openvmm/snapshots.md#restoring-a-snapshot) describes. diff --git a/Guide/src/user_guide/openvmm/snapshots.md b/Guide/src/user_guide/openvmm/snapshots.md index 731cbd54d3a..17ae8f42f21 100644 --- a/Guide/src/user_guide/openvmm/snapshots.md +++ b/Guide/src/user_guide/openvmm/snapshots.md @@ -179,12 +179,12 @@ Restore completes the restoration of every device before any vCPU runs, even when the VM stays paused: invalid device state, or a host resource that a device's state refers to and that OpenVMM cannot reacquire, fails the restore itself. Restoring starts no virtqueue. Each device starts its queues when the -VM first runs, and neither the resume nor the readiness event waits for them. -A queue that fails to start then, such as one whose network backend cannot -create its queues, fails neither: its transport sets `DEVICE_NEEDS_RESET` -instead, with a configuration-change notification if the driver has set -`DRIVER_OK`, and OpenVMM logs the failure. Once the guest resets the device, -setting `DRIVER_OK` starts its queues again. +VM first runs, and the resume waits for them before any vCPU runs. A queue that +fails to start then, such as one whose network backend cannot create its +queues, fails the resume: OpenVMM stops the devices that started again and +logs the failure, no vCPU runs, and the restore publishes no readiness event. +The devices keep their restored state, so a later resume starts their queues +again. For a tiered microVM restore, and for a restore that replaces the network policy of a snapshot without a tier, OpenVMM starts device workers with diff --git a/vm/chipset_device_resources/src/lib.rs b/vm/chipset_device_resources/src/lib.rs index 8b93378f358..e3dab1affbd 100644 --- a/vm/chipset_device_resources/src/lib.rs +++ b/vm/chipset_device_resources/src/lib.rs @@ -92,6 +92,7 @@ pub trait ConfigureChipsetDevice: Send { #[async_trait] trait DynChipsetDevice: ChipsetDevice + AsyncProtobufSaveRestore + InspectMut { fn start(&mut self); + async fn confirm_start(&mut self) -> anyhow::Result<()>; async fn quiesce_input(&mut self) -> anyhow::Result<()>; async fn resume_input(&mut self) -> anyhow::Result<()>; async fn stop(&mut self); @@ -108,6 +109,9 @@ impl anyhow::Result<()> { + ChangeDeviceState::confirm_start(self).await + } async fn quiesce_input(&mut self) -> anyhow::Result<()> { ChangeDeviceState::quiesce_input(self).await } @@ -152,6 +156,10 @@ impl ChangeDeviceState for ErasedChipsetDevice { self.0.start() } + async fn confirm_start(&mut self) -> anyhow::Result<()> { + self.0.confirm_start().await + } + async fn quiesce_input(&mut self) -> anyhow::Result<()> { self.0.quiesce_input().await } diff --git a/vm/devices/virtio/virtio/src/device.rs b/vm/devices/virtio/virtio/src/device.rs index b19cbed2835..fe684411e54 100644 --- a/vm/devices/virtio/virtio/src/device.rs +++ b/vm/devices/virtio/virtio/src/device.rs @@ -90,8 +90,9 @@ pub trait VirtioDevice: InspectMut + Send { /// (indices at 0). /// /// An error fails the start. When the guest sets DRIVER_OK, the - /// transport fails DRIVER_OK; when the VM resumes, which does not wait - /// for queues to start, the transport marks the device as needing reset. + /// transport fails DRIVER_OK; when the VM starts, the transport fails the + /// confirmation of the start, and the VM stops the device again before + /// any vCPU runs, which keeps the progress of the queues that did start. /// Acquire the host resources that restored state refers to in /// [`restore_device`](Self::restore_device) instead, so that a missing /// one fails the restore. diff --git a/vm/devices/virtio/virtio/src/tests.rs b/vm/devices/virtio/virtio/src/tests.rs index 0ca85c3b71d..a9a9292b9b4 100644 --- a/vm/devices/virtio/virtio/src/tests.rs +++ b/vm/devices/virtio/virtio/src/tests.rs @@ -4335,6 +4335,8 @@ trait TestTransport { fn stop(&mut self) -> impl Future; /// Drive ChangeDeviceState::start(). fn start(&mut self); + /// Drive ChangeDeviceState::confirm_start(). + fn confirm_start(&mut self) -> impl Future>; fn teardown(self) -> impl Future; } @@ -4410,6 +4412,9 @@ impl TestTransport for MmioTestTransport { fn start(&mut self) { ChangeDeviceState::start(&mut self.dev); } + async fn confirm_start(&mut self) -> anyhow::Result<()> { + ChangeDeviceState::confirm_start(&mut self.dev).await + } async fn teardown(self) { self.dev.teardown().await; } @@ -4551,6 +4556,9 @@ impl TestTransport for PciTestTransport { fn start(&mut self) { ChangeDeviceState::start(&mut self.dev); } + async fn confirm_start(&mut self) -> anyhow::Result<()> { + ChangeDeviceState::confirm_start(&mut self.dev).await + } async fn teardown(self) { self.dev.teardown().await; } diff --git a/vm/devices/virtio/virtio/src/tests/saved_state.rs b/vm/devices/virtio/virtio/src/tests/saved_state.rs index 3118de02056..0767a77e0e1 100644 --- a/vm/devices/virtio/virtio/src/tests/saved_state.rs +++ b/vm/devices/virtio/virtio/src/tests/saved_state.rs @@ -877,29 +877,36 @@ async fn fail_resume(transport: &mut impl TestTransport, probe: &PrivateStatePro transport.start(); } -async fn verify_failed_resume_marks_device_needs_reset( +/// A resume whose second queue fails to start fails its confirmation and +/// leaves the device as it was: the queue that started is stopped again, and +/// the device is not marked as needing reset, since the guest does not run +/// until a start succeeds. The VM's next start then starts both queues. +async fn verify_failed_resume_fails_its_confirmation( transport: &mut impl TestTransport, probe: &PrivateStateProbe, ) { fail_resume(transport, probe).await; - yield_and_poll(transport).await; - let status = transport.read_status(); - assert_ne!( - status & VIRTIO_DEVICE_NEEDS_RESET, - 0, - "a failed start must mark the device as needing reset" + let error = transport.confirm_start().await.unwrap_err(); + assert!( + format!("{error:#}").contains("intentional queue start failure"), + "{error:#}" ); + let status = transport.read_status(); + assert_eq!(status & VIRTIO_DEVICE_NEEDS_RESET, 0); assert_ne!(status & VIRTIO_DRIVER_OK, 0); - // The queue that started before the failure is stopped again. assert_eq!(probe.running_count.load(Ordering::Relaxed), 0); - // The guest's reset recovers the device. + // The VM stops the device again, and its next start succeeds. + transport.stop().await; + assert_eq!(transport.read_status() & VIRTIO_DEVICE_NEEDS_RESET, 0); probe.fail_start_at.store(usize::MAX, Ordering::Relaxed); - transport.write_status_zero(); - yield_and_poll(transport).await; - assert_eq!(transport.read_status(), 0); + transport.start(); + transport.confirm_start().await.unwrap(); + assert_eq!(probe.running_count.load(Ordering::Relaxed), 2); } +/// A resume that nothing confirms leaves the failure to the next stop, which +/// marks the device as needing reset. async fn verify_stop_applies_failed_start( transport: &mut impl TestTransport, probe: &PrivateStateProbe, @@ -911,17 +918,17 @@ async fn verify_stop_applies_failed_start( } #[async_test] -async fn failed_resume_marks_device_needs_reset_mmio(driver: DefaultDriver) { +async fn failed_resume_fails_its_confirmation_mmio(driver: DefaultDriver) { let (device, probe) = PrivateStateTestDevice::new(0, 2, false); let mut transport = MmioTestTransport::new(Box::new(device), &driver, 2); - verify_failed_resume_marks_device_needs_reset(&mut transport, &probe).await; + verify_failed_resume_fails_its_confirmation(&mut transport, &probe).await; } #[async_test] -async fn failed_resume_marks_device_needs_reset_pci(driver: DefaultDriver) { +async fn failed_resume_fails_its_confirmation_pci(driver: DefaultDriver) { let (device, probe) = PrivateStateTestDevice::new(0, 2, false); let mut transport = PciTestTransport::new(Box::new(device), &driver, 2); - verify_failed_resume_marks_device_needs_reset(&mut transport, &probe).await; + verify_failed_resume_fails_its_confirmation(&mut transport, &probe).await; } #[async_test] diff --git a/vm/devices/virtio/virtio/src/tests/start.rs b/vm/devices/virtio/virtio/src/tests/start.rs index 912a5d09c75..4dd3cd5adb4 100644 --- a/vm/devices/virtio/virtio/src/tests/start.rs +++ b/vm/devices/virtio/virtio/src/tests/start.rs @@ -2,8 +2,8 @@ // Licensed under the MIT License. //! Tests for restarting the queues of an active device when the VM starts: -//! a queue that fails to start, and the progress of the queues that started -//! before it. +//! the confirmation of the start, a queue that fails to start, and the +//! progress of the queues that started before it. use super::*; use test_with_tracing::test; @@ -198,3 +198,91 @@ async fn failed_start_keeps_queue_progress_pci(driver: DefaultDriver) { let mut transport = PciTestTransport::new(Box::new(device), &driver, 2); verify_failed_start_keeps_queue_progress(&mut transport, &probe).await; } + +/// Confirms the starts of an inactive device, which starts nothing, and of +/// the same device once its queues are active. +async fn verify_start_confirmation(transport: &mut impl TestTransport, probe: &ProgressProbe) { + transport.start(); + transport.confirm_start().await.unwrap(); + transport.stop().await; + assert_eq!(probe.last_start(0), None); + + transport.write_driver_ok(); + yield_and_poll(transport).await; + transport.stop().await; + transport.start(); + transport.confirm_start().await.unwrap(); + assert_eq!( + probe.last_start(0), + Some(QueueState { + avail_index: 1, + used_index: 1 + }) + ); + // A confirmed start leaves no answer for the stop to apply. + transport.stop().await; + assert_eq!(transport.read_status() & VIRTIO_DEVICE_NEEDS_RESET, 0); +} + +#[async_test] +async fn start_confirmation_mmio(driver: DefaultDriver) { + let (device, probe) = ProgressDevice::new(); + let mut transport = MmioTestTransport::new(Box::new(device), &driver, 2); + verify_start_confirmation(&mut transport, &probe).await; +} + +#[async_test] +async fn start_confirmation_pci(driver: DefaultDriver) { + let (device, probe) = ProgressDevice::new(); + let mut transport = PciTestTransport::new(Box::new(device), &driver, 2); + verify_start_confirmation(&mut transport, &probe).await; +} + +/// Fails the confirmation of a start whose second queue fails to start, then +/// stops and starts the device again, as the VM does when it rolls the start +/// back and later resumes: the device is never marked as needing reset, and +/// the first queue resumes after the request that it completed. +async fn verify_failed_confirmation_keeps_queue_progress( + transport: &mut impl TestTransport, + probe: &ProgressProbe, +) { + transport.write_driver_ok(); + yield_and_poll(transport).await; + transport.stop().await; + + probe.fail_at(Some(1)); + transport.start(); + let error = transport.confirm_start().await.unwrap_err(); + assert!( + format!("{error:#}").contains("intentional queue start failure"), + "{error:#}" + ); + transport.stop().await; + + probe.fail_at(None); + transport.start(); + transport.confirm_start().await.unwrap(); + assert_eq!( + probe.last_start(0), + Some(QueueState { + avail_index: 2, + used_index: 2 + }) + ); + transport.stop().await; + assert_eq!(transport.read_status() & VIRTIO_DEVICE_NEEDS_RESET, 0); +} + +#[async_test] +async fn failed_confirmation_keeps_queue_progress_mmio(driver: DefaultDriver) { + let (device, probe) = ProgressDevice::new(); + let mut transport = MmioTestTransport::new(Box::new(device), &driver, 2); + verify_failed_confirmation_keeps_queue_progress(&mut transport, &probe).await; +} + +#[async_test] +async fn failed_confirmation_keeps_queue_progress_pci(driver: DefaultDriver) { + let (device, probe) = ProgressDevice::new(); + let mut transport = PciTestTransport::new(Box::new(device), &driver, 2); + verify_failed_confirmation_keeps_queue_progress(&mut transport, &probe).await; +} diff --git a/vm/devices/virtio/virtio/src/transport/core.rs b/vm/devices/virtio/virtio/src/transport/core.rs index 0cc72019c1e..76f529cf327 100644 --- a/vm/devices/virtio/virtio/src/transport/core.rs +++ b/vm/devices/virtio/virtio/src/transport/core.rs @@ -18,8 +18,8 @@ use crate::queue::QueueParams; use crate::queue::QueueState; use crate::spec::VirtioDeviceFeatures; use crate::spec::VirtioDeviceStatus; +use anyhow::Context as _; use chipset_device::io::deferred::DeferredWrite; -use futures::FutureExt; use futures::task::AtomicWaker; use guestmem::DoorbellRegistration; use guestmem::GuestMemory; @@ -141,7 +141,7 @@ pub(crate) struct VirtioTransportCore { pub pending_status_deferred: Option, #[inspect(with = "Vec::len")] pub stalled_io: Vec, - /// The device task's answer to the last start, until `poll_device` or + /// The device task's answer to the last start, until `confirm_start()` or /// `stop()` takes it. #[inspect(skip)] pending_start: Option>, @@ -489,35 +489,43 @@ impl VirtioTransportCore { self.mark_needs_reset(ops); } } - - if let Some(start) = &mut self.pending_start { - if let Poll::Ready(result) = start.poll_unpin(cx) { - self.pending_start = None; - self.complete_start(ops, result); - } - } } /// `ChangeDeviceState::start()` implementation. /// - /// `start()` cannot wait for the device task, so `poll_device` or the next - /// `stop()` takes its answer. + /// `start()` cannot wait for the device task, so `confirm_start()`, or the + /// next `stop()` when nothing confirms the start, takes its answer. pub fn start(&mut self, ops: &mut dyn TransportOps) { if let Some(params) = self.start_params(ops) { self.pending_start = Some( self.device_sender .call_failable(DeviceCommand::Start, params), ); - if let Some(waker) = self.poll_waker.take() { - waker.wake(); - } } } - /// Applies the device task's answer to a start: a device whose queues - /// did not all start processes nothing, so the transport marks it as - /// needing reset, as virtio 1.2 section 2.1.2 requires of a device in an - /// error state, rather than let the guest wait on it. + /// `ChangeDeviceState::confirm_start()` implementation. + /// + /// Waits for the device task's answer to the last start and fails if a + /// queue did not start. The device task has then stopped the queues that + /// did and kept their progress, so the caller's stop leaves the device as + /// it was before the start. The transport does not mark the device as + /// needing reset, since the caller stops it again rather than let the + /// guest use it. + pub async fn confirm_start(&mut self) -> anyhow::Result<()> { + if let Some(start) = self.pending_start.take() { + start + .await + .context("virtio device failed to start its queues")?; + } + Ok(()) + } + + /// Applies the device task's answer to a start that nothing confirmed: a + /// device whose queues did not all start processes nothing, so the + /// transport marks it as needing reset, as virtio 1.2 section 2.1.2 + /// requires of a device in an error state, rather than let the guest wait + /// on it. fn complete_start( &mut self, ops: &mut dyn TransportOps, diff --git a/vm/devices/virtio/virtio/src/transport/mmio.rs b/vm/devices/virtio/virtio/src/transport/mmio.rs index 314006ec500..f716a31c2d3 100644 --- a/vm/devices/virtio/virtio/src/transport/mmio.rs +++ b/vm/devices/virtio/virtio/src/transport/mmio.rs @@ -418,6 +418,10 @@ impl ChangeDeviceState for VirtioMmioDevice { self.core.start(&mut self.mmio); } + async fn confirm_start(&mut self) -> anyhow::Result<()> { + self.core.confirm_start().await + } + async fn quiesce_input(&mut self) -> anyhow::Result<()> { self.core.quiesce_input().await } diff --git a/vm/devices/virtio/virtio/src/transport/pci.rs b/vm/devices/virtio/virtio/src/transport/pci.rs index f265a0427a7..fcbccd55337 100644 --- a/vm/devices/virtio/virtio/src/transport/pci.rs +++ b/vm/devices/virtio/virtio/src/transport/pci.rs @@ -831,6 +831,10 @@ impl ChangeDeviceState for VirtioPciDevice { self.core.start(&mut self.pci); } + async fn confirm_start(&mut self) -> anyhow::Result<()> { + self.core.confirm_start().await + } + async fn quiesce_input(&mut self) -> anyhow::Result<()> { self.core.quiesce_input().await } diff --git a/vm/vmcore/src/device_state.rs b/vm/vmcore/src/device_state.rs index c1d9578615c..6eaa01c28e3 100644 --- a/vm/vmcore/src/device_state.rs +++ b/vm/vmcore/src/device_state.rs @@ -19,15 +19,29 @@ pub trait ChangeDeviceState { /// this method. /// /// This is a synchronous method instead of an asynchronous one because it - /// is a notification only--callers do not need to wait for the device to - /// finish starting to consider the VM started. Devices should kick off any - /// tasks that need to run any return, without waiting. + /// is a notification only: devices should kick off any tasks that need to + /// run and return, without waiting. A device whose start can still fail + /// once it returns reports that failure from [`Self::confirm_start`], + /// which callers await before they consider the device started. /// // FUTURE: make this asynchronous if any device has a good reason to need // it. This may also require changing state units to wait for device start // to finish, which may require a bunch of other changes. fn start(&mut self); + /// Waits for the start that [`Self::start`] began to finish, and fails if + /// the device did not start. + /// + /// Callers call this after each [`Self::start`], before any other state + /// change, and stop the device again if it fails. A device that fails + /// keeps the state that it had before the start, so that stopping it + /// leaves it as it was and a later start can try again. + /// + /// The default reports success, for devices whose start cannot fail. + fn confirm_start(&mut self) -> impl Send + Future> { + async { Ok(()) } + } + /// Stops accepting new host input before a snapshot vCPU boundary. fn quiesce_input(&mut self) -> impl Send + Future> { async { Ok(()) } diff --git a/vmm_core/state_unit/src/lib.rs b/vmm_core/state_unit/src/lib.rs index 7cef7bd4f84..df8d2ffaff9 100644 --- a/vmm_core/state_unit/src/lib.rs +++ b/vmm_core/state_unit/src/lib.rs @@ -78,6 +78,9 @@ pub enum StateRequest { /// Start asynchronous operations. Start(Rpc<(), ()>), + /// Confirm that the start of a just-started unit succeeded. + ConfirmStart(FailableRpc<(), ()>), + /// Stop asynchronous operations. Stop(Rpc<(), ()>), @@ -113,6 +116,14 @@ pub trait StateUnit: InspectMut { /// Start asynchronous processing. async fn start(&mut self); + /// Waits for the start that [`Self::start`] began to finish, and fails if + /// the unit did not start. [`StateUnits::start`] sends this request right + /// after each start, starts the unit's dependents only once it succeeds, + /// and stops the started units again when it fails. + async fn confirm_start(&mut self) -> anyhow::Result<()> { + Ok(()) + } + /// Stop asynchronous processing. async fn stop(&mut self); @@ -197,6 +208,7 @@ impl StateRequest { } StateRequest::Start(_) + | StateRequest::ConfirmStart(_) | StateRequest::Stop(_) | StateRequest::QuiesceInput(_) | StateRequest::ResumeInput(_) @@ -228,6 +240,10 @@ impl StateRequest { pub async fn apply(self, unit: &mut impl StateUnit) { match self { StateRequest::Start(rpc) => rpc.handle(async |()| unit.start().await).await, + StateRequest::ConfirmStart(rpc) => { + rpc.handle_failable(async |()| unit.confirm_start().await) + .await + } StateRequest::Stop(rpc) => rpc.handle(async |()| unit.stop().await).await, StateRequest::QuiesceInput(rpc) => { rpc.handle_failable(async |()| unit.quiesce_input().await) diff --git a/vmm_core/state_unit/src/start.rs b/vmm_core/state_unit/src/start.rs index 468d0525d48..dedb26ec200 100644 --- a/vmm_core/state_unit/src/start.rs +++ b/vmm_core/state_unit/src/start.rs @@ -1,9 +1,10 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -//! Start with rollback. A unit's start cannot fail, but its task can stop -//! answering: then its dependents do not start, and the units that did start -//! are stopped again, rather than the whole process panicking. +//! Start with rollback. A unit's start cannot fail, but its confirmation can, +//! and its task can stop answering: then its dependents do not start, and the +//! units that did start are stopped again, rather than the whole process +//! panicking. use super::State; use super::StateRequest; @@ -15,14 +16,21 @@ use std::time::Instant; impl StateUnits { /// Starts all stopped units in dependency order. /// - /// Fails without starting any unit if one is in a terminal state. A unit - /// does not start if one of its dependencies - /// did not start. If a unit's task does not answer its start request, the - /// units that did start are stopped again, and the unit is left in a - /// terminal state that fails later starts. + /// Fails without starting any unit if one is in a terminal state. Each + /// unit confirms its start, with [`StateUnit::confirm_start`], before its + /// dependents start. A unit does not start if one of its dependencies did + /// not start or did not confirm its start. If a unit fails to confirm its + /// start, or its task does not answer its start or confirmation request, + /// the units that did start, the failed unit among them, are stopped + /// again. A unit whose task does not answer is left in a terminal state + /// that fails later starts; a unit that failed to confirm its start can + /// start again. + /// + /// [`StateUnit::confirm_start`]: crate::StateUnit::confirm_start pub(super) async fn start_with_rollback(&mut self) -> anyhow::Result<()> { enum StartResult { Started, + NotConfirmed(anyhow::Error), DependencyFailed, Uncertain(anyhow::Error), } @@ -59,6 +67,8 @@ impl StateUnits { let dependencies = unit.dependencies.iter().copied().collect::>(); let ready_set = ready_set.clone(); let start = state_change(name.clone(), unit, StateRequest::Start, Some(())); + let confirm = + state_change(name.clone(), unit, StateRequest::ConfirmStart, Some(())); operations.push(async move { if !ready_set.wait("start", id, &dependencies).await { ready_set.done(id, false); @@ -66,7 +76,14 @@ impl StateUnits { } let result = match start.await { - Ok(Some(())) => StartResult::Started, + Ok(Some(())) => match confirm.await { + Ok(Some(Ok(()))) => StartResult::Started, + Ok(Some(Err(error))) => StartResult::NotConfirmed(error.into()), + Ok(None) => StartResult::Uncertain(anyhow::anyhow!( + "confirmation request was not sent" + )), + Err(error) => StartResult::Uncertain(error.into()), + }, Ok(None) => { StartResult::Uncertain(anyhow::anyhow!("start request was not sent")) } @@ -96,6 +113,11 @@ impl StateUnits { unit.state = State::Running; started_ids.push(id); } + StartResult::NotConfirmed(error) => { + unit.state = State::Running; + started_ids.push(id); + failures.push(format!("{name}: {error:#}")); + } StartResult::DependencyFailed => { unit.state = State::Stopped; failures.push(format!("{name}: a dependency did not start")); diff --git a/vmm_core/state_unit/src/tests/start.rs b/vmm_core/state_unit/src/tests/start.rs index 7cd277b288a..d41d23f3bc9 100644 --- a/vmm_core/state_unit/src/tests/start.rs +++ b/vmm_core/state_unit/src/tests/start.rs @@ -11,9 +11,11 @@ use futures::StreamExt; use inspect::InspectMut; use pal_async::DefaultDriver; use pal_async::async_test; +use parking_lot::Mutex; use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; +use std::time::Duration; use test_with_tracing::test; use vmcore::save_restore::RestoreError; use vmcore::save_restore::SaveError; @@ -105,3 +107,219 @@ async fn an_unanswered_start_rolls_back_and_blocks_dependents(driver: DefaultDri let retry_error = units.start().await.unwrap_err(); assert!(retry_error.to_string().contains("terminal state")); } + +/// The record of the starts, confirmations, and stops of [`ConfirmUnit`]s. +#[derive(Clone, Default)] +struct Events(Arc>>); + +impl Events { + fn push(&self, unit: &str, event: &str) { + self.0.lock().push(format!("{unit} {event}")); + } + + fn take(&self) -> Vec { + std::mem::take(&mut *self.0.lock()) + } +} + +/// A unit whose start confirmation takes `delay`, then fails while `fail` +/// is set, as one whose device failed to start would. +struct ConfirmUnit { + name: &'static str, + driver: DefaultDriver, + delay: Duration, + fail: Arc, + events: Events, +} + +impl StateUnit for ConfirmUnit { + async fn start(&mut self) { + self.events.push(self.name, "started"); + } + + async fn confirm_start(&mut self) -> anyhow::Result<()> { + pal_async::timer::PolledTimer::new(&self.driver) + .sleep(self.delay) + .await; + anyhow::ensure!( + !self.fail.load(Ordering::Relaxed), + "the unit's device did not start" + ); + self.events.push(self.name, "confirmed"); + Ok(()) + } + + async fn stop(&mut self) { + self.events.push(self.name, "stopped"); + } + + async fn reset(&mut self) -> anyhow::Result<()> { + Ok(()) + } + + async fn save(&mut self) -> Result, SaveError> { + Ok(None) + } + + async fn restore(&mut self, _state: SavedStateBlob) -> Result<(), RestoreError> { + Err(RestoreError::SavedStateNotSupported) + } +} + +impl InspectMut for ConfirmUnit { + fn inspect_mut(&mut self, req: inspect::Request<'_>) { + req.respond(); + } +} + +/// Adds a [`ConfirmUnit`] named `name`, which depends on `dependency` if +/// one is given. +fn add_confirm_unit( + units: &StateUnits, + driver: &DefaultDriver, + name: &'static str, + dependency: Option<&crate::UnitHandle>, + delay: Duration, + fail: &Arc, + events: &Events, +) -> crate::SpawnedUnit { + let mut builder = units.add(name); + if let Some(dependency) = dependency { + builder = builder.depends_on(dependency); + } + let unit = ConfirmUnit { + name, + driver: driver.clone(), + delay, + fail: fail.clone(), + events: events.clone(), + }; + builder.spawn(driver, |recv| run_unit(unit, recv)).unwrap() +} + +#[async_test] +async fn a_failed_confirmation_rolls_back_and_blocks_dependents(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let events = Events::default(); + let no_failure = Arc::new(AtomicBool::new(false)); + let fail = Arc::new(AtomicBool::new(true)); + let _independent = add_confirm_unit( + &units, + &driver, + "independent", + None, + Duration::ZERO, + &no_failure, + &events, + ); + let failing = add_confirm_unit( + &units, + &driver, + "failing", + None, + Duration::ZERO, + &fail, + &events, + ); + let _dependent = add_confirm_unit( + &units, + &driver, + "dependent", + Some(failing.handle()), + Duration::ZERO, + &no_failure, + &events, + ); + + let error = format!("{:#}", units.start().await.unwrap_err()); + assert!( + error.contains("failing: the unit's device did not start"), + "{error}" + ); + assert!( + error.contains("dependent: a dependency did not start"), + "{error}" + ); + assert!(!units.is_running()); + // The unit whose confirmation failed did start, so it stops again with + // the others, and its dependent never starts. + let first = events.take(); + for event in ["independent stopped", "failing started", "failing stopped"] { + assert!(first.iter().any(|e| e == event), "{first:?}"); + } + assert!( + !first.iter().any(|e| e.starts_with("dependent")), + "{first:?}" + ); + + // Unlike a unit that does not answer, the unit can start again. + fail.store(false, Ordering::Relaxed); + units.start().await.unwrap(); + assert!(units.is_running()); + let second = events.take(); + assert!( + second.iter().any(|e| e == "dependent confirmed"), + "{second:?}" + ); +} + +#[async_test] +async fn dependents_start_once_their_dependencies_confirm(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let events = Events::default(); + let no_failure = Arc::new(AtomicBool::new(false)); + let dependency = add_confirm_unit( + &units, + &driver, + "dependency", + None, + Duration::from_millis(50), + &no_failure, + &events, + ); + let _dependent = add_confirm_unit( + &units, + &driver, + "dependent", + Some(dependency.handle()), + Duration::ZERO, + &no_failure, + &events, + ); + + units.start().await.unwrap(); + assert_eq!( + events.take(), + [ + "dependency started", + "dependency confirmed", + "dependent started", + "dependent confirmed" + ] + ); +} + +#[async_test] +async fn an_unanswered_confirmation_leaves_a_terminal_unit(driver: DefaultDriver) { + let mut units = StateUnits::new(); + // A unit whose task starts, then drops its confirmation request without + // answering, as a task that has failed would. + let _unanswered = units + .add("unanswered") + .spawn(&driver, |mut recv| async move { + while let Some(request) = recv.next().await { + match request { + StateRequest::Start(rpc) => rpc.complete(()), + StateRequest::ConfirmStart(_) => break, + _ => {} + } + } + }) + .unwrap(); + + let error = units.start().await.unwrap_err(); + assert!(error.to_string().contains("unanswered"), "{error:#}"); + assert!(!units.is_running()); + let retry_error = units.start().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); +} diff --git a/vmm_core/vmotherboard/src/chipset/backing/arc_mutex/state_unit.rs b/vmm_core/vmotherboard/src/chipset/backing/arc_mutex/state_unit.rs index 6a9b382f67a..0a69e7c4773 100644 --- a/vmm_core/vmotherboard/src/chipset/backing/arc_mutex/state_unit.rs +++ b/vmm_core/vmotherboard/src/chipset/backing/arc_mutex/state_unit.rs @@ -96,6 +96,7 @@ impl InspectMut for ArcMutexChipsetDeviceUnit { #[async_trait] trait DynDevice: InspectMut + Send { fn start(&mut self); + async fn confirm_start(&mut self) -> anyhow::Result<()>; async fn stop(&mut self); async fn quiesce_input(&mut self) -> anyhow::Result<()>; async fn resume_input(&mut self) -> anyhow::Result<()>; @@ -112,6 +113,10 @@ impl DynDevice for T { self.start() } + async fn confirm_start(&mut self) -> anyhow::Result<()> { + vmcore::device_state::ChangeDeviceState::confirm_start(self).await + } + async fn stop(&mut self) { self.stop().await } @@ -233,6 +238,10 @@ impl StateUnit for ArcMutexChipsetDeviceUnit { device.poll_device(&mut Context::from_waker(&waker_ref(&self.poll_event))); } + async fn confirm_start(&mut self) -> anyhow::Result<()> { + self.device.clone().close().confirm_start().await + } + async fn stop(&mut self) { self.device.clone().close().stop().await; self.running = false; @@ -415,6 +424,57 @@ mod tests { assert!(!device.lock().restored); } + /// A device whose start fails once `gate` opens. + #[derive(InspectMut)] + #[inspect(skip)] + struct StartFailingDevice { + gate: Option>, + } + + impl ChipsetDevice for StartFailingDevice {} + + impl ChangeDeviceState for StartFailingDevice { + fn start(&mut self) {} + async fn confirm_start(&mut self) -> anyhow::Result<()> { + if let Some(gate) = self.gate.take() { + gate.await?; + } + anyhow::bail!("the device did not start") + } + async fn stop(&mut self) {} + async fn reset(&mut self) {} + } + + impl AsyncSaveRestore for StartFailingDevice { + type SavedState = NoSavedState; + + async fn save(&mut self) -> Result { + Ok(NoSavedState) + } + + async fn restore(&mut self, _: NoSavedState) -> Result<(), RestoreError> { + Ok(()) + } + } + + #[async_test] + async fn start_confirmation_waits_with_exclusive_device_access() { + let (send, recv) = mesh::oneshot(); + let device = Arc::new(CloseableMutex::new(StartFailingDevice { gate: Some(recv) })); + let mut unit = ArcMutexChipsetDeviceUnit::new(device.clone(), false); + unit.start().await; + { + let mut confirm = std::pin::pin!(unit.confirm_start()); + assert!(poll!(&mut confirm).is_pending()); + assert!(device.lock_if_open().is_none()); + send.send(()); + let error = confirm.await.unwrap_err(); + assert!(error.to_string().contains("did not start"), "{error:#}"); + } + assert!(device.lock_if_open().is_some()); + unit.stop().await; + } + struct DropMarker(Arc); impl Drop for DropMarker { diff --git a/workers/chipset_device_worker/src/protocol.rs b/workers/chipset_device_worker/src/protocol.rs index 3a479f9dc5b..31152264f11 100644 --- a/workers/chipset_device_worker/src/protocol.rs +++ b/workers/chipset_device_worker/src/protocol.rs @@ -61,6 +61,8 @@ pub(crate) enum DeviceRequest { PciConfigWrite(WriteRequest, PciConfigByteEnable), /// Start the device Start, + /// Confirm that the start of the device succeeded. + ConfirmStart(mesh::rpc::FailableRpc<(), ()>), /// Stop accepting new host input before snapshot. QuiesceInput(mesh::rpc::FailableRpc<(), ()>), /// Resume host input after a failed snapshot. diff --git a/workers/chipset_device_worker/src/proxy.rs b/workers/chipset_device_worker/src/proxy.rs index f750268646f..dbf7854ed19 100644 --- a/workers/chipset_device_worker/src/proxy.rs +++ b/workers/chipset_device_worker/src/proxy.rs @@ -275,6 +275,13 @@ impl ChangeDeviceState for ChipsetDeviceProxy { self.req_send.send(DeviceRequest::Start) } + async fn confirm_start(&mut self) -> anyhow::Result<()> { + self.req_send + .call_failable(DeviceRequest::ConfirmStart, ()) + .await?; + Ok(()) + } + async fn quiesce_input(&mut self) -> anyhow::Result<()> { self.req_send .call_failable(DeviceRequest::QuiesceInput, ()) @@ -402,6 +409,10 @@ mod tests { impl ChangeDeviceState for TestDevice { fn start(&mut self) {} + async fn confirm_start(&mut self) -> anyhow::Result<()> { + anyhow::ensure!(self.value != 0xbad, "the device did not start"); + Ok(()) + } async fn stop(&mut self) { self.access_memory(); } @@ -604,4 +615,23 @@ mod tests { ); stop_worker(&mut proxy).await; } + + #[async_test] + async fn remote_start_confirmations_are_reported(driver: DefaultDriver) { + let vmtime = VmTimeKeeper::new(&driver, VmTime::from_100ns(0)); + let mut proxy = new_proxy(&vmtime, &driver).await; + proxy.start(); + proxy.confirm_start().await.unwrap(); + proxy.stop().await; + + proxy.restore(state(0xbad)).await.unwrap(); + proxy.start(); + let error = proxy.confirm_start().await.unwrap_err(); + assert!( + format!("{error:#}").contains("the device did not start"), + "{error:#}" + ); + proxy.stop().await; + stop_worker(&mut proxy).await; + } } diff --git a/workers/chipset_device_worker/src/worker.rs b/workers/chipset_device_worker/src/worker.rs index eb62d7ecb1f..3fb8ecde75f 100644 --- a/workers/chipset_device_worker/src/worker.rs +++ b/workers/chipset_device_worker/src/worker.rs @@ -265,6 +265,10 @@ impl Worker for RemoteChipsetDeviceWorker { self.device.start(); self.running = true; } + DeviceRequest::ConfirmStart(rpc) => { + rpc.handle_failable(async |()| self.device.confirm_start().await) + .await + } DeviceRequest::Teardown(rpc) => { assert!( !self.running,