From 5ee6b9b4d762301e88537710553b6d604ce8a6eb Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Thu, 24 Sep 2026 23:50:18 -0700 Subject: [PATCH 1/2] state_unit: roll back a partial start instead of panicking `StateUnits::start` panics when a unit's task does not answer its start request, as when the task has exited, so a unit failure takes the whole process down instead of failing the start. It also always considers every unit running. Start the units in dependency order without panicking. A unit whose start request is not delivered or answered is left in a terminal state, its dependents do not start, and the units that did start are stopped again in reverse dependency order, so that no partially started VM keeps running. `StateUnits::start` and `start_stopped_units` return the failures; a unit's own start still cannot fail. A unit that was not stopped, such as one that was already running or one whose start is uncertain, keeps the units that it depends on running. A started unit that is kept running, or whose stop fails, is left in the terminal state, the failure is reported, and the units stay running so that a later stop can stop it. While a unit is in a terminal state, a later start, reset, save, or restore fails, and a stop also stops the units in a terminal state and keeps their terminal markers. The callers of `StateUnits::start` and `start_stopped_units` handle the error: - VMBus, PCIe, and relayed vPCI hot-add, and the dynamic vPCI device builder, remove the units they added and return it. VF hot-add removes the network it added and returns it. - After a failed worker restart, a failure to start the VM again is reported to the requester, and the worker tears the VM down. - PCI resource assignment returns it. - OpenHCL reports it to the requester of a resume or a diagnostic save, and tears the VM down, as it does when it cannot resume the VM after a failed restart or servicing request. A failed autostart tears the VM down and fails the worker. - VM resume cannot report a failure to its callers yet, so it treats a unit that fails to start as an unrecoverable unit failure, like the other state changes do. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- openhcl/underhill_core/src/dispatch/mod.rs | 143 +++++--- openhcl/underhill_core/src/lib.rs | 11 +- openhcl/underhill_core/src/worker.rs | 1 + openvmm/openvmm_core/src/worker/dispatch.rs | 39 +- vm/devices/pci/vpci_relay/src/lib.rs | 10 +- vmm_core/src/device_builder.rs | 6 +- vmm_core/state_unit/src/lib.rs | 86 +++-- vmm_core/state_unit/src/start.rs | 196 ++++++++++ vmm_core/state_unit/src/tests/start.rs | 380 ++++++++++++++++++++ 9 files changed, 795 insertions(+), 77 deletions(-) create mode 100644 vmm_core/state_unit/src/start.rs create mode 100644 vmm_core/state_unit/src/tests/start.rs diff --git a/openhcl/underhill_core/src/dispatch/mod.rs b/openhcl/underhill_core/src/dispatch/mod.rs index 20eebc55c06..2c6274a3389 100644 --- a/openhcl/underhill_core/src/dispatch/mod.rs +++ b/openhcl/underhill_core/src/dispatch/mod.rs @@ -79,7 +79,7 @@ use vtl2_settings_worker::handle_vtl2_config_rpc; #[derive(MeshPayload)] pub enum UhVmRpc { Pause(Rpc<(), bool>), - Resume(Rpc<(), bool>), + Resume(FailableRpc<(), bool>), Save(FailableRpc<(), Vec>), ClearHalt(Rpc<(), bool>), // TODO: remove this, and use DebugRequest::Resume PacketCapture(FailableRpc, PacketCaptureParams>), @@ -233,9 +233,12 @@ impl LoadedVm { correlation_id: Option, mut vm_rpc: mesh::Receiver, mut worker_rpc: mesh::Receiver>, - ) -> Option> { + ) -> anyhow::Result>> { if autostart_vps { - self.start(correlation_id).await; + if let Err(error) = self.start(correlation_id).await { + self.teardown().await; + return Err(error.context("failed to autostart VM")); + } } // VTL2 settings services @@ -323,7 +326,17 @@ impl LoadedVm { Event::WorkerRpc(message) => match message { WorkerRpc::Stop => break None, WorkerRpc::Restart(rpc) => { - let state = async { + enum RestartOutcome { + Restart(R, S), + // The restart failed, but the VM is back in its + // previous state, so this worker can keep serving. + Recovered, + // The restart failed and the VM could not be + // resumed, so this worker cannot serve any more. + Unrecoverable, + } + + let outcome = async { let running = self.stop().await; match self .save( @@ -334,26 +347,40 @@ impl LoadedVm { ) .await { - Ok(servicing_state) => Some((rpc, servicing_state)), + Ok(servicing_state) => RestartOutcome::Restart(rpc, servicing_state), Err(err) => { if running { - self.start(None).await; + if let Err(start_error) = self.start(None).await { + rpc.complete(Err(RemoteError::new( + start_error.context(format!( + "worker restart failed ({err:#}) and the VM could not resume" + )), + ))); + return RestartOutcome::Unrecoverable; + } } rpc.complete(Err(RemoteError::new(err))); - None + RestartOutcome::Recovered } } } .instrument(tracing::info_span!("restart", CVM_ALLOWED)) .await; - if let Some((rpc, servicing_state)) = state { - break Some(LoadedVmState { - restart_rpc: rpc, - servicing_state, - vm_rpc, - control_send: self.control_send.lock().take().unwrap(), - }); + match outcome { + RestartOutcome::Restart(rpc, servicing_state) => { + break Some(LoadedVmState { + restart_rpc: rpc, + servicing_state, + vm_rpc, + control_send: self.control_send.lock().take().unwrap(), + }); + } + RestartOutcome::Recovered => {} + // The error has already completed the RPC. Tear + // down through the common path rather than keep a + // worker whose state units can no longer run. + RestartOutcome::Unrecoverable => break None, } } WorkerRpc::Inspect(deferred) => deferred.respond(|resp| { @@ -403,34 +430,39 @@ impl LoadedVm { } Event::UhVmRpc(msg) => match msg { UhVmRpc::Resume(rpc) => { - rpc.handle(async |()| { - if !self.state_units.is_running() { - self.start(None).await; - true - } else { - false - } - }) - .await + if self.state_units.is_running() { + rpc.complete(Ok(false)); + } else if let Err(error) = self.start(None).await { + rpc.complete(Err(RemoteError::new( + error.context("failed to resume VM"), + ))); + // A failed start may leave state units terminal; + // this worker cannot safely continue serving. + break None; + } else { + rpc.complete(Ok(true)); + } } UhVmRpc::Pause(rpc) => rpc.handle(async |()| self.stop().await).await, UhVmRpc::Save(rpc) => { - rpc.handle_failable(async |()| { - let running = self.stop().await; - let r = self - .save( - SaveReason::Diagnostic, - None, - KeepAliveConfig::Disabled, - KeepAliveConfig::Disabled, - ) - .await; - if running { - self.start(None).await; - } - r.map(mesh::payload::encode) - }) - .await + let running = self.stop().await; + let result = self + .save( + SaveReason::Diagnostic, + None, + KeepAliveConfig::Disabled, + KeepAliveConfig::Disabled, + ) + .await; + if running && let Err(error) = self.start(None).await { + rpc.complete(Err(RemoteError::new( + error.context("failed to resume VM after save"), + ))); + // The state units rolled back and cannot run + // again, so tear down rather than keep this worker. + break None; + } + rpc.complete(result.map(mesh::payload::encode).map_err(RemoteError::new)); } UhVmRpc::ClearHalt(rpc) => { rpc.handle(async |()| self.partition_unit.clear_halt().await) @@ -582,6 +614,12 @@ impl LoadedVm { } }; + self.teardown().await; + + Ok(state) + } + + async fn teardown(mut self) { // Drain resealing before tearing down any storage transport. Removing // the unit closes its request stream; in-flight I/O finishes first. if let Some(resealer) = self.hardware_reseal.take() { @@ -599,8 +637,6 @@ impl LoadedVm { if let Some(vmbus) = self.vmbus_server { vmbus.remove().await.shutdown().await; } - - state } /// Handles a servicing request from the host. @@ -651,7 +687,9 @@ impl LoadedVm { .context("failed to notify host of servicing-while-paused failure")?; if running { - self.start(Some(correlation_id)).await; + self.start(Some(correlation_id)) + .await + .context("failed to resume VM after servicing failure")?; } false } @@ -881,8 +919,8 @@ impl LoadedVm { } } - async fn start(&mut self, correlation_id: Option) { - self.state_units.start().await; + async fn start(&mut self, correlation_id: Option) -> anyhow::Result<()> { + self.state_units.start().await?; // Log the boot/blackout time. let reference_time = ReferenceTime::new(self.partition.reference_time()); @@ -908,6 +946,8 @@ impl LoadedVm { "starting VM" ) } + + Ok(()) } /// Returns true if the VM was previously running. @@ -1134,7 +1174,20 @@ impl LoadedVm { ) .await?; - self.state_units.start_stopped_units().await; + if let Err(error) = self.state_units.start_stopped_units().await { + if let Err(remove_error) = self + .network_settings + .as_mut() + .unwrap() + .remove_network(instance_id) + .await + { + return Err(error.context(format!( + "failed to remove network after start failure: {remove_error:#}" + ))); + } + return Err(error); + } self.emuplat_servicing.netvsp_state.push(save_state); Ok(()) diff --git a/openhcl/underhill_core/src/lib.rs b/openhcl/underhill_core/src/lib.rs index 329d1c119f8..69cb928ad5c 100644 --- a/openhcl/underhill_core/src/lib.rs +++ b/openhcl/underhill_core/src/lib.rs @@ -686,11 +686,18 @@ async fn run_control( continue; }; - let was_resumed = workers + let was_resumed = match workers .vm_rpc .call(UhVmRpc::Resume, ()) .await - .context("failed to resumed VM")?; + .context("failed to resume VM")? + { + Ok(was_resumed) => was_resumed, + Err(error) => { + rpc.complete(Err(error)); + continue; + } + }; let was_halted = workers .vm_rpc diff --git a/openhcl/underhill_core/src/worker.rs b/openhcl/underhill_core/src/worker.rs index 702894bd524..694653db74e 100644 --- a/openhcl/underhill_core/src/worker.rs +++ b/openhcl/underhill_core/src/worker.rs @@ -465,6 +465,7 @@ impl Worker for UnderhillVmWorker { )); tracing::info!(CVM_ALLOWED, "terminating worker"); self.get_thread.join().unwrap(); + let state = state?; if let Some(state) = state { let params = UnderhillWorkerParameters { env_cfg: self.env_cfg, diff --git a/openvmm/openvmm_core/src/worker/dispatch.rs b/openvmm/openvmm_core/src/worker/dispatch.rs index ce8e17f1d79..2ae6541952f 100644 --- a/openvmm/openvmm_core/src/worker/dispatch.rs +++ b/openvmm/openvmm_core/src/worker/dispatch.rs @@ -3697,7 +3697,12 @@ impl LoadedVm { if self.running { return false; } - self.state_units.start().await; + // Resume cannot report a failure to its callers, so treat a unit that + // fails to start like any other unrecoverable state unit failure. + self.state_units + .start() + .await + .expect("state units failed to start"); self.running = true; true } @@ -3738,7 +3743,10 @@ impl LoadedVm { let stop_guard = self.inner.partition_unit.temporarily_stop_vps().await; // Start state units so device config space is accessible. - self.state_units.start().await; + self.state_units + .start() + .await + .context("failed to start devices for PCI resource assignment")?; let result = ecam_config_access::assign_pci_resources_for_root_complexes( &self.inner.chipset, @@ -3824,7 +3832,12 @@ impl LoadedVm { } Err(err) => { if stopped { - self.state_units.start().await; + if let Err(start_error) = self.state_units.start().await { + rpc.complete(Err(RemoteError::new(start_error.context( + "worker restart failed and the VM could not resume", + )))); + break; + } } rpc.complete(Err(RemoteError::new(err))); } @@ -3897,7 +3910,15 @@ impl LoadedVm { ) .await?; self.inner.vmbus_devices.push(device); - self.state_units.start_stopped_units().await; + if let Err(error) = self.state_units.start_stopped_units().await { + let device = self + .inner + .vmbus_devices + .pop() + .expect("device was just pushed"); + drop(device.remove().await); + return Err(error); + } anyhow::Ok(()) }) .await @@ -4043,7 +4064,15 @@ impl LoadedVm { // MSI. The guest may begin probing config space // immediately after receiving the interrupt, so // the device must be ready first. - self.state_units.start_stopped_units().await; + if let Err(error) = self.state_units.start_stopped_units().await { + let (_, unit, _) = self + .inner + .pcie_hotplug_devices + .pop() + .expect("device was just pushed"); + unit.remove().await; + return Err(error); + } // Now attach the device and notify the guest. if let Err(e) = rc.lock().hotplug_add_device( diff --git a/vm/devices/pci/vpci_relay/src/lib.rs b/vm/devices/pci/vpci_relay/src/lib.rs index 22e971f1f90..76bae759c18 100644 --- a/vm/devices/pci/vpci_relay/src/lib.rs +++ b/vm/devices/pci/vpci_relay/src/lib.rs @@ -490,7 +490,7 @@ impl VpciRelay { .await? }; - entry.insert(RelayedDevice { + let device = RelayedDevice { bus_instance_id: instance_id, bus_client: vpci_client, vpci_device: vpci_device.clone(), @@ -498,9 +498,13 @@ impl VpciRelay { bus_unit, device_unit, ready_to_remove: false, - }); + }; - state_units.start_stopped_units().await; + if let Err(error) = state_units.start_stopped_units().await { + device.remove().await; + return Err(error); + } + entry.insert(device); Ok(()) } } diff --git a/vmm_core/src/device_builder.rs b/vmm_core/src/device_builder.rs index 089ac1444c9..440d98a9dc4 100644 --- a/vmm_core/src/device_builder.rs +++ b/vmm_core/src/device_builder.rs @@ -144,7 +144,11 @@ pub async fn build_dynamic_vpci_device( }; let pending_offer = pending_offer.context("missing deferred VPCI channel offer")?; - state_units.start_stopped_units().await; + if let Err(error) = state_units.start_stopped_units().await { + vpci_unit.remove().await; + pci_unit.remove().await; + return Err(error); + } if let Err(error) = pending_offer .offer_registered(&vpci_bus, driver_source, vmbus, state_units.is_running()) .await diff --git a/vmm_core/state_unit/src/lib.rs b/vmm_core/state_unit/src/lib.rs index 87a9525e73d..d20ba75d256 100644 --- a/vmm_core/state_unit/src/lib.rs +++ b/vmm_core/state_unit/src/lib.rs @@ -30,6 +30,8 @@ #![forbid(unsafe_code)] +mod start; + use futures::FutureExt; use futures::StreamExt; use futures::future::join_all; @@ -70,9 +72,14 @@ use vmcore::save_restore::SavedStateBlob; #[derive(Debug, MeshPayload)] pub enum StateRequest { /// Start asynchronous operations. + /// + /// Once this request is delivered, a [`StateRequest::Stop`] may follow + /// even if its response is not observed. Start(Rpc<(), ()>), /// Stop asynchronous operations. + /// + /// The handler must be safe after any delivered start attempt. Stop(Rpc<(), ()>), /// Reset a stopped unit to initial state. @@ -222,6 +229,7 @@ enum State { Resetting, Saving, Restoring, + StartUncertain, } #[derive(Debug)] @@ -486,36 +494,60 @@ impl StateUnits { self.running } + fn check_no_terminal_units(&self, op: &'static str) -> Result<(), StateTransitionError> { + let inner = self.inner.lock(); + check( + op, + inner + .units + .values() + .filter(|unit| matches!(unit.state, State::StartUncertain)) + .map(|unit| { + ( + unit.name.clone(), + Err::<(), _>(anyhow::anyhow!( + "unit is in terminal state {:?}", + unit.state + )), + ) + }), + ) + } + /// Starts any units that are individually stopped, because they were added /// via [`StateUnits::add`] while the VM was running. /// /// Does nothing if all units are stopped, via [`StateUnits::stop`]. - pub async fn start_stopped_units(&mut self) { + pub async fn start_stopped_units(&mut self) -> anyhow::Result<()> { if self.is_running() { - self.start().await; + self.start().await?; } + Ok(()) } /// Starts all the state units. - pub async fn start(&mut self) { + pub async fn start(&mut self) -> anyhow::Result<()> { self.restore_allowed = false; - self.run_op( - "start", - None, - State::Stopped, - State::Starting, - State::Running, - StateRequest::Start, - |_, _| Some(()), - |unit| Some(&unit.dependencies), - ) - .await; - self.running = true; + self.start_with_rollback().await } /// Stops all the state units. pub async fn stop(&mut self) { assert!(self.running); + // Make units whose start is uncertain receive Stop, then restore their + // terminal markers after the ordered operation. + let terminal_states = { + let mut inner = self.inner.lock(); + inner + .units + .iter_mut() + .filter(|(_, unit)| unit.state == State::StartUncertain) + .map(|(&id, unit)| { + unit.state = State::Running; + (id, State::StartUncertain) + }) + .collect::>() + }; // Stop units in reverse dependency order so that a dependency is not // stopped before its dependant. self.run_op( @@ -529,6 +561,13 @@ impl StateUnits { |unit| Some(&unit.dependents), ) .await; + let mut inner = self.inner.lock(); + for (id, state) in terminal_states { + if let Some(unit) = inner.units.get_mut(&id) { + assert_eq!(unit.state, State::Stopped); + unit.state = state; + } + } self.running = false; } @@ -537,6 +576,7 @@ impl StateUnits { /// Panics if running. pub async fn reset(&mut self) -> Result<(), StateTransitionError> { assert!(!self.running); + self.check_no_terminal_units("reset")?; self.restore_allowed = false; // Reset in dependency order so that dependants observe their // dependencies' reset state. @@ -563,6 +603,7 @@ impl StateUnits { /// Panics if running. pub async fn save(&mut self) -> Result, StateTransitionError> { assert!(!self.running); + self.check_no_terminal_units("save")?; // Save can occur in any order since it will not observably mutate // state. let r = self @@ -601,6 +642,7 @@ impl StateUnits { states: Vec, ) -> Result<(), StateTransitionError> { assert!(!self.running); + self.check_no_terminal_units("restore")?; if !self.restore_allowed { return Err(StateTransitionError { op: "restore", @@ -1047,6 +1089,8 @@ impl Ready { #[cfg(test)] mod tests { + mod start; + use super::StateTransitionErrorKind; use super::StateUnit; use super::StateUnits; @@ -1219,7 +1263,7 @@ mod tests { let state = units.save().await.unwrap(); units.restore(state).await.unwrap(); assert_eq!(calls.load(Ordering::Relaxed), 2); - units.start().await; + units.start().await.unwrap(); units.stop().await; let state = units.save().await.unwrap(); assert!(units.restore(state).await.is_err()); @@ -1293,13 +1337,13 @@ mod tests { .add("b") .spawn(&driver, |recv| run_unit(TestUnit::default(), recv)) .unwrap(); - units.start().await; + units.start().await.unwrap(); let _c = units .add("c") .spawn(&driver, |recv| run_unit(TestUnit::default(), recv)); units.stop().await; - units.start().await; + units.start().await.unwrap(); units.stop().await; @@ -1347,7 +1391,7 @@ mod tests { ) }) .unwrap(); - units.start().await; + units.start().await.unwrap(); units.stop().await; let state = units.save().await.unwrap(); @@ -1457,7 +1501,7 @@ mod tests { ); } - units.start().await; + units.start().await.unwrap(); units.stop().await; a.remove().await; @@ -1467,7 +1511,7 @@ mod tests { assert!(inner.units[&c.handle().id.id].dependencies.is_empty()); } - units.start().await; + units.start().await.unwrap(); units.stop().await; // Removing the remaining unit leaves the set empty. diff --git a/vmm_core/state_unit/src/start.rs b/vmm_core/state_unit/src/start.rs new file mode 100644 index 00000000000..cc4f8a114bf --- /dev/null +++ b/vmm_core/state_unit/src/start.rs @@ -0,0 +1,196 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Start with rollback. A unit's start cannot fail, but its task can stop +//! answering: then its dependents do not start, and the units that did start +//! are stopped again, rather than the whole process panicking. + +use super::State; +use super::StateRequest; +use super::StateUnits; +use super::state_change; +use futures::future::join_all; +use std::collections::BTreeSet; +use std::time::Instant; + +impl StateUnits { + /// Starts all stopped units in dependency order. + /// + /// Fails without starting any unit if one is in a terminal state. A unit + /// does not start if one of its dependencies + /// did not start. If a unit's task does not answer its start request, the + /// units that did start are stopped again, and the unit is left in a + /// terminal state that fails later starts. A started unit that cannot be + /// stopped again, because a unit that depends on it was not stopped, is + /// left running in that terminal state too, and the units stay running so + /// that [`StateUnits::stop`] can stop them later. + pub(super) async fn start_with_rollback(&mut self) -> anyhow::Result<()> { + enum StartResult { + Started, + DependencyFailed, + Uncertain(anyhow::Error), + } + + let was_running = self.running; + let mut operations = Vec::new(); + let ready_set; + { + let mut inner = self.inner.lock(); + if let Some(unit) = inner + .units + .values() + .find(|unit| !matches!(unit.state, State::Stopped | State::Running)) + { + anyhow::bail!( + "state unit '{}' cannot be started from terminal state {:?}", + unit.name, + unit.state + ); + } + ready_set = inner.ready_set(None); + for (&id, unit) in &mut inner.units { + if unit.state == State::Running { + ready_set.done(id, true); + continue; + } + assert_eq!( + unit.state, + State::Stopped, + "unit {} is not stopped before start", + unit.name, + ); + let name = unit.name.clone(); + let dependencies = unit.dependencies.iter().copied().collect::>(); + let ready_set = ready_set.clone(); + let start = state_change(name.clone(), unit, StateRequest::Start, Some(())); + operations.push(async move { + if !ready_set.wait("start", id, &dependencies).await { + ready_set.done(id, false); + return (name, id, StartResult::DependencyFailed); + } + + let result = match start.await { + Ok(Some(())) => StartResult::Started, + Ok(None) => { + StartResult::Uncertain(anyhow::anyhow!("start request was not sent")) + } + Err(error) => StartResult::Uncertain(error.into()), + }; + ready_set.done(id, matches!(result, StartResult::Started)); + (name, id, result) + }); + unit.state = State::Starting; + } + } + + let start = Instant::now(); + let results = join_all(operations).await; + tracing::info!(duration = ?Instant::now() - start, "state-unit start complete"); + + let mut failures = Vec::new(); + let mut started_ids = Vec::new(); + { + let mut inner = self.inner.lock(); + for (name, id, result) in results { + let Some(unit) = inner.units.get_mut(&id) else { + continue; + }; + match result { + StartResult::Started => { + unit.state = State::Running; + started_ids.push(id); + } + StartResult::DependencyFailed => { + unit.state = State::Stopped; + failures.push(format!("{name}: a dependency did not start")); + } + StartResult::Uncertain(error) => { + unit.state = State::StartUncertain; + failures.push(format!("{name}: {error:#}")); + } + } + } + } + + if failures.is_empty() { + self.running = true; + Ok(()) + } else { + let cleanup_failures = self.stop_after_failed_start(&started_ids).await; + self.running = was_running || !cleanup_failures.is_empty(); + failures.extend(cleanup_failures); + anyhow::bail!("state units could not start: {}", failures.join("; ")) + } + } + + /// Stops the units in `started_ids` again in reverse dependency order. + /// + /// Every other unit that is not stopped, such as one that was already + /// running or one whose start is uncertain, blocks the stop of its + /// dependencies without receiving a stop request. A unit whose stop is + /// blocked or not confirmed is left in a terminal state. + async fn stop_after_failed_start(&self, started_ids: &[u64]) -> Vec { + const OP: &str = "failed_start_rollback"; + let mut operations = Vec::new(); + let ready_set; + { + let mut inner = self.inner.lock(); + let started_ids = started_ids.iter().copied().collect::>(); + ready_set = inner.ready_set(None); + for (&id, unit) in &inner.units { + if !started_ids.contains(&id) { + ready_set.done(id, unit.state == State::Stopped); + } + } + for (&id, unit) in inner + .units + .iter_mut() + .filter(|(id, _)| started_ids.contains(id)) + { + assert_eq!(unit.state, State::Running); + let name = unit.name.clone(); + let dependents = unit.dependents.iter().copied().collect::>(); + let ready_set = ready_set.clone(); + let stop = state_change(name.clone(), unit, StateRequest::Stop, Some(())); + operations.push(async move { + if !ready_set.wait(OP, id, &dependents).await { + ready_set.done(id, false); + return (name, id, None); + } + let result = stop.await; + ready_set.done(id, matches!(&result, Ok(Some(())))); + (name, id, Some(result)) + }); + unit.state = State::Stopping; + } + } + + let results = join_all(operations).await; + let mut failures = Vec::new(); + let mut inner = self.inner.lock(); + for (name, id, result) in results { + let Some(unit) = inner.units.get_mut(&id) else { + continue; + }; + assert_eq!(unit.state, State::Stopping); + match result { + Some(Ok(Some(()))) => unit.state = State::Stopped, + Some(Ok(None)) => { + unit.state = State::StartUncertain; + failures.push(format!("{name}: {OP} stop request was not sent")); + } + Some(Err(error)) => { + unit.state = State::StartUncertain; + failures.push(format!("{name}: {OP} stop failed: {error:#}")); + } + None => { + unit.state = State::StartUncertain; + failures.push(format!( + "{name}: {OP} stop skipped because a dependent was not stopped" + )); + } + } + } + failures + } +} diff --git a/vmm_core/state_unit/src/tests/start.rs b/vmm_core/state_unit/src/tests/start.rs new file mode 100644 index 00000000000..2aaa39424dc --- /dev/null +++ b/vmm_core/state_unit/src/tests/start.rs @@ -0,0 +1,380 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Tests for start with rollback. + +use crate::State; +use crate::StateRequest; +use crate::StateUnit; +use crate::StateUnits; +use crate::run_unit; +use futures::StreamExt; +use inspect::InspectMut; +use mesh::Receiver; +use pal_async::DefaultDriver; +use pal_async::async_test; +use std::sync::Arc; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; +use test_with_tracing::test; +use vmcore::save_restore::RestoreError; +use vmcore::save_restore::SaveError; +use vmcore::save_restore::SavedStateBlob; + +struct RecordingUnit { + started: Arc, + stopped: Arc, +} + +impl StateUnit for RecordingUnit { + async fn start(&mut self) { + self.started.store(true, Ordering::Relaxed); + } + + async fn stop(&mut self) { + self.stopped.store(true, Ordering::Relaxed); + } + + async fn reset(&mut self) -> anyhow::Result<()> { + Ok(()) + } + + async fn save(&mut self) -> Result, SaveError> { + Ok(None) + } + + async fn restore(&mut self, _state: SavedStateBlob) -> Result<(), RestoreError> { + Err(RestoreError::SavedStateNotSupported) + } +} + +impl InspectMut for RecordingUnit { + fn inspect_mut(&mut self, req: inspect::Request<'_>) { + req.respond(); + } +} + +/// Runs `unit`, but drops each start request after starting the unit, without +/// answering it, so that the start is uncertain while the unit still answers +/// later requests. +async fn run_unanswering_start( + mut unit: RecordingUnit, + mut recv: Receiver, +) -> RecordingUnit { + while let Some(request) = recv.next().await { + match request { + StateRequest::Start(response) => { + unit.start().await; + drop(response); + } + request => request.apply(&mut unit).await, + } + } + unit +} + +fn recording_unit(started: &Arc, stopped: &Arc) -> RecordingUnit { + RecordingUnit { + started: started.clone(), + stopped: stopped.clone(), + } +} + +fn unit_state(units: &StateUnits, name: &str) -> State { + units + .inner + .lock() + .units + .values() + .find(|unit| unit.name.as_ref() == name) + .unwrap() + .state +} + +#[async_test] +async fn an_unanswered_start_rolls_back_and_blocks_dependents(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let independent_started = Arc::new(AtomicBool::new(false)); + let independent_stopped = Arc::new(AtomicBool::new(false)); + let _independent = units + .add("independent") + .spawn(&driver, |recv| { + run_unit( + RecordingUnit { + started: independent_started.clone(), + stopped: independent_stopped.clone(), + }, + recv, + ) + }) + .unwrap(); + // A unit whose task drops its start request without answering, as a + // task that has failed would. + let unanswered = units + .add("unanswered") + .spawn(&driver, |mut recv| async move { + while let Some(request) = recv.next().await { + if matches!(request, StateRequest::Start(_)) { + break; + } + } + }) + .unwrap(); + let dependent_started = Arc::new(AtomicBool::new(false)); + let _dependent = units + .add("dependent") + .depends_on(unanswered.handle()) + .spawn(&driver, |recv| { + run_unit( + RecordingUnit { + started: dependent_started.clone(), + stopped: Arc::new(AtomicBool::new(false)), + }, + recv, + ) + }) + .unwrap(); + + let error = units.start().await.unwrap_err(); + assert!(error.to_string().contains("unanswered"), "{error:#}"); + assert!(independent_started.load(Ordering::Relaxed)); + assert!(independent_stopped.load(Ordering::Relaxed)); + assert!(!dependent_started.load(Ordering::Relaxed)); + assert!(!units.is_running()); + let retry_error = units.start().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); +} + +#[async_test] +async fn test_failed_hot_add_does_not_stop_dependency_of_running_unit(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let running_dependent_started = Arc::new(AtomicBool::new(false)); + let running_dependent_stopped = Arc::new(AtomicBool::new(false)); + let running_dependent = units + .add("running-dependent") + .spawn(&driver, |recv| { + run_unit( + recording_unit(&running_dependent_started, &running_dependent_stopped), + recv, + ) + }) + .unwrap(); + units.start().await.unwrap(); + + let dependency_started = Arc::new(AtomicBool::new(false)); + let dependency_stopped = Arc::new(AtomicBool::new(false)); + let _dependency = units + .add("new-dependency") + .dependency_of(running_dependent.handle()) + .spawn(&driver, |recv| { + run_unit( + recording_unit(&dependency_started, &dependency_stopped), + recv, + ) + }) + .unwrap(); + let unanswered_started = Arc::new(AtomicBool::new(false)); + let unanswered_stopped = Arc::new(AtomicBool::new(false)); + let _unanswered = units + .add("unanswered") + .spawn(&driver, |recv| { + run_unanswering_start( + recording_unit(&unanswered_started, &unanswered_stopped), + recv, + ) + }) + .unwrap(); + + let error = units.start_stopped_units().await.unwrap_err().to_string(); + assert!( + error.contains("failed_start_rollback stop skipped"), + "{error}" + ); + assert!(dependency_started.load(Ordering::Relaxed)); + assert!(!dependency_stopped.load(Ordering::Relaxed)); + assert!(!running_dependent_stopped.load(Ordering::Relaxed)); + assert!(unanswered_started.load(Ordering::Relaxed)); + assert!(!unanswered_stopped.load(Ordering::Relaxed)); + assert!(units.is_running()); + assert_eq!(unit_state(&units, "new-dependency"), State::StartUncertain); + assert_eq!(unit_state(&units, "unanswered"), State::StartUncertain); + + units.stop().await; + assert!(dependency_stopped.load(Ordering::Relaxed)); + assert!(running_dependent_stopped.load(Ordering::Relaxed)); + assert!(unanswered_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); + assert_eq!(unit_state(&units, "new-dependency"), State::StartUncertain); + let retry_error = units.start().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); + assert!(units.reset().await.is_err()); + assert!(units.save().await.is_err()); + assert!(units.restore(Vec::new()).await.is_err()); +} + +#[async_test] +async fn test_started_unit_exit_during_rollback_returns_error(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let dependency_started = Arc::new(AtomicBool::new(false)); + let dependency_stopped = Arc::new(AtomicBool::new(false)); + let dependency = units + .add("dependency") + .spawn(&driver, |recv| { + run_unit( + recording_unit(&dependency_started, &dependency_stopped), + recv, + ) + }) + .unwrap(); + // A unit whose task answers its start request and then exits. + let exited = units + .add("exited") + .depends_on(dependency.handle()) + .spawn(&driver, |mut recv| async move { + let Some(StateRequest::Start(response)) = recv.next().await else { + panic!("unexpected request"); + }; + response.complete(()); + }) + .unwrap(); + let unanswered_stopped = Arc::new(AtomicBool::new(false)); + let _unanswered = units + .add("unanswered") + .spawn(&driver, |recv| { + run_unanswering_start( + recording_unit(&Arc::new(AtomicBool::new(false)), &unanswered_stopped), + recv, + ) + }) + .unwrap(); + + let error = units.start().await.unwrap_err().to_string(); + assert!( + error.contains("failed_start_rollback stop failed"), + "{error}" + ); + assert!( + error.contains("failed_start_rollback stop skipped"), + "{error}" + ); + assert!(dependency_started.load(Ordering::Relaxed)); + assert!(!dependency_stopped.load(Ordering::Relaxed)); + // The dependency is still running, so a later stop must be able to stop it. + assert!(units.is_running()); + + exited.remove().await; + units.stop().await; + assert!(dependency_stopped.load(Ordering::Relaxed)); + assert!(unanswered_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); + let retry_error = units.start().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); + assert!(units.reset().await.is_err()); + assert!(units.save().await.is_err()); + assert!(units.restore(Vec::new()).await.is_err()); +} + +#[async_test] +async fn test_uncertain_start_blocks_dependency_rollback(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let dependency_started = Arc::new(AtomicBool::new(false)); + let dependency_stopped = Arc::new(AtomicBool::new(false)); + let dependency = units + .add("dependency") + .spawn(&driver, |recv| { + run_unit( + recording_unit(&dependency_started, &dependency_stopped), + recv, + ) + }) + .unwrap(); + let unanswered_started = Arc::new(AtomicBool::new(false)); + let unanswered_stopped = Arc::new(AtomicBool::new(false)); + let _unanswered = units + .add("unanswered") + .depends_on(dependency.handle()) + .spawn(&driver, |recv| { + run_unanswering_start( + recording_unit(&unanswered_started, &unanswered_stopped), + recv, + ) + }) + .unwrap(); + + let error = units.start().await.unwrap_err().to_string(); + assert!( + error.contains("failed_start_rollback stop skipped"), + "{error}" + ); + assert!(dependency_started.load(Ordering::Relaxed)); + assert!(unanswered_started.load(Ordering::Relaxed)); + assert!(!dependency_stopped.load(Ordering::Relaxed)); + assert!(units.is_running()); + assert_eq!(unit_state(&units, "dependency"), State::StartUncertain); + + units.stop().await; + assert!(unanswered_stopped.load(Ordering::Relaxed)); + assert!(dependency_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); + assert_eq!(unit_state(&units, "dependency"), State::StartUncertain); + assert!(units.reset().await.is_err()); + assert!(units.save().await.is_err()); + assert!(units.restore(Vec::new()).await.is_err()); +} + +#[async_test] +async fn test_initial_start_rollback_can_be_retried(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let started = Arc::new(AtomicBool::new(false)); + let stopped = Arc::new(AtomicBool::new(false)); + let stop_requests = Arc::new(AtomicUsize::new(0)); + // A unit that loses the response to its first stop request without + // stopping. + let _unit = units + .add("rollback-retry") + .spawn(&driver, |mut recv| { + let mut unit = recording_unit(&started, &stopped); + let stop_requests = stop_requests.clone(); + async move { + while let Some(request) = recv.next().await { + match request { + StateRequest::Stop(response) + if stop_requests.fetch_add(1, Ordering::Relaxed) == 0 => + { + drop(response); + } + request => request.apply(&mut unit).await, + } + } + unit + } + }) + .unwrap(); + let unanswered_stopped = Arc::new(AtomicBool::new(false)); + let _unanswered = units + .add("unanswered") + .spawn(&driver, |recv| { + run_unanswering_start( + recording_unit(&Arc::new(AtomicBool::new(false)), &unanswered_stopped), + recv, + ) + }) + .unwrap(); + + let error = units.start().await.unwrap_err().to_string(); + assert!( + error.contains("failed_start_rollback stop failed"), + "{error}" + ); + assert!(started.load(Ordering::Relaxed)); + assert!(!stopped.load(Ordering::Relaxed)); + assert_eq!(stop_requests.load(Ordering::Relaxed), 1); + assert!(units.is_running()); + + units.stop().await; + assert!(stopped.load(Ordering::Relaxed)); + assert_eq!(stop_requests.load(Ordering::Relaxed), 2); + assert!(unanswered_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); +} From d894c0f48630c7688cea0a6f7e3921c72700737c Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Tue, 3 Nov 2026 10:28:13 -0800 Subject: [PATCH 2/2] state_unit: add bounded quiesce transactions with rollback classification Saving a running VM for a snapshot first stops all of its state units. `StateUnits::stop` waits indefinitely and cannot tell its caller whether a failure left the VM in a state that it can resume from. Add `StateUnits::quiesce_for_save`, which stops all units in reverse dependency order within a timeout. A unit one of whose dependents did not stop is not stopped and keeps running. A unit whose stop timed out or could not be delivered may be partially stopped: it is left in a terminal state, and the returned `QuiesceForSaveError` reports that the VM must be terminated instead of resumed. A completed stop followed by another unit's failure is classified as uncertain too, so a clean error guarantees that no unit changed state. A stop whose response timed out is retained, even if its unit is removed, and `StateUnits::stop` awaits it instead of sending an overlapping stop. Add `StateUnits::resume_after_failed_save`, which restarts all units within a timeout when saving fails after a fully successful quiesce. It refuses to run unless every unit is cleanly stopped and every dependency of a unit it restarts was quiesced, and it leaves units that fail to restart in a terminal state. Neither transaction sends a request after its deadline has expired. Like a unit whose start is uncertain, a unit that a failed quiesce leaves in a terminal state fails a later start, reset, save, or restore. A stop stops the units whose quiesce is uncertain and skips the ones that the quiesce already stopped, keeping their terminal markers. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- vmm_core/state_unit/src/lib.rs | 148 ++++- vmm_core/state_unit/src/quiesce.rs | 364 +++++++++++++ vmm_core/state_unit/src/tests/quiesce.rs | 661 +++++++++++++++++++++++ 3 files changed, 1162 insertions(+), 11 deletions(-) create mode 100644 vmm_core/state_unit/src/quiesce.rs create mode 100644 vmm_core/state_unit/src/tests/quiesce.rs diff --git a/vmm_core/state_unit/src/lib.rs b/vmm_core/state_unit/src/lib.rs index d20ba75d256..4a542fce8f3 100644 --- a/vmm_core/state_unit/src/lib.rs +++ b/vmm_core/state_unit/src/lib.rs @@ -30,10 +30,12 @@ #![forbid(unsafe_code)] +pub mod quiesce; mod start; use futures::FutureExt; use futures::StreamExt; +use futures::future::BoxFuture; use futures::future::join_all; use futures_concurrency::stream::Merge; use inspect::Inspect; @@ -79,7 +81,9 @@ pub enum StateRequest { /// Stop asynchronous operations. /// - /// The handler must be safe after any delivered start attempt. + /// The handler must be safe after any delivered start attempt. An + /// outstanding stop request is retained until its response is observed, so + /// another stop request will not overlap it. Stop(Rpc<(), ()>), /// Reset a stopped unit to initial state. @@ -106,6 +110,9 @@ pub trait StateUnit: InspectMut { async fn start(&mut self); /// Stop asynchronous processing. + /// + /// [`StateUnits`] retains an outstanding stop request until its response + /// is observed instead of issuing an overlapping stop. async fn stop(&mut self); /// Reset to initial state. @@ -217,6 +224,7 @@ impl StateRequest { pub struct StateUnits { inner: Arc>, running: bool, + save_quiesced_units: Option>, restore_allowed: bool, } @@ -230,6 +238,8 @@ enum State { Saving, Restoring, StartUncertain, + QuiesceUncertain, + QuiesceFailedStopped, } #[derive(Debug)] @@ -237,6 +247,7 @@ struct Inner { next_id: u64, units: BTreeMap, names: HashMap, u64>, + orphaned_pending_stops: Vec<(Arc, PendingStop)>, } #[derive(Debug)] @@ -246,9 +257,20 @@ struct Unit { dependencies: BTreeSet, dependents: BTreeSet, state: State, + pending_stop: Option, inspect_sensitivity: inspect::SensitivityLevel, } +struct PendingStop { + future: BoxFuture<'static, Result, UnitRecvError>>, +} + +impl Debug for PendingStop { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PendingStop").finish_non_exhaustive() + } +} + /// An error returned when a state unit name is already in use. #[derive(Debug, Error)] #[error("state unit name {0} is in use")] @@ -317,6 +339,11 @@ impl UnitHandle { .dependencies .remove(&id); } + if let Some(pending_stop) = unit.pending_stop { + inner + .orphaned_pending_stops + .push((unit.name.clone(), pending_stop)); + } } } } @@ -459,8 +486,10 @@ impl StateUnits { next_id: 0, units: BTreeMap::new(), names: HashMap::new(), + orphaned_pending_stops: Vec::new(), })), running: false, + save_quiesced_units: None, restore_allowed: true, } } @@ -501,7 +530,14 @@ impl StateUnits { inner .units .values() - .filter(|unit| matches!(unit.state, State::StartUncertain)) + .filter(|unit| { + matches!( + unit.state, + State::StartUncertain + | State::QuiesceUncertain + | State::QuiesceFailedStopped + ) + }) .map(|unit| { ( unit.name.clone(), @@ -528,26 +564,65 @@ impl StateUnits { /// Starts all the state units. pub async fn start(&mut self) -> anyhow::Result<()> { self.restore_allowed = false; - self.start_with_rollback().await + self.start_with_rollback().await?; + self.save_quiesced_units = None; + Ok(()) } /// Stops all the state units. pub async fn stop(&mut self) { assert!(self.running); - // Make units whose start is uncertain receive Stop, then restore their - // terminal markers after the ordered operation. let terminal_states = { - let mut inner = self.inner.lock(); + let inner = self.inner.lock(); inner .units - .iter_mut() - .filter(|(_, unit)| unit.state == State::StartUncertain) - .map(|(&id, unit)| { - unit.state = State::Running; - (id, State::StartUncertain) + .iter() + .filter_map(|(&id, unit)| { + matches!( + unit.state, + State::StartUncertain + | State::QuiesceUncertain + | State::QuiesceFailedStopped + ) + .then_some((id, unit.state)) }) .collect::>() }; + + // A bounded quiesce may stop waiting for an RPC response while the + // handler is still running. Complete those RPCs before deciding + // whether a cleanup Stop still needs to be sent. + self.finish_pending_stops().await; + + // Make uncertain transitions receive Stop and units that a failed + // quiesce already stopped skip it, then restore the terminal markers + // after the ordered operation. + { + let mut inner = self.inner.lock(); + for &(id, state) in &terminal_states { + let Some(unit) = inner.units.get_mut(&id) else { + continue; + }; + match state { + State::StartUncertain => { + assert_eq!(unit.state, state); + unit.state = State::Running; + } + State::QuiesceUncertain => match unit.state { + State::QuiesceUncertain => unit.state = State::Running, + State::Stopped => {} + current => { + panic!("unit {} in {current:?} state after pending stop", unit.name); + } + }, + State::QuiesceFailedStopped => { + assert_eq!(unit.state, state); + unit.state = State::Stopped; + } + _ => unreachable!(), + } + } + } // Stop units in reverse dependency order so that a dependency is not // stopped before its dependant. self.run_op( @@ -571,6 +646,55 @@ impl StateUnits { self.running = false; } + async fn finish_pending_stops(&self) { + let pending = { + let mut inner = self.inner.lock(); + let mut pending = std::mem::take(&mut inner.orphaned_pending_stops) + .into_iter() + .map(|(name, pending)| (None, name, pending)) + .collect::>(); + pending.extend(inner.units.iter_mut().filter_map(|(&id, unit)| { + unit.pending_stop.take().map(|pending| { + assert_eq!(unit.state, State::QuiesceUncertain); + (Some(id), unit.name.clone(), pending) + }) + })); + pending + }; + + let results = join_all( + pending + .into_iter() + .map(|(id, name, pending)| async move { (id, name, pending.future.await) }), + ) + .await; + + let mut inner = self.inner.lock(); + for (id, name, result) in results { + match result { + Ok(Some(())) => { + if let Some(id) = id { + if let Some(unit) = inner.units.get_mut(&id) { + assert_eq!(unit.state, State::QuiesceUncertain); + unit.state = State::Stopped; + } + } + } + Ok(None) => unreachable!("pending stop did not contain an RPC"), + Err(error) => { + if id.is_some_and(|id| inner.units.contains_key(&id)) { + panic!("{error:?}"); + } + tracing::warn!( + unit = %name, + error = ?error, + "removed state unit failed to finish a pending stop" + ); + } + } + } + } + /// Resets all the state units. /// /// Panics if running. @@ -986,6 +1110,7 @@ impl UnitBuilder<'_> { dependencies: self.dependencies, dependents: self.dependents, state: State::Stopped, + pending_stop: None, inspect_sensitivity: self.inspect_sensitivity, }, ); @@ -1089,6 +1214,7 @@ impl Ready { #[cfg(test)] mod tests { + mod quiesce; mod start; use super::StateTransitionErrorKind; diff --git a/vmm_core/state_unit/src/quiesce.rs b/vmm_core/state_unit/src/quiesce.rs new file mode 100644 index 00000000000..1d21afa0789 --- /dev/null +++ b/vmm_core/state_unit/src/quiesce.rs @@ -0,0 +1,364 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Snapshot quiesce transactions: bounded quiesce of every unit for save, and +//! restart after a failed save. + +use super::PendingStop; +use super::State; +use super::StateRequest; +use super::StateUnits; +use super::state_change; +use futures::FutureExt; +use futures::future::join_all; +use mesh::CancelContext; +use std::collections::BTreeSet; +use std::time::Duration; +use std::time::Instant; +use thiserror::Error; + +/// A bounded quiesce failed and the VM must not be saved. +#[derive(Debug, Error)] +#[error("failed to quiesce state units for save: {failures:?}")] +pub struct QuiesceForSaveError { + failures: Vec<(String, String)>, + uncertain: bool, +} + +impl QuiesceForSaveError { + /// Returns whether any unit changed state or may have partially changed + /// ownership or state. + /// + /// When true, the VM cannot be resumed safely and must be terminated. + /// When false, no unit changed state during the failed transaction. + pub fn has_uncertain_state(&self) -> bool { + self.uncertain + } +} + +impl StateUnits { + /// Stops all units in reverse dependency order within `timeout`. + /// + /// A timeout or communication failure after a stop request is sent leaves + /// ownership uncertain. The returned error reports this via + /// [`QuiesceForSaveError::has_uncertain_state`]; callers must terminate the + /// VM rather than attempt rollback in that case. A completed stop followed + /// by another unit's failure is also classified as uncertain, so a clean + /// error guarantees that no unit changed state. + pub async fn quiesce_for_save(&mut self, timeout: Duration) -> Result<(), QuiesceForSaveError> { + assert!(self.running); + + enum UnitResult { + Stopped, + DependencyFailed, + NotSent(anyhow::Error), + Uncertain(anyhow::Error, Option), + } + + let context = CancelContext::new().with_timeout(timeout); + let mut operations = Vec::new(); + let mut failures = Vec::new(); + let mut uncertain = false; + let mut quiesced_units = Vec::new(); + let ready_set; + { + let mut inner = self.inner.lock(); + ready_set = inner.ready_set(None); + for (&id, unit) in &mut inner.units { + match unit.state { + State::Stopped => { + ready_set.done(id, true); + continue; + } + State::Running => {} + State::QuiesceFailedStopped => { + ready_set.done(id, false); + uncertain = true; + failures.push(( + unit.name.to_string(), + "unit is in terminal state QuiesceFailedStopped".to_owned(), + )); + continue; + } + State::StartUncertain | State::QuiesceUncertain => { + ready_set.done(id, false); + uncertain = true; + failures.push(( + unit.name.to_string(), + format!("unit is in terminal state {:?}", unit.state), + )); + continue; + } + state => { + ready_set.done(id, false); + uncertain = true; + failures.push(( + unit.name.to_string(), + format!("unit is in unexpected state {state:?}"), + )); + continue; + } + } + quiesced_units.push(id); + let name = unit.name.clone(); + let dependencies = unit.dependents.iter().copied().collect::>(); + let ready_set = ready_set.clone(); + let mut context = context.clone(); + let mut stop = PendingStop { + future: state_change(name.clone(), unit, StateRequest::Stop, Some(())).boxed(), + }; + operations.push(async move { + let ready = match context + .until_cancelled(ready_set.wait("quiesce_for_save", id, &dependencies)) + .await + { + Ok(ready) => ready, + Err(reason) => { + ready_set.done(id, false); + return (name, id, UnitResult::NotSent(reason.into())); + } + }; + if !ready { + ready_set.done(id, false); + return (name, id, UnitResult::DependencyFailed); + } + if context.is_cancelled() { + ready_set.done(id, false); + return ( + name, + id, + UnitResult::NotSent(anyhow::anyhow!( + "quiesce deadline expired before stop request was sent" + )), + ); + } + + let result = match context.until_cancelled(stop.future.as_mut()).await { + Ok(Ok(Some(()))) => UnitResult::Stopped, + Ok(Ok(None)) => UnitResult::Uncertain( + anyhow::anyhow!("stop request was not sent"), + None, + ), + Ok(Err(error)) => UnitResult::Uncertain(error.into(), None), + Err(reason) => UnitResult::Uncertain(reason.into(), Some(stop)), + }; + ready_set.done(id, matches!(result, UnitResult::Stopped)); + (name, id, result) + }); + unit.state = State::Stopping; + } + } + + let start = Instant::now(); + let results = join_all(operations).await; + tracing::info!(duration = ?Instant::now() - start, "save quiesce complete"); + + let mut inner = self.inner.lock(); + let mut stopped_ids = Vec::new(); + for (name, id, result) in results { + match result { + UnitResult::Stopped => { + if let Some(unit) = inner.units.get_mut(&id) { + unit.state = State::Stopped; + } + stopped_ids.push(id); + } + UnitResult::DependencyFailed => { + if let Some(unit) = inner.units.get_mut(&id) { + unit.state = State::Running; + } + failures.push(( + name.to_string(), + "a dependent unit did not quiesce".to_owned(), + )); + } + UnitResult::NotSent(error) => { + if let Some(unit) = inner.units.get_mut(&id) { + unit.state = State::Running; + } + failures.push((name.to_string(), format!("{error:#}"))); + } + UnitResult::Uncertain(error, mut pending_stop) => { + if let Some(unit) = inner.units.get_mut(&id) { + unit.state = State::QuiesceUncertain; + assert!(unit.pending_stop.is_none()); + unit.pending_stop = pending_stop.take(); + } else if let Some(pending_stop) = pending_stop { + inner + .orphaned_pending_stops + .push((name.clone(), pending_stop)); + } + uncertain = true; + failures.push((name.to_string(), format!("{error:#}"))); + } + } + } + if !failures.is_empty() { + uncertain |= !stopped_ids.is_empty(); + for id in stopped_ids { + let Some(unit) = inner.units.get_mut(&id) else { + continue; + }; + assert_eq!(unit.state, State::Stopped); + unit.state = State::QuiesceFailedStopped; + } + } + drop(inner); + + if failures.is_empty() { + self.running = false; + self.save_quiesced_units = Some(quiesced_units); + Ok(()) + } else { + Err(QuiesceForSaveError { + failures, + uncertain, + }) + } + } + + /// Restarts every unit after a save failure that occurred after a fully + /// successful quiesce but before publication committed. + pub async fn resume_after_failed_save(&mut self, timeout: Duration) -> anyhow::Result<()> { + anyhow::ensure!(!self.running, "VM was not successfully quiesced for save"); + let quiesced_units = self + .save_quiesced_units + .clone() + .ok_or_else(|| anyhow::anyhow!("VM was not successfully quiesced for save"))?; + { + let inner = self.inner.lock(); + anyhow::ensure!( + inner + .units + .values() + .all(|unit| unit.state == State::Stopped), + "state-unit rollback is unsafe because not every unit is cleanly stopped" + ); + let quiesced_units = quiesced_units.iter().copied().collect::>(); + for id in &quiesced_units { + let Some(unit) = inner.units.get(id) else { + continue; + }; + for dependency in &unit.dependencies { + if !quiesced_units.contains(dependency) + && inner + .units + .get(dependency) + .is_some_and(|dependency| dependency.state == State::Stopped) + { + anyhow::bail!( + "state-unit rollback cannot restart '{}' because dependency '{}' was not quiesced", + unit.name, + inner.units[dependency].name + ); + } + } + } + } + + enum StartResult { + Started, + DependencyFailed, + NotSent(anyhow::Error), + Uncertain(anyhow::Error), + } + + let context = CancelContext::new().with_timeout(timeout); + let mut operations = Vec::new(); + let ready_set; + { + let mut inner = self.inner.lock(); + ready_set = inner.ready_set(None); + for (&id, unit) in &mut inner.units { + if !quiesced_units.contains(&id) { + ready_set.done(id, true); + continue; + } + let name = unit.name.clone(); + let dependencies = unit.dependencies.iter().copied().collect::>(); + let ready_set = ready_set.clone(); + let mut context = context.clone(); + let start = state_change(name.clone(), unit, StateRequest::Start, Some(())); + operations.push(async move { + let ready = match context + .until_cancelled(ready_set.wait("snapshot_rollback", id, &dependencies)) + .await + { + Ok(ready) => ready, + Err(reason) => { + ready_set.done(id, false); + return (name, id, StartResult::NotSent(reason.into())); + } + }; + if !ready { + ready_set.done(id, false); + return (name, id, StartResult::DependencyFailed); + } + if context.is_cancelled() { + ready_set.done(id, false); + return ( + name, + id, + StartResult::NotSent(anyhow::anyhow!( + "snapshot rollback deadline expired before start request was sent" + )), + ); + } + + let result = match context.until_cancelled(start).await { + Ok(Ok(Some(()))) => StartResult::Started, + Ok(Ok(None)) => { + StartResult::Uncertain(anyhow::anyhow!("start request was not sent")) + } + Ok(Err(error)) => StartResult::Uncertain(error.into()), + Err(reason) => StartResult::Uncertain(reason.into()), + }; + ready_set.done(id, matches!(result, StartResult::Started)); + (name, id, result) + }); + unit.state = State::Starting; + } + } + + let results = join_all(operations).await; + let mut failures = Vec::new(); + let mut inner = self.inner.lock(); + for (name, id, result) in results { + match result { + StartResult::Started => { + if let Some(unit) = inner.units.get_mut(&id) { + unit.state = State::Running; + } + } + StartResult::DependencyFailed => { + if let Some(unit) = inner.units.get_mut(&id) { + unit.state = State::Stopped; + } + failures.push(format!("{name}: a dependency did not restart")); + } + StartResult::NotSent(error) => { + if let Some(unit) = inner.units.get_mut(&id) { + unit.state = State::Stopped; + } + failures.push(format!("{name}: {error:#}")); + } + StartResult::Uncertain(error) => { + if let Some(unit) = inner.units.get_mut(&id) { + unit.state = State::QuiesceUncertain; + } + failures.push(format!("{name}: {error:#}")); + } + } + } + drop(inner); + + anyhow::ensure!( + failures.is_empty(), + "snapshot rollback could not safely restart all state units: {}", + failures.join("; ") + ); + self.running = true; + self.save_quiesced_units = None; + Ok(()) + } +} diff --git a/vmm_core/state_unit/src/tests/quiesce.rs b/vmm_core/state_unit/src/tests/quiesce.rs new file mode 100644 index 00000000000..978c7ce683f --- /dev/null +++ b/vmm_core/state_unit/src/tests/quiesce.rs @@ -0,0 +1,661 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Tests for snapshot quiesce and for restart after a failed save. + +use super::TestUnit; +use crate::State; +use crate::StateRequest; +use crate::StateUnit; +use crate::StateUnits; +use crate::run_async_unit; +use crate::run_unit; +use futures::StreamExt; +use inspect::InspectMut; +use pal_async::DefaultDriver; +use pal_async::async_test; +use std::sync::Arc; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; +use std::time::Duration; +use test_with_tracing::test; +use vmcore::save_restore::RestoreError; +use vmcore::save_restore::SaveError; +use vmcore::save_restore::SavedStateBlob; + +struct SlowStopUnit { + driver: DefaultDriver, + delay: Duration, +} + +impl StateUnit for SlowStopUnit { + async fn start(&mut self) {} + + async fn stop(&mut self) { + pal_async::timer::PolledTimer::new(&self.driver) + .sleep(self.delay) + .await; + } + + async fn reset(&mut self) -> anyhow::Result<()> { + Ok(()) + } + + async fn save(&mut self) -> Result, SaveError> { + Ok(None) + } + + async fn restore(&mut self, _state: SavedStateBlob) -> Result<(), RestoreError> { + Err(RestoreError::SavedStateNotSupported) + } +} + +impl InspectMut for SlowStopUnit { + fn inspect_mut(&mut self, req: inspect::Request<'_>) { + req.respond(); + } +} + +struct AsyncSlowStopUnit { + driver: DefaultDriver, + delay: Duration, + stops: Arc, + completed_stops: Arc, +} + +impl StateUnit for &'_ AsyncSlowStopUnit { + async fn start(&mut self) {} + + async fn stop(&mut self) { + self.stops.fetch_add(1, Ordering::Relaxed); + pal_async::timer::PolledTimer::new(&self.driver) + .sleep(self.delay) + .await; + self.completed_stops.fetch_add(1, Ordering::Relaxed); + } + + async fn reset(&mut self) -> anyhow::Result<()> { + Ok(()) + } + + async fn save(&mut self) -> Result, SaveError> { + Ok(None) + } + + async fn restore(&mut self, _state: SavedStateBlob) -> Result<(), RestoreError> { + Err(RestoreError::SavedStateNotSupported) + } +} + +impl InspectMut for &'_ AsyncSlowStopUnit { + fn inspect_mut(&mut self, req: inspect::Request<'_>) { + req.respond(); + } +} + +struct SlowRollbackStartUnit { + driver: DefaultDriver, + starts: Arc, +} + +impl StateUnit for SlowRollbackStartUnit { + async fn start(&mut self) { + if self.starts.fetch_add(1, Ordering::Relaxed) != 0 { + pal_async::timer::PolledTimer::new(&self.driver) + .sleep(Duration::from_secs(1)) + .await; + } + } + + async fn stop(&mut self) {} + + async fn reset(&mut self) -> anyhow::Result<()> { + Ok(()) + } + + async fn save(&mut self) -> Result, SaveError> { + Ok(None) + } + + async fn restore(&mut self, _state: SavedStateBlob) -> Result<(), RestoreError> { + Err(RestoreError::SavedStateNotSupported) + } +} + +impl InspectMut for SlowRollbackStartUnit { + fn inspect_mut(&mut self, req: inspect::Request<'_>) { + req.respond(); + } +} + +struct StopCountingUnit { + stops: Arc, +} + +impl StateUnit for StopCountingUnit { + async fn start(&mut self) {} + + async fn stop(&mut self) { + self.stops.fetch_add(1, Ordering::Relaxed); + } + + async fn reset(&mut self) -> anyhow::Result<()> { + Ok(()) + } + + async fn save(&mut self) -> Result, SaveError> { + Ok(None) + } + + async fn restore(&mut self, _state: SavedStateBlob) -> Result<(), RestoreError> { + Err(RestoreError::SavedStateNotSupported) + } +} + +impl InspectMut for StopCountingUnit { + fn inspect_mut(&mut self, req: inspect::Request<'_>) { + req.respond(); + } +} + +#[async_test] +async fn test_quiesce_and_resume_after_failed_save(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let _unit = units + .add("unit") + .spawn(&driver, |recv| run_unit(TestUnit::default(), recv)) + .unwrap(); + units.start().await.unwrap(); + + units + .quiesce_for_save(Duration::from_secs(1)) + .await + .unwrap(); + assert!(!units.is_running()); + + units + .resume_after_failed_save(Duration::from_secs(1)) + .await + .unwrap(); + assert!(units.is_running()); +} + +#[async_test] +async fn test_quiesce_skips_stopped_units(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let _running = units + .add("running") + .spawn(&driver, |recv| run_unit(TestUnit::default(), recv)) + .unwrap(); + units.start().await.unwrap(); + + let stops = Arc::new(AtomicUsize::new(0)); + let _stopped = units + .add("stopped") + .spawn(&driver, |recv| { + run_unit( + StopCountingUnit { + stops: stops.clone(), + }, + recv, + ) + }) + .unwrap(); + + units + .quiesce_for_save(Duration::from_secs(1)) + .await + .unwrap(); + assert_eq!(stops.load(Ordering::Relaxed), 0); + units + .resume_after_failed_save(Duration::from_secs(1)) + .await + .unwrap(); + assert!( + units + .inner + .lock() + .units + .values() + .any(|unit| unit.name.as_ref() == "stopped" && unit.state == State::Stopped) + ); +} + +#[async_test] +async fn test_quiesce_timeout_is_uncertain(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let _unit = units + .add("slow") + .spawn(&driver, { + let driver = driver.clone(); + |recv| { + run_unit( + SlowStopUnit { + driver, + delay: Duration::from_secs(1), + }, + recv, + ) + } + }) + .unwrap(); + units.start().await.unwrap(); + + let started = std::time::Instant::now(); + let error = units + .quiesce_for_save(Duration::from_millis(10)) + .await + .unwrap_err(); + assert!(error.has_uncertain_state()); + assert!(started.elapsed() < Duration::from_millis(500)); + assert!( + units + .resume_after_failed_save(Duration::from_secs(1)) + .await + .is_err() + ); +} + +#[async_test] +async fn test_stop_cleans_up_uncertain_quiesce(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let stops = Arc::new(AtomicUsize::new(0)); + let stop_requests = Arc::new(AtomicUsize::new(0)); + let _unit = units + .add("uncertain") + .spawn(&driver, |mut recv| { + let stop_requests = stop_requests.clone(); + let mut unit = StopCountingUnit { + stops: stops.clone(), + }; + async move { + while let Some(request) = recv.next().await { + match request { + StateRequest::Stop(response) => { + if stop_requests.fetch_add(1, Ordering::Relaxed) == 0 { + drop(response); + } else { + StateRequest::Stop(response).apply(&mut unit).await; + } + } + request => request.apply(&mut unit).await, + } + } + unit + } + }) + .unwrap(); + units.start().await.unwrap(); + + let error = units + .quiesce_for_save(Duration::from_secs(1)) + .await + .unwrap_err(); + assert!(error.has_uncertain_state()); + assert_eq!(stop_requests.load(Ordering::Relaxed), 1); + assert_eq!(stops.load(Ordering::Relaxed), 0); + + units.stop().await; + assert_eq!(stop_requests.load(Ordering::Relaxed), 2); + assert_eq!(stops.load(Ordering::Relaxed), 1); + assert!(!units.is_running()); + assert!( + units + .inner + .lock() + .units + .values() + .any(|unit| unit.state == State::QuiesceUncertain) + ); +} + +#[async_test] +async fn test_stop_awaits_timed_out_async_stop(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let stops = Arc::new(AtomicUsize::new(0)); + let completed_stops = Arc::new(AtomicUsize::new(0)); + let _unit = units + .add("async-slow") + .spawn(&driver, { + let driver = driver.clone(); + let stops = stops.clone(); + let completed_stops = completed_stops.clone(); + |recv| { + run_async_unit( + AsyncSlowStopUnit { + driver, + delay: Duration::from_millis(100), + stops, + completed_stops, + }, + recv, + ) + } + }) + .unwrap(); + units.start().await.unwrap(); + + let error = units + .quiesce_for_save(Duration::from_millis(10)) + .await + .unwrap_err(); + assert!(error.has_uncertain_state()); + assert_eq!(stops.load(Ordering::Relaxed), 1); + + units.stop().await; + assert_eq!(stops.load(Ordering::Relaxed), 1); + assert_eq!(completed_stops.load(Ordering::Relaxed), 1); + assert!(!units.is_running()); +} + +#[async_test] +async fn test_stop_awaits_timed_out_stop_after_unit_removal(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let completed_stops = Arc::new(AtomicUsize::new(0)); + let unit = units + .add("removed-async-slow") + .spawn(&driver, { + let driver = driver.clone(); + let completed_stops = completed_stops.clone(); + |recv| { + run_async_unit( + AsyncSlowStopUnit { + driver, + delay: Duration::from_millis(100), + stops: Arc::new(AtomicUsize::new(0)), + completed_stops, + }, + recv, + ) + } + }) + .unwrap(); + units.start().await.unwrap(); + + let error = units + .quiesce_for_save(Duration::from_millis(10)) + .await + .unwrap_err(); + assert!(error.has_uncertain_state()); + + let (_, ()) = futures::join!(unit.remove(), units.stop()); + + assert_eq!(completed_stops.load(Ordering::Relaxed), 1); + assert!(units.inner.lock().orphaned_pending_stops.is_empty()); + assert!(!units.is_running()); +} + +#[async_test] +async fn test_expired_quiesce_deadline_does_not_send_stop(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let stops = Arc::new(AtomicUsize::new(0)); + let _unit = units + .add("stop-counting") + .spawn(&driver, |recv| { + run_unit( + StopCountingUnit { + stops: stops.clone(), + }, + recv, + ) + }) + .unwrap(); + units.start().await.unwrap(); + + let error = units.quiesce_for_save(Duration::ZERO).await.unwrap_err(); + + assert!(!error.has_uncertain_state()); + assert_eq!(stops.load(Ordering::Relaxed), 0); + assert!(units.is_running()); +} + +#[async_test] +async fn test_quiesce_readiness_timeout_is_bounded(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let a = units + .add("a") + .spawn(&driver, |recv| run_unit(TestUnit::default(), recv)) + .unwrap(); + units.start().await.unwrap(); + let _b = units + .add("b") + .depends_on(a.handle()) + .dependency_of(a.handle()) + .spawn(&driver, |recv| run_unit(TestUnit::default(), recv)) + .unwrap(); + units.start_stopped_units().await.unwrap(); + + let started = std::time::Instant::now(); + let error = units + .quiesce_for_save(Duration::from_millis(10)) + .await + .unwrap_err(); + assert!(!error.has_uncertain_state()); + assert!(started.elapsed() < Duration::from_millis(500)); + assert!(units.is_running()); + assert!( + units + .inner + .lock() + .units + .values() + .all(|unit| unit.state == State::Running) + ); +} + +#[async_test] +async fn test_quiesce_partial_success_is_uncertain(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let a = units + .add("a") + .spawn(&driver, |recv| run_unit(TestUnit::default(), recv)) + .unwrap(); + units.start().await.unwrap(); + let _b = units + .add("b") + .depends_on(a.handle()) + .dependency_of(a.handle()) + .spawn(&driver, |recv| run_unit(TestUnit::default(), recv)) + .unwrap(); + let stops = Arc::new(AtomicUsize::new(0)); + let _independent = units + .add("independent") + .spawn(&driver, |recv| { + run_unit( + StopCountingUnit { + stops: stops.clone(), + }, + recv, + ) + }) + .unwrap(); + units.start_stopped_units().await.unwrap(); + + let started = std::time::Instant::now(); + let error = units + .quiesce_for_save(Duration::from_millis(10)) + .await + .unwrap_err(); + assert!(error.has_uncertain_state()); + assert!(started.elapsed() < Duration::from_millis(500)); + assert_eq!(stops.load(Ordering::Relaxed), 1); + assert!(units.check_no_terminal_units("test").is_err()); + assert!(units.inner.lock().units.values().any(|unit| { + unit.name.as_ref() == "independent" && unit.state == State::QuiesceFailedStopped + })); +} + +#[async_test] +async fn test_rollback_start_timeout_is_bounded(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let starts = Arc::new(AtomicUsize::new(0)); + let _unit = units + .add("slow-rollback") + .spawn(&driver, { + let driver = driver.clone(); + let starts = starts.clone(); + |recv| run_unit(SlowRollbackStartUnit { driver, starts }, recv) + }) + .unwrap(); + units.start().await.unwrap(); + units + .quiesce_for_save(Duration::from_secs(1)) + .await + .unwrap(); + + let started = std::time::Instant::now(); + let error = units + .resume_after_failed_save(Duration::from_millis(10)) + .await + .unwrap_err(); + assert!(error.to_string().contains("snapshot rollback")); + assert!(started.elapsed() < Duration::from_millis(500)); + assert!(!units.is_running()); +} + +#[async_test] +async fn test_rollback_reports_uncertain_start_after_unit_removal(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let starts = Arc::new(AtomicUsize::new(0)); + let unit = units + .add("removed-during-rollback") + .spawn(&driver, { + let driver = driver.clone(); + let starts = starts.clone(); + |recv| run_unit(SlowRollbackStartUnit { driver, starts }, recv) + }) + .unwrap(); + units.start().await.unwrap(); + units + .quiesce_for_save(Duration::from_secs(1)) + .await + .unwrap(); + + let (result, _) = futures::join!( + units.resume_after_failed_save(Duration::from_millis(10)), + unit.remove() + ); + + let error = result.unwrap_err(); + assert!(error.to_string().contains("removed-during-rollback")); + assert!(!units.is_running()); +} + +#[async_test] +async fn test_expired_rollback_deadline_does_not_send_start(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let starts = Arc::new(AtomicUsize::new(0)); + let _unit = units + .add("start-counting") + .spawn(&driver, { + let driver = driver.clone(); + let starts = starts.clone(); + |recv| run_unit(SlowRollbackStartUnit { driver, starts }, recv) + }) + .unwrap(); + units.start().await.unwrap(); + units + .quiesce_for_save(Duration::from_secs(1)) + .await + .unwrap(); + + let error = units + .resume_after_failed_save(Duration::ZERO) + .await + .unwrap_err(); + + assert!(error.to_string().contains("snapshot rollback")); + assert_eq!(starts.load(Ordering::Relaxed), 1); + assert!(!units.is_running()); +} + +#[async_test] +async fn test_rollback_rejects_dependency_added_after_quiesce(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let starts = Arc::new(AtomicUsize::new(0)); + let unit = units + .add("quiesced") + .spawn(&driver, { + let driver = driver.clone(); + let starts = starts.clone(); + |recv| run_unit(SlowRollbackStartUnit { driver, starts }, recv) + }) + .unwrap(); + units.start().await.unwrap(); + units + .quiesce_for_save(Duration::from_secs(1)) + .await + .unwrap(); + + let added_dependency_starts = Arc::new(AtomicUsize::new(0)); + let _added_dependency = units + .add("added-dependency") + .dependency_of(unit.handle()) + .spawn(&driver, { + let driver = driver.clone(); + let starts = added_dependency_starts.clone(); + |recv| run_unit(SlowRollbackStartUnit { driver, starts }, recv) + }) + .unwrap(); + + let error = units + .resume_after_failed_save(Duration::from_secs(1)) + .await + .unwrap_err(); + + assert!(error.to_string().contains("was not quiesced")); + assert_eq!(starts.load(Ordering::Relaxed), 1); + assert_eq!(added_dependency_starts.load(Ordering::Relaxed), 0); + assert!(!units.is_running()); + assert!( + units + .inner + .lock() + .units + .values() + .all(|unit| unit.state == State::Stopped) + ); +} + +#[async_test] +async fn test_rollback_readiness_timeout_is_bounded(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let a = units + .add("a") + .spawn(&driver, |recv| run_unit(TestUnit::default(), recv)) + .unwrap(); + units.start().await.unwrap(); + units + .quiesce_for_save(Duration::from_secs(1)) + .await + .unwrap(); + let b = units + .add("b") + .depends_on(a.handle()) + .dependency_of(a.handle()) + .spawn(&driver, |recv| run_unit(TestUnit::default(), recv)) + .unwrap(); + // Exercise the deadline against a cyclic rollback set directly. + units + .save_quiesced_units + .as_mut() + .unwrap() + .push(b.handle().id.id); + + let started = std::time::Instant::now(); + let error = units + .resume_after_failed_save(Duration::from_millis(10)) + .await + .unwrap_err(); + assert!(error.to_string().contains("snapshot rollback")); + assert!(started.elapsed() < Duration::from_millis(500)); + assert!(!units.is_running()); + assert!( + units + .inner + .lock() + .units + .values() + .all(|unit| unit.state == State::Stopped) + ); +}