From ffff9aa1064fd4110bb9e0ba304761eb180ab606 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Thu, 24 Sep 2026 23:50:18 -0700 Subject: [PATCH 1/7] state_unit: roll back a partial start instead of panicking `StateUnits::start` panics when a unit's task does not answer its start request, as when the task has exited, so a unit failure takes the whole process down instead of failing the start. It also always considers every unit running. Start the units in dependency order without panicking. A unit whose start request is not delivered or answered is left in a terminal state, its dependents do not start, and the units that did start are stopped again in reverse dependency order, so that no partially started VM keeps running. `StateUnits::start` and `start_stopped_units` return the failures; a unit's own start still cannot fail. A unit that was not stopped, such as one that was already running or one whose start is uncertain, keeps the units that it depends on running. A started unit that is kept running, or whose stop fails, is left in the terminal state, the failure is reported, and the units stay running so that a later stop can stop it. While a unit is in a terminal state, a later start, reset, save, or restore fails, and a stop also stops the units in a terminal state and keeps their terminal markers. The task of such a unit may have exited, so `StateUnits::stop_for_teardown`, which the worker uses when it tears the VM down, logs a stop that the unit does not answer instead of panicking, and stops the other units. The worker's teardown stops the state units whenever they run, since a failed start can leave them running while the VM is paused. The callers of `StateUnits::start` and `start_stopped_units` handle the error: - VMBus, PCIe, and relayed vPCI hot-add, and the dynamic vPCI device builder, remove the units they added and return it. VF hot-add removes the network it added and returns it. - After a failed worker restart, a failure to start the VM again is reported to the requester, and the worker tears the VM down. - PCI resource assignment returns it. - OpenHCL reports it to the requester of a resume or a diagnostic save, and tears the VM down, as it does when it cannot resume the VM after a failed restart or servicing request. A failed autostart tears the VM down and fails the worker. - VM resume cannot report a failure to its callers yet, so it treats a unit that fails to start as an unrecoverable unit failure, like the other state changes do. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- openhcl/underhill_core/src/dispatch/mod.rs | 143 +++++-- openhcl/underhill_core/src/lib.rs | 11 +- openhcl/underhill_core/src/worker.rs | 1 + openvmm/openvmm_core/src/worker/dispatch.rs | 48 ++- vm/devices/pci/vpci_relay/src/lib.rs | 10 +- vmm_core/src/device_builder.rs | 6 +- vmm_core/state_unit/src/lib.rs | 147 +++++-- vmm_core/state_unit/src/start.rs | 194 +++++++++ vmm_core/state_unit/src/tests/start.rs | 428 ++++++++++++++++++++ 9 files changed, 895 insertions(+), 93 deletions(-) create mode 100644 vmm_core/state_unit/src/start.rs create mode 100644 vmm_core/state_unit/src/tests/start.rs diff --git a/openhcl/underhill_core/src/dispatch/mod.rs b/openhcl/underhill_core/src/dispatch/mod.rs index e91a38fb8a4..b7a0916189d 100644 --- a/openhcl/underhill_core/src/dispatch/mod.rs +++ b/openhcl/underhill_core/src/dispatch/mod.rs @@ -80,7 +80,7 @@ use vtl2_settings_worker::handle_vtl2_config_rpc; #[derive(MeshPayload)] pub enum UhVmRpc { Pause(Rpc<(), bool>), - Resume(Rpc<(), bool>), + Resume(FailableRpc<(), bool>), Save(FailableRpc<(), Vec>), ClearHalt(Rpc<(), bool>), // TODO: remove this, and use DebugRequest::Resume PacketCapture(FailableRpc, PacketCaptureParams>), @@ -235,9 +235,12 @@ impl LoadedVm { correlation_id: Option, mut vm_rpc: mesh::Receiver, mut worker_rpc: mesh::Receiver>, - ) -> Option> { + ) -> anyhow::Result>> { if autostart_vps { - self.start(correlation_id).await; + if let Err(error) = self.start(correlation_id).await { + self.teardown().await; + return Err(error.context("failed to autostart VM")); + } } // VTL2 settings services @@ -325,7 +328,17 @@ impl LoadedVm { Event::WorkerRpc(message) => match message { WorkerRpc::Stop => break None, WorkerRpc::Restart(rpc) => { - let state = async { + enum RestartOutcome { + Restart(R, S), + // The restart failed, but the VM is back in its + // previous state, so this worker can keep serving. + Recovered, + // The restart failed and the VM could not be + // resumed, so this worker cannot serve any more. + Unrecoverable, + } + + let outcome = async { let running = self.stop().await; match self .save( @@ -336,26 +349,40 @@ impl LoadedVm { ) .await { - Ok(servicing_state) => Some((rpc, servicing_state)), + Ok(servicing_state) => RestartOutcome::Restart(rpc, servicing_state), Err(err) => { if running { - self.start(None).await; + if let Err(start_error) = self.start(None).await { + rpc.complete(Err(RemoteError::new( + start_error.context(format!( + "worker restart failed ({err:#}) and the VM could not resume" + )), + ))); + return RestartOutcome::Unrecoverable; + } } rpc.complete(Err(RemoteError::new(err))); - None + RestartOutcome::Recovered } } } .instrument(tracing::info_span!("restart", CVM_ALLOWED)) .await; - if let Some((rpc, servicing_state)) = state { - break Some(LoadedVmState { - restart_rpc: rpc, - servicing_state, - vm_rpc, - control_send: self.control_send.lock().take().unwrap(), - }); + match outcome { + RestartOutcome::Restart(rpc, servicing_state) => { + break Some(LoadedVmState { + restart_rpc: rpc, + servicing_state, + vm_rpc, + control_send: self.control_send.lock().take().unwrap(), + }); + } + RestartOutcome::Recovered => {} + // The error has already completed the RPC. Tear + // down through the common path rather than keep a + // worker whose state units can no longer run. + RestartOutcome::Unrecoverable => break None, } } WorkerRpc::Inspect(deferred) => deferred.respond(|resp| { @@ -406,34 +433,39 @@ impl LoadedVm { } Event::UhVmRpc(msg) => match msg { UhVmRpc::Resume(rpc) => { - rpc.handle(async |()| { - if !self.state_units.is_running() { - self.start(None).await; - true - } else { - false - } - }) - .await + if self.state_units.is_running() { + rpc.complete(Ok(false)); + } else if let Err(error) = self.start(None).await { + rpc.complete(Err(RemoteError::new( + error.context("failed to resume VM"), + ))); + // A failed start may leave state units terminal; + // this worker cannot safely continue serving. + break None; + } else { + rpc.complete(Ok(true)); + } } UhVmRpc::Pause(rpc) => rpc.handle(async |()| self.stop().await).await, UhVmRpc::Save(rpc) => { - rpc.handle_failable(async |()| { - let running = self.stop().await; - let r = self - .save( - SaveReason::Diagnostic, - None, - KeepAliveConfig::Disabled, - KeepAliveConfig::Disabled, - ) - .await; - if running { - self.start(None).await; - } - r.map(mesh::payload::encode) - }) - .await + let running = self.stop().await; + let result = self + .save( + SaveReason::Diagnostic, + None, + KeepAliveConfig::Disabled, + KeepAliveConfig::Disabled, + ) + .await; + if running && let Err(error) = self.start(None).await { + rpc.complete(Err(RemoteError::new( + error.context("failed to resume VM after save"), + ))); + // The state units rolled back and cannot run + // again, so tear down rather than keep this worker. + break None; + } + rpc.complete(result.map(mesh::payload::encode).map_err(RemoteError::new)); } UhVmRpc::ClearHalt(rpc) => { rpc.handle(async |()| self.partition_unit.clear_halt().await) @@ -585,6 +617,12 @@ impl LoadedVm { } }; + self.teardown().await; + + Ok(state) + } + + async fn teardown(mut self) { // Drain resealing before tearing down any storage transport. Removing // the unit closes its request stream; in-flight I/O finishes first. if let Some(resealer) = self.hardware_reseal.take() { @@ -602,8 +640,6 @@ impl LoadedVm { if let Some(vmbus) = self.vmbus_server { vmbus.remove().await.shutdown().await; } - - state } /// Handles a servicing request from the host. @@ -654,7 +690,9 @@ impl LoadedVm { .context("failed to notify host of servicing-while-paused failure")?; if running { - self.start(Some(correlation_id)).await; + self.start(Some(correlation_id)) + .await + .context("failed to resume VM after servicing failure")?; } false } @@ -884,8 +922,8 @@ impl LoadedVm { } } - async fn start(&mut self, correlation_id: Option) { - self.state_units.start().await; + async fn start(&mut self, correlation_id: Option) -> anyhow::Result<()> { + self.state_units.start().await?; // Log the boot/blackout time. let reference_time = ReferenceTime::new(self.partition.reference_time()); @@ -911,6 +949,8 @@ impl LoadedVm { "starting VM" ) } + + Ok(()) } /// Returns true if the VM was previously running. @@ -1137,7 +1177,20 @@ impl LoadedVm { ) .await?; - self.state_units.start_stopped_units().await; + if let Err(error) = self.state_units.start_stopped_units().await { + if let Err(remove_error) = self + .network_settings + .as_mut() + .unwrap() + .remove_network(instance_id) + .await + { + return Err(error.context(format!( + "failed to remove network after start failure: {remove_error:#}" + ))); + } + return Err(error); + } self.emuplat_servicing.netvsp_state.push(save_state); Ok(()) diff --git a/openhcl/underhill_core/src/lib.rs b/openhcl/underhill_core/src/lib.rs index 329d1c119f8..69cb928ad5c 100644 --- a/openhcl/underhill_core/src/lib.rs +++ b/openhcl/underhill_core/src/lib.rs @@ -686,11 +686,18 @@ async fn run_control( continue; }; - let was_resumed = workers + let was_resumed = match workers .vm_rpc .call(UhVmRpc::Resume, ()) .await - .context("failed to resumed VM")?; + .context("failed to resume VM")? + { + Ok(was_resumed) => was_resumed, + Err(error) => { + rpc.complete(Err(error)); + continue; + } + }; let was_halted = workers .vm_rpc diff --git a/openhcl/underhill_core/src/worker.rs b/openhcl/underhill_core/src/worker.rs index 1039d11cfaa..df433f73e24 100644 --- a/openhcl/underhill_core/src/worker.rs +++ b/openhcl/underhill_core/src/worker.rs @@ -468,6 +468,7 @@ impl Worker for UnderhillVmWorker { )); tracing::info!(CVM_ALLOWED, "terminating worker"); self.get_thread.join().unwrap(); + let state = state?; if let Some(state) = state { let params = UnderhillWorkerParameters { env_cfg: self.env_cfg, diff --git a/openvmm/openvmm_core/src/worker/dispatch.rs b/openvmm/openvmm_core/src/worker/dispatch.rs index 6566a94a8ec..6b6b558aa58 100644 --- a/openvmm/openvmm_core/src/worker/dispatch.rs +++ b/openvmm/openvmm_core/src/worker/dispatch.rs @@ -3682,7 +3682,12 @@ impl LoadedVm { if self.running { return false; } - self.state_units.start().await; + // Resume cannot report a failure to its callers, so treat a unit that + // fails to start like any other unrecoverable state unit failure. + self.state_units + .start() + .await + .expect("state units failed to start"); self.running = true; true } @@ -3724,8 +3729,10 @@ impl LoadedVm { // Start state units so device config space is accessible. This also // freezes SMMU capabilities before firmware reads them. - self.state_units.start().await; - + self.state_units + .start() + .await + .context("failed to start devices for PCI resource assignment")?; let result = ecam_config_access::assign_pci_resources_for_root_complexes( &self.inner.chipset, &self.inner.pcie_host_bridges, @@ -3832,7 +3839,12 @@ impl LoadedVm { } Err(err) => { if stopped { - self.state_units.start().await; + if let Err(start_error) = self.state_units.start().await { + rpc.complete(Err(RemoteError::new(start_error.context( + "worker restart failed and the VM could not resume", + )))); + break; + } } rpc.complete(Err(RemoteError::new(err))); } @@ -3906,7 +3918,15 @@ impl LoadedVm { ) .await?; self.inner.vmbus_devices.push(device); - self.state_units.start_stopped_units().await; + if let Err(error) = self.state_units.start_stopped_units().await { + let device = self + .inner + .vmbus_devices + .pop() + .expect("device was just pushed"); + drop(device.remove().await); + return Err(error); + } anyhow::Ok(()) }) .await @@ -4072,7 +4092,15 @@ impl LoadedVm { // MSI. The guest may begin probing config space // immediately after receiving the interrupt, so // the device must be ready first. - self.state_units.start_stopped_units().await; + if let Err(error) = self.state_units.start_stopped_units().await { + let (_, unit, _) = self + .inner + .pcie_hotplug_devices + .pop() + .expect("device was just pushed"); + unit.remove().await; + return Err(error); + } // Now attach the device and notify the guest. if let Err(e) = rc.lock().hotplug_add_device( @@ -4243,7 +4271,13 @@ impl LoadedVm { } device_removal.drain().await; - self.pause().await; + // Stop the state units whenever they run, not only when the VM does: a + // failed start, as in a reset, can leave units running while the VM is + // paused, and a unit whose start is uncertain may not answer its stop. + if self.state_units.is_running() { + self.state_units.stop_for_teardown().await; + } + self.running = false; self.teardown_chipset_devices().await; self.inner.partition_unit.teardown().await; if let Some(vmbus) = self.inner.vmbus_server { diff --git a/vm/devices/pci/vpci_relay/src/lib.rs b/vm/devices/pci/vpci_relay/src/lib.rs index 22e971f1f90..76bae759c18 100644 --- a/vm/devices/pci/vpci_relay/src/lib.rs +++ b/vm/devices/pci/vpci_relay/src/lib.rs @@ -490,7 +490,7 @@ impl VpciRelay { .await? }; - entry.insert(RelayedDevice { + let device = RelayedDevice { bus_instance_id: instance_id, bus_client: vpci_client, vpci_device: vpci_device.clone(), @@ -498,9 +498,13 @@ impl VpciRelay { bus_unit, device_unit, ready_to_remove: false, - }); + }; - state_units.start_stopped_units().await; + if let Err(error) = state_units.start_stopped_units().await { + device.remove().await; + return Err(error); + } + entry.insert(device); Ok(()) } } diff --git a/vmm_core/src/device_builder.rs b/vmm_core/src/device_builder.rs index 089ac1444c9..440d98a9dc4 100644 --- a/vmm_core/src/device_builder.rs +++ b/vmm_core/src/device_builder.rs @@ -144,7 +144,11 @@ pub async fn build_dynamic_vpci_device( }; let pending_offer = pending_offer.context("missing deferred VPCI channel offer")?; - state_units.start_stopped_units().await; + if let Err(error) = state_units.start_stopped_units().await { + vpci_unit.remove().await; + pci_unit.remove().await; + return Err(error); + } if let Err(error) = pending_offer .offer_registered(&vpci_bus, driver_source, vmbus, state_units.is_running()) .await diff --git a/vmm_core/state_unit/src/lib.rs b/vmm_core/state_unit/src/lib.rs index 87a9525e73d..d6e9ef772a5 100644 --- a/vmm_core/state_unit/src/lib.rs +++ b/vmm_core/state_unit/src/lib.rs @@ -30,6 +30,8 @@ #![forbid(unsafe_code)] +mod start; + use futures::FutureExt; use futures::StreamExt; use futures::future::join_all; @@ -70,9 +72,14 @@ use vmcore::save_restore::SavedStateBlob; #[derive(Debug, MeshPayload)] pub enum StateRequest { /// Start asynchronous operations. + /// + /// Once this request is delivered, a [`StateRequest::Stop`] may follow + /// even if its response is not observed. Start(Rpc<(), ()>), /// Stop asynchronous operations. + /// + /// The handler must be safe after any delivered start attempt. Stop(Rpc<(), ()>), /// Reset a stopped unit to initial state. @@ -222,6 +229,7 @@ enum State { Resetting, Saving, Restoring, + StartUncertain, } #[derive(Debug)] @@ -486,41 +494,82 @@ impl StateUnits { self.running } + fn check_no_terminal_units(&self, op: &'static str) -> Result<(), StateTransitionError> { + let inner = self.inner.lock(); + check( + op, + inner + .units + .values() + .filter(|unit| matches!(unit.state, State::StartUncertain)) + .map(|unit| { + ( + unit.name.clone(), + Err::<(), _>(anyhow::anyhow!( + "unit is in terminal state {:?}", + unit.state + )), + ) + }), + ) + } + /// Starts any units that are individually stopped, because they were added /// via [`StateUnits::add`] while the VM was running. /// /// Does nothing if all units are stopped, via [`StateUnits::stop`]. - pub async fn start_stopped_units(&mut self) { + pub async fn start_stopped_units(&mut self) -> anyhow::Result<()> { if self.is_running() { - self.start().await; + self.start().await?; } + Ok(()) } /// Starts all the state units. - pub async fn start(&mut self) { + pub async fn start(&mut self) -> anyhow::Result<()> { self.restore_allowed = false; - self.run_op( - "start", - None, - State::Stopped, - State::Starting, - State::Running, - StateRequest::Start, - |_, _| Some(()), - |unit| Some(&unit.dependencies), - ) - .await; - self.running = true; + self.start_with_rollback().await } /// Stops all the state units. + /// + /// This also stops the units whose start is uncertain. pub async fn stop(&mut self) { + self.stop_units(false).await + } + + /// Stops all the state units before the VM is torn down. + /// + /// Unlike [`StateUnits::stop`], this does not panic when a unit whose start + /// is uncertain does not answer its stop, as when its task has exited. It + /// logs the failure and stops the other units, including the units that + /// such a unit depends on, since the VM is not run again. + pub async fn stop_for_teardown(&mut self) { + self.stop_units(true).await + } + + async fn stop_units(&mut self, for_teardown: bool) { assert!(self.running); + // Make units whose start is uncertain receive Stop, then restore their + // terminal markers after the ordered operation. + let uncertain_ids = { + let mut inner = self.inner.lock(); + inner + .units + .iter_mut() + .filter(|(_, unit)| unit.state == State::StartUncertain) + .map(|(&id, unit)| { + unit.state = State::Running; + id + }) + .collect::>() + }; // Stop units in reverse dependency order so that a dependency is not // stopped before its dependant. self.run_op( "stop", None, + if for_teardown { &uncertain_ids } else { &[] }, State::Running, State::Stopping, State::Stopped, @@ -529,6 +578,13 @@ impl StateUnits { |unit| Some(&unit.dependents), ) .await; + let mut inner = self.inner.lock(); + for id in uncertain_ids { + if let Some(unit) = inner.units.get_mut(&id) { + assert_eq!(unit.state, State::Stopped); + unit.state = State::StartUncertain; + } + } self.running = false; } @@ -537,6 +593,7 @@ impl StateUnits { /// Panics if running. pub async fn reset(&mut self) -> Result<(), StateTransitionError> { assert!(!self.running); + self.check_no_terminal_units("reset")?; self.restore_allowed = false; // Reset in dependency order so that dependants observe their // dependencies' reset state. @@ -544,6 +601,7 @@ impl StateUnits { .run_op( "reset", None, + &[], State::Stopped, State::Resetting, State::Stopped, @@ -563,12 +621,14 @@ impl StateUnits { /// Panics if running. pub async fn save(&mut self) -> Result, StateTransitionError> { assert!(!self.running); + self.check_no_terminal_units("save")?; // Save can occur in any order since it will not observably mutate // state. let r = self .run_op( "save", None, + &[], State::Stopped, State::Saving, State::Stopped, @@ -601,6 +661,7 @@ impl StateUnits { states: Vec, ) -> Result<(), StateTransitionError> { assert!(!self.running); + self.check_no_terminal_units("restore")?; if !self.restore_allowed { return Err(StateTransitionError { op: "restore", @@ -644,6 +705,7 @@ impl StateUnits { .run_op( "restore", None, + &[], State::Stopped, State::Restoring, State::Stopped, @@ -675,6 +737,10 @@ impl StateUnits { /// `unit_ids` is the set of the units whose state should be changed. If /// `unit_ids` is `None`, all units change states. /// + /// `tolerated_ids` is the set of the units whose failure to answer the + /// request is logged instead of treated as an unrecoverable unit failure, + /// such as the units whose start is uncertain when the VM is torn down. + /// /// The old state for each unit must be `old_state`. During the operation, /// the unit is temporarily places in `interim_state`. When complete, the /// unit is placed in `new_state`. @@ -693,6 +759,7 @@ impl StateUnits { &self, op: &str, unit_ids: Option<&[u64]>, + tolerated_ids: &[u64], old_state: State, interim_state: State, new_state: State, @@ -751,20 +818,12 @@ impl StateUnits { let mut inner = self.inner.lock(); let r = results .into_iter() - .filter_map(|(name, id, r)| { - match r { - Ok(Some(r)) => Some((name, r)), - Ok(None) => None, - Err(err) => { - // If the unit was removed during the operation, then - // ignore the failure. Otherwise, panic because unit - // failure is not recoverable. FUTURE: reconsider this - // position. - if inner.units.contains_key(&id) { - panic!("{:?}", err); - } - None - } + .filter_map(|(name, id, r)| match r { + Ok(Some(r)) => Some((name, r)), + Ok(None) => None, + Err(err) => { + inner.unanswered_request(id, tolerated_ids, err); + None } }) .collect(); @@ -808,6 +867,22 @@ impl Inner { }; ReadySet(Arc::new(units)) } + + /// Handles a state change request that unit `id` did not answer. + /// + /// If the unit is in `tolerated_ids`, log the failure. If the unit was + /// removed during the operation, ignore it. Otherwise, panic because unit + /// failure is not recoverable. FUTURE: reconsider this position. + fn unanswered_request(&self, id: u64, tolerated_ids: &[u64], err: UnitRecvError) { + if tolerated_ids.contains(&id) { + tracing::error!( + error = &err as &dyn std::error::Error, + "unit whose start is uncertain did not answer" + ); + } else if self.units.contains_key(&id) { + panic!("{:?}", err); + } + } } #[derive(Clone)] @@ -1047,6 +1122,8 @@ impl Ready { #[cfg(test)] mod tests { + mod start; + use super::StateTransitionErrorKind; use super::StateUnit; use super::StateUnits; @@ -1219,7 +1296,7 @@ mod tests { let state = units.save().await.unwrap(); units.restore(state).await.unwrap(); assert_eq!(calls.load(Ordering::Relaxed), 2); - units.start().await; + units.start().await.unwrap(); units.stop().await; let state = units.save().await.unwrap(); assert!(units.restore(state).await.is_err()); @@ -1293,13 +1370,13 @@ mod tests { .add("b") .spawn(&driver, |recv| run_unit(TestUnit::default(), recv)) .unwrap(); - units.start().await; + units.start().await.unwrap(); let _c = units .add("c") .spawn(&driver, |recv| run_unit(TestUnit::default(), recv)); units.stop().await; - units.start().await; + units.start().await.unwrap(); units.stop().await; @@ -1347,7 +1424,7 @@ mod tests { ) }) .unwrap(); - units.start().await; + units.start().await.unwrap(); units.stop().await; let state = units.save().await.unwrap(); @@ -1457,7 +1534,7 @@ mod tests { ); } - units.start().await; + units.start().await.unwrap(); units.stop().await; a.remove().await; @@ -1467,7 +1544,7 @@ mod tests { assert!(inner.units[&c.handle().id.id].dependencies.is_empty()); } - units.start().await; + units.start().await.unwrap(); units.stop().await; // Removing the remaining unit leaves the set empty. diff --git a/vmm_core/state_unit/src/start.rs b/vmm_core/state_unit/src/start.rs new file mode 100644 index 00000000000..13c18f7693b --- /dev/null +++ b/vmm_core/state_unit/src/start.rs @@ -0,0 +1,194 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Start with rollback. A unit's start cannot fail, but its task can stop +//! answering: then its dependents do not start, and the units that did start +//! are stopped again, rather than the whole process panicking. + +use super::State; +use super::StateRequest; +use super::StateUnits; +use super::state_change; +use futures::future::join_all; +use std::time::Instant; + +impl StateUnits { + /// Starts all stopped units in dependency order. + /// + /// Fails without starting any unit if one is in a terminal state. A unit + /// does not start if one of its dependencies + /// did not start. If a unit's task does not answer its start request, the + /// units that did start are stopped again, and the unit is left in a + /// terminal state that fails later starts. A started unit that cannot be + /// stopped again, because a unit that depends on it was not stopped, is + /// left running in that terminal state too, and the units stay running so + /// that [`StateUnits::stop`] can stop them later. + pub(super) async fn start_with_rollback(&mut self) -> anyhow::Result<()> { + enum StartResult { + Started, + DependencyFailed, + Uncertain(anyhow::Error), + } + + let was_running = self.running; + let mut operations = Vec::new(); + let ready_set; + { + let mut inner = self.inner.lock(); + if let Some(unit) = inner + .units + .values() + .find(|unit| !matches!(unit.state, State::Stopped | State::Running)) + { + anyhow::bail!( + "state unit '{}' cannot be started from terminal state {:?}", + unit.name, + unit.state + ); + } + ready_set = inner.ready_set(None); + for (&id, unit) in &mut inner.units { + if unit.state == State::Running { + ready_set.done(id, true); + continue; + } + assert_eq!( + unit.state, + State::Stopped, + "unit {} is not stopped before start", + unit.name, + ); + let name = unit.name.clone(); + let dependencies = unit.dependencies.iter().copied().collect::>(); + let ready_set = ready_set.clone(); + let start = state_change(name.clone(), unit, StateRequest::Start, Some(())); + operations.push(async move { + if !ready_set.wait("start", id, &dependencies).await { + ready_set.done(id, false); + return (name, id, StartResult::DependencyFailed); + } + + let result = match start.await { + Ok(Some(())) => StartResult::Started, + Ok(None) => { + StartResult::Uncertain(anyhow::anyhow!("start request was not sent")) + } + Err(error) => StartResult::Uncertain(error.into()), + }; + ready_set.done(id, matches!(result, StartResult::Started)); + (name, id, result) + }); + unit.state = State::Starting; + } + } + + let start = Instant::now(); + let results = join_all(operations).await; + tracing::info!(duration = ?Instant::now() - start, "state-unit start complete"); + + let mut failures = Vec::new(); + let mut started_ids = Vec::new(); + { + let mut inner = self.inner.lock(); + for (name, id, result) in results { + let Some(unit) = inner.units.get_mut(&id) else { + continue; + }; + match result { + StartResult::Started => { + unit.state = State::Running; + started_ids.push(id); + } + StartResult::DependencyFailed => { + unit.state = State::Stopped; + failures.push(format!("{name}: a dependency did not start")); + } + StartResult::Uncertain(error) => { + unit.state = State::StartUncertain; + failures.push(format!("{name}: {error:#}")); + } + } + } + } + + if failures.is_empty() { + self.running = true; + Ok(()) + } else { + let cleanup_failures = self.stop_after_failed_start(&started_ids).await; + self.running = was_running || !cleanup_failures.is_empty(); + failures.extend(cleanup_failures); + anyhow::bail!("state units could not start: {}", failures.join("; ")) + } + } + + /// Stops the units in `started_ids` again in reverse dependency order. + /// + /// Every other unit that is not stopped, such as one that was already + /// running or one whose start is uncertain, blocks the stop of its + /// dependencies without receiving a stop request. A unit whose stop is + /// blocked or not confirmed is left in a terminal state. + async fn stop_after_failed_start(&self, started_ids: &[u64]) -> Vec { + const OP: &str = "failed_start_rollback"; + let mut operations = Vec::new(); + let ready_set; + { + let mut inner = self.inner.lock(); + ready_set = inner.ready_set(None); + for (&id, unit) in &inner.units { + if !started_ids.contains(&id) { + ready_set.done(id, unit.state == State::Stopped); + } + } + for (&id, unit) in inner + .units + .iter_mut() + .filter(|(id, _)| started_ids.contains(id)) + { + assert_eq!(unit.state, State::Running); + let name = unit.name.clone(); + let dependents = unit.dependents.iter().copied().collect::>(); + let ready_set = ready_set.clone(); + let stop = state_change(name.clone(), unit, StateRequest::Stop, Some(())); + operations.push(async move { + if !ready_set.wait(OP, id, &dependents).await { + ready_set.done(id, false); + return (name, id, None); + } + let result = stop.await; + ready_set.done(id, matches!(&result, Ok(Some(())))); + (name, id, Some(result)) + }); + unit.state = State::Stopping; + } + } + + let results = join_all(operations).await; + let mut failures = Vec::new(); + let mut inner = self.inner.lock(); + for (name, id, result) in results { + let Some(unit) = inner.units.get_mut(&id) else { + continue; + }; + assert_eq!(unit.state, State::Stopping); + match result { + Some(Ok(Some(()))) => unit.state = State::Stopped, + Some(Ok(None)) => { + unit.state = State::StartUncertain; + failures.push(format!("{name}: {OP} stop request was not sent")); + } + Some(Err(error)) => { + unit.state = State::StartUncertain; + failures.push(format!("{name}: {OP} stop failed: {error:#}")); + } + None => { + unit.state = State::StartUncertain; + failures.push(format!( + "{name}: {OP} stop skipped because a dependent was not stopped" + )); + } + } + } + failures + } +} diff --git a/vmm_core/state_unit/src/tests/start.rs b/vmm_core/state_unit/src/tests/start.rs new file mode 100644 index 00000000000..41bd5d203b1 --- /dev/null +++ b/vmm_core/state_unit/src/tests/start.rs @@ -0,0 +1,428 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Tests for start with rollback. + +use crate::State; +use crate::StateRequest; +use crate::StateUnit; +use crate::StateUnits; +use crate::run_unit; +use futures::StreamExt; +use inspect::InspectMut; +use mesh::Receiver; +use pal_async::DefaultDriver; +use pal_async::async_test; +use std::sync::Arc; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; +use test_with_tracing::test; +use vmcore::save_restore::RestoreError; +use vmcore::save_restore::SaveError; +use vmcore::save_restore::SavedStateBlob; + +struct RecordingUnit { + started: Arc, + stopped: Arc, +} + +impl StateUnit for RecordingUnit { + async fn start(&mut self) { + self.started.store(true, Ordering::Relaxed); + } + + async fn stop(&mut self) { + self.stopped.store(true, Ordering::Relaxed); + } + + async fn reset(&mut self) -> anyhow::Result<()> { + Ok(()) + } + + async fn save(&mut self) -> Result, SaveError> { + Ok(None) + } + + async fn restore(&mut self, _state: SavedStateBlob) -> Result<(), RestoreError> { + Err(RestoreError::SavedStateNotSupported) + } +} + +impl InspectMut for RecordingUnit { + fn inspect_mut(&mut self, req: inspect::Request<'_>) { + req.respond(); + } +} + +/// Runs `unit`, but drops each start request after starting the unit, without +/// answering it, so that the start is uncertain while the unit still answers +/// later requests. +async fn run_unanswering_start( + mut unit: RecordingUnit, + mut recv: Receiver, +) -> RecordingUnit { + while let Some(request) = recv.next().await { + match request { + StateRequest::Start(response) => { + unit.start().await; + drop(response); + } + request => request.apply(&mut unit).await, + } + } + unit +} + +fn recording_unit(started: &Arc, stopped: &Arc) -> RecordingUnit { + RecordingUnit { + started: started.clone(), + stopped: stopped.clone(), + } +} + +fn unit_state(units: &StateUnits, name: &str) -> State { + units + .inner + .lock() + .units + .values() + .find(|unit| unit.name.as_ref() == name) + .unwrap() + .state +} + +#[async_test] +async fn an_unanswered_start_rolls_back_and_blocks_dependents(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let independent_started = Arc::new(AtomicBool::new(false)); + let independent_stopped = Arc::new(AtomicBool::new(false)); + let _independent = units + .add("independent") + .spawn(&driver, |recv| { + run_unit( + RecordingUnit { + started: independent_started.clone(), + stopped: independent_stopped.clone(), + }, + recv, + ) + }) + .unwrap(); + // A unit whose task drops its start request without answering, as a + // task that has failed would. + let unanswered = units + .add("unanswered") + .spawn(&driver, |mut recv| async move { + while let Some(request) = recv.next().await { + if matches!(request, StateRequest::Start(_)) { + break; + } + } + }) + .unwrap(); + let dependent_started = Arc::new(AtomicBool::new(false)); + let _dependent = units + .add("dependent") + .depends_on(unanswered.handle()) + .spawn(&driver, |recv| { + run_unit( + RecordingUnit { + started: dependent_started.clone(), + stopped: Arc::new(AtomicBool::new(false)), + }, + recv, + ) + }) + .unwrap(); + + let error = units.start().await.unwrap_err(); + assert!(error.to_string().contains("unanswered"), "{error:#}"); + assert!(independent_started.load(Ordering::Relaxed)); + assert!(independent_stopped.load(Ordering::Relaxed)); + assert!(!dependent_started.load(Ordering::Relaxed)); + assert!(!units.is_running()); + let retry_error = units.start().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); +} + +#[async_test] +async fn test_failed_hot_add_does_not_stop_dependency_of_running_unit(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let running_dependent_started = Arc::new(AtomicBool::new(false)); + let running_dependent_stopped = Arc::new(AtomicBool::new(false)); + let running_dependent = units + .add("running-dependent") + .spawn(&driver, |recv| { + run_unit( + recording_unit(&running_dependent_started, &running_dependent_stopped), + recv, + ) + }) + .unwrap(); + units.start().await.unwrap(); + + let dependency_started = Arc::new(AtomicBool::new(false)); + let dependency_stopped = Arc::new(AtomicBool::new(false)); + let _dependency = units + .add("new-dependency") + .dependency_of(running_dependent.handle()) + .spawn(&driver, |recv| { + run_unit( + recording_unit(&dependency_started, &dependency_stopped), + recv, + ) + }) + .unwrap(); + let unanswered_started = Arc::new(AtomicBool::new(false)); + let unanswered_stopped = Arc::new(AtomicBool::new(false)); + let _unanswered = units + .add("unanswered") + .spawn(&driver, |recv| { + run_unanswering_start( + recording_unit(&unanswered_started, &unanswered_stopped), + recv, + ) + }) + .unwrap(); + + let error = units.start_stopped_units().await.unwrap_err().to_string(); + assert!( + error.contains("failed_start_rollback stop skipped"), + "{error}" + ); + assert!(dependency_started.load(Ordering::Relaxed)); + assert!(!dependency_stopped.load(Ordering::Relaxed)); + assert!(!running_dependent_stopped.load(Ordering::Relaxed)); + assert!(unanswered_started.load(Ordering::Relaxed)); + assert!(!unanswered_stopped.load(Ordering::Relaxed)); + assert!(units.is_running()); + assert_eq!(unit_state(&units, "new-dependency"), State::StartUncertain); + assert_eq!(unit_state(&units, "unanswered"), State::StartUncertain); + + units.stop().await; + assert!(dependency_stopped.load(Ordering::Relaxed)); + assert!(running_dependent_stopped.load(Ordering::Relaxed)); + assert!(unanswered_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); + assert_eq!(unit_state(&units, "new-dependency"), State::StartUncertain); + let retry_error = units.start().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); + assert!(units.reset().await.is_err()); + assert!(units.save().await.is_err()); + assert!(units.restore(Vec::new()).await.is_err()); +} + +#[async_test] +async fn test_started_unit_exit_during_rollback_returns_error(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let dependency_started = Arc::new(AtomicBool::new(false)); + let dependency_stopped = Arc::new(AtomicBool::new(false)); + let dependency = units + .add("dependency") + .spawn(&driver, |recv| { + run_unit( + recording_unit(&dependency_started, &dependency_stopped), + recv, + ) + }) + .unwrap(); + // A unit whose task answers its start request and then exits. + let exited = units + .add("exited") + .depends_on(dependency.handle()) + .spawn(&driver, |mut recv| async move { + let Some(StateRequest::Start(response)) = recv.next().await else { + panic!("unexpected request"); + }; + response.complete(()); + }) + .unwrap(); + let unanswered_stopped = Arc::new(AtomicBool::new(false)); + let _unanswered = units + .add("unanswered") + .spawn(&driver, |recv| { + run_unanswering_start( + recording_unit(&Arc::new(AtomicBool::new(false)), &unanswered_stopped), + recv, + ) + }) + .unwrap(); + + let error = units.start().await.unwrap_err().to_string(); + assert!( + error.contains("failed_start_rollback stop failed"), + "{error}" + ); + assert!( + error.contains("failed_start_rollback stop skipped"), + "{error}" + ); + assert!(dependency_started.load(Ordering::Relaxed)); + assert!(!dependency_stopped.load(Ordering::Relaxed)); + // The dependency is still running, so a later stop must be able to stop it. + assert!(units.is_running()); + + exited.remove().await; + units.stop().await; + assert!(dependency_stopped.load(Ordering::Relaxed)); + assert!(unanswered_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); + let retry_error = units.start().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); + assert!(units.reset().await.is_err()); + assert!(units.save().await.is_err()); + assert!(units.restore(Vec::new()).await.is_err()); +} + +#[async_test] +async fn test_uncertain_start_blocks_dependency_rollback(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let dependency_started = Arc::new(AtomicBool::new(false)); + let dependency_stopped = Arc::new(AtomicBool::new(false)); + let dependency = units + .add("dependency") + .spawn(&driver, |recv| { + run_unit( + recording_unit(&dependency_started, &dependency_stopped), + recv, + ) + }) + .unwrap(); + let unanswered_started = Arc::new(AtomicBool::new(false)); + let unanswered_stopped = Arc::new(AtomicBool::new(false)); + let _unanswered = units + .add("unanswered") + .depends_on(dependency.handle()) + .spawn(&driver, |recv| { + run_unanswering_start( + recording_unit(&unanswered_started, &unanswered_stopped), + recv, + ) + }) + .unwrap(); + + let error = units.start().await.unwrap_err().to_string(); + assert!( + error.contains("failed_start_rollback stop skipped"), + "{error}" + ); + assert!(dependency_started.load(Ordering::Relaxed)); + assert!(unanswered_started.load(Ordering::Relaxed)); + assert!(!dependency_stopped.load(Ordering::Relaxed)); + assert!(units.is_running()); + assert_eq!(unit_state(&units, "dependency"), State::StartUncertain); + + units.stop().await; + assert!(unanswered_stopped.load(Ordering::Relaxed)); + assert!(dependency_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); + assert_eq!(unit_state(&units, "dependency"), State::StartUncertain); + assert!(units.reset().await.is_err()); + assert!(units.save().await.is_err()); + assert!(units.restore(Vec::new()).await.is_err()); +} + +#[async_test] +async fn test_initial_start_rollback_can_be_retried(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let started = Arc::new(AtomicBool::new(false)); + let stopped = Arc::new(AtomicBool::new(false)); + let stop_requests = Arc::new(AtomicUsize::new(0)); + // A unit that loses the response to its first stop request without + // stopping. + let _unit = units + .add("rollback-retry") + .spawn(&driver, |mut recv| { + let mut unit = recording_unit(&started, &stopped); + let stop_requests = stop_requests.clone(); + async move { + while let Some(request) = recv.next().await { + match request { + StateRequest::Stop(response) + if stop_requests.fetch_add(1, Ordering::Relaxed) == 0 => + { + drop(response); + } + request => request.apply(&mut unit).await, + } + } + unit + } + }) + .unwrap(); + let unanswered_stopped = Arc::new(AtomicBool::new(false)); + let _unanswered = units + .add("unanswered") + .spawn(&driver, |recv| { + run_unanswering_start( + recording_unit(&Arc::new(AtomicBool::new(false)), &unanswered_stopped), + recv, + ) + }) + .unwrap(); + + let error = units.start().await.unwrap_err().to_string(); + assert!( + error.contains("failed_start_rollback stop failed"), + "{error}" + ); + assert!(started.load(Ordering::Relaxed)); + assert!(!stopped.load(Ordering::Relaxed)); + assert_eq!(stop_requests.load(Ordering::Relaxed), 1); + assert!(units.is_running()); + + units.stop().await; + assert!(stopped.load(Ordering::Relaxed)); + assert_eq!(stop_requests.load(Ordering::Relaxed), 2); + assert!(unanswered_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); +} + +#[async_test] +async fn test_teardown_stop_after_unit_exits_before_answering_start(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let dependency_started = Arc::new(AtomicBool::new(false)); + let dependency_stopped = Arc::new(AtomicBool::new(false)); + let dependency = units + .add("dependency") + .spawn(&driver, |recv| { + run_unit( + recording_unit(&dependency_started, &dependency_stopped), + recv, + ) + }) + .unwrap(); + // A unit whose task answers a start and a stop, and then exits before it + // answers the next start, as a task that fails while the VM restarts after + // a failed save would. + let _exited = units + .add("exited") + .depends_on(dependency.handle()) + .spawn(&driver, |mut recv| async move { + for _ in 0..2 { + match recv.next().await { + Some(StateRequest::Start(response)) => response.complete(()), + Some(StateRequest::Stop(response)) => response.complete(()), + _ => panic!("unexpected request"), + } + } + }) + .unwrap(); + units.start().await.unwrap(); + units.stop().await; + dependency_stopped.store(false, Ordering::Relaxed); + + let error = units.start().await.unwrap_err().to_string(); + assert!(error.contains("exited"), "{error}"); + assert!(dependency_started.load(Ordering::Relaxed)); + assert!(!dependency_stopped.load(Ordering::Relaxed)); + assert!(units.is_running()); + + // Tearing the VM down stops the units without the exited unit answering. + units.stop_for_teardown().await; + assert!(dependency_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); + assert_eq!(unit_state(&units, "exited"), State::StartUncertain); + assert_eq!(unit_state(&units, "dependency"), State::StartUncertain); +} From 7a954ee45cf28c26cdc578eaa594123e8a1cd466 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Tue, 3 Nov 2026 16:10:43 -0800 Subject: [PATCH 2/7] state_unit: split the start into functions `start_with_rollback` builds the start operations and records their outcomes in two long blocks. Move each into a function of its own, `start_operations` and `record_start_results`, so that the function reads as the steps of the start. Behavior does not change. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- vmm_core/state_unit/src/start.rs | 195 +++++++++++++++++-------------- 1 file changed, 106 insertions(+), 89 deletions(-) diff --git a/vmm_core/state_unit/src/start.rs b/vmm_core/state_unit/src/start.rs index 13c18f7693b..a2503efe02a 100644 --- a/vmm_core/state_unit/src/start.rs +++ b/vmm_core/state_unit/src/start.rs @@ -10,107 +10,39 @@ use super::StateRequest; use super::StateUnits; use super::state_change; use futures::future::join_all; +use std::future::Future; +use std::sync::Arc; use std::time::Instant; +/// The outcome of a unit's start. +enum StartResult { + Started, + DependencyFailed, + Uncertain(anyhow::Error), +} + +/// The outcome of a unit's start, with the unit's name and ID. +type UnitStartResult = (Arc, u64, StartResult); + impl StateUnits { /// Starts all stopped units in dependency order. /// /// Fails without starting any unit if one is in a terminal state. A unit - /// does not start if one of its dependencies - /// did not start. If a unit's task does not answer its start request, the - /// units that did start are stopped again, and the unit is left in a - /// terminal state that fails later starts. A started unit that cannot be - /// stopped again, because a unit that depends on it was not stopped, is - /// left running in that terminal state too, and the units stay running so - /// that [`StateUnits::stop`] can stop them later. + /// does not start if one of its dependencies did not start. If a unit's + /// task does not answer its start request, the units that did start are + /// stopped again, and the unit is left in a terminal state that fails later + /// starts. A started unit that cannot be stopped again, because a unit that + /// depends on it was not stopped, is left running in that terminal state + /// too, and the units stay running so that [`StateUnits::stop`] can stop + /// them later. pub(super) async fn start_with_rollback(&mut self) -> anyhow::Result<()> { - enum StartResult { - Started, - DependencyFailed, - Uncertain(anyhow::Error), - } - let was_running = self.running; - let mut operations = Vec::new(); - let ready_set; - { - let mut inner = self.inner.lock(); - if let Some(unit) = inner - .units - .values() - .find(|unit| !matches!(unit.state, State::Stopped | State::Running)) - { - anyhow::bail!( - "state unit '{}' cannot be started from terminal state {:?}", - unit.name, - unit.state - ); - } - ready_set = inner.ready_set(None); - for (&id, unit) in &mut inner.units { - if unit.state == State::Running { - ready_set.done(id, true); - continue; - } - assert_eq!( - unit.state, - State::Stopped, - "unit {} is not stopped before start", - unit.name, - ); - let name = unit.name.clone(); - let dependencies = unit.dependencies.iter().copied().collect::>(); - let ready_set = ready_set.clone(); - let start = state_change(name.clone(), unit, StateRequest::Start, Some(())); - operations.push(async move { - if !ready_set.wait("start", id, &dependencies).await { - ready_set.done(id, false); - return (name, id, StartResult::DependencyFailed); - } - - let result = match start.await { - Ok(Some(())) => StartResult::Started, - Ok(None) => { - StartResult::Uncertain(anyhow::anyhow!("start request was not sent")) - } - Err(error) => StartResult::Uncertain(error.into()), - }; - ready_set.done(id, matches!(result, StartResult::Started)); - (name, id, result) - }); - unit.state = State::Starting; - } - } - + let operations = self.start_operations()?; let start = Instant::now(); let results = join_all(operations).await; tracing::info!(duration = ?Instant::now() - start, "state-unit start complete"); - let mut failures = Vec::new(); - let mut started_ids = Vec::new(); - { - let mut inner = self.inner.lock(); - for (name, id, result) in results { - let Some(unit) = inner.units.get_mut(&id) else { - continue; - }; - match result { - StartResult::Started => { - unit.state = State::Running; - started_ids.push(id); - } - StartResult::DependencyFailed => { - unit.state = State::Stopped; - failures.push(format!("{name}: a dependency did not start")); - } - StartResult::Uncertain(error) => { - unit.state = State::StartUncertain; - failures.push(format!("{name}: {error:#}")); - } - } - } - } - + let (started_ids, mut failures) = self.record_start_results(results); if failures.is_empty() { self.running = true; Ok(()) @@ -122,6 +54,91 @@ impl StateUnits { } } + /// Marks the stopped units as starting, and returns the operations that + /// start them in dependency order. + /// + /// Fails without changing any unit if one is in a terminal state. + fn start_operations( + &self, + ) -> anyhow::Result + use<>>> { + let mut inner = self.inner.lock(); + if let Some(unit) = inner + .units + .values() + .find(|unit| !matches!(unit.state, State::Stopped | State::Running)) + { + anyhow::bail!( + "state unit '{}' cannot be started from terminal state {:?}", + unit.name, + unit.state + ); + } + let ready_set = inner.ready_set(None); + let mut operations = Vec::new(); + for (&id, unit) in &mut inner.units { + if unit.state == State::Running { + ready_set.done(id, true); + continue; + } + assert_eq!( + unit.state, + State::Stopped, + "unit {} is not stopped before start", + unit.name, + ); + let name = unit.name.clone(); + let dependencies = unit.dependencies.iter().copied().collect::>(); + let ready_set = ready_set.clone(); + let start = state_change(name.clone(), unit, StateRequest::Start, Some(())); + operations.push(async move { + if !ready_set.wait("start", id, &dependencies).await { + ready_set.done(id, false); + return (name, id, StartResult::DependencyFailed); + } + + let result = match start.await { + Ok(Some(())) => StartResult::Started, + Ok(None) => { + StartResult::Uncertain(anyhow::anyhow!("start request was not sent")) + } + Err(error) => StartResult::Uncertain(error.into()), + }; + ready_set.done(id, matches!(result, StartResult::Started)); + (name, id, result) + }); + unit.state = State::Starting; + } + Ok(operations) + } + + /// Moves each unit to the state that the outcome of its start leaves it in, + /// and returns the IDs of the units that started and the failures. + fn record_start_results(&self, results: Vec) -> (Vec, Vec) { + let mut started_ids = Vec::new(); + let mut failures = Vec::new(); + let mut inner = self.inner.lock(); + for (name, id, result) in results { + let Some(unit) = inner.units.get_mut(&id) else { + continue; + }; + match result { + StartResult::Started => { + unit.state = State::Running; + started_ids.push(id); + } + StartResult::DependencyFailed => { + unit.state = State::Stopped; + failures.push(format!("{name}: a dependency did not start")); + } + StartResult::Uncertain(error) => { + unit.state = State::StartUncertain; + failures.push(format!("{name}: {error:#}")); + } + } + } + (started_ids, failures) + } + /// Stops the units in `started_ids` again in reverse dependency order. /// /// Every other unit that is not stopped, such as one that was already From fa3c129dd19d92cd848fe25f598c25719af4d5de Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Tue, 3 Nov 2026 16:12:25 -0800 Subject: [PATCH 3/7] state_unit: keep the units running while a start is uncertain A unit whose task drops its start request without answering it may still be running. After an initial start fails, the rollback stops the units that did start, but the units are then reported as stopped even though that unit may still run, and a later `StateUnits::stop` panics on its running assertion instead of stopping it. After a failed start, keep the units running while any of them is not stopped, as is a unit whose start is uncertain, so that `StateUnits::stop` can stop it. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- vmm_core/state_unit/src/lib.rs | 5 +++ vmm_core/state_unit/src/start.rs | 15 ++++++--- vmm_core/state_unit/src/tests/start.rs | 45 ++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 5 deletions(-) diff --git a/vmm_core/state_unit/src/lib.rs b/vmm_core/state_unit/src/lib.rs index d6e9ef772a5..d4ea5dbdbd0 100644 --- a/vmm_core/state_unit/src/lib.rs +++ b/vmm_core/state_unit/src/lib.rs @@ -526,6 +526,11 @@ impl StateUnits { } /// Starts all the state units. + /// + /// If a unit's task does not answer its start request, the start fails, + /// and the units that did start are stopped again. If a unit may still be + /// running after that, [`StateUnits::is_running`] returns true, so that + /// [`StateUnits::stop`] can stop it. pub async fn start(&mut self) -> anyhow::Result<()> { self.restore_allowed = false; self.start_with_rollback().await diff --git a/vmm_core/state_unit/src/start.rs b/vmm_core/state_unit/src/start.rs index a2503efe02a..896a6fa0559 100644 --- a/vmm_core/state_unit/src/start.rs +++ b/vmm_core/state_unit/src/start.rs @@ -33,8 +33,8 @@ impl StateUnits { /// stopped again, and the unit is left in a terminal state that fails later /// starts. A started unit that cannot be stopped again, because a unit that /// depends on it was not stopped, is left running in that terminal state - /// too, and the units stay running so that [`StateUnits::stop`] can stop - /// them later. + /// too. Because a unit in that state may be running, the units then stay + /// running, so that [`StateUnits::stop`] can stop it. pub(super) async fn start_with_rollback(&mut self) -> anyhow::Result<()> { let was_running = self.running; let operations = self.start_operations()?; @@ -47,9 +47,14 @@ impl StateUnits { self.running = true; Ok(()) } else { - let cleanup_failures = self.stop_after_failed_start(&started_ids).await; - self.running = was_running || !cleanup_failures.is_empty(); - failures.extend(cleanup_failures); + failures.extend(self.stop_after_failed_start(&started_ids).await); + self.running = was_running + || self + .inner + .lock() + .units + .values() + .any(|unit| unit.state != State::Stopped); anyhow::bail!("state units could not start: {}", failures.join("; ")) } } diff --git a/vmm_core/state_unit/src/tests/start.rs b/vmm_core/state_unit/src/tests/start.rs index 41bd5d203b1..c85a88e90f7 100644 --- a/vmm_core/state_unit/src/tests/start.rs +++ b/vmm_core/state_unit/src/tests/start.rs @@ -141,7 +141,52 @@ async fn an_unanswered_start_rolls_back_and_blocks_dependents(driver: DefaultDri assert!(independent_started.load(Ordering::Relaxed)); assert!(independent_stopped.load(Ordering::Relaxed)); assert!(!dependent_started.load(Ordering::Relaxed)); + // The unanswered unit may be running. + assert!(units.is_running()); + let retry_error = units.start().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); +} + +#[async_test] +async fn test_uncertain_start_keeps_units_running_until_stopped(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let independent_started = Arc::new(AtomicBool::new(false)); + let independent_stopped = Arc::new(AtomicBool::new(false)); + let _independent = units + .add("independent") + .spawn(&driver, |recv| { + run_unit( + recording_unit(&independent_started, &independent_stopped), + recv, + ) + }) + .unwrap(); + let unanswered_started = Arc::new(AtomicBool::new(false)); + let unanswered_stopped = Arc::new(AtomicBool::new(false)); + let _unanswered = units + .add("unanswered") + .spawn(&driver, |recv| { + run_unanswering_start( + recording_unit(&unanswered_started, &unanswered_stopped), + recv, + ) + }) + .unwrap(); + + let error = units.start().await.unwrap_err().to_string(); + assert!(error.contains("unanswered"), "{error}"); + assert!(independent_started.load(Ordering::Relaxed)); + assert!(independent_stopped.load(Ordering::Relaxed)); + assert!(unanswered_started.load(Ordering::Relaxed)); + assert!(!unanswered_stopped.load(Ordering::Relaxed)); + // The unanswered unit is running, so a stop must still be able to stop it. + assert!(units.is_running()); + assert_eq!(unit_state(&units, "unanswered"), State::StartUncertain); + + units.stop().await; + assert!(unanswered_stopped.load(Ordering::Relaxed)); assert!(!units.is_running()); + assert_eq!(unit_state(&units, "unanswered"), State::StartUncertain); let retry_error = units.start().await.unwrap_err(); assert!(retry_error.to_string().contains("terminal state")); } From 118a0337f756af29b55d87cf8c67e3b017204d99 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Tue, 3 Nov 2026 16:19:03 -0800 Subject: [PATCH 4/7] openvmm_core: tear the VM down when a reset cannot start its devices A reset with firmware reload starts the state units to assign PCI resources. If that start fails, it returns before stopping the units, which a unit whose start is uncertain keeps running, while the reset has already paused the VM. The worker then keeps serving with the state units and the VM disagreeing on whether the VM runs, and the next reset skips stopping the units and panics on the running assertion of `StateUnits::reset`. The reset pauses the VM, so state units that still run after a failed reset were left by a failed start, and they can never start again. Report the failure and tear the VM down then, as after a failed restart. The teardown stops those units, since it stops the state units whenever they run. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- openvmm/openvmm_core/src/worker/dispatch.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/openvmm/openvmm_core/src/worker/dispatch.rs b/openvmm/openvmm_core/src/worker/dispatch.rs index 6b6b558aa58..5c0ea9b3297 100644 --- a/openvmm/openvmm_core/src/worker/dispatch.rs +++ b/openvmm/openvmm_core/src/worker/dispatch.rs @@ -3864,7 +3864,16 @@ impl LoadedVm { Event::VmRpc(Err(_)) => break, Event::VmRpc(Ok(message)) => match message { VmRpc::Reset(rpc) => { - rpc.handle_failable(async |()| self.reset(true).await).await + let result = self.reset(true).await; + // The reset pauses the VM, so state units that still + // run after it fails were left by a failed start, and + // can never start again. Tear the VM down, as after a + // failed restart. + let start_failed = result.is_err() && self.state_units.is_running(); + rpc.complete(result.map_err(RemoteError::new)); + if start_failed { + break; + } } VmRpc::ClearHalt(rpc) => { rpc.handle(async |()| self.inner.partition_unit.clear_halt().await) From a4b12f216fea1af5f18ba2f2c98ec3df22afca44 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Tue, 3 Nov 2026 16:19:18 -0800 Subject: [PATCH 5/7] openvmm_core: add a hot-added VMBus device once it starts VMBus hot-add pushes the new device before starting it, and pops it again if the start fails. Push it only once it has started, as relayed vPCI hot-add does, so that the failure path only removes the unit and drops its channel unit, which revokes the channel. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- openvmm/openvmm_core/src/worker/dispatch.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/openvmm/openvmm_core/src/worker/dispatch.rs b/openvmm/openvmm_core/src/worker/dispatch.rs index 5c0ea9b3297..e2297dec9a8 100644 --- a/openvmm/openvmm_core/src/worker/dispatch.rs +++ b/openvmm/openvmm_core/src/worker/dispatch.rs @@ -3926,16 +3926,12 @@ impl LoadedVm { resource, ) .await?; - self.inner.vmbus_devices.push(device); if let Err(error) = self.state_units.start_stopped_units().await { - let device = self - .inner - .vmbus_devices - .pop() - .expect("device was just pushed"); + // Dropping the channel unit revokes the channel. drop(device.remove().await); return Err(error); } + self.inner.vmbus_devices.push(device); anyhow::Ok(()) }) .await From 7ccd934571b6d98935eac461026f6bdaa6ace5a4 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Tue, 3 Nov 2026 16:19:37 -0800 Subject: [PATCH 6/7] underhill_core: define the restart outcome at file scope `RestartOutcome` is defined inside the handler of a worker restart, deep in the request loop of `LoadedVm::run`. Define it at file scope, next to the state that a restart produces, and give its variants their concrete types. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- openhcl/underhill_core/src/dispatch/mod.rs | 23 ++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/openhcl/underhill_core/src/dispatch/mod.rs b/openhcl/underhill_core/src/dispatch/mod.rs index b7a0916189d..4b52cdfbf33 100644 --- a/openhcl/underhill_core/src/dispatch/mod.rs +++ b/openhcl/underhill_core/src/dispatch/mod.rs @@ -226,6 +226,19 @@ pub struct LoadedVmState { pub control_send: mesh::Sender, } +/// The outcome of a worker restart request. +#[expect(clippy::large_enum_variant)] +enum RestartOutcome { + /// The VM state was saved, so the worker can restart. + Restart(FailableRpc<(), T>, ServicingState), + /// The restart failed, but the VM is back in its previous state, so the + /// worker can keep serving. + Recovered, + /// The restart failed and the VM could not be resumed, so the worker + /// cannot serve any more. + Unrecoverable, +} + impl LoadedVm { /// Start running the VM which will start running VTL0. pub async fn run( @@ -328,16 +341,6 @@ impl LoadedVm { Event::WorkerRpc(message) => match message { WorkerRpc::Stop => break None, WorkerRpc::Restart(rpc) => { - enum RestartOutcome { - Restart(R, S), - // The restart failed, but the VM is back in its - // previous state, so this worker can keep serving. - Recovered, - // The restart failed and the VM could not be - // resumed, so this worker cannot serve any more. - Unrecoverable, - } - let outcome = async { let running = self.stop().await; match self From 3db2c7495029745e2fcbee00cf88220dc2afaf26 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Tue, 3 Nov 2026 10:39:37 -0800 Subject: [PATCH 7/7] vmcore: add device input quiesce and resume A snapshot has to be taken at a vCPU boundary where no host input, such as console or network input, is still being delivered to the guest. Nothing lets the VMM tell a device to stop accepting host input while the VM keeps running. Add `quiesce_input` and `resume_input` to `ChangeDeviceState` and to `StateUnit`. A device or unit that implements them stops accepting new host input while keeping its runtime state, and resumes it after a failed snapshot transaction. Both default to doing nothing. The hooks run synchronously and return an `InputHookCompletion`, a future for the work that they must wait for, such as a request to a device task. vCPUs still run while input is quiesced, so the chipset device state unit runs a device's hook with the device lock held but not closed, and awaits the completion only after it releases the lock. A guest MMIO or PIO access that contends for the lock then waits only for the synchronous hook, never for asynchronous work that might need the executor that the access blocks, as when a VP and the unit share a CPU's thread in OpenHCL. While the completion is pending, the unit keeps polling the device, and the chipset device worker keeps serving guest requests and polling its device, so a completion may wait for the device to progress. A transaction sends a unit no other request before its hook is answered, but a dropped transaction can. The unit then defers any request but an inspection until the completion finishes, as it defers its exit when its channel closes, and keeps polling the device meanwhile. Since the hooks are not `async`, the object-safe wrappers of every chipset device forward them without boxing a future. Add the corresponding `QuiesceInput` and `ResumeInput` state requests, and `StateUnits::quiesce_input_for_save` and `resume_input_after_save`, which send them to every running unit, wait for every unit to answer, and fail if any unit fails, cannot be reached, or is in a terminal state. Like a stop, they have no deadline: a hook that never completes stalls the transaction until the process that drives the VMM ends it. The erased chipset device, and the chipset device worker and its proxy, forward the requests to the device, and the proxy keeps serving guest memory requests while it waits for a remote hook. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- vm/chipset_device_resources/src/lib.rs | 17 + vm/vmcore/src/device_state.rs | 60 +++ vmm_core/state_unit/src/lib.rs | 49 ++ vmm_core/state_unit/src/quiesce.rs | 91 ++++ vmm_core/state_unit/src/tests/quiesce.rs | 286 ++++++++++++ vmm_core/vmotherboard/Cargo.toml | 1 + .../chipset/backing/arc_mutex/state_unit.rs | 429 +++++++++++++++++- workers/chipset_device_worker/src/protocol.rs | 4 + workers/chipset_device_worker/src/proxy.rs | 122 ++++- workers/chipset_device_worker/src/worker.rs | 21 + 10 files changed, 1067 insertions(+), 13 deletions(-) create mode 100644 vmm_core/state_unit/src/quiesce.rs create mode 100644 vmm_core/state_unit/src/tests/quiesce.rs diff --git a/vm/chipset_device_resources/src/lib.rs b/vm/chipset_device_resources/src/lib.rs index 9d2bab63d58..8abd0872803 100644 --- a/vm/chipset_device_resources/src/lib.rs +++ b/vm/chipset_device_resources/src/lib.rs @@ -13,6 +13,7 @@ use std::ops::RangeInclusive; use vm_resource::CanResolveTo; use vm_resource::kind::ChipsetDeviceHandleKind; use vmcore::device_state::ChangeDeviceState; +use vmcore::device_state::InputHookCompletion; use vmcore::line_interrupt::LineInterrupt; use vmcore::save_restore::AsyncProtobufSaveRestore; use vmcore::vm_task::VmTaskDriverSource; @@ -92,6 +93,8 @@ pub trait ConfigureChipsetDevice: Send { #[async_trait] trait DynChipsetDevice: ChipsetDevice + AsyncProtobufSaveRestore + InspectMut { fn start(&mut self); + fn quiesce_input(&mut self) -> InputHookCompletion; + fn resume_input(&mut self) -> InputHookCompletion; async fn stop(&mut self); async fn reset(&mut self); async fn teardown(self: Box); @@ -104,6 +107,12 @@ impl InputHookCompletion { + self.quiesce_input() + } + fn resume_input(&mut self) -> InputHookCompletion { + self.resume_input() + } async fn stop(&mut self) { self.stop().await } @@ -136,6 +145,14 @@ impl ChangeDeviceState for ErasedChipsetDevice { self.0.start() } + fn quiesce_input(&mut self) -> InputHookCompletion { + self.0.quiesce_input() + } + + fn resume_input(&mut self) -> InputHookCompletion { + self.0.resume_input() + } + async fn stop(&mut self) { self.0.stop().await } diff --git a/vm/vmcore/src/device_state.rs b/vm/vmcore/src/device_state.rs index 4dcdcb24156..92bf2ea5941 100644 --- a/vm/vmcore/src/device_state.rs +++ b/vm/vmcore/src/device_state.rs @@ -6,6 +6,41 @@ #![forbid(unsafe_code)] use std::future::Future; +use std::pin::Pin; +use std::task::Context; +use std::task::Poll; + +/// The completion of a device's input hook, such as +/// [`ChangeDeviceState::quiesce_input`]. +/// +/// An input hook runs synchronously, with exclusive access to the device. The +/// work that it must wait for, such as a request to a device task, completes in +/// this future instead, which the caller awaits after it gives up that access. +#[must_use = "an input hook is complete only once its completion is awaited"] +pub struct InputHookCompletion(Option> + Send>>>); + +impl InputHookCompletion { + /// Returns the completion of a hook that has nothing left to wait for. + pub fn ready() -> Self { + Self(None) + } + + /// Returns a completion that waits for `future`. + pub fn new(future: impl 'static + Send + Future>) -> Self { + Self(Some(Box::pin(future))) + } +} + +impl Future for InputHookCompletion { + type Output = anyhow::Result<()>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + match &mut self.get_mut().0 { + Some(future) => future.as_mut().poll(cx), + None => Poll::Ready(Ok(())), + } + } +} /// Trait for transitioning device state. pub trait ChangeDeviceState { @@ -27,6 +62,31 @@ pub trait ChangeDeviceState { // to finish, which may require a bunch of other changes. fn start(&mut self); + /// Stops accepting new host input before a snapshot vCPU boundary. + /// + /// This runs with exclusive access to the device, which for a chipset + /// device means with its lock held while vCPUs may still access the device, + /// so it must not block. The work that stopping input must wait for, such + /// as a request to a device task, belongs in the returned completion, which + /// the caller awaits after it releases the device. Meanwhile, the device + /// keeps serving vCPUs and being polled. + /// + /// [`ChangeDeviceState::resume_input`] may be called even if the completion + /// fails or is not awaited. + fn quiesce_input(&mut self) -> InputHookCompletion { + InputHookCompletion::ready() + } + + /// Resumes host input after a failed snapshot transaction. + /// + /// This must be safe whether or not [`ChangeDeviceState::quiesce_input`] + /// completed and idempotent across repeated recovery attempts. Like that + /// hook, it must not block, and the caller awaits its completion after it + /// releases the device. + fn resume_input(&mut self) -> InputHookCompletion { + InputHookCompletion::ready() + } + /// Pauses guest-request processing in a save-consistent state. /// /// After this returns, the device must not process additional guest requests diff --git a/vmm_core/state_unit/src/lib.rs b/vmm_core/state_unit/src/lib.rs index d4ea5dbdbd0..8a8192967c1 100644 --- a/vmm_core/state_unit/src/lib.rs +++ b/vmm_core/state_unit/src/lib.rs @@ -30,6 +30,7 @@ #![forbid(unsafe_code)] +mod quiesce; mod start; use futures::FutureExt; @@ -41,6 +42,7 @@ use inspect::InspectMut; use mesh::MeshPayload; use mesh::Receiver; use mesh::Sender; +use mesh::error::RemoteError; use mesh::payload::Protobuf; use mesh::rpc::FailableRpc; use mesh::rpc::Rpc; @@ -64,6 +66,7 @@ use std::sync::atomic::Ordering; use std::time::Instant; use thiserror::Error; use tracing::Instrument; +use vmcore::device_state::InputHookCompletion; use vmcore::save_restore::RestoreError; use vmcore::save_restore::SaveError; use vmcore::save_restore::SavedStateBlob; @@ -82,6 +85,18 @@ pub enum StateRequest { /// The handler must be safe after any delivered start attempt. Stop(Rpc<(), ()>), + /// Stop accepting new host input before a snapshot vCPU boundary. + /// + /// A [`StateRequest::ResumeInput`] may follow even if this request returns + /// an error or its response is not observed. + QuiesceInput(FailableRpc<(), ()>), + + /// Resume host input after a failed snapshot transaction. + /// + /// The handler must be safe whether or not input quiesce completed and + /// idempotent across repeated recovery attempts. + ResumeInput(FailableRpc<(), ()>), + /// Reset a stopped unit to initial state. Reset(FailableRpc<(), ()>), @@ -108,6 +123,26 @@ pub trait StateUnit: InspectMut { /// Stop asynchronous processing. async fn stop(&mut self); + /// Stops accepting new host input while preserving runtime state. + /// + /// This runs synchronously, and the work that it must wait for completes + /// in the returned completion, which is awaited before the request is + /// answered. [`StateUnit::resume_input`] may be called even if the + /// completion fails or is not awaited. + fn quiesce_input(&mut self) -> InputHookCompletion { + InputHookCompletion::ready() + } + + /// Resumes host input after a failed snapshot transaction. + /// + /// This must be safe whether or not [`StateUnit::quiesce_input`] completed + /// and idempotent across repeated recovery attempts. Like that hook, it + /// runs synchronously, and its completion is awaited before the request is + /// answered. + fn resume_input(&mut self) -> InputHookCompletion { + InputHookCompletion::ready() + } + /// Reset to initial state. /// /// Must only be called while stopped. @@ -173,6 +208,8 @@ impl StateRequest { StateRequest::Start(_) | StateRequest::Stop(_) + | StateRequest::QuiesceInput(_) + | StateRequest::ResumeInput(_) | StateRequest::Reset(_) | StateRequest::Save(_) | StateRequest::Restore(_) => { @@ -201,6 +238,8 @@ impl StateRequest { match self { StateRequest::Start(rpc) => rpc.handle(async |()| unit.start().await).await, StateRequest::Stop(rpc) => rpc.handle(async |()| unit.stop().await).await, + StateRequest::QuiesceInput(rpc) => complete_input_hook(rpc, unit.quiesce_input()).await, + StateRequest::ResumeInput(rpc) => complete_input_hook(rpc, unit.resume_input()).await, StateRequest::Reset(rpc) => rpc.handle_failable(async |()| unit.reset().await).await, StateRequest::Save(rpc) => rpc.handle_failable(async |()| unit.save().await).await, StateRequest::Restore(rpc) => { @@ -212,6 +251,15 @@ impl StateRequest { } } +/// Answers `rpc` once the completion of an input hook finishes. +/// +/// This is not generic, so that the state requests of every unit type share +/// it. +async fn complete_input_hook(rpc: FailableRpc<(), ()>, completion: InputHookCompletion) { + let result = completion.await; + rpc.complete(result.map_err(RemoteError::new)); +} + /// A set of state units. #[derive(Debug)] pub struct StateUnits { @@ -1127,6 +1175,7 @@ impl Ready { #[cfg(test)] mod tests { + mod quiesce; mod start; use super::StateTransitionErrorKind; diff --git a/vmm_core/state_unit/src/quiesce.rs b/vmm_core/state_unit/src/quiesce.rs new file mode 100644 index 00000000000..fd27ef627f7 --- /dev/null +++ b/vmm_core/state_unit/src/quiesce.rs @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Snapshot input gating: stopping host input before the snapshot vCPU +//! boundary, and resuming it after a failed snapshot transaction. + +use super::State; +use super::StateRequest; +use super::StateUnits; +use super::state_change; +use futures::future::join_all; +use mesh::rpc::FailableRpc; + +impl StateUnits { + /// Stops host-input producers in running units before establishing a + /// snapshot vCPU boundary. + /// + /// Waits for every running unit, without a deadline, so a unit that does + /// not answer stalls the transaction. Stopped units are skipped, and + /// terminal lifecycle states fail the transaction. Drive the returned + /// future to completion: if it is dropped, the units still run the hooks + /// that it requested, and a unit's next request may arrive before its hook + /// completes. + pub async fn quiesce_input_for_save(&mut self) -> anyhow::Result<()> { + self.run_input_op("quiesce_input", StateRequest::QuiesceInput) + .await + } + + /// Resumes host-input producers in running units before releasing a failed + /// snapshot boundary. + /// + /// Waits for every running unit, without a deadline, so a unit that does + /// not answer stalls the transaction. Stopped units are skipped, and + /// terminal lifecycle states fail the transaction. As for + /// [`StateUnits::quiesce_input_for_save`], drive the returned future to + /// completion. + pub async fn resume_input_after_save(&mut self) -> anyhow::Result<()> { + self.run_input_op("resume_input", StateRequest::ResumeInput) + .await + } + + async fn run_input_op( + &mut self, + operation: &'static str, + request: impl Copy + FnOnce(FailableRpc<(), ()>) -> StateRequest, + ) -> anyhow::Result<()> { + let mut operations = Vec::new(); + let mut failures = Vec::new(); + { + let inner = self.inner.lock(); + for unit in inner.units.values() { + match unit.state { + State::Running => { + let name = unit.name.clone(); + let operation = state_change(name.clone(), unit, request, Some(())); + operations.push(async move { (name, operation.await) }); + } + State::Stopped => {} + State::StartUncertain => { + failures.push(format!( + "{}: unit is in terminal state {:?}", + unit.name, unit.state + )); + } + state => { + failures.push(format!( + "{}: unit is in unexpected state {state:?}", + unit.name + )); + } + } + } + } + + for (name, result) in join_all(operations).await { + match result { + Ok(Some(Ok(()))) => {} + Ok(Some(Err(error))) => failures.push(format!("{name}: {error}")), + Ok(None) => failures.push(format!("{name}: request was not sent")), + Err(error) => failures.push(format!("{name}: {error}")), + } + } + + anyhow::ensure!( + failures.is_empty(), + "{operation} failed: {}", + failures.join("; ") + ); + Ok(()) + } +} diff --git a/vmm_core/state_unit/src/tests/quiesce.rs b/vmm_core/state_unit/src/tests/quiesce.rs new file mode 100644 index 00000000000..8ca9fa62a3c --- /dev/null +++ b/vmm_core/state_unit/src/tests/quiesce.rs @@ -0,0 +1,286 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Tests for gating host input around the snapshot boundary. + +use crate::StateRequest; +use crate::StateUnit; +use crate::StateUnits; +use crate::run_unit; +use futures::StreamExt; +use inspect::InspectMut; +use mesh::Receiver; +use pal_async::DefaultDriver; +use pal_async::async_test; +use std::sync::Arc; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; +use std::time::Duration; +use std::time::Instant; +use test_with_tracing::test; +use vmcore::device_state::InputHookCompletion; +use vmcore::save_restore::RestoreError; +use vmcore::save_restore::SaveError; +use vmcore::save_restore::SavedStateBlob; + +#[derive(Default)] +struct InputCounts { + quiesces: AtomicUsize, + resumes: AtomicUsize, +} + +struct InputTestUnit { + driver: DefaultDriver, + counts: Arc, + delay: Option, + fail_quiesce: bool, + fail_resume: bool, +} + +impl StateUnit for InputTestUnit { + async fn start(&mut self) {} + + async fn stop(&mut self) {} + + fn quiesce_input(&mut self) -> InputHookCompletion { + self.counts.quiesces.fetch_add(1, Ordering::Relaxed); + let fail = self.fail_quiesce; + let delay = self.delay.map(|delay| (self.driver.clone(), delay)); + InputHookCompletion::new(async move { + if let Some((driver, delay)) = delay { + pal_async::timer::PolledTimer::new(&driver) + .sleep(delay) + .await; + } + anyhow::ensure!(!fail, "intentional input quiesce failure"); + Ok(()) + }) + } + + fn resume_input(&mut self) -> InputHookCompletion { + self.counts.resumes.fetch_add(1, Ordering::Relaxed); + let fail = self.fail_resume; + InputHookCompletion::new(async move { + anyhow::ensure!(!fail, "intentional input resume failure"); + Ok(()) + }) + } + + async fn reset(&mut self) -> anyhow::Result<()> { + Ok(()) + } + + async fn save(&mut self) -> Result, SaveError> { + Ok(None) + } + + async fn restore(&mut self, _state: SavedStateBlob) -> Result<(), RestoreError> { + Err(RestoreError::SavedStateNotSupported) + } +} + +impl InspectMut for InputTestUnit { + fn inspect_mut(&mut self, req: inspect::Request<'_>) { + req.respond(); + } +} + +fn input_unit(driver: &DefaultDriver, counts: Arc) -> InputTestUnit { + InputTestUnit { + driver: driver.clone(), + counts, + delay: None, + fail_quiesce: false, + fail_resume: false, + } +} + +/// Runs `unit` like [`run_unit`], but after answering `answered_starts` start +/// requests, starts the unit for each later one without answering it, so that +/// the start is uncertain while the unit still answers other requests. +async fn run_unit_leaving_starts_unanswered( + mut unit: T, + mut recv: Receiver, + answered_starts: usize, +) -> T { + let mut starts = 0; + while let Some(request) = recv.next().await { + match request { + StateRequest::Start(response) if starts >= answered_starts => { + unit.start().await; + drop(response); + } + request => { + if matches!(request, StateRequest::Start(_)) { + starts += 1; + } + request.apply(&mut unit).await; + } + } + } + unit +} + +#[async_test] +async fn test_input_hooks_run_only_for_running_units(driver: DefaultDriver) { + let units = StateUnits::new(); + let running_counts = Arc::new(InputCounts::default()); + let _running = units + .add("running") + .spawn(&driver, |recv| { + run_unit(input_unit(&driver, running_counts.clone()), recv) + }) + .unwrap(); + let mut units = units; + units.start().await.unwrap(); + + let stopped_counts = Arc::new(InputCounts::default()); + let _stopped = units + .add("stopped") + .spawn(&driver, |recv| { + run_unit(input_unit(&driver, stopped_counts.clone()), recv) + }) + .unwrap(); + + units.quiesce_input_for_save().await.unwrap(); + units.resume_input_after_save().await.unwrap(); + + assert_eq!(running_counts.quiesces.load(Ordering::Relaxed), 1); + assert_eq!(running_counts.resumes.load(Ordering::Relaxed), 1); + assert_eq!(stopped_counts.quiesces.load(Ordering::Relaxed), 0); + assert_eq!(stopped_counts.resumes.load(Ordering::Relaxed), 0); +} + +#[async_test] +async fn test_input_hook_errors_are_reported_per_unit(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let successful_counts = Arc::new(InputCounts::default()); + let _successful = units + .add("successful") + .spawn(&driver, |recv| { + run_unit(input_unit(&driver, successful_counts.clone()), recv) + }) + .unwrap(); + let failing_counts = Arc::new(InputCounts::default()); + let _failing = units + .add("failing") + .spawn(&driver, |recv| { + let mut unit = input_unit(&driver, failing_counts.clone()); + unit.fail_quiesce = true; + unit.fail_resume = true; + run_unit(unit, recv) + }) + .unwrap(); + units.start().await.unwrap(); + + let error = units.quiesce_input_for_save().await.unwrap_err(); + assert!( + error + .to_string() + .contains("failing: intentional input quiesce failure") + ); + let error = units.resume_input_after_save().await.unwrap_err(); + assert!( + error + .to_string() + .contains("failing: intentional input resume failure") + ); + assert_eq!(successful_counts.quiesces.load(Ordering::Relaxed), 1); + assert_eq!(successful_counts.resumes.load(Ordering::Relaxed), 1); + assert_eq!(failing_counts.quiesces.load(Ordering::Relaxed), 1); + assert_eq!(failing_counts.resumes.load(Ordering::Relaxed), 1); +} + +#[async_test] +async fn test_input_hook_closed_channel_is_reported(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let _closed = units + .add("closed") + .spawn(&driver, |mut recv| async move { + let request = recv.next().await.unwrap(); + let StateRequest::Start(response) = request else { + panic!("unexpected request"); + }; + response.complete(()); + }) + .unwrap(); + units.start().await.unwrap(); + + let error = units.quiesce_input_for_save().await.unwrap_err(); + assert!(error.to_string().contains("quiesce_input failed: closed:")); +} + +#[async_test] +async fn test_slow_input_hook_is_awaited(driver: DefaultDriver) { + let delay = Duration::from_millis(200); + let mut units = StateUnits::new(); + let failing_counts = Arc::new(InputCounts::default()); + let _failing = units + .add("failing") + .spawn(&driver, |recv| { + let mut unit = input_unit(&driver, failing_counts.clone()); + unit.fail_quiesce = true; + run_unit(unit, recv) + }) + .unwrap(); + let counts = Arc::new(InputCounts::default()); + let _slow = units + .add("slow") + .spawn(&driver, |recv| { + let mut unit = input_unit(&driver, counts.clone()); + unit.delay = Some(delay); + run_unit(unit, recv) + }) + .unwrap(); + units.start().await.unwrap(); + + // The transaction has no deadline: it waits for the slow hook, which + // succeeds, and reports only the hook that failed. + let started = Instant::now(); + let error = units.quiesce_input_for_save().await.unwrap_err(); + assert!(started.elapsed() >= delay, "{:?}", started.elapsed()); + let error = error.to_string(); + assert!(error.contains("failing: intentional input quiesce failure")); + assert!(!error.contains("slow"), "{error}"); + assert_eq!(failing_counts.quiesces.load(Ordering::Relaxed), 1); + assert_eq!(counts.quiesces.load(Ordering::Relaxed), 1); + + // Resuming input reaches both units, and a stop follows. + units.resume_input_after_save().await.unwrap(); + assert_eq!(failing_counts.resumes.load(Ordering::Relaxed), 1); + assert_eq!(counts.resumes.load(Ordering::Relaxed), 1); + units.stop().await; +} + +#[async_test] +async fn test_input_hooks_reject_terminal_units(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let _running = units + .add("running") + .spawn(&driver, |recv| { + run_unit(input_unit(&driver, Arc::new(InputCounts::default())), recv) + }) + .unwrap(); + units.start().await.unwrap(); + + let terminal_counts = Arc::new(InputCounts::default()); + let _terminal = units + .add("terminal") + .spawn(&driver, |recv| { + run_unit_leaving_starts_unanswered( + input_unit(&driver, terminal_counts.clone()), + recv, + 0, + ) + }) + .unwrap(); + units.start_stopped_units().await.unwrap_err(); + + let error = units.quiesce_input_for_save().await.unwrap_err(); + assert!( + error + .to_string() + .contains("terminal: unit is in terminal state StartUncertain") + ); + assert_eq!(terminal_counts.quiesces.load(Ordering::Relaxed), 0); +} diff --git a/vmm_core/vmotherboard/Cargo.toml b/vmm_core/vmotherboard/Cargo.toml index b71f155d9e1..a834bf12c51 100644 --- a/vmm_core/vmotherboard/Cargo.toml +++ b/vmm_core/vmotherboard/Cargo.toml @@ -58,6 +58,7 @@ thiserror.workspace = true tracing.workspace = true [dev-dependencies] +inspect = { workspace = true, features = ["initiate"] } test_with_tracing.workspace = true [lints] diff --git a/vmm_core/vmotherboard/src/chipset/backing/arc_mutex/state_unit.rs b/vmm_core/vmotherboard/src/chipset/backing/arc_mutex/state_unit.rs index 15783878967..155adf83ef3 100644 --- a/vmm_core/vmotherboard/src/chipset/backing/arc_mutex/state_unit.rs +++ b/vmm_core/vmotherboard/src/chipset/backing/arc_mutex/state_unit.rs @@ -9,16 +9,20 @@ use closeable_mutex::CloseableMutex; use futures::FutureExt; use futures::StreamExt; use futures::future::BoxFuture; +use futures::future::Fuse; use futures::task::ArcWake; use futures::task::WakerRef; use futures::task::waker_ref; use inspect::InspectMut; +use mesh::error::RemoteError; +use mesh::rpc::FailableRpc; use pal_async::task::Spawn; use state_unit::SpawnedUnit; use state_unit::StateRequest; use state_unit::StateUnit; use std::sync::Arc; use std::task::Context; +use vmcore::device_state::InputHookCompletion; use vmcore::save_restore::RestoreError; use vmcore::save_restore::SaveError; use vmcore::save_restore::SavedStateBlob; @@ -97,6 +101,8 @@ impl InspectMut for ArcMutexChipsetDeviceUnit { trait DynDevice: InspectMut + Send { fn start(&mut self); async fn stop(&mut self); + fn quiesce_input(&mut self) -> InputHookCompletion; + fn resume_input(&mut self) -> InputHookCompletion; async fn reset(&mut self); fn poll_device(&mut self, cx: &mut Context<'_>); async fn save(&mut self) -> Result; @@ -113,6 +119,14 @@ impl DynDevice for T { self.stop().await } + fn quiesce_input(&mut self) -> InputHookCompletion { + self.quiesce_input() + } + + fn resume_input(&mut self) -> InputHookCompletion { + self.resume_input() + } + async fn reset(&mut self) { self.reset().await } @@ -166,12 +180,50 @@ impl ArcMutexChipsetDeviceUnit { } pub async fn run(mut self, mut state_change: mesh::Receiver) -> Self { + // An input hook request, whose answer waits for the hook's completion. + // The unit keeps polling the device meanwhile, since vCPUs may still + // access it and the completion may wait for the device to progress. + let mut input_hook: Option<(FailableRpc<(), ()>, InputHookCompletion)> = None; + // A state request that the unit has received but not applied. A + // transaction sends no request before the input hook is answered, but + // a dropped one can. Since only an inspection may overlap the hook, any + // other request waits for it, and the unit receives no other request + // until it has applied this one. + let mut request = None; + let mut closed = false; loop { + if let Some(req) = request + .take_if(|req| input_hook.is_none() || matches!(req, StateRequest::Inspect(_))) + { + match req { + StateRequest::QuiesceInput(rpc) => { + input_hook = Some((rpc, StateUnit::quiesce_input(&mut self))); + } + StateRequest::ResumeInput(rpc) => { + input_hook = Some((rpc, StateUnit::resume_input(&mut self))); + } + req => req.apply(&mut self).await, + } + continue; + } + if closed && input_hook.is_none() { + break; + } + enum Event<'a> { - StateChange(StateRequest), + StateChange(Option), Poll(WakerRef<'a>), + InputHookDone(anyhow::Result<()>), } + // Wait for state requests until the channel closes, but not while a + // request waits for the input hook. + let mut state_change_fut = if closed || request.is_some() { + Fuse::terminated() + } else { + state_change.next().fuse() + }; + // Wait for poll requests. let poll_fut = async { if self.running { @@ -183,27 +235,35 @@ impl ArcMutexChipsetDeviceUnit { std::future::pending().await }; + // Wait for the completion of the pending input hook. + let mut input_hook_fut = match &mut input_hook { + Some((_, completion)) => completion.fuse(), + None => Fuse::terminated(), + }; + let event = futures::select! { - req = state_change.next() => { - if let Some(req) = req { - Event::StateChange(req) - } else { - break; - } + req = state_change_fut => { + Event::StateChange(req) } waker = poll_fut.fuse() => { Event::Poll(waker) } + result = input_hook_fut => { + Event::InputHookDone(result) + } }; match event { - Event::StateChange(req) => { - req.apply(&mut self).await; - } + Event::StateChange(None) => closed = true, + Event::StateChange(Some(req)) => request = Some(req), Event::Poll(waker) => { let mut device = self.device.lock(); device.poll_device(&mut Context::from_waker(&waker)); } + Event::InputHookDone(result) => { + let (rpc, _) = input_hook.take().expect("input hook is pending"); + rpc.complete(result.map_err(RemoteError::new)); + } } } self @@ -234,6 +294,20 @@ impl StateUnit for ArcMutexChipsetDeviceUnit { // These are currently not true. } + fn quiesce_input(&mut self) -> InputHookCompletion { + // VPs are still running during input transitions, so keep the mutex + // open: a concurrent guest access waits for the hook, as for any other + // device call, instead of panicking on a closed mutex. The hook is + // synchronous, and the unit awaits its completion after the lock is + // released, so a guest access never waits for asynchronous work, which + // might need the executor that the guest access would block. + self.device.lock().quiesce_input() + } + + fn resume_input(&mut self) -> InputHookCompletion { + self.device.lock().resume_input() + } + async fn reset(&mut self) -> anyhow::Result<()> { self.device.clone().close().reset().await; Ok(()) @@ -264,21 +338,36 @@ mod tests { use chipset_device::io::IoResult; use chipset_device::mmio::MmioIntercept; use chipset_device::pio::PortIoIntercept; + use chipset_device::poll_device::PollDevice; use chipset_device_resources::ResolvedChipsetDevice; use closeable_mutex::CloseableMutex; + use futures::future::Either; use futures::poll; use inspect::InspectMut; + use mesh::rpc::FailableRpc; + use mesh::rpc::PendingFailableRpc; + use mesh::rpc::RpcSend; use pal_async::DefaultDriver; + use pal_async::DefaultPool; use pal_async::async_test; + use pal_async::task::Spawn; + use pal_async::timer::PolledTimer; + use state_unit::StateRequest; use state_unit::StateUnit; use state_unit::StateUnits; + use std::pin::pin; use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; + use std::sync::mpsc; + use std::task::Context; use std::task::Poll; + use std::task::Waker; + use std::time::Duration; use test_with_tracing::test; use vmcore::device_state::ChangeDeviceState; + use vmcore::device_state::InputHookCompletion; use vmcore::save_restore::AsyncSaveRestore; use vmcore::save_restore::NoSavedState; use vmcore::save_restore::RestoreError; @@ -756,4 +845,324 @@ mod tests { } } } + + /// A device whose input hooks signal that they ran and complete only when + /// the test releases them. + struct InputHookDevice { + started: Option>, + release: Option>, + } + + impl InputHookDevice { + fn hook(&mut self) -> InputHookCompletion { + self.started.take().expect("hook runs once").send(()); + let release = self.release.take().expect("hook runs once"); + InputHookCompletion::new(async move { + release + .await + .map_err(|_| anyhow::anyhow!("hook release dropped")) + }) + } + } + + impl ChipsetDevice for InputHookDevice {} + + impl InspectMut for InputHookDevice { + fn inspect_mut(&mut self, req: inspect::Request<'_>) { + req.respond(); + } + } + + impl ChangeDeviceState for InputHookDevice { + fn start(&mut self) {} + + fn quiesce_input(&mut self) -> InputHookCompletion { + self.hook() + } + + fn resume_input(&mut self) -> InputHookCompletion { + self.hook() + } + + async fn stop(&mut self) {} + + async fn reset(&mut self) {} + } + + impl AsyncSaveRestore for InputHookDevice { + type SavedState = NoSavedState; + + async fn save(&mut self) -> Result { + Ok(NoSavedState) + } + + async fn restore(&mut self, _: NoSavedState) -> Result<(), RestoreError> { + Ok(()) + } + } + + /// Sends `request` to the unit of an [`InputHookDevice`] whose hook does not + /// complete until a guest access to the device, on the same single-threaded + /// executor, releases it, as when a VP and the unit share a CPU's thread in + /// OpenHCL's threadpool. + /// + /// The guest access locks the device as MMIO and PIO dispatch do. If the + /// hook held the lock until its completion, the guest access would block + /// the executor that must complete the hook, so this fails after a timeout + /// instead of hanging. + fn input_hook_with_guest_access_on_one_executor( + request: fn(FailableRpc<(), ()>) -> StateRequest, + ) { + let (done_send, done_recv) = mpsc::channel(); + std::thread::spawn(move || { + DefaultPool::run_with(async |driver| { + let (started_send, started_recv) = mesh::oneshot(); + let (release_send, release_recv) = mesh::oneshot(); + let device = Arc::new(CloseableMutex::new(InputHookDevice { + started: Some(started_send), + release: Some(release_recv), + })); + let unit = ArcMutexChipsetDeviceUnit::new(device.clone(), false); + let (send, recv) = mesh::channel(); + let unit_task = driver.spawn("unit", unit.run(recv)); + let guest_access = driver.spawn("guest-access", async move { + started_recv.await.unwrap(); + drop(device.lock_if_open().expect("device is open")); + release_send.send(()); + }); + send.call_failable(request, ()).await.unwrap(); + guest_access.await; + drop(send); + unit_task.await; + }); + done_send.send(()).unwrap(); + }); + done_recv + .recv_timeout(Duration::from_secs(10)) + .expect("a guest access deadlocked with an input hook on its executor"); + } + + #[test] + fn input_hooks_do_not_block_guest_access_on_their_executor() { + input_hook_with_guest_access_on_one_executor(StateRequest::QuiesceInput); + input_hook_with_guest_access_on_one_executor(StateRequest::ResumeInput); + } + + /// A device whose input hooks complete only once the unit polls the device + /// after the hook ran, as a device that finishes reading host input when it + /// is polled would. + #[derive(Default)] + struct PolledHookDevice { + waker: Option, + hook_started: Option>, + polled: Option>, + } + + impl PolledHookDevice { + fn hook(&mut self) -> InputHookCompletion { + if let Some(hook_started) = self.hook_started.take() { + hook_started.send(()); + } + let (send, recv) = mesh::oneshot(); + self.polled = Some(send); + InputHookCompletion::new(async move { + recv.await + .map_err(|_| anyhow::anyhow!("poll signal dropped")) + }) + } + } + + impl ChipsetDevice for PolledHookDevice { + fn supports_poll_device(&mut self) -> Option<&mut dyn PollDevice> { + Some(self) + } + } + + impl PollDevice for PolledHookDevice { + fn poll_device(&mut self, cx: &mut Context<'_>) { + self.waker = Some(cx.waker().clone()); + if let Some(polled) = self.polled.take() { + polled.send(()); + } + } + } + + impl InspectMut for PolledHookDevice { + fn inspect_mut(&mut self, req: inspect::Request<'_>) { + req.respond(); + } + } + + impl ChangeDeviceState for PolledHookDevice { + fn start(&mut self) {} + + fn quiesce_input(&mut self) -> InputHookCompletion { + self.hook() + } + + fn resume_input(&mut self) -> InputHookCompletion { + self.hook() + } + + async fn stop(&mut self) {} + + async fn reset(&mut self) {} + } + + impl AsyncSaveRestore for PolledHookDevice { + type SavedState = NoSavedState; + + async fn save(&mut self) -> Result { + Ok(NoSavedState) + } + + async fn restore(&mut self, _: NoSavedState) -> Result<(), RestoreError> { + Ok(()) + } + } + + /// Starts the unit of a new [`PolledHookDevice`], with the device. + async fn start_polled_hook_unit( + driver: &DefaultDriver, + ) -> ( + Arc>, + mesh::Sender, + pal_async::task::Task, + ) { + let device = Arc::new(CloseableMutex::new(PolledHookDevice::default())); + let unit = ArcMutexChipsetDeviceUnit::new(device.clone(), false); + let (send, recv) = mesh::channel(); + let unit_task = driver.spawn("unit", unit.run(recv)); + send.call(StateRequest::Start, ()).await.unwrap(); + (device, send, unit_task) + } + + /// Sends `request` to the unit of `device`, without waiting for its answer, + /// and returns once the device's hook has run. + async fn start_polled_hook( + device: &CloseableMutex, + send: &mesh::Sender, + request: fn(FailableRpc<(), ()>) -> StateRequest, + ) -> PendingFailableRpc<()> { + let (started_send, started_recv) = mesh::oneshot(); + device.lock().hook_started = Some(started_send); + let hook = send.call_failable(request, ()); + started_recv.await.unwrap(); + hook + } + + /// Asks the unit of `device` to poll it. + fn request_poll(device: &CloseableMutex) { + device + .lock() + .waker + .as_ref() + .expect("device was polled") + .wake_by_ref(); + } + + /// Waits for `fut`, failing after a timeout with `message`. + async fn with_timeout( + driver: &DefaultDriver, + fut: impl Future, + message: &str, + ) -> T { + let mut timer = PolledTimer::new(driver); + let timeout = timer.sleep(Duration::from_secs(10)); + match futures::future::select(pin!(fut), pin!(timeout)).await { + Either::Left((output, _)) => output, + Either::Right(_) => panic!("{message}"), + } + } + + #[async_test] + async fn input_hooks_keep_polling_the_device(driver: DefaultDriver) { + let (device, send, unit_task) = start_polled_hook_unit(&driver).await; + + let requests: [fn(FailableRpc<(), ()>) -> StateRequest; 2] = + [StateRequest::QuiesceInput, StateRequest::ResumeInput]; + for request in requests { + let hook = start_polled_hook(&device, &send, request).await; + request_poll(&device); + with_timeout( + &driver, + hook, + "the unit did not poll the device during an input hook", + ) + .await + .unwrap(); + } + + send.call(StateRequest::Stop, ()).await.unwrap(); + drop(send); + unit_task.await; + } + + #[async_test] + async fn queued_request_waits_for_a_polled_input_hook(driver: DefaultDriver) { + let (device, send, unit_task) = start_polled_hook_unit(&driver).await; + + // Drop the answer to the hook, as a dropped input transaction would, + // and stop the unit while the hook is pending. + drop(start_polled_hook(&device, &send, StateRequest::QuiesceInput).await); + let mut stop = pin!(send.call(StateRequest::Stop, ())); + PolledTimer::new(&driver) + .sleep(Duration::from_millis(100)) + .await; + assert!(poll!(stop.as_mut()).is_pending()); + + // The stop waits for the hook, which the unit completes when it polls + // the device. + request_poll(&device); + with_timeout( + &driver, + stop, + "the unit did not poll the device before a stop", + ) + .await + .unwrap(); + drop(send); + unit_task.await; + } + + #[async_test] + async fn closed_channel_waits_for_a_polled_input_hook(driver: DefaultDriver) { + let (device, send, unit_task) = start_polled_hook_unit(&driver).await; + + // Drop the answer to the hook and close the unit's channel, as a + // removal that follows a dropped input transaction would. + drop(start_polled_hook(&device, &send, StateRequest::QuiesceInput).await); + drop(send); + PolledTimer::new(&driver) + .sleep(Duration::from_millis(100)) + .await; + + // The unit exits once the hook completes, when it polls the device. + request_poll(&device); + with_timeout( + &driver, + unit_task, + "the unit did not poll the device before exiting", + ) + .await; + } + + #[async_test] + async fn inspection_does_not_wait_for_a_polled_input_hook(driver: DefaultDriver) { + let (device, send, unit_task) = start_polled_hook_unit(&driver).await; + + let hook = start_polled_hook(&device, &send, StateRequest::QuiesceInput).await; + let mut inspection = inspect::inspect("", &inspect::send(&send, StateRequest::Inspect)); + with_timeout( + &driver, + inspection.resolve(), + "the inspection waited for the input hook", + ) + .await; + + request_poll(&device); + hook.await.unwrap(); + drop(send); + unit_task.await; + } } diff --git a/workers/chipset_device_worker/src/protocol.rs b/workers/chipset_device_worker/src/protocol.rs index 52ab0e9dd0b..3a479f9dc5b 100644 --- a/workers/chipset_device_worker/src/protocol.rs +++ b/workers/chipset_device_worker/src/protocol.rs @@ -61,6 +61,10 @@ pub(crate) enum DeviceRequest { PciConfigWrite(WriteRequest, PciConfigByteEnable), /// Start the device Start, + /// Stop accepting new host input before snapshot. + QuiesceInput(mesh::rpc::FailableRpc<(), ()>), + /// Resume host input after a failed snapshot. + ResumeInput(mesh::rpc::FailableRpc<(), ()>), /// Stop the device Stop(Rpc<(), ()>), /// Reset the device diff --git a/workers/chipset_device_worker/src/proxy.rs b/workers/chipset_device_worker/src/proxy.rs index 05ded9079da..119ef81b38f 100644 --- a/workers/chipset_device_worker/src/proxy.rs +++ b/workers/chipset_device_worker/src/proxy.rs @@ -26,12 +26,14 @@ use chipset_device_resources::ResolveChipsetDeviceHandleParams; use futures::StreamExt; use inspect::Inspect; use inspect::InspectMut; +use mesh::rpc::FailableRpc; use mesh::rpc::RpcSend; use mesh_worker::WorkerHandle; use slab::Slab; use std::task::Poll; use task_control::TaskControl; use vmcore::device_state::ChangeDeviceState; +use vmcore::device_state::InputHookCompletion; use vmcore::save_restore::AsyncProtobufSaveRestore; use vmcore::save_restore::RestoreError; use vmcore::save_restore::SaveError; @@ -130,6 +132,13 @@ impl ChipsetDeviceProxy { _guest_memory: guest_memory, }) } + + /// Sends an input hook request to the remote device, and returns a + /// completion that waits for its answer. + fn input_hook(&self, request: fn(FailableRpc<(), ()>) -> DeviceRequest) -> InputHookCompletion { + let call = self.req_send.call_failable(request, ()); + InputHookCompletion::new(async move { Ok(call.await?) }) + } } impl ChipsetDevice for ChipsetDeviceProxy { @@ -275,6 +284,14 @@ impl ChangeDeviceState for ChipsetDeviceProxy { self.req_send.send(DeviceRequest::Start) } + fn quiesce_input(&mut self) -> InputHookCompletion { + self.input_hook(DeviceRequest::QuiesceInput) + } + + fn resume_input(&mut self) -> InputHookCompletion { + self.input_hook(DeviceRequest::ResumeInput) + } + async fn stop(&mut self) { self.req_send .call(DeviceRequest::Stop, ()) @@ -317,6 +334,7 @@ impl AsyncProtobufSaveRestore for ChipsetDeviceProxy { #[cfg(test)] mod tests { use super::ChipsetDeviceProxy; + use super::PioProxy; use crate::RemoteDynamicResolvers; use crate::guestmem::GuestMemoryTask; use crate::protocol::DeviceInit; @@ -324,17 +342,25 @@ mod tests { use crate::worker::RemoteChipsetDeviceWorker; use crate::worker::RemoteChipsetDeviceWorkerParameters; use chipset_device::ChipsetDevice; + use chipset_device::io::IoResult; + use chipset_device::pio::PortIoIntercept; + use chipset_device::poll_device::PollDevice; use chipset_device_resources::ResolveChipsetDeviceHandleParams; use chipset_device_resources::ResolvedChipsetDevice; + use futures::future::Either; use guestmem::GuestMemory; use inspect::InspectMut; use mesh::MeshPayload; use mesh::payload::Protobuf; use pal_async::DefaultDriver; use pal_async::async_test; + use pal_async::timer::PolledTimer; use slab::Slab; use std::convert::Infallible; + use std::pin::Pin; + use std::pin::pin; use std::task::Poll; + use std::time::Duration; use task_control::TaskControl; use test_with_tracing::test; use vm_resource::IntoResource; @@ -343,6 +369,7 @@ mod tests { use vm_resource::ResourceResolver; use vm_resource::kind::ChipsetDeviceHandleKind; use vmcore::device_state::ChangeDeviceState; + use vmcore::device_state::InputHookCompletion; use vmcore::save_restore::AsyncProtobufSaveRestore; use vmcore::save_restore::AsyncSaveRestore; use vmcore::save_restore::RestoreError; @@ -366,6 +393,8 @@ mod tests { guest_memory: GuestMemory, #[inspect(skip)] encrypted_guest_memory: GuestMemory, + #[inspect(skip)] + port_write: Option>, } impl TestDevice { @@ -382,12 +411,51 @@ mod tests { assert_eq!(actual, expected); } } + + /// Accesses guest memory, and returns a completion that waits for the + /// next guest port write. + fn input_hook(&mut self) -> InputHookCompletion { + self.access_memory(); + let (send, recv) = mesh::oneshot(); + self.port_write = Some(send); + InputHookCompletion::new(async move { + recv.await + .map_err(|_| anyhow::anyhow!("port write signal dropped")) + }) + } } - impl ChipsetDevice for TestDevice {} + impl ChipsetDevice for TestDevice { + fn supports_pio(&mut self) -> Option<&mut dyn PortIoIntercept> { + Some(self) + } + } + + impl PortIoIntercept for TestDevice { + fn io_read(&mut self, _io_port: u16, data: &mut [u8]) -> IoResult { + data.fill(0); + IoResult::Ok + } + + fn io_write(&mut self, _io_port: u16, _data: &[u8]) -> IoResult { + if let Some(port_write) = self.port_write.take() { + port_write.send(()); + } + IoResult::Ok + } + } impl ChangeDeviceState for TestDevice { fn start(&mut self) {} + + fn quiesce_input(&mut self) -> InputHookCompletion { + self.input_hook() + } + + fn resume_input(&mut self) -> InputHookCompletion { + self.input_hook() + } + async fn stop(&mut self) { self.access_memory(); } @@ -456,6 +524,7 @@ mod tests { value: 0, guest_memory: input.guest_memory.clone(), encrypted_guest_memory: input.encrypted_guest_memory.clone(), + port_write: None, }; device.access_memory(); Ok(device.into()) @@ -506,7 +575,7 @@ mod tests { .unwrap(); let DeviceInit { mmio, pio, pci } = cap_recv.await.unwrap(); - assert!(mmio.is_none() && pio.is_none() && pci.is_none()); + assert!(mmio.is_none() && pci.is_none()); ChipsetDeviceProxy { req_send, resp_recv, @@ -515,7 +584,9 @@ mod tests { in_flight_writes: Slab::new(), _guest_memory: guest_memory, mmio: None, - pio: None, + pio: pio.map(|_| PioProxy { + static_regions: Vec::new(), + }), pci: None, } } @@ -590,4 +661,49 @@ mod tests { ); stop_worker(&mut proxy).await; } + + /// Runs `hook` on the proxy of a device that completes the hook only after a + /// guest port write, and writes to the port while the hook is pending, as a + /// vCPU may. This fails after a timeout if the worker stops serving guest + /// requests, or the proxy stops completing them, while the hook is pending. + async fn input_hook_with_guest_write( + driver: &DefaultDriver, + proxy: &mut ChipsetDeviceProxy, + hook: fn(&mut ChipsetDeviceProxy) -> InputHookCompletion, + ) { + let mut completion = hook(proxy); + let IoResult::Defer(mut write) = proxy.io_write(0, &[1]) else { + panic!("proxy port writes are deferred"); + }; + let mut written = false; + let round_trip = std::future::poll_fn(|cx| { + proxy.poll_device(cx); + if !written && let Poll::Ready(result) = write.poll_write(cx) { + result.unwrap(); + written = true; + } + if written { + Pin::new(&mut completion).poll(cx) + } else { + Poll::Pending + } + }); + let mut timer = PolledTimer::new(driver); + let timeout = timer.sleep(Duration::from_secs(10)); + match futures::future::select(pin!(round_trip), pin!(timeout)).await { + Either::Left((result, _)) => result.unwrap(), + Either::Right(_) => panic!("a guest request was not served during an input hook"), + } + } + + #[async_test] + async fn input_hooks_serve_guest_requests(driver: DefaultDriver) { + let vmtime = VmTimeKeeper::new(&driver, VmTime::from_100ns(0)); + let mut proxy = new_proxy(&vmtime, &driver).await; + + input_hook_with_guest_write(&driver, &mut proxy, ChangeDeviceState::quiesce_input).await; + input_hook_with_guest_write(&driver, &mut proxy, ChangeDeviceState::resume_input).await; + + stop_worker(&mut proxy).await; + } } diff --git a/workers/chipset_device_worker/src/worker.rs b/workers/chipset_device_worker/src/worker.rs index e3b90a96c17..572f61ed2ad 100644 --- a/workers/chipset_device_worker/src/worker.rs +++ b/workers/chipset_device_worker/src/worker.rs @@ -24,15 +24,19 @@ use chipset_device_resources::ErasedChipsetDevice; use chipset_device_resources::ResolveChipsetDeviceHandleParams; use mesh::MeshPayload; use mesh::error::RemoteError; +use mesh::rpc::FailableRpc; use mesh_worker::Worker; use mesh_worker::WorkerId; use mesh_worker::WorkerRpc; use pal_async::DefaultPool; +use std::future::Future; +use std::pin::Pin; use std::task::Poll; use vm_resource::Resource; use vm_resource::ResourceResolver; use vm_resource::kind::ChipsetDeviceHandleKind; use vmcore::device_state::ChangeDeviceState; +use vmcore::device_state::InputHookCompletion; use vmcore::save_restore::AsyncProtobufSaveRestore; /// Worker ID for ChipsetDevice workers. @@ -190,6 +194,10 @@ impl Worker for RemoteChipsetDeviceWorker { fn run(mut self, mut rpc_recv: mesh::Receiver>) -> anyhow::Result<()> { self.pool.take().unwrap().run_until(async move { + // An input hook request, whose answer waits for the hook's + // completion. The worker keeps serving guest requests and polling + // the device meanwhile, since vCPUs may still access the device. + let mut input_hook: Option<(FailableRpc<(), ()>, InputHookCompletion)> = None; loop { enum WorkerEvent { Rpc(WorkerRpc<()>), @@ -203,6 +211,13 @@ impl Worker for RemoteChipsetDeviceWorker { poll_device.poll_device(cx); } + if let Some((_, completion)) = &mut input_hook + && let Poll::Ready(result) = Pin::new(completion).poll(cx) + { + let (rpc, _) = input_hook.take().unwrap(); + rpc.complete(result.map_err(RemoteError::new)); + } + self.deferred_reads .extract_if(.., |read| { let mut data = vec![0; read.size]; @@ -265,6 +280,12 @@ impl Worker for RemoteChipsetDeviceWorker { self.device.start(); self.running = true; } + DeviceRequest::QuiesceInput(rpc) => { + input_hook = Some((rpc, self.device.quiesce_input())); + } + DeviceRequest::ResumeInput(rpc) => { + input_hook = Some((rpc, self.device.resume_input())); + } DeviceRequest::Teardown(rpc) => { assert!( !self.running,