From ffff9aa1064fd4110bb9e0ba304761eb180ab606 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Thu, 24 Sep 2026 23:50:18 -0700 Subject: [PATCH 1/6] state_unit: roll back a partial start instead of panicking `StateUnits::start` panics when a unit's task does not answer its start request, as when the task has exited, so a unit failure takes the whole process down instead of failing the start. It also always considers every unit running. Start the units in dependency order without panicking. A unit whose start request is not delivered or answered is left in a terminal state, its dependents do not start, and the units that did start are stopped again in reverse dependency order, so that no partially started VM keeps running. `StateUnits::start` and `start_stopped_units` return the failures; a unit's own start still cannot fail. A unit that was not stopped, such as one that was already running or one whose start is uncertain, keeps the units that it depends on running. A started unit that is kept running, or whose stop fails, is left in the terminal state, the failure is reported, and the units stay running so that a later stop can stop it. While a unit is in a terminal state, a later start, reset, save, or restore fails, and a stop also stops the units in a terminal state and keeps their terminal markers. The task of such a unit may have exited, so `StateUnits::stop_for_teardown`, which the worker uses when it tears the VM down, logs a stop that the unit does not answer instead of panicking, and stops the other units. The worker's teardown stops the state units whenever they run, since a failed start can leave them running while the VM is paused. The callers of `StateUnits::start` and `start_stopped_units` handle the error: - VMBus, PCIe, and relayed vPCI hot-add, and the dynamic vPCI device builder, remove the units they added and return it. VF hot-add removes the network it added and returns it. - After a failed worker restart, a failure to start the VM again is reported to the requester, and the worker tears the VM down. - PCI resource assignment returns it. - OpenHCL reports it to the requester of a resume or a diagnostic save, and tears the VM down, as it does when it cannot resume the VM after a failed restart or servicing request. A failed autostart tears the VM down and fails the worker. - VM resume cannot report a failure to its callers yet, so it treats a unit that fails to start as an unrecoverable unit failure, like the other state changes do. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- openhcl/underhill_core/src/dispatch/mod.rs | 143 +++++-- openhcl/underhill_core/src/lib.rs | 11 +- openhcl/underhill_core/src/worker.rs | 1 + openvmm/openvmm_core/src/worker/dispatch.rs | 48 ++- vm/devices/pci/vpci_relay/src/lib.rs | 10 +- vmm_core/src/device_builder.rs | 6 +- vmm_core/state_unit/src/lib.rs | 147 +++++-- vmm_core/state_unit/src/start.rs | 194 +++++++++ vmm_core/state_unit/src/tests/start.rs | 428 ++++++++++++++++++++ 9 files changed, 895 insertions(+), 93 deletions(-) create mode 100644 vmm_core/state_unit/src/start.rs create mode 100644 vmm_core/state_unit/src/tests/start.rs diff --git a/openhcl/underhill_core/src/dispatch/mod.rs b/openhcl/underhill_core/src/dispatch/mod.rs index e91a38fb8a4..b7a0916189d 100644 --- a/openhcl/underhill_core/src/dispatch/mod.rs +++ b/openhcl/underhill_core/src/dispatch/mod.rs @@ -80,7 +80,7 @@ use vtl2_settings_worker::handle_vtl2_config_rpc; #[derive(MeshPayload)] pub enum UhVmRpc { Pause(Rpc<(), bool>), - Resume(Rpc<(), bool>), + Resume(FailableRpc<(), bool>), Save(FailableRpc<(), Vec>), ClearHalt(Rpc<(), bool>), // TODO: remove this, and use DebugRequest::Resume PacketCapture(FailableRpc, PacketCaptureParams>), @@ -235,9 +235,12 @@ impl LoadedVm { correlation_id: Option, mut vm_rpc: mesh::Receiver, mut worker_rpc: mesh::Receiver>, - ) -> Option> { + ) -> anyhow::Result>> { if autostart_vps { - self.start(correlation_id).await; + if let Err(error) = self.start(correlation_id).await { + self.teardown().await; + return Err(error.context("failed to autostart VM")); + } } // VTL2 settings services @@ -325,7 +328,17 @@ impl LoadedVm { Event::WorkerRpc(message) => match message { WorkerRpc::Stop => break None, WorkerRpc::Restart(rpc) => { - let state = async { + enum RestartOutcome { + Restart(R, S), + // The restart failed, but the VM is back in its + // previous state, so this worker can keep serving. + Recovered, + // The restart failed and the VM could not be + // resumed, so this worker cannot serve any more. + Unrecoverable, + } + + let outcome = async { let running = self.stop().await; match self .save( @@ -336,26 +349,40 @@ impl LoadedVm { ) .await { - Ok(servicing_state) => Some((rpc, servicing_state)), + Ok(servicing_state) => RestartOutcome::Restart(rpc, servicing_state), Err(err) => { if running { - self.start(None).await; + if let Err(start_error) = self.start(None).await { + rpc.complete(Err(RemoteError::new( + start_error.context(format!( + "worker restart failed ({err:#}) and the VM could not resume" + )), + ))); + return RestartOutcome::Unrecoverable; + } } rpc.complete(Err(RemoteError::new(err))); - None + RestartOutcome::Recovered } } } .instrument(tracing::info_span!("restart", CVM_ALLOWED)) .await; - if let Some((rpc, servicing_state)) = state { - break Some(LoadedVmState { - restart_rpc: rpc, - servicing_state, - vm_rpc, - control_send: self.control_send.lock().take().unwrap(), - }); + match outcome { + RestartOutcome::Restart(rpc, servicing_state) => { + break Some(LoadedVmState { + restart_rpc: rpc, + servicing_state, + vm_rpc, + control_send: self.control_send.lock().take().unwrap(), + }); + } + RestartOutcome::Recovered => {} + // The error has already completed the RPC. Tear + // down through the common path rather than keep a + // worker whose state units can no longer run. + RestartOutcome::Unrecoverable => break None, } } WorkerRpc::Inspect(deferred) => deferred.respond(|resp| { @@ -406,34 +433,39 @@ impl LoadedVm { } Event::UhVmRpc(msg) => match msg { UhVmRpc::Resume(rpc) => { - rpc.handle(async |()| { - if !self.state_units.is_running() { - self.start(None).await; - true - } else { - false - } - }) - .await + if self.state_units.is_running() { + rpc.complete(Ok(false)); + } else if let Err(error) = self.start(None).await { + rpc.complete(Err(RemoteError::new( + error.context("failed to resume VM"), + ))); + // A failed start may leave state units terminal; + // this worker cannot safely continue serving. + break None; + } else { + rpc.complete(Ok(true)); + } } UhVmRpc::Pause(rpc) => rpc.handle(async |()| self.stop().await).await, UhVmRpc::Save(rpc) => { - rpc.handle_failable(async |()| { - let running = self.stop().await; - let r = self - .save( - SaveReason::Diagnostic, - None, - KeepAliveConfig::Disabled, - KeepAliveConfig::Disabled, - ) - .await; - if running { - self.start(None).await; - } - r.map(mesh::payload::encode) - }) - .await + let running = self.stop().await; + let result = self + .save( + SaveReason::Diagnostic, + None, + KeepAliveConfig::Disabled, + KeepAliveConfig::Disabled, + ) + .await; + if running && let Err(error) = self.start(None).await { + rpc.complete(Err(RemoteError::new( + error.context("failed to resume VM after save"), + ))); + // The state units rolled back and cannot run + // again, so tear down rather than keep this worker. + break None; + } + rpc.complete(result.map(mesh::payload::encode).map_err(RemoteError::new)); } UhVmRpc::ClearHalt(rpc) => { rpc.handle(async |()| self.partition_unit.clear_halt().await) @@ -585,6 +617,12 @@ impl LoadedVm { } }; + self.teardown().await; + + Ok(state) + } + + async fn teardown(mut self) { // Drain resealing before tearing down any storage transport. Removing // the unit closes its request stream; in-flight I/O finishes first. if let Some(resealer) = self.hardware_reseal.take() { @@ -602,8 +640,6 @@ impl LoadedVm { if let Some(vmbus) = self.vmbus_server { vmbus.remove().await.shutdown().await; } - - state } /// Handles a servicing request from the host. @@ -654,7 +690,9 @@ impl LoadedVm { .context("failed to notify host of servicing-while-paused failure")?; if running { - self.start(Some(correlation_id)).await; + self.start(Some(correlation_id)) + .await + .context("failed to resume VM after servicing failure")?; } false } @@ -884,8 +922,8 @@ impl LoadedVm { } } - async fn start(&mut self, correlation_id: Option) { - self.state_units.start().await; + async fn start(&mut self, correlation_id: Option) -> anyhow::Result<()> { + self.state_units.start().await?; // Log the boot/blackout time. let reference_time = ReferenceTime::new(self.partition.reference_time()); @@ -911,6 +949,8 @@ impl LoadedVm { "starting VM" ) } + + Ok(()) } /// Returns true if the VM was previously running. @@ -1137,7 +1177,20 @@ impl LoadedVm { ) .await?; - self.state_units.start_stopped_units().await; + if let Err(error) = self.state_units.start_stopped_units().await { + if let Err(remove_error) = self + .network_settings + .as_mut() + .unwrap() + .remove_network(instance_id) + .await + { + return Err(error.context(format!( + "failed to remove network after start failure: {remove_error:#}" + ))); + } + return Err(error); + } self.emuplat_servicing.netvsp_state.push(save_state); Ok(()) diff --git a/openhcl/underhill_core/src/lib.rs b/openhcl/underhill_core/src/lib.rs index 329d1c119f8..69cb928ad5c 100644 --- a/openhcl/underhill_core/src/lib.rs +++ b/openhcl/underhill_core/src/lib.rs @@ -686,11 +686,18 @@ async fn run_control( continue; }; - let was_resumed = workers + let was_resumed = match workers .vm_rpc .call(UhVmRpc::Resume, ()) .await - .context("failed to resumed VM")?; + .context("failed to resume VM")? + { + Ok(was_resumed) => was_resumed, + Err(error) => { + rpc.complete(Err(error)); + continue; + } + }; let was_halted = workers .vm_rpc diff --git a/openhcl/underhill_core/src/worker.rs b/openhcl/underhill_core/src/worker.rs index 1039d11cfaa..df433f73e24 100644 --- a/openhcl/underhill_core/src/worker.rs +++ b/openhcl/underhill_core/src/worker.rs @@ -468,6 +468,7 @@ impl Worker for UnderhillVmWorker { )); tracing::info!(CVM_ALLOWED, "terminating worker"); self.get_thread.join().unwrap(); + let state = state?; if let Some(state) = state { let params = UnderhillWorkerParameters { env_cfg: self.env_cfg, diff --git a/openvmm/openvmm_core/src/worker/dispatch.rs b/openvmm/openvmm_core/src/worker/dispatch.rs index 6566a94a8ec..6b6b558aa58 100644 --- a/openvmm/openvmm_core/src/worker/dispatch.rs +++ b/openvmm/openvmm_core/src/worker/dispatch.rs @@ -3682,7 +3682,12 @@ impl LoadedVm { if self.running { return false; } - self.state_units.start().await; + // Resume cannot report a failure to its callers, so treat a unit that + // fails to start like any other unrecoverable state unit failure. + self.state_units + .start() + .await + .expect("state units failed to start"); self.running = true; true } @@ -3724,8 +3729,10 @@ impl LoadedVm { // Start state units so device config space is accessible. This also // freezes SMMU capabilities before firmware reads them. - self.state_units.start().await; - + self.state_units + .start() + .await + .context("failed to start devices for PCI resource assignment")?; let result = ecam_config_access::assign_pci_resources_for_root_complexes( &self.inner.chipset, &self.inner.pcie_host_bridges, @@ -3832,7 +3839,12 @@ impl LoadedVm { } Err(err) => { if stopped { - self.state_units.start().await; + if let Err(start_error) = self.state_units.start().await { + rpc.complete(Err(RemoteError::new(start_error.context( + "worker restart failed and the VM could not resume", + )))); + break; + } } rpc.complete(Err(RemoteError::new(err))); } @@ -3906,7 +3918,15 @@ impl LoadedVm { ) .await?; self.inner.vmbus_devices.push(device); - self.state_units.start_stopped_units().await; + if let Err(error) = self.state_units.start_stopped_units().await { + let device = self + .inner + .vmbus_devices + .pop() + .expect("device was just pushed"); + drop(device.remove().await); + return Err(error); + } anyhow::Ok(()) }) .await @@ -4072,7 +4092,15 @@ impl LoadedVm { // MSI. The guest may begin probing config space // immediately after receiving the interrupt, so // the device must be ready first. - self.state_units.start_stopped_units().await; + if let Err(error) = self.state_units.start_stopped_units().await { + let (_, unit, _) = self + .inner + .pcie_hotplug_devices + .pop() + .expect("device was just pushed"); + unit.remove().await; + return Err(error); + } // Now attach the device and notify the guest. if let Err(e) = rc.lock().hotplug_add_device( @@ -4243,7 +4271,13 @@ impl LoadedVm { } device_removal.drain().await; - self.pause().await; + // Stop the state units whenever they run, not only when the VM does: a + // failed start, as in a reset, can leave units running while the VM is + // paused, and a unit whose start is uncertain may not answer its stop. + if self.state_units.is_running() { + self.state_units.stop_for_teardown().await; + } + self.running = false; self.teardown_chipset_devices().await; self.inner.partition_unit.teardown().await; if let Some(vmbus) = self.inner.vmbus_server { diff --git a/vm/devices/pci/vpci_relay/src/lib.rs b/vm/devices/pci/vpci_relay/src/lib.rs index 22e971f1f90..76bae759c18 100644 --- a/vm/devices/pci/vpci_relay/src/lib.rs +++ b/vm/devices/pci/vpci_relay/src/lib.rs @@ -490,7 +490,7 @@ impl VpciRelay { .await? }; - entry.insert(RelayedDevice { + let device = RelayedDevice { bus_instance_id: instance_id, bus_client: vpci_client, vpci_device: vpci_device.clone(), @@ -498,9 +498,13 @@ impl VpciRelay { bus_unit, device_unit, ready_to_remove: false, - }); + }; - state_units.start_stopped_units().await; + if let Err(error) = state_units.start_stopped_units().await { + device.remove().await; + return Err(error); + } + entry.insert(device); Ok(()) } } diff --git a/vmm_core/src/device_builder.rs b/vmm_core/src/device_builder.rs index 089ac1444c9..440d98a9dc4 100644 --- a/vmm_core/src/device_builder.rs +++ b/vmm_core/src/device_builder.rs @@ -144,7 +144,11 @@ pub async fn build_dynamic_vpci_device( }; let pending_offer = pending_offer.context("missing deferred VPCI channel offer")?; - state_units.start_stopped_units().await; + if let Err(error) = state_units.start_stopped_units().await { + vpci_unit.remove().await; + pci_unit.remove().await; + return Err(error); + } if let Err(error) = pending_offer .offer_registered(&vpci_bus, driver_source, vmbus, state_units.is_running()) .await diff --git a/vmm_core/state_unit/src/lib.rs b/vmm_core/state_unit/src/lib.rs index 87a9525e73d..d6e9ef772a5 100644 --- a/vmm_core/state_unit/src/lib.rs +++ b/vmm_core/state_unit/src/lib.rs @@ -30,6 +30,8 @@ #![forbid(unsafe_code)] +mod start; + use futures::FutureExt; use futures::StreamExt; use futures::future::join_all; @@ -70,9 +72,14 @@ use vmcore::save_restore::SavedStateBlob; #[derive(Debug, MeshPayload)] pub enum StateRequest { /// Start asynchronous operations. + /// + /// Once this request is delivered, a [`StateRequest::Stop`] may follow + /// even if its response is not observed. Start(Rpc<(), ()>), /// Stop asynchronous operations. + /// + /// The handler must be safe after any delivered start attempt. Stop(Rpc<(), ()>), /// Reset a stopped unit to initial state. @@ -222,6 +229,7 @@ enum State { Resetting, Saving, Restoring, + StartUncertain, } #[derive(Debug)] @@ -486,41 +494,82 @@ impl StateUnits { self.running } + fn check_no_terminal_units(&self, op: &'static str) -> Result<(), StateTransitionError> { + let inner = self.inner.lock(); + check( + op, + inner + .units + .values() + .filter(|unit| matches!(unit.state, State::StartUncertain)) + .map(|unit| { + ( + unit.name.clone(), + Err::<(), _>(anyhow::anyhow!( + "unit is in terminal state {:?}", + unit.state + )), + ) + }), + ) + } + /// Starts any units that are individually stopped, because they were added /// via [`StateUnits::add`] while the VM was running. /// /// Does nothing if all units are stopped, via [`StateUnits::stop`]. - pub async fn start_stopped_units(&mut self) { + pub async fn start_stopped_units(&mut self) -> anyhow::Result<()> { if self.is_running() { - self.start().await; + self.start().await?; } + Ok(()) } /// Starts all the state units. - pub async fn start(&mut self) { + pub async fn start(&mut self) -> anyhow::Result<()> { self.restore_allowed = false; - self.run_op( - "start", - None, - State::Stopped, - State::Starting, - State::Running, - StateRequest::Start, - |_, _| Some(()), - |unit| Some(&unit.dependencies), - ) - .await; - self.running = true; + self.start_with_rollback().await } /// Stops all the state units. + /// + /// This also stops the units whose start is uncertain. pub async fn stop(&mut self) { + self.stop_units(false).await + } + + /// Stops all the state units before the VM is torn down. + /// + /// Unlike [`StateUnits::stop`], this does not panic when a unit whose start + /// is uncertain does not answer its stop, as when its task has exited. It + /// logs the failure and stops the other units, including the units that + /// such a unit depends on, since the VM is not run again. + pub async fn stop_for_teardown(&mut self) { + self.stop_units(true).await + } + + async fn stop_units(&mut self, for_teardown: bool) { assert!(self.running); + // Make units whose start is uncertain receive Stop, then restore their + // terminal markers after the ordered operation. + let uncertain_ids = { + let mut inner = self.inner.lock(); + inner + .units + .iter_mut() + .filter(|(_, unit)| unit.state == State::StartUncertain) + .map(|(&id, unit)| { + unit.state = State::Running; + id + }) + .collect::>() + }; // Stop units in reverse dependency order so that a dependency is not // stopped before its dependant. self.run_op( "stop", None, + if for_teardown { &uncertain_ids } else { &[] }, State::Running, State::Stopping, State::Stopped, @@ -529,6 +578,13 @@ impl StateUnits { |unit| Some(&unit.dependents), ) .await; + let mut inner = self.inner.lock(); + for id in uncertain_ids { + if let Some(unit) = inner.units.get_mut(&id) { + assert_eq!(unit.state, State::Stopped); + unit.state = State::StartUncertain; + } + } self.running = false; } @@ -537,6 +593,7 @@ impl StateUnits { /// Panics if running. pub async fn reset(&mut self) -> Result<(), StateTransitionError> { assert!(!self.running); + self.check_no_terminal_units("reset")?; self.restore_allowed = false; // Reset in dependency order so that dependants observe their // dependencies' reset state. @@ -544,6 +601,7 @@ impl StateUnits { .run_op( "reset", None, + &[], State::Stopped, State::Resetting, State::Stopped, @@ -563,12 +621,14 @@ impl StateUnits { /// Panics if running. pub async fn save(&mut self) -> Result, StateTransitionError> { assert!(!self.running); + self.check_no_terminal_units("save")?; // Save can occur in any order since it will not observably mutate // state. let r = self .run_op( "save", None, + &[], State::Stopped, State::Saving, State::Stopped, @@ -601,6 +661,7 @@ impl StateUnits { states: Vec, ) -> Result<(), StateTransitionError> { assert!(!self.running); + self.check_no_terminal_units("restore")?; if !self.restore_allowed { return Err(StateTransitionError { op: "restore", @@ -644,6 +705,7 @@ impl StateUnits { .run_op( "restore", None, + &[], State::Stopped, State::Restoring, State::Stopped, @@ -675,6 +737,10 @@ impl StateUnits { /// `unit_ids` is the set of the units whose state should be changed. If /// `unit_ids` is `None`, all units change states. /// + /// `tolerated_ids` is the set of the units whose failure to answer the + /// request is logged instead of treated as an unrecoverable unit failure, + /// such as the units whose start is uncertain when the VM is torn down. + /// /// The old state for each unit must be `old_state`. During the operation, /// the unit is temporarily places in `interim_state`. When complete, the /// unit is placed in `new_state`. @@ -693,6 +759,7 @@ impl StateUnits { &self, op: &str, unit_ids: Option<&[u64]>, + tolerated_ids: &[u64], old_state: State, interim_state: State, new_state: State, @@ -751,20 +818,12 @@ impl StateUnits { let mut inner = self.inner.lock(); let r = results .into_iter() - .filter_map(|(name, id, r)| { - match r { - Ok(Some(r)) => Some((name, r)), - Ok(None) => None, - Err(err) => { - // If the unit was removed during the operation, then - // ignore the failure. Otherwise, panic because unit - // failure is not recoverable. FUTURE: reconsider this - // position. - if inner.units.contains_key(&id) { - panic!("{:?}", err); - } - None - } + .filter_map(|(name, id, r)| match r { + Ok(Some(r)) => Some((name, r)), + Ok(None) => None, + Err(err) => { + inner.unanswered_request(id, tolerated_ids, err); + None } }) .collect(); @@ -808,6 +867,22 @@ impl Inner { }; ReadySet(Arc::new(units)) } + + /// Handles a state change request that unit `id` did not answer. + /// + /// If the unit is in `tolerated_ids`, log the failure. If the unit was + /// removed during the operation, ignore it. Otherwise, panic because unit + /// failure is not recoverable. FUTURE: reconsider this position. + fn unanswered_request(&self, id: u64, tolerated_ids: &[u64], err: UnitRecvError) { + if tolerated_ids.contains(&id) { + tracing::error!( + error = &err as &dyn std::error::Error, + "unit whose start is uncertain did not answer" + ); + } else if self.units.contains_key(&id) { + panic!("{:?}", err); + } + } } #[derive(Clone)] @@ -1047,6 +1122,8 @@ impl Ready { #[cfg(test)] mod tests { + mod start; + use super::StateTransitionErrorKind; use super::StateUnit; use super::StateUnits; @@ -1219,7 +1296,7 @@ mod tests { let state = units.save().await.unwrap(); units.restore(state).await.unwrap(); assert_eq!(calls.load(Ordering::Relaxed), 2); - units.start().await; + units.start().await.unwrap(); units.stop().await; let state = units.save().await.unwrap(); assert!(units.restore(state).await.is_err()); @@ -1293,13 +1370,13 @@ mod tests { .add("b") .spawn(&driver, |recv| run_unit(TestUnit::default(), recv)) .unwrap(); - units.start().await; + units.start().await.unwrap(); let _c = units .add("c") .spawn(&driver, |recv| run_unit(TestUnit::default(), recv)); units.stop().await; - units.start().await; + units.start().await.unwrap(); units.stop().await; @@ -1347,7 +1424,7 @@ mod tests { ) }) .unwrap(); - units.start().await; + units.start().await.unwrap(); units.stop().await; let state = units.save().await.unwrap(); @@ -1457,7 +1534,7 @@ mod tests { ); } - units.start().await; + units.start().await.unwrap(); units.stop().await; a.remove().await; @@ -1467,7 +1544,7 @@ mod tests { assert!(inner.units[&c.handle().id.id].dependencies.is_empty()); } - units.start().await; + units.start().await.unwrap(); units.stop().await; // Removing the remaining unit leaves the set empty. diff --git a/vmm_core/state_unit/src/start.rs b/vmm_core/state_unit/src/start.rs new file mode 100644 index 00000000000..13c18f7693b --- /dev/null +++ b/vmm_core/state_unit/src/start.rs @@ -0,0 +1,194 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Start with rollback. A unit's start cannot fail, but its task can stop +//! answering: then its dependents do not start, and the units that did start +//! are stopped again, rather than the whole process panicking. + +use super::State; +use super::StateRequest; +use super::StateUnits; +use super::state_change; +use futures::future::join_all; +use std::time::Instant; + +impl StateUnits { + /// Starts all stopped units in dependency order. + /// + /// Fails without starting any unit if one is in a terminal state. A unit + /// does not start if one of its dependencies + /// did not start. If a unit's task does not answer its start request, the + /// units that did start are stopped again, and the unit is left in a + /// terminal state that fails later starts. A started unit that cannot be + /// stopped again, because a unit that depends on it was not stopped, is + /// left running in that terminal state too, and the units stay running so + /// that [`StateUnits::stop`] can stop them later. + pub(super) async fn start_with_rollback(&mut self) -> anyhow::Result<()> { + enum StartResult { + Started, + DependencyFailed, + Uncertain(anyhow::Error), + } + + let was_running = self.running; + let mut operations = Vec::new(); + let ready_set; + { + let mut inner = self.inner.lock(); + if let Some(unit) = inner + .units + .values() + .find(|unit| !matches!(unit.state, State::Stopped | State::Running)) + { + anyhow::bail!( + "state unit '{}' cannot be started from terminal state {:?}", + unit.name, + unit.state + ); + } + ready_set = inner.ready_set(None); + for (&id, unit) in &mut inner.units { + if unit.state == State::Running { + ready_set.done(id, true); + continue; + } + assert_eq!( + unit.state, + State::Stopped, + "unit {} is not stopped before start", + unit.name, + ); + let name = unit.name.clone(); + let dependencies = unit.dependencies.iter().copied().collect::>(); + let ready_set = ready_set.clone(); + let start = state_change(name.clone(), unit, StateRequest::Start, Some(())); + operations.push(async move { + if !ready_set.wait("start", id, &dependencies).await { + ready_set.done(id, false); + return (name, id, StartResult::DependencyFailed); + } + + let result = match start.await { + Ok(Some(())) => StartResult::Started, + Ok(None) => { + StartResult::Uncertain(anyhow::anyhow!("start request was not sent")) + } + Err(error) => StartResult::Uncertain(error.into()), + }; + ready_set.done(id, matches!(result, StartResult::Started)); + (name, id, result) + }); + unit.state = State::Starting; + } + } + + let start = Instant::now(); + let results = join_all(operations).await; + tracing::info!(duration = ?Instant::now() - start, "state-unit start complete"); + + let mut failures = Vec::new(); + let mut started_ids = Vec::new(); + { + let mut inner = self.inner.lock(); + for (name, id, result) in results { + let Some(unit) = inner.units.get_mut(&id) else { + continue; + }; + match result { + StartResult::Started => { + unit.state = State::Running; + started_ids.push(id); + } + StartResult::DependencyFailed => { + unit.state = State::Stopped; + failures.push(format!("{name}: a dependency did not start")); + } + StartResult::Uncertain(error) => { + unit.state = State::StartUncertain; + failures.push(format!("{name}: {error:#}")); + } + } + } + } + + if failures.is_empty() { + self.running = true; + Ok(()) + } else { + let cleanup_failures = self.stop_after_failed_start(&started_ids).await; + self.running = was_running || !cleanup_failures.is_empty(); + failures.extend(cleanup_failures); + anyhow::bail!("state units could not start: {}", failures.join("; ")) + } + } + + /// Stops the units in `started_ids` again in reverse dependency order. + /// + /// Every other unit that is not stopped, such as one that was already + /// running or one whose start is uncertain, blocks the stop of its + /// dependencies without receiving a stop request. A unit whose stop is + /// blocked or not confirmed is left in a terminal state. + async fn stop_after_failed_start(&self, started_ids: &[u64]) -> Vec { + const OP: &str = "failed_start_rollback"; + let mut operations = Vec::new(); + let ready_set; + { + let mut inner = self.inner.lock(); + ready_set = inner.ready_set(None); + for (&id, unit) in &inner.units { + if !started_ids.contains(&id) { + ready_set.done(id, unit.state == State::Stopped); + } + } + for (&id, unit) in inner + .units + .iter_mut() + .filter(|(id, _)| started_ids.contains(id)) + { + assert_eq!(unit.state, State::Running); + let name = unit.name.clone(); + let dependents = unit.dependents.iter().copied().collect::>(); + let ready_set = ready_set.clone(); + let stop = state_change(name.clone(), unit, StateRequest::Stop, Some(())); + operations.push(async move { + if !ready_set.wait(OP, id, &dependents).await { + ready_set.done(id, false); + return (name, id, None); + } + let result = stop.await; + ready_set.done(id, matches!(&result, Ok(Some(())))); + (name, id, Some(result)) + }); + unit.state = State::Stopping; + } + } + + let results = join_all(operations).await; + let mut failures = Vec::new(); + let mut inner = self.inner.lock(); + for (name, id, result) in results { + let Some(unit) = inner.units.get_mut(&id) else { + continue; + }; + assert_eq!(unit.state, State::Stopping); + match result { + Some(Ok(Some(()))) => unit.state = State::Stopped, + Some(Ok(None)) => { + unit.state = State::StartUncertain; + failures.push(format!("{name}: {OP} stop request was not sent")); + } + Some(Err(error)) => { + unit.state = State::StartUncertain; + failures.push(format!("{name}: {OP} stop failed: {error:#}")); + } + None => { + unit.state = State::StartUncertain; + failures.push(format!( + "{name}: {OP} stop skipped because a dependent was not stopped" + )); + } + } + } + failures + } +} diff --git a/vmm_core/state_unit/src/tests/start.rs b/vmm_core/state_unit/src/tests/start.rs new file mode 100644 index 00000000000..41bd5d203b1 --- /dev/null +++ b/vmm_core/state_unit/src/tests/start.rs @@ -0,0 +1,428 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Tests for start with rollback. + +use crate::State; +use crate::StateRequest; +use crate::StateUnit; +use crate::StateUnits; +use crate::run_unit; +use futures::StreamExt; +use inspect::InspectMut; +use mesh::Receiver; +use pal_async::DefaultDriver; +use pal_async::async_test; +use std::sync::Arc; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; +use test_with_tracing::test; +use vmcore::save_restore::RestoreError; +use vmcore::save_restore::SaveError; +use vmcore::save_restore::SavedStateBlob; + +struct RecordingUnit { + started: Arc, + stopped: Arc, +} + +impl StateUnit for RecordingUnit { + async fn start(&mut self) { + self.started.store(true, Ordering::Relaxed); + } + + async fn stop(&mut self) { + self.stopped.store(true, Ordering::Relaxed); + } + + async fn reset(&mut self) -> anyhow::Result<()> { + Ok(()) + } + + async fn save(&mut self) -> Result, SaveError> { + Ok(None) + } + + async fn restore(&mut self, _state: SavedStateBlob) -> Result<(), RestoreError> { + Err(RestoreError::SavedStateNotSupported) + } +} + +impl InspectMut for RecordingUnit { + fn inspect_mut(&mut self, req: inspect::Request<'_>) { + req.respond(); + } +} + +/// Runs `unit`, but drops each start request after starting the unit, without +/// answering it, so that the start is uncertain while the unit still answers +/// later requests. +async fn run_unanswering_start( + mut unit: RecordingUnit, + mut recv: Receiver, +) -> RecordingUnit { + while let Some(request) = recv.next().await { + match request { + StateRequest::Start(response) => { + unit.start().await; + drop(response); + } + request => request.apply(&mut unit).await, + } + } + unit +} + +fn recording_unit(started: &Arc, stopped: &Arc) -> RecordingUnit { + RecordingUnit { + started: started.clone(), + stopped: stopped.clone(), + } +} + +fn unit_state(units: &StateUnits, name: &str) -> State { + units + .inner + .lock() + .units + .values() + .find(|unit| unit.name.as_ref() == name) + .unwrap() + .state +} + +#[async_test] +async fn an_unanswered_start_rolls_back_and_blocks_dependents(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let independent_started = Arc::new(AtomicBool::new(false)); + let independent_stopped = Arc::new(AtomicBool::new(false)); + let _independent = units + .add("independent") + .spawn(&driver, |recv| { + run_unit( + RecordingUnit { + started: independent_started.clone(), + stopped: independent_stopped.clone(), + }, + recv, + ) + }) + .unwrap(); + // A unit whose task drops its start request without answering, as a + // task that has failed would. + let unanswered = units + .add("unanswered") + .spawn(&driver, |mut recv| async move { + while let Some(request) = recv.next().await { + if matches!(request, StateRequest::Start(_)) { + break; + } + } + }) + .unwrap(); + let dependent_started = Arc::new(AtomicBool::new(false)); + let _dependent = units + .add("dependent") + .depends_on(unanswered.handle()) + .spawn(&driver, |recv| { + run_unit( + RecordingUnit { + started: dependent_started.clone(), + stopped: Arc::new(AtomicBool::new(false)), + }, + recv, + ) + }) + .unwrap(); + + let error = units.start().await.unwrap_err(); + assert!(error.to_string().contains("unanswered"), "{error:#}"); + assert!(independent_started.load(Ordering::Relaxed)); + assert!(independent_stopped.load(Ordering::Relaxed)); + assert!(!dependent_started.load(Ordering::Relaxed)); + assert!(!units.is_running()); + let retry_error = units.start().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); +} + +#[async_test] +async fn test_failed_hot_add_does_not_stop_dependency_of_running_unit(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let running_dependent_started = Arc::new(AtomicBool::new(false)); + let running_dependent_stopped = Arc::new(AtomicBool::new(false)); + let running_dependent = units + .add("running-dependent") + .spawn(&driver, |recv| { + run_unit( + recording_unit(&running_dependent_started, &running_dependent_stopped), + recv, + ) + }) + .unwrap(); + units.start().await.unwrap(); + + let dependency_started = Arc::new(AtomicBool::new(false)); + let dependency_stopped = Arc::new(AtomicBool::new(false)); + let _dependency = units + .add("new-dependency") + .dependency_of(running_dependent.handle()) + .spawn(&driver, |recv| { + run_unit( + recording_unit(&dependency_started, &dependency_stopped), + recv, + ) + }) + .unwrap(); + let unanswered_started = Arc::new(AtomicBool::new(false)); + let unanswered_stopped = Arc::new(AtomicBool::new(false)); + let _unanswered = units + .add("unanswered") + .spawn(&driver, |recv| { + run_unanswering_start( + recording_unit(&unanswered_started, &unanswered_stopped), + recv, + ) + }) + .unwrap(); + + let error = units.start_stopped_units().await.unwrap_err().to_string(); + assert!( + error.contains("failed_start_rollback stop skipped"), + "{error}" + ); + assert!(dependency_started.load(Ordering::Relaxed)); + assert!(!dependency_stopped.load(Ordering::Relaxed)); + assert!(!running_dependent_stopped.load(Ordering::Relaxed)); + assert!(unanswered_started.load(Ordering::Relaxed)); + assert!(!unanswered_stopped.load(Ordering::Relaxed)); + assert!(units.is_running()); + assert_eq!(unit_state(&units, "new-dependency"), State::StartUncertain); + assert_eq!(unit_state(&units, "unanswered"), State::StartUncertain); + + units.stop().await; + assert!(dependency_stopped.load(Ordering::Relaxed)); + assert!(running_dependent_stopped.load(Ordering::Relaxed)); + assert!(unanswered_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); + assert_eq!(unit_state(&units, "new-dependency"), State::StartUncertain); + let retry_error = units.start().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); + assert!(units.reset().await.is_err()); + assert!(units.save().await.is_err()); + assert!(units.restore(Vec::new()).await.is_err()); +} + +#[async_test] +async fn test_started_unit_exit_during_rollback_returns_error(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let dependency_started = Arc::new(AtomicBool::new(false)); + let dependency_stopped = Arc::new(AtomicBool::new(false)); + let dependency = units + .add("dependency") + .spawn(&driver, |recv| { + run_unit( + recording_unit(&dependency_started, &dependency_stopped), + recv, + ) + }) + .unwrap(); + // A unit whose task answers its start request and then exits. + let exited = units + .add("exited") + .depends_on(dependency.handle()) + .spawn(&driver, |mut recv| async move { + let Some(StateRequest::Start(response)) = recv.next().await else { + panic!("unexpected request"); + }; + response.complete(()); + }) + .unwrap(); + let unanswered_stopped = Arc::new(AtomicBool::new(false)); + let _unanswered = units + .add("unanswered") + .spawn(&driver, |recv| { + run_unanswering_start( + recording_unit(&Arc::new(AtomicBool::new(false)), &unanswered_stopped), + recv, + ) + }) + .unwrap(); + + let error = units.start().await.unwrap_err().to_string(); + assert!( + error.contains("failed_start_rollback stop failed"), + "{error}" + ); + assert!( + error.contains("failed_start_rollback stop skipped"), + "{error}" + ); + assert!(dependency_started.load(Ordering::Relaxed)); + assert!(!dependency_stopped.load(Ordering::Relaxed)); + // The dependency is still running, so a later stop must be able to stop it. + assert!(units.is_running()); + + exited.remove().await; + units.stop().await; + assert!(dependency_stopped.load(Ordering::Relaxed)); + assert!(unanswered_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); + let retry_error = units.start().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); + assert!(units.reset().await.is_err()); + assert!(units.save().await.is_err()); + assert!(units.restore(Vec::new()).await.is_err()); +} + +#[async_test] +async fn test_uncertain_start_blocks_dependency_rollback(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let dependency_started = Arc::new(AtomicBool::new(false)); + let dependency_stopped = Arc::new(AtomicBool::new(false)); + let dependency = units + .add("dependency") + .spawn(&driver, |recv| { + run_unit( + recording_unit(&dependency_started, &dependency_stopped), + recv, + ) + }) + .unwrap(); + let unanswered_started = Arc::new(AtomicBool::new(false)); + let unanswered_stopped = Arc::new(AtomicBool::new(false)); + let _unanswered = units + .add("unanswered") + .depends_on(dependency.handle()) + .spawn(&driver, |recv| { + run_unanswering_start( + recording_unit(&unanswered_started, &unanswered_stopped), + recv, + ) + }) + .unwrap(); + + let error = units.start().await.unwrap_err().to_string(); + assert!( + error.contains("failed_start_rollback stop skipped"), + "{error}" + ); + assert!(dependency_started.load(Ordering::Relaxed)); + assert!(unanswered_started.load(Ordering::Relaxed)); + assert!(!dependency_stopped.load(Ordering::Relaxed)); + assert!(units.is_running()); + assert_eq!(unit_state(&units, "dependency"), State::StartUncertain); + + units.stop().await; + assert!(unanswered_stopped.load(Ordering::Relaxed)); + assert!(dependency_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); + assert_eq!(unit_state(&units, "dependency"), State::StartUncertain); + assert!(units.reset().await.is_err()); + assert!(units.save().await.is_err()); + assert!(units.restore(Vec::new()).await.is_err()); +} + +#[async_test] +async fn test_initial_start_rollback_can_be_retried(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let started = Arc::new(AtomicBool::new(false)); + let stopped = Arc::new(AtomicBool::new(false)); + let stop_requests = Arc::new(AtomicUsize::new(0)); + // A unit that loses the response to its first stop request without + // stopping. + let _unit = units + .add("rollback-retry") + .spawn(&driver, |mut recv| { + let mut unit = recording_unit(&started, &stopped); + let stop_requests = stop_requests.clone(); + async move { + while let Some(request) = recv.next().await { + match request { + StateRequest::Stop(response) + if stop_requests.fetch_add(1, Ordering::Relaxed) == 0 => + { + drop(response); + } + request => request.apply(&mut unit).await, + } + } + unit + } + }) + .unwrap(); + let unanswered_stopped = Arc::new(AtomicBool::new(false)); + let _unanswered = units + .add("unanswered") + .spawn(&driver, |recv| { + run_unanswering_start( + recording_unit(&Arc::new(AtomicBool::new(false)), &unanswered_stopped), + recv, + ) + }) + .unwrap(); + + let error = units.start().await.unwrap_err().to_string(); + assert!( + error.contains("failed_start_rollback stop failed"), + "{error}" + ); + assert!(started.load(Ordering::Relaxed)); + assert!(!stopped.load(Ordering::Relaxed)); + assert_eq!(stop_requests.load(Ordering::Relaxed), 1); + assert!(units.is_running()); + + units.stop().await; + assert!(stopped.load(Ordering::Relaxed)); + assert_eq!(stop_requests.load(Ordering::Relaxed), 2); + assert!(unanswered_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); +} + +#[async_test] +async fn test_teardown_stop_after_unit_exits_before_answering_start(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let dependency_started = Arc::new(AtomicBool::new(false)); + let dependency_stopped = Arc::new(AtomicBool::new(false)); + let dependency = units + .add("dependency") + .spawn(&driver, |recv| { + run_unit( + recording_unit(&dependency_started, &dependency_stopped), + recv, + ) + }) + .unwrap(); + // A unit whose task answers a start and a stop, and then exits before it + // answers the next start, as a task that fails while the VM restarts after + // a failed save would. + let _exited = units + .add("exited") + .depends_on(dependency.handle()) + .spawn(&driver, |mut recv| async move { + for _ in 0..2 { + match recv.next().await { + Some(StateRequest::Start(response)) => response.complete(()), + Some(StateRequest::Stop(response)) => response.complete(()), + _ => panic!("unexpected request"), + } + } + }) + .unwrap(); + units.start().await.unwrap(); + units.stop().await; + dependency_stopped.store(false, Ordering::Relaxed); + + let error = units.start().await.unwrap_err().to_string(); + assert!(error.contains("exited"), "{error}"); + assert!(dependency_started.load(Ordering::Relaxed)); + assert!(!dependency_stopped.load(Ordering::Relaxed)); + assert!(units.is_running()); + + // Tearing the VM down stops the units without the exited unit answering. + units.stop_for_teardown().await; + assert!(dependency_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); + assert_eq!(unit_state(&units, "exited"), State::StartUncertain); + assert_eq!(unit_state(&units, "dependency"), State::StartUncertain); +} From 7a954ee45cf28c26cdc578eaa594123e8a1cd466 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Tue, 3 Nov 2026 16:10:43 -0800 Subject: [PATCH 2/6] state_unit: split the start into functions `start_with_rollback` builds the start operations and records their outcomes in two long blocks. Move each into a function of its own, `start_operations` and `record_start_results`, so that the function reads as the steps of the start. Behavior does not change. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- vmm_core/state_unit/src/start.rs | 195 +++++++++++++++++-------------- 1 file changed, 106 insertions(+), 89 deletions(-) diff --git a/vmm_core/state_unit/src/start.rs b/vmm_core/state_unit/src/start.rs index 13c18f7693b..a2503efe02a 100644 --- a/vmm_core/state_unit/src/start.rs +++ b/vmm_core/state_unit/src/start.rs @@ -10,107 +10,39 @@ use super::StateRequest; use super::StateUnits; use super::state_change; use futures::future::join_all; +use std::future::Future; +use std::sync::Arc; use std::time::Instant; +/// The outcome of a unit's start. +enum StartResult { + Started, + DependencyFailed, + Uncertain(anyhow::Error), +} + +/// The outcome of a unit's start, with the unit's name and ID. +type UnitStartResult = (Arc, u64, StartResult); + impl StateUnits { /// Starts all stopped units in dependency order. /// /// Fails without starting any unit if one is in a terminal state. A unit - /// does not start if one of its dependencies - /// did not start. If a unit's task does not answer its start request, the - /// units that did start are stopped again, and the unit is left in a - /// terminal state that fails later starts. A started unit that cannot be - /// stopped again, because a unit that depends on it was not stopped, is - /// left running in that terminal state too, and the units stay running so - /// that [`StateUnits::stop`] can stop them later. + /// does not start if one of its dependencies did not start. If a unit's + /// task does not answer its start request, the units that did start are + /// stopped again, and the unit is left in a terminal state that fails later + /// starts. A started unit that cannot be stopped again, because a unit that + /// depends on it was not stopped, is left running in that terminal state + /// too, and the units stay running so that [`StateUnits::stop`] can stop + /// them later. pub(super) async fn start_with_rollback(&mut self) -> anyhow::Result<()> { - enum StartResult { - Started, - DependencyFailed, - Uncertain(anyhow::Error), - } - let was_running = self.running; - let mut operations = Vec::new(); - let ready_set; - { - let mut inner = self.inner.lock(); - if let Some(unit) = inner - .units - .values() - .find(|unit| !matches!(unit.state, State::Stopped | State::Running)) - { - anyhow::bail!( - "state unit '{}' cannot be started from terminal state {:?}", - unit.name, - unit.state - ); - } - ready_set = inner.ready_set(None); - for (&id, unit) in &mut inner.units { - if unit.state == State::Running { - ready_set.done(id, true); - continue; - } - assert_eq!( - unit.state, - State::Stopped, - "unit {} is not stopped before start", - unit.name, - ); - let name = unit.name.clone(); - let dependencies = unit.dependencies.iter().copied().collect::>(); - let ready_set = ready_set.clone(); - let start = state_change(name.clone(), unit, StateRequest::Start, Some(())); - operations.push(async move { - if !ready_set.wait("start", id, &dependencies).await { - ready_set.done(id, false); - return (name, id, StartResult::DependencyFailed); - } - - let result = match start.await { - Ok(Some(())) => StartResult::Started, - Ok(None) => { - StartResult::Uncertain(anyhow::anyhow!("start request was not sent")) - } - Err(error) => StartResult::Uncertain(error.into()), - }; - ready_set.done(id, matches!(result, StartResult::Started)); - (name, id, result) - }); - unit.state = State::Starting; - } - } - + let operations = self.start_operations()?; let start = Instant::now(); let results = join_all(operations).await; tracing::info!(duration = ?Instant::now() - start, "state-unit start complete"); - let mut failures = Vec::new(); - let mut started_ids = Vec::new(); - { - let mut inner = self.inner.lock(); - for (name, id, result) in results { - let Some(unit) = inner.units.get_mut(&id) else { - continue; - }; - match result { - StartResult::Started => { - unit.state = State::Running; - started_ids.push(id); - } - StartResult::DependencyFailed => { - unit.state = State::Stopped; - failures.push(format!("{name}: a dependency did not start")); - } - StartResult::Uncertain(error) => { - unit.state = State::StartUncertain; - failures.push(format!("{name}: {error:#}")); - } - } - } - } - + let (started_ids, mut failures) = self.record_start_results(results); if failures.is_empty() { self.running = true; Ok(()) @@ -122,6 +54,91 @@ impl StateUnits { } } + /// Marks the stopped units as starting, and returns the operations that + /// start them in dependency order. + /// + /// Fails without changing any unit if one is in a terminal state. + fn start_operations( + &self, + ) -> anyhow::Result + use<>>> { + let mut inner = self.inner.lock(); + if let Some(unit) = inner + .units + .values() + .find(|unit| !matches!(unit.state, State::Stopped | State::Running)) + { + anyhow::bail!( + "state unit '{}' cannot be started from terminal state {:?}", + unit.name, + unit.state + ); + } + let ready_set = inner.ready_set(None); + let mut operations = Vec::new(); + for (&id, unit) in &mut inner.units { + if unit.state == State::Running { + ready_set.done(id, true); + continue; + } + assert_eq!( + unit.state, + State::Stopped, + "unit {} is not stopped before start", + unit.name, + ); + let name = unit.name.clone(); + let dependencies = unit.dependencies.iter().copied().collect::>(); + let ready_set = ready_set.clone(); + let start = state_change(name.clone(), unit, StateRequest::Start, Some(())); + operations.push(async move { + if !ready_set.wait("start", id, &dependencies).await { + ready_set.done(id, false); + return (name, id, StartResult::DependencyFailed); + } + + let result = match start.await { + Ok(Some(())) => StartResult::Started, + Ok(None) => { + StartResult::Uncertain(anyhow::anyhow!("start request was not sent")) + } + Err(error) => StartResult::Uncertain(error.into()), + }; + ready_set.done(id, matches!(result, StartResult::Started)); + (name, id, result) + }); + unit.state = State::Starting; + } + Ok(operations) + } + + /// Moves each unit to the state that the outcome of its start leaves it in, + /// and returns the IDs of the units that started and the failures. + fn record_start_results(&self, results: Vec) -> (Vec, Vec) { + let mut started_ids = Vec::new(); + let mut failures = Vec::new(); + let mut inner = self.inner.lock(); + for (name, id, result) in results { + let Some(unit) = inner.units.get_mut(&id) else { + continue; + }; + match result { + StartResult::Started => { + unit.state = State::Running; + started_ids.push(id); + } + StartResult::DependencyFailed => { + unit.state = State::Stopped; + failures.push(format!("{name}: a dependency did not start")); + } + StartResult::Uncertain(error) => { + unit.state = State::StartUncertain; + failures.push(format!("{name}: {error:#}")); + } + } + } + (started_ids, failures) + } + /// Stops the units in `started_ids` again in reverse dependency order. /// /// Every other unit that is not stopped, such as one that was already From fa3c129dd19d92cd848fe25f598c25719af4d5de Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Tue, 3 Nov 2026 16:12:25 -0800 Subject: [PATCH 3/6] state_unit: keep the units running while a start is uncertain A unit whose task drops its start request without answering it may still be running. After an initial start fails, the rollback stops the units that did start, but the units are then reported as stopped even though that unit may still run, and a later `StateUnits::stop` panics on its running assertion instead of stopping it. After a failed start, keep the units running while any of them is not stopped, as is a unit whose start is uncertain, so that `StateUnits::stop` can stop it. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- vmm_core/state_unit/src/lib.rs | 5 +++ vmm_core/state_unit/src/start.rs | 15 ++++++--- vmm_core/state_unit/src/tests/start.rs | 45 ++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 5 deletions(-) diff --git a/vmm_core/state_unit/src/lib.rs b/vmm_core/state_unit/src/lib.rs index d6e9ef772a5..d4ea5dbdbd0 100644 --- a/vmm_core/state_unit/src/lib.rs +++ b/vmm_core/state_unit/src/lib.rs @@ -526,6 +526,11 @@ impl StateUnits { } /// Starts all the state units. + /// + /// If a unit's task does not answer its start request, the start fails, + /// and the units that did start are stopped again. If a unit may still be + /// running after that, [`StateUnits::is_running`] returns true, so that + /// [`StateUnits::stop`] can stop it. pub async fn start(&mut self) -> anyhow::Result<()> { self.restore_allowed = false; self.start_with_rollback().await diff --git a/vmm_core/state_unit/src/start.rs b/vmm_core/state_unit/src/start.rs index a2503efe02a..896a6fa0559 100644 --- a/vmm_core/state_unit/src/start.rs +++ b/vmm_core/state_unit/src/start.rs @@ -33,8 +33,8 @@ impl StateUnits { /// stopped again, and the unit is left in a terminal state that fails later /// starts. A started unit that cannot be stopped again, because a unit that /// depends on it was not stopped, is left running in that terminal state - /// too, and the units stay running so that [`StateUnits::stop`] can stop - /// them later. + /// too. Because a unit in that state may be running, the units then stay + /// running, so that [`StateUnits::stop`] can stop it. pub(super) async fn start_with_rollback(&mut self) -> anyhow::Result<()> { let was_running = self.running; let operations = self.start_operations()?; @@ -47,9 +47,14 @@ impl StateUnits { self.running = true; Ok(()) } else { - let cleanup_failures = self.stop_after_failed_start(&started_ids).await; - self.running = was_running || !cleanup_failures.is_empty(); - failures.extend(cleanup_failures); + failures.extend(self.stop_after_failed_start(&started_ids).await); + self.running = was_running + || self + .inner + .lock() + .units + .values() + .any(|unit| unit.state != State::Stopped); anyhow::bail!("state units could not start: {}", failures.join("; ")) } } diff --git a/vmm_core/state_unit/src/tests/start.rs b/vmm_core/state_unit/src/tests/start.rs index 41bd5d203b1..c85a88e90f7 100644 --- a/vmm_core/state_unit/src/tests/start.rs +++ b/vmm_core/state_unit/src/tests/start.rs @@ -141,7 +141,52 @@ async fn an_unanswered_start_rolls_back_and_blocks_dependents(driver: DefaultDri assert!(independent_started.load(Ordering::Relaxed)); assert!(independent_stopped.load(Ordering::Relaxed)); assert!(!dependent_started.load(Ordering::Relaxed)); + // The unanswered unit may be running. + assert!(units.is_running()); + let retry_error = units.start().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); +} + +#[async_test] +async fn test_uncertain_start_keeps_units_running_until_stopped(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let independent_started = Arc::new(AtomicBool::new(false)); + let independent_stopped = Arc::new(AtomicBool::new(false)); + let _independent = units + .add("independent") + .spawn(&driver, |recv| { + run_unit( + recording_unit(&independent_started, &independent_stopped), + recv, + ) + }) + .unwrap(); + let unanswered_started = Arc::new(AtomicBool::new(false)); + let unanswered_stopped = Arc::new(AtomicBool::new(false)); + let _unanswered = units + .add("unanswered") + .spawn(&driver, |recv| { + run_unanswering_start( + recording_unit(&unanswered_started, &unanswered_stopped), + recv, + ) + }) + .unwrap(); + + let error = units.start().await.unwrap_err().to_string(); + assert!(error.contains("unanswered"), "{error}"); + assert!(independent_started.load(Ordering::Relaxed)); + assert!(independent_stopped.load(Ordering::Relaxed)); + assert!(unanswered_started.load(Ordering::Relaxed)); + assert!(!unanswered_stopped.load(Ordering::Relaxed)); + // The unanswered unit is running, so a stop must still be able to stop it. + assert!(units.is_running()); + assert_eq!(unit_state(&units, "unanswered"), State::StartUncertain); + + units.stop().await; + assert!(unanswered_stopped.load(Ordering::Relaxed)); assert!(!units.is_running()); + assert_eq!(unit_state(&units, "unanswered"), State::StartUncertain); let retry_error = units.start().await.unwrap_err(); assert!(retry_error.to_string().contains("terminal state")); } From 118a0337f756af29b55d87cf8c67e3b017204d99 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Tue, 3 Nov 2026 16:19:03 -0800 Subject: [PATCH 4/6] openvmm_core: tear the VM down when a reset cannot start its devices A reset with firmware reload starts the state units to assign PCI resources. If that start fails, it returns before stopping the units, which a unit whose start is uncertain keeps running, while the reset has already paused the VM. The worker then keeps serving with the state units and the VM disagreeing on whether the VM runs, and the next reset skips stopping the units and panics on the running assertion of `StateUnits::reset`. The reset pauses the VM, so state units that still run after a failed reset were left by a failed start, and they can never start again. Report the failure and tear the VM down then, as after a failed restart. The teardown stops those units, since it stops the state units whenever they run. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- openvmm/openvmm_core/src/worker/dispatch.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/openvmm/openvmm_core/src/worker/dispatch.rs b/openvmm/openvmm_core/src/worker/dispatch.rs index 6b6b558aa58..5c0ea9b3297 100644 --- a/openvmm/openvmm_core/src/worker/dispatch.rs +++ b/openvmm/openvmm_core/src/worker/dispatch.rs @@ -3864,7 +3864,16 @@ impl LoadedVm { Event::VmRpc(Err(_)) => break, Event::VmRpc(Ok(message)) => match message { VmRpc::Reset(rpc) => { - rpc.handle_failable(async |()| self.reset(true).await).await + let result = self.reset(true).await; + // The reset pauses the VM, so state units that still + // run after it fails were left by a failed start, and + // can never start again. Tear the VM down, as after a + // failed restart. + let start_failed = result.is_err() && self.state_units.is_running(); + rpc.complete(result.map_err(RemoteError::new)); + if start_failed { + break; + } } VmRpc::ClearHalt(rpc) => { rpc.handle(async |()| self.inner.partition_unit.clear_halt().await) From a4b12f216fea1af5f18ba2f2c98ec3df22afca44 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Tue, 3 Nov 2026 16:19:18 -0800 Subject: [PATCH 5/6] openvmm_core: add a hot-added VMBus device once it starts VMBus hot-add pushes the new device before starting it, and pops it again if the start fails. Push it only once it has started, as relayed vPCI hot-add does, so that the failure path only removes the unit and drops its channel unit, which revokes the channel. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- openvmm/openvmm_core/src/worker/dispatch.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/openvmm/openvmm_core/src/worker/dispatch.rs b/openvmm/openvmm_core/src/worker/dispatch.rs index 5c0ea9b3297..e2297dec9a8 100644 --- a/openvmm/openvmm_core/src/worker/dispatch.rs +++ b/openvmm/openvmm_core/src/worker/dispatch.rs @@ -3926,16 +3926,12 @@ impl LoadedVm { resource, ) .await?; - self.inner.vmbus_devices.push(device); if let Err(error) = self.state_units.start_stopped_units().await { - let device = self - .inner - .vmbus_devices - .pop() - .expect("device was just pushed"); + // Dropping the channel unit revokes the channel. drop(device.remove().await); return Err(error); } + self.inner.vmbus_devices.push(device); anyhow::Ok(()) }) .await From 7ccd934571b6d98935eac461026f6bdaa6ace5a4 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Tue, 3 Nov 2026 16:19:37 -0800 Subject: [PATCH 6/6] underhill_core: define the restart outcome at file scope `RestartOutcome` is defined inside the handler of a worker restart, deep in the request loop of `LoadedVm::run`. Define it at file scope, next to the state that a restart produces, and give its variants their concrete types. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- openhcl/underhill_core/src/dispatch/mod.rs | 23 ++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/openhcl/underhill_core/src/dispatch/mod.rs b/openhcl/underhill_core/src/dispatch/mod.rs index b7a0916189d..4b52cdfbf33 100644 --- a/openhcl/underhill_core/src/dispatch/mod.rs +++ b/openhcl/underhill_core/src/dispatch/mod.rs @@ -226,6 +226,19 @@ pub struct LoadedVmState { pub control_send: mesh::Sender, } +/// The outcome of a worker restart request. +#[expect(clippy::large_enum_variant)] +enum RestartOutcome { + /// The VM state was saved, so the worker can restart. + Restart(FailableRpc<(), T>, ServicingState), + /// The restart failed, but the VM is back in its previous state, so the + /// worker can keep serving. + Recovered, + /// The restart failed and the VM could not be resumed, so the worker + /// cannot serve any more. + Unrecoverable, +} + impl LoadedVm { /// Start running the VM which will start running VTL0. pub async fn run( @@ -328,16 +341,6 @@ impl LoadedVm { Event::WorkerRpc(message) => match message { WorkerRpc::Stop => break None, WorkerRpc::Restart(rpc) => { - enum RestartOutcome { - Restart(R, S), - // The restart failed, but the VM is back in its - // previous state, so this worker can keep serving. - Recovered, - // The restart failed and the VM could not be - // resumed, so this worker cannot serve any more. - Unrecoverable, - } - let outcome = async { let running = self.stop().await; match self