From 54eb2e27615f5ad254e856332271f4b828745f74 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Thu, 24 Sep 2026 23:50:18 -0700 Subject: [PATCH 01/17] state_unit: make unit start fallible and roll back partial starts `StateUnit::start` could not fail, so a unit had no way to report that it could not start, and `StateUnits::start` always considered every unit running. Units that restore process-local state when they start need to report such a failure before their dependents start. Make `StateUnit::start` return `anyhow::Result<()>` and carry the result in `StateRequest::Start`. `StateUnits::start` still starts the units in dependency order, but it no longer starts a unit one of whose dependencies failed to start. If any unit fails, the units that did start are stopped again and `StateUnits::start` returns the failures. A unit whose start failed, or whose start request could not be delivered, is left in a terminal state, so that a later start fails instead of running a partially started VM. `start_stopped_units` propagates the error. Every existing `StateUnit` implementation starts infallibly and now returns `Ok(())`. The callers of `StateUnits::start` and `start_stopped_units` handle the error: - VMBus, PCIe, VF, and relayed vPCI hot-add return it, and the dynamic vPCI device builder also removes the units it added. - After a failed worker restart, a failure to start the VM again is reported to the requester. - PCI resource assignment returns it. - OpenHCL logs it and does not proceed with the start. - VM resume cannot report a failure to its callers yet, so it treats a unit that fails to start as an unrecoverable unit failure, like the other state changes do. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: ef640993-0003-4e49-b1ed-1b8b4776d0ef --- openhcl/underhill_core/src/dispatch/mod.rs | 11 +- .../underhill_core/src/vmbus_relay_unit.rs | 3 +- openvmm/openvmm_core/src/worker/dispatch.rs | 23 ++- vm/devices/pci/vpci_relay/src/lib.rs | 2 +- vmm_core/src/device_builder.rs | 6 +- vmm_core/src/input_distributor.rs | 3 +- vmm_core/src/partition_unit.rs | 3 +- vmm_core/src/partition_unit/tests.rs | 12 +- vmm_core/src/vmbus_unit.rs | 9 +- vmm_core/src/vmtime_unit.rs | 3 +- vmm_core/state_unit/src/lib.rs | 49 ++++--- vmm_core/state_unit/src/start.rs | 136 ++++++++++++++++++ vmm_core/state_unit/src/tests/start.rs | 111 ++++++++++++++ .../chipset/backing/arc_mutex/state_unit.rs | 3 +- .../vmotherboard/src/chipset/builder/mod.rs | 4 +- .../vmotherboard/src/chipset/line_sets.rs | 4 +- 16 files changed, 332 insertions(+), 50 deletions(-) create mode 100644 vmm_core/state_unit/src/start.rs create mode 100644 vmm_core/state_unit/src/tests/start.rs diff --git a/openhcl/underhill_core/src/dispatch/mod.rs b/openhcl/underhill_core/src/dispatch/mod.rs index 20eebc55c06..288c5b4ab41 100644 --- a/openhcl/underhill_core/src/dispatch/mod.rs +++ b/openhcl/underhill_core/src/dispatch/mod.rs @@ -882,7 +882,14 @@ impl LoadedVm { } async fn start(&mut self, correlation_id: Option) { - self.state_units.start().await; + if let Err(error) = self.state_units.start().await { + tracing::error!( + CVM_ALLOWED, + error = error.as_ref() as &dyn std::error::Error, + "VM state units failed to start" + ); + return; + } // Log the boot/blackout time. let reference_time = ReferenceTime::new(self.partition.reference_time()); @@ -1134,7 +1141,7 @@ impl LoadedVm { ) .await?; - self.state_units.start_stopped_units().await; + self.state_units.start_stopped_units().await?; self.emuplat_servicing.netvsp_state.push(save_state); Ok(()) diff --git a/openhcl/underhill_core/src/vmbus_relay_unit.rs b/openhcl/underhill_core/src/vmbus_relay_unit.rs index bce72bc9b17..21a172e45d9 100644 --- a/openhcl/underhill_core/src/vmbus_relay_unit.rs +++ b/openhcl/underhill_core/src/vmbus_relay_unit.rs @@ -43,8 +43,9 @@ impl VmbusRelayHandle { struct VmbusRelayUnit(HostVmbusTransport); impl StateUnit for &'_ VmbusRelayUnit { - async fn start(&mut self) { + async fn start(&mut self) -> anyhow::Result<()> { self.0.start(); + Ok(()) } async fn stop(&mut self) { diff --git a/openvmm/openvmm_core/src/worker/dispatch.rs b/openvmm/openvmm_core/src/worker/dispatch.rs index ce8e17f1d79..a125d9346ab 100644 --- a/openvmm/openvmm_core/src/worker/dispatch.rs +++ b/openvmm/openvmm_core/src/worker/dispatch.rs @@ -3697,7 +3697,12 @@ impl LoadedVm { if self.running { return false; } - self.state_units.start().await; + // Resume cannot report a failure to its callers, so treat a unit that + // fails to start like any other unrecoverable state unit failure. + self.state_units + .start() + .await + .expect("state units failed to start"); self.running = true; true } @@ -3738,7 +3743,10 @@ impl LoadedVm { let stop_guard = self.inner.partition_unit.temporarily_stop_vps().await; // Start state units so device config space is accessible. - self.state_units.start().await; + self.state_units + .start() + .await + .context("failed to start devices for PCI resource assignment")?; let result = ecam_config_access::assign_pci_resources_for_root_complexes( &self.inner.chipset, @@ -3824,7 +3832,12 @@ impl LoadedVm { } Err(err) => { if stopped { - self.state_units.start().await; + if let Err(start_error) = self.state_units.start().await { + rpc.complete(Err(RemoteError::new(start_error.context( + "worker restart failed and the VM could not resume", + )))); + continue; + } } rpc.complete(Err(RemoteError::new(err))); } @@ -3897,7 +3910,7 @@ impl LoadedVm { ) .await?; self.inner.vmbus_devices.push(device); - self.state_units.start_stopped_units().await; + self.state_units.start_stopped_units().await?; anyhow::Ok(()) }) .await @@ -4043,7 +4056,7 @@ impl LoadedVm { // MSI. The guest may begin probing config space // immediately after receiving the interrupt, so // the device must be ready first. - self.state_units.start_stopped_units().await; + self.state_units.start_stopped_units().await?; // Now attach the device and notify the guest. if let Err(e) = rc.lock().hotplug_add_device( diff --git a/vm/devices/pci/vpci_relay/src/lib.rs b/vm/devices/pci/vpci_relay/src/lib.rs index 22e971f1f90..7222dcdbf5f 100644 --- a/vm/devices/pci/vpci_relay/src/lib.rs +++ b/vm/devices/pci/vpci_relay/src/lib.rs @@ -500,7 +500,7 @@ impl VpciRelay { ready_to_remove: false, }); - state_units.start_stopped_units().await; + state_units.start_stopped_units().await?; Ok(()) } } diff --git a/vmm_core/src/device_builder.rs b/vmm_core/src/device_builder.rs index 089ac1444c9..440d98a9dc4 100644 --- a/vmm_core/src/device_builder.rs +++ b/vmm_core/src/device_builder.rs @@ -144,7 +144,11 @@ pub async fn build_dynamic_vpci_device( }; let pending_offer = pending_offer.context("missing deferred VPCI channel offer")?; - state_units.start_stopped_units().await; + if let Err(error) = state_units.start_stopped_units().await { + vpci_unit.remove().await; + pci_unit.remove().await; + return Err(error); + } if let Err(error) = pending_offer .offer_registered(&vpci_bus, driver_source, vmbus, state_units.is_running()) .await diff --git a/vmm_core/src/input_distributor.rs b/vmm_core/src/input_distributor.rs index 6aafd50d85d..7d6aef61e1c 100644 --- a/vmm_core/src/input_distributor.rs +++ b/vmm_core/src/input_distributor.rs @@ -195,8 +195,9 @@ struct Inner { } impl StateUnit for Inner { - async fn start(&mut self) { + async fn start(&mut self) -> anyhow::Result<()> { self.running = true; + Ok(()) } async fn stop(&mut self) { diff --git a/vmm_core/src/partition_unit.rs b/vmm_core/src/partition_unit.rs index 3dd1ea5f802..962d9db6dab 100644 --- a/vmm_core/src/partition_unit.rs +++ b/vmm_core/src/partition_unit.rs @@ -594,10 +594,11 @@ impl Drop for StopGuard { } impl StateUnit for PartitionUnitRunner { - async fn start(&mut self) { + async fn start(&mut self) -> anyhow::Result<()> { self.partition.thaw_time(); self.unit_started = true; self.try_start(); + Ok(()) } async fn stop(&mut self) { diff --git a/vmm_core/src/partition_unit/tests.rs b/vmm_core/src/partition_unit/tests.rs index 2c62663243c..29e0690ec37 100644 --- a/vmm_core/src/partition_unit/tests.rs +++ b/vmm_core/src/partition_unit/tests.rs @@ -131,7 +131,7 @@ fn time_stays_frozen_across_initial_restore_and_reset() { StateUnit::reset(&mut runner).await.unwrap(); assert!(clock.lock().frozen); assert!(clock.lock().transitions.is_empty()); - StateUnit::start(&mut runner).await; + StateUnit::start(&mut runner).await.unwrap(); assert_eq!(clock.lock().transitions, [false]); }); } @@ -141,7 +141,7 @@ fn full_stop_freezes_time_until_resume() { block_on(async { let (mut runner, clock, _) = new_runner(); for _ in 0..2 { - StateUnit::start(&mut runner).await; + StateUnit::start(&mut runner).await.unwrap(); assert!(!clock.lock().frozen); StateUnit::stop(&mut runner).await; let state = StateUnit::save(&mut runner).await.unwrap().unwrap(); @@ -157,7 +157,7 @@ fn full_stop_freezes_time_until_resume() { fn temporary_stops_and_debugger_halts_leave_time_running() { block_on(async { let (mut runner, clock, halt) = new_runner(); - StateUnit::start(&mut runner).await; + StateUnit::start(&mut runner).await.unwrap(); runner.stop_vps().await; runner.stop_vps().await; assert!(!clock.lock().frozen); @@ -178,14 +178,14 @@ fn temporary_stops_and_debugger_halts_leave_time_running() { fn temporary_stop_guards_do_not_override_full_stop() { block_on(async { let (mut runner, clock, _) = new_runner(); - StateUnit::start(&mut runner).await; + StateUnit::start(&mut runner).await.unwrap(); runner.stop_vps().await; runner.stop_vps().await; StateUnit::stop(&mut runner).await; runner.resume_vps(); assert!(clock.lock().frozen); - StateUnit::start(&mut runner).await; + StateUnit::start(&mut runner).await.unwrap(); assert!(!clock.lock().frozen); runner.resume_vps(); assert_eq!(clock.lock().transitions, [false, true, false]); @@ -196,7 +196,7 @@ fn temporary_stop_guards_do_not_override_full_stop() { fn temporary_scrub_thaws_time_before_restarting_vps() { block_on(async { let (mut runner, clock, _) = new_runner(); - StateUnit::start(&mut runner).await; + StateUnit::start(&mut runner).await.unwrap(); runner.stop_vps().await; runner.partition.scrub_vtl(Vtl::Vtl2).unwrap(); assert!(clock.lock().frozen); diff --git a/vmm_core/src/vmbus_unit.rs b/vmm_core/src/vmbus_unit.rs index 5d6d9b0f398..76024683b0d 100644 --- a/vmm_core/src/vmbus_unit.rs +++ b/vmm_core/src/vmbus_unit.rs @@ -78,8 +78,9 @@ impl VmbusServerHandle { struct VmbusServerUnit(VmbusServer); impl StateUnit for &'_ VmbusServerUnit { - async fn start(&mut self) { + async fn start(&mut self) -> anyhow::Result<()> { self.0.start(); + Ok(()) } async fn stop(&mut self) { @@ -134,8 +135,9 @@ impl ChannelUnit { } impl StateUnit for &'_ ChannelUnit { - async fn start(&mut self) { + async fn start(&mut self) -> anyhow::Result<()> { self.0.start(); + Ok(()) } async fn stop(&mut self) { @@ -205,8 +207,9 @@ impl Inspect for SimpleChannelUnit { } impl StateUnit for &'_ SimpleChannelUnit { - async fn start(&mut self) { + async fn start(&mut self) -> anyhow::Result<()> { self.0.start(); + Ok(()) } async fn stop(&mut self) { diff --git a/vmm_core/src/vmtime_unit.rs b/vmm_core/src/vmtime_unit.rs index a53688cb527..9fa55a3c138 100644 --- a/vmm_core/src/vmtime_unit.rs +++ b/vmm_core/src/vmtime_unit.rs @@ -17,8 +17,9 @@ use vmcore::vmtime::VmTimeKeeper; struct KeeperUnit<'a>(#[inspect(mut)] &'a mut VmTimeKeeper); impl StateUnit for KeeperUnit<'_> { - async fn start(&mut self) { + async fn start(&mut self) -> anyhow::Result<()> { self.0.start().await; + Ok(()) } async fn stop(&mut self) { diff --git a/vmm_core/state_unit/src/lib.rs b/vmm_core/state_unit/src/lib.rs index 2d050514d1d..26b98af8594 100644 --- a/vmm_core/state_unit/src/lib.rs +++ b/vmm_core/state_unit/src/lib.rs @@ -30,6 +30,8 @@ #![forbid(unsafe_code)] +mod start; + use futures::FutureExt; use futures::StreamExt; use futures::future::join_all; @@ -70,7 +72,7 @@ use vmcore::save_restore::SavedStateBlob; #[derive(Debug, MeshPayload)] pub enum StateRequest { /// Start asynchronous operations. - Start(Rpc<(), ()>), + Start(FailableRpc<(), ()>), /// Stop asynchronous operations. Stop(Rpc<(), ()>), @@ -96,7 +98,7 @@ pub enum StateRequest { #[expect(async_fn_in_trait)] // Don't need Send bounds pub trait StateUnit: InspectMut { /// Start asynchronous processing. - async fn start(&mut self); + async fn start(&mut self) -> anyhow::Result<()>; /// Stop asynchronous processing. async fn stop(&mut self); @@ -190,7 +192,7 @@ impl StateRequest { /// Runs this state request against `unit`. pub async fn apply(self, unit: &mut impl StateUnit) { match self { - StateRequest::Start(rpc) => rpc.handle(async |()| unit.start().await).await, + StateRequest::Start(rpc) => rpc.handle_failable(async |()| unit.start().await).await, StateRequest::Stop(rpc) => rpc.handle(async |()| unit.stop().await).await, StateRequest::Reset(rpc) => rpc.handle_failable(async |()| unit.reset().await).await, StateRequest::Save(rpc) => rpc.handle_failable(async |()| unit.save().await).await, @@ -219,6 +221,7 @@ enum State { Resetting, Saving, Restoring, + StartUncertain, } #[derive(Debug)] @@ -478,26 +481,16 @@ impl StateUnits { /// via [`StateUnits::add`] while the VM was running. /// /// Does nothing if all units are stopped, via [`StateUnits::stop`]. - pub async fn start_stopped_units(&mut self) { + pub async fn start_stopped_units(&mut self) -> anyhow::Result<()> { if self.is_running() { - self.start().await; + self.start().await?; } + Ok(()) } /// Starts all the state units. - pub async fn start(&mut self) { - self.run_op( - "start", - None, - State::Stopped, - State::Starting, - State::Running, - StateRequest::Start, - |_, _| Some(()), - |unit| Some(&unit.dependencies), - ) - .await; - self.running = true; + pub async fn start(&mut self) -> anyhow::Result<()> { + self.start_with_rollback().await } /// Stops all the state units. @@ -1020,6 +1013,8 @@ impl Ready { #[cfg(test)] mod tests { + mod start; + use super::StateUnit; use super::StateUnits; use crate::run_unit; @@ -1050,7 +1045,9 @@ mod tests { struct SavedState(bool); impl StateUnit for TestUnit { - async fn start(&mut self) {} + async fn start(&mut self) -> anyhow::Result<()> { + Ok(()) + } async fn stop(&mut self) {} @@ -1092,7 +1089,9 @@ mod tests { } impl StateUnit for TestUnitSetDep { - async fn start(&mut self) {} + async fn start(&mut self) -> anyhow::Result<()> { + Ok(()) + } async fn stop(&mut self) {} @@ -1144,13 +1143,13 @@ mod tests { .add("b") .spawn(&driver, |recv| run_unit(TestUnit::default(), recv)) .unwrap(); - units.start().await; + units.start().await.unwrap(); let _c = units .add("c") .spawn(&driver, |recv| run_unit(TestUnit::default(), recv)); units.stop().await; - units.start().await; + units.start().await.unwrap(); units.stop().await; @@ -1197,7 +1196,7 @@ mod tests { ) }) .unwrap(); - units.start().await; + units.start().await.unwrap(); units.stop().await; let state = units.save().await.unwrap(); @@ -1306,7 +1305,7 @@ mod tests { ); } - units.start().await; + units.start().await.unwrap(); units.stop().await; a.remove().await; @@ -1316,7 +1315,7 @@ mod tests { assert!(inner.units[&c.handle().id.id].dependencies.is_empty()); } - units.start().await; + units.start().await.unwrap(); units.stop().await; // Removing the remaining unit leaves the set empty. diff --git a/vmm_core/state_unit/src/start.rs b/vmm_core/state_unit/src/start.rs new file mode 100644 index 00000000000..ed7b00a3cde --- /dev/null +++ b/vmm_core/state_unit/src/start.rs @@ -0,0 +1,136 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Fallible start with rollback. If a unit fails to start, its dependents do +//! not start, and the units that did start are stopped again. + +use super::State; +use super::StateRequest; +use super::StateUnits; +use super::state_change; +use futures::future::join_all; +use std::time::Instant; + +impl StateUnits { + /// Starts all stopped units in dependency order. + /// + /// A unit does not start if one of its dependencies fails to start. On any + /// failure, the units that did start are stopped again, and a unit whose + /// start failed or is uncertain is left in a terminal state that fails + /// later starts. + pub(super) async fn start_with_rollback(&mut self) -> anyhow::Result<()> { + enum StartResult { + Started, + DependencyFailed, + Failed(anyhow::Error), + Uncertain(anyhow::Error), + } + + let was_running = self.running; + let mut operations = Vec::new(); + let ready_set; + { + let mut inner = self.inner.lock(); + if let Some(unit) = inner + .units + .values() + .find(|unit| !matches!(unit.state, State::Stopped | State::Running)) + { + anyhow::bail!( + "state unit '{}' cannot be started from terminal state {:?}", + unit.name, + unit.state + ); + } + ready_set = inner.ready_set(None); + for (&id, unit) in &mut inner.units { + if unit.state == State::Running { + ready_set.done(id, true); + continue; + } + assert_eq!( + unit.state, + State::Stopped, + "unit {} is not stopped before start", + unit.name, + ); + let name = unit.name.clone(); + let dependencies = unit.dependencies.iter().copied().collect::>(); + let ready_set = ready_set.clone(); + let start = state_change(name.clone(), unit, StateRequest::Start, Some(())); + operations.push(async move { + if !ready_set.wait("start", id, &dependencies).await { + ready_set.done(id, false); + return (name, id, StartResult::DependencyFailed); + } + + let result = match start.await { + Ok(Some(Ok(()))) => StartResult::Started, + Ok(Some(Err(error))) => StartResult::Failed(error.into()), + Ok(None) => { + StartResult::Uncertain(anyhow::anyhow!("start request was not sent")) + } + Err(error) => StartResult::Uncertain(error.into()), + }; + ready_set.done(id, matches!(result, StartResult::Started)); + (name, id, result) + }); + unit.state = State::Starting; + } + } + + let start = Instant::now(); + let results = join_all(operations).await; + tracing::info!(duration = ?Instant::now() - start, "state-unit start complete"); + + let mut failures = Vec::new(); + let mut started_ids = Vec::new(); + { + let mut inner = self.inner.lock(); + for (name, id, result) in results { + let Some(unit) = inner.units.get_mut(&id) else { + continue; + }; + match result { + StartResult::Started => { + unit.state = State::Running; + started_ids.push(id); + } + StartResult::DependencyFailed => { + unit.state = State::Stopped; + failures.push(format!("{name}: a dependency did not start")); + } + StartResult::Failed(error) => { + unit.state = State::StartUncertain; + failures.push(format!("{name}: {error:#}")); + } + StartResult::Uncertain(error) => { + unit.state = State::StartUncertain; + failures.push(format!("{name}: {error:#}")); + } + } + } + } + + if failures.is_empty() { + self.running = true; + Ok(()) + } else { + if !started_ids.is_empty() { + self.run_op( + "failed_start_rollback", + Some(&started_ids), + State::Running, + State::Stopping, + State::Stopped, + StateRequest::Stop, + |_, _| Some(()), + |unit| Some(&unit.dependents), + ) + .await; + } + self.running = was_running; + anyhow::bail!("state units could not start: {}", failures.join("; ")) + } + } +} diff --git a/vmm_core/state_unit/src/tests/start.rs b/vmm_core/state_unit/src/tests/start.rs new file mode 100644 index 00000000000..f08063f4cae --- /dev/null +++ b/vmm_core/state_unit/src/tests/start.rs @@ -0,0 +1,111 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Tests for fallible start. + +use crate::StateUnit; +use crate::StateUnits; +use crate::run_unit; +use inspect::InspectMut; +use pal_async::DefaultDriver; +use pal_async::async_test; +use std::sync::Arc; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; +use test_with_tracing::test; +use vmcore::save_restore::RestoreError; +use vmcore::save_restore::SaveError; +use vmcore::save_restore::SavedStateBlob; + +struct StartResultUnit { + started: Arc, + stopped: Arc, + fail: bool, +} + +impl StateUnit for StartResultUnit { + async fn start(&mut self) -> anyhow::Result<()> { + anyhow::ensure!(!self.fail, "intentional start failure"); + self.started.store(true, Ordering::Relaxed); + Ok(()) + } + + async fn stop(&mut self) { + self.stopped.store(true, Ordering::Relaxed); + } + + async fn reset(&mut self) -> anyhow::Result<()> { + Ok(()) + } + + async fn save(&mut self) -> Result, SaveError> { + Ok(None) + } + + async fn restore(&mut self, _state: SavedStateBlob) -> Result<(), RestoreError> { + Err(RestoreError::SavedStateNotSupported) + } +} + +impl InspectMut for StartResultUnit { + fn inspect_mut(&mut self, req: inspect::Request<'_>) { + req.respond(); + } +} + +#[async_test] +async fn test_start_failure_blocks_dependent(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let independent_started = Arc::new(AtomicBool::new(false)); + let independent_stopped = Arc::new(AtomicBool::new(false)); + let _independent = units + .add("independent") + .spawn(&driver, |recv| { + run_unit( + StartResultUnit { + started: independent_started.clone(), + stopped: independent_stopped.clone(), + fail: false, + }, + recv, + ) + }) + .unwrap(); + let failed = units + .add("failed") + .spawn(&driver, |recv| { + run_unit( + StartResultUnit { + started: Arc::new(AtomicBool::new(false)), + stopped: Arc::new(AtomicBool::new(false)), + fail: true, + }, + recv, + ) + }) + .unwrap(); + let dependent_started = Arc::new(AtomicBool::new(false)); + let _dependent = units + .add("dependent") + .depends_on(failed.handle()) + .spawn(&driver, |recv| { + run_unit( + StartResultUnit { + started: dependent_started.clone(), + stopped: Arc::new(AtomicBool::new(false)), + fail: false, + }, + recv, + ) + }) + .unwrap(); + + let error = units.start().await.unwrap_err(); + assert!(error.to_string().contains("intentional start failure")); + assert!(independent_started.load(Ordering::Relaxed)); + assert!(independent_stopped.load(Ordering::Relaxed)); + assert!(!dependent_started.load(Ordering::Relaxed)); + assert!(!units.is_running()); + let retry_error = units.start().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); +} diff --git a/vmm_core/vmotherboard/src/chipset/backing/arc_mutex/state_unit.rs b/vmm_core/vmotherboard/src/chipset/backing/arc_mutex/state_unit.rs index 5f296e7e28f..2c9550410fd 100644 --- a/vmm_core/vmotherboard/src/chipset/backing/arc_mutex/state_unit.rs +++ b/vmm_core/vmotherboard/src/chipset/backing/arc_mutex/state_unit.rs @@ -139,13 +139,14 @@ impl ArcMutexChipsetDeviceUnit { } impl StateUnit for ArcMutexChipsetDeviceUnit { - async fn start(&mut self) { + async fn start(&mut self) -> anyhow::Result<()> { self.running = true; // Poll the device at least once. let mut device = self.device.lock(); device.start(); device.poll_device(&mut Context::from_waker(&waker_ref(&self.poll_event))); + Ok(()) } async fn stop(&mut self) { diff --git a/vmm_core/vmotherboard/src/chipset/builder/mod.rs b/vmm_core/vmotherboard/src/chipset/builder/mod.rs index 788e80f98b1..a8e47c0fae0 100644 --- a/vmm_core/vmotherboard/src/chipset/builder/mod.rs +++ b/vmm_core/vmotherboard/src/chipset/builder/mod.rs @@ -379,7 +379,9 @@ mod chipset_unit { pub struct ChipsetUnit<'a>(pub &'a Chipset); impl StateUnit for ChipsetUnit<'_> { - async fn start(&mut self) {} + async fn start(&mut self) -> anyhow::Result<()> { + Ok(()) + } async fn stop(&mut self) {} diff --git a/vmm_core/vmotherboard/src/chipset/line_sets.rs b/vmm_core/vmotherboard/src/chipset/line_sets.rs index dd6d73f7d7b..9d8517a8176 100644 --- a/vmm_core/vmotherboard/src/chipset/line_sets.rs +++ b/vmm_core/vmotherboard/src/chipset/line_sets.rs @@ -60,7 +60,9 @@ impl LineSets { struct LineSetUnit<'a>(&'a LineSet); impl StateUnit for LineSetUnit<'_> { - async fn start(&mut self) {} + async fn start(&mut self) -> anyhow::Result<()> { + Ok(()) + } async fn stop(&mut self) {} From 18ff657b31a1d9bcdc921e83d35eb70378ae0227 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Mon, 12 Oct 2026 17:53:44 -0300 Subject: [PATCH 02/17] state_unit: clean up failed starts Allow a full stop to clean up units whose start result is uncertain, while returning transition errors instead of panicking when reset, save, or restore cannot operate on such a unit. Remove failed dynamic VMBus, PCIe, VPCI relay, and OpenHCL network additions. Propagate OpenHCL resume failures and terminate workers whose recovery start fails so their running state cannot become stale. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- openhcl/underhill_core/src/dispatch/mod.rs | 111 ++++++++++++-------- openhcl/underhill_core/src/lib.rs | 2 +- openhcl/underhill_core/src/worker.rs | 1 + openvmm/openvmm_core/src/worker/dispatch.rs | 22 +++- vm/devices/pci/vpci_relay/src/lib.rs | 10 +- vmm_core/state_unit/src/lib.rs | 31 ++++++ vmm_core/state_unit/src/tests/start.rs | 43 ++++++++ 7 files changed, 169 insertions(+), 51 deletions(-) diff --git a/openhcl/underhill_core/src/dispatch/mod.rs b/openhcl/underhill_core/src/dispatch/mod.rs index 288c5b4ab41..e8a8e5678ff 100644 --- a/openhcl/underhill_core/src/dispatch/mod.rs +++ b/openhcl/underhill_core/src/dispatch/mod.rs @@ -79,7 +79,7 @@ use vtl2_settings_worker::handle_vtl2_config_rpc; #[derive(MeshPayload)] pub enum UhVmRpc { Pause(Rpc<(), bool>), - Resume(Rpc<(), bool>), + Resume(FailableRpc<(), bool>), Save(FailableRpc<(), Vec>), ClearHalt(Rpc<(), bool>), // TODO: remove this, and use DebugRequest::Resume PacketCapture(FailableRpc, PacketCaptureParams>), @@ -233,9 +233,11 @@ impl LoadedVm { correlation_id: Option, mut vm_rpc: mesh::Receiver, mut worker_rpc: mesh::Receiver>, - ) -> Option> { + ) -> anyhow::Result>> { if autostart_vps { - self.start(correlation_id).await; + self.start(correlation_id) + .await + .context("failed to autostart VM")?; } // VTL2 settings services @@ -323,7 +325,7 @@ impl LoadedVm { Event::WorkerRpc(message) => match message { WorkerRpc::Stop => break None, WorkerRpc::Restart(rpc) => { - let state = async { + let (state, terminate) = async { let running = self.stop().await; match self .save( @@ -334,19 +336,29 @@ impl LoadedVm { ) .await { - Ok(servicing_state) => Some((rpc, servicing_state)), + Ok(servicing_state) => (Some((rpc, servicing_state)), false), Err(err) => { if running { - self.start(None).await; + if let Err(start_error) = self.start(None).await { + rpc.complete(Err(RemoteError::new( + start_error.context(format!( + "worker restart failed ({err:#}) and the VM could not resume" + )), + ))); + return (None, true); + } } rpc.complete(Err(RemoteError::new(err))); - None + (None, false) } } } .instrument(tracing::info_span!("restart", CVM_ALLOWED)) .await; + if terminate { + break None; + } if let Some((rpc, servicing_state)) = state { break Some(LoadedVmState { restart_rpc: rpc, @@ -403,34 +415,35 @@ impl LoadedVm { } Event::UhVmRpc(msg) => match msg { UhVmRpc::Resume(rpc) => { - rpc.handle(async |()| { - if !self.state_units.is_running() { - self.start(None).await; - true - } else { - false - } - }) - .await + if self.state_units.is_running() { + rpc.complete(Ok(false)); + } else if let Err(error) = self.start(None).await { + rpc.complete(Err(RemoteError::new( + error.context("failed to resume VM"), + ))); + break None; + } else { + rpc.complete(Ok(true)); + } } UhVmRpc::Pause(rpc) => rpc.handle(async |()| self.stop().await).await, UhVmRpc::Save(rpc) => { - rpc.handle_failable(async |()| { - let running = self.stop().await; - let r = self - .save( - SaveReason::Diagnostic, - None, - KeepAliveConfig::Disabled, - KeepAliveConfig::Disabled, - ) - .await; - if running { - self.start(None).await; - } - r.map(mesh::payload::encode) - }) - .await + let running = self.stop().await; + let result = self + .save( + SaveReason::Diagnostic, + None, + KeepAliveConfig::Disabled, + KeepAliveConfig::Disabled, + ) + .await; + if running && let Err(error) = self.start(None).await { + rpc.complete(Err(RemoteError::new( + error.context("failed to resume VM after save"), + ))); + break None; + } + rpc.complete(result.map(mesh::payload::encode).map_err(RemoteError::new)); } UhVmRpc::ClearHalt(rpc) => { rpc.handle(async |()| self.partition_unit.clear_halt().await) @@ -600,7 +613,7 @@ impl LoadedVm { vmbus.remove().await.shutdown().await; } - state + Ok(state) } /// Handles a servicing request from the host. @@ -651,7 +664,9 @@ impl LoadedVm { .context("failed to notify host of servicing-while-paused failure")?; if running { - self.start(Some(correlation_id)).await; + self.start(Some(correlation_id)) + .await + .context("failed to resume VM after servicing failure")?; } false } @@ -881,15 +896,8 @@ impl LoadedVm { } } - async fn start(&mut self, correlation_id: Option) { - if let Err(error) = self.state_units.start().await { - tracing::error!( - CVM_ALLOWED, - error = error.as_ref() as &dyn std::error::Error, - "VM state units failed to start" - ); - return; - } + async fn start(&mut self, correlation_id: Option) -> anyhow::Result<()> { + self.state_units.start().await?; // Log the boot/blackout time. let reference_time = ReferenceTime::new(self.partition.reference_time()); @@ -915,6 +923,8 @@ impl LoadedVm { "starting VM" ) } + + Ok(()) } /// Returns true if the VM was previously running. @@ -1141,7 +1151,20 @@ impl LoadedVm { ) .await?; - self.state_units.start_stopped_units().await?; + if let Err(error) = self.state_units.start_stopped_units().await { + if let Err(remove_error) = self + .network_settings + .as_mut() + .unwrap() + .remove_network(instance_id) + .await + { + return Err(error.context(format!( + "failed to remove network after start failure: {remove_error:#}" + ))); + } + return Err(error); + } self.emuplat_servicing.netvsp_state.push(save_state); Ok(()) diff --git a/openhcl/underhill_core/src/lib.rs b/openhcl/underhill_core/src/lib.rs index 329d1c119f8..f36b578149e 100644 --- a/openhcl/underhill_core/src/lib.rs +++ b/openhcl/underhill_core/src/lib.rs @@ -690,7 +690,7 @@ async fn run_control( .vm_rpc .call(UhVmRpc::Resume, ()) .await - .context("failed to resumed VM")?; + .context("failed to resume VM")??; let was_halted = workers .vm_rpc diff --git a/openhcl/underhill_core/src/worker.rs b/openhcl/underhill_core/src/worker.rs index 702894bd524..694653db74e 100644 --- a/openhcl/underhill_core/src/worker.rs +++ b/openhcl/underhill_core/src/worker.rs @@ -465,6 +465,7 @@ impl Worker for UnderhillVmWorker { )); tracing::info!(CVM_ALLOWED, "terminating worker"); self.get_thread.join().unwrap(); + let state = state?; if let Some(state) = state { let params = UnderhillWorkerParameters { env_cfg: self.env_cfg, diff --git a/openvmm/openvmm_core/src/worker/dispatch.rs b/openvmm/openvmm_core/src/worker/dispatch.rs index a125d9346ab..8eacf856af9 100644 --- a/openvmm/openvmm_core/src/worker/dispatch.rs +++ b/openvmm/openvmm_core/src/worker/dispatch.rs @@ -3836,7 +3836,7 @@ impl LoadedVm { rpc.complete(Err(RemoteError::new(start_error.context( "worker restart failed and the VM could not resume", )))); - continue; + return; } } rpc.complete(Err(RemoteError::new(err))); @@ -3910,7 +3910,15 @@ impl LoadedVm { ) .await?; self.inner.vmbus_devices.push(device); - self.state_units.start_stopped_units().await?; + if let Err(error) = self.state_units.start_stopped_units().await { + let device = self + .inner + .vmbus_devices + .pop() + .expect("device was just pushed"); + drop(device.remove().await); + return Err(error); + } anyhow::Ok(()) }) .await @@ -4056,7 +4064,15 @@ impl LoadedVm { // MSI. The guest may begin probing config space // immediately after receiving the interrupt, so // the device must be ready first. - self.state_units.start_stopped_units().await?; + if let Err(error) = self.state_units.start_stopped_units().await { + let (_, unit, _) = self + .inner + .pcie_hotplug_devices + .pop() + .expect("device was just pushed"); + unit.remove().await; + return Err(error); + } // Now attach the device and notify the guest. if let Err(e) = rc.lock().hotplug_add_device( diff --git a/vm/devices/pci/vpci_relay/src/lib.rs b/vm/devices/pci/vpci_relay/src/lib.rs index 7222dcdbf5f..76bae759c18 100644 --- a/vm/devices/pci/vpci_relay/src/lib.rs +++ b/vm/devices/pci/vpci_relay/src/lib.rs @@ -490,7 +490,7 @@ impl VpciRelay { .await? }; - entry.insert(RelayedDevice { + let device = RelayedDevice { bus_instance_id: instance_id, bus_client: vpci_client, vpci_device: vpci_device.clone(), @@ -498,9 +498,13 @@ impl VpciRelay { bus_unit, device_unit, ready_to_remove: false, - }); + }; - state_units.start_stopped_units().await?; + if let Err(error) = state_units.start_stopped_units().await { + device.remove().await; + return Err(error); + } + entry.insert(device); Ok(()) } } diff --git a/vmm_core/state_unit/src/lib.rs b/vmm_core/state_unit/src/lib.rs index 26b98af8594..fa4915f89dc 100644 --- a/vmm_core/state_unit/src/lib.rs +++ b/vmm_core/state_unit/src/lib.rs @@ -477,6 +477,24 @@ impl StateUnits { self.running } + fn check_no_terminal_units(&self, op: &'static str) -> Result<(), StateTransitionError> { + let inner = self.inner.lock(); + check( + op, + inner.units.values().filter_map(|unit| { + (unit.state == State::StartUncertain).then(|| { + ( + unit.name.clone(), + Err::<(), _>(anyhow::anyhow!( + "unit is in terminal state {:?}", + unit.state + )), + ) + }) + }), + ) + } + /// Starts any units that are individually stopped, because they were added /// via [`StateUnits::add`] while the VM was running. /// @@ -496,6 +514,16 @@ impl StateUnits { /// Stops all the state units. pub async fn stop(&mut self) { assert!(self.running); + { + let mut inner = self.inner.lock(); + for unit in inner.units.values_mut() { + if unit.state == State::StartUncertain { + // Start may have changed the unit before failing, so it + // must receive a stop request with the running units. + unit.state = State::Running; + } + } + } // Stop units in reverse dependency order so that a dependency is not // stopped before its dependant. self.run_op( @@ -517,6 +545,7 @@ impl StateUnits { /// Panics if running. pub async fn reset(&mut self) -> Result<(), StateTransitionError> { assert!(!self.running); + self.check_no_terminal_units("reset")?; // Reset in dependency order so that dependants observe their // dependencies' reset state. let r = self @@ -541,6 +570,7 @@ impl StateUnits { /// Panics if running. pub async fn save(&mut self) -> Result, StateTransitionError> { assert!(!self.running); + self.check_no_terminal_units("save")?; // Save can occur in any order since it will not observably mutate // state. let r = self @@ -574,6 +604,7 @@ impl StateUnits { states: Vec, ) -> Result<(), StateTransitionError> { assert!(!self.running); + self.check_no_terminal_units("restore")?; #[derive(Debug, Error)] enum RestoreUnitError { diff --git a/vmm_core/state_unit/src/tests/start.rs b/vmm_core/state_unit/src/tests/start.rs index f08063f4cae..db93798328a 100644 --- a/vmm_core/state_unit/src/tests/start.rs +++ b/vmm_core/state_unit/src/tests/start.rs @@ -108,4 +108,47 @@ async fn test_start_failure_blocks_dependent(driver: DefaultDriver) { assert!(!units.is_running()); let retry_error = units.start().await.unwrap_err(); assert!(retry_error.to_string().contains("terminal state")); + assert!(units.reset().await.is_err()); + assert!(units.save().await.is_err()); + assert!(units.restore(Vec::new()).await.is_err()); +} + +#[async_test] +async fn test_stop_cleans_up_failed_start_while_running(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let _running = units + .add("running") + .spawn(&driver, |recv| { + run_unit( + StartResultUnit { + started: Arc::new(AtomicBool::new(false)), + stopped: Arc::new(AtomicBool::new(false)), + fail: false, + }, + recv, + ) + }) + .unwrap(); + units.start().await.unwrap(); + + let failed_stopped = Arc::new(AtomicBool::new(false)); + let _failed = units + .add("failed") + .spawn(&driver, |recv| { + run_unit( + StartResultUnit { + started: Arc::new(AtomicBool::new(false)), + stopped: failed_stopped.clone(), + fail: true, + }, + recv, + ) + }) + .unwrap(); + + units.start_stopped_units().await.unwrap_err(); + assert!(units.is_running()); + units.stop().await; + assert!(failed_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); } From c598d975874a1de8fcf508140f3c0fb9c444d0b9 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Mon, 12 Oct 2026 18:20:47 -0300 Subject: [PATCH 03/17] openvmm: preserve teardown after start failures Exit the loaded-VM loop, rather than the worker function, when restart recovery cannot start the state units so normal OpenVMM teardown still runs. Factor Underhill's common teardown into a helper and invoke it before returning an autostart failure, ensuring partially started resources are released on that path as well. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- openhcl/underhill_core/src/dispatch/mod.rs | 15 ++++++++++----- openvmm/openvmm_core/src/worker/dispatch.rs | 2 +- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/openhcl/underhill_core/src/dispatch/mod.rs b/openhcl/underhill_core/src/dispatch/mod.rs index e8a8e5678ff..06c6f173e83 100644 --- a/openhcl/underhill_core/src/dispatch/mod.rs +++ b/openhcl/underhill_core/src/dispatch/mod.rs @@ -235,9 +235,10 @@ impl LoadedVm { mut worker_rpc: mesh::Receiver>, ) -> anyhow::Result>> { if autostart_vps { - self.start(correlation_id) - .await - .context("failed to autostart VM")?; + if let Err(error) = self.start(correlation_id).await { + self.teardown().await; + return Err(error.context("failed to autostart VM")); + } } // VTL2 settings services @@ -595,6 +596,12 @@ impl LoadedVm { } }; + self.teardown().await; + + Ok(state) + } + + async fn teardown(mut self) { // Drain resealing before tearing down any storage transport. Removing // the unit closes its request stream; in-flight I/O finishes first. if let Some(resealer) = self.hardware_reseal.take() { @@ -612,8 +619,6 @@ impl LoadedVm { if let Some(vmbus) = self.vmbus_server { vmbus.remove().await.shutdown().await; } - - Ok(state) } /// Handles a servicing request from the host. diff --git a/openvmm/openvmm_core/src/worker/dispatch.rs b/openvmm/openvmm_core/src/worker/dispatch.rs index 8eacf856af9..2ae6541952f 100644 --- a/openvmm/openvmm_core/src/worker/dispatch.rs +++ b/openvmm/openvmm_core/src/worker/dispatch.rs @@ -3836,7 +3836,7 @@ impl LoadedVm { rpc.complete(Err(RemoteError::new(start_error.context( "worker restart failed and the VM could not resume", )))); - return; + break; } } rpc.complete(Err(RemoteError::new(err))); From c6499e3ed0c9f4776a63bc196f5c226db9f72686 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Mon, 12 Oct 2026 18:41:43 -0300 Subject: [PATCH 04/17] state_unit: stop units whose start fails A unit can change runtime state before returning a start error. Send stop to such units before rolling back successfully started dependencies, while retaining their terminal state so an immediate retry remains prohibited. Extend the start rollback test with a failed unit that records both its start side effect and subsequent cleanup. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- vmm_core/state_unit/src/lib.rs | 10 ++++++---- vmm_core/state_unit/src/start.rs | 15 +++++++++++++++ vmm_core/state_unit/src/tests/start.rs | 10 +++++++--- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/vmm_core/state_unit/src/lib.rs b/vmm_core/state_unit/src/lib.rs index fa4915f89dc..b1732809111 100644 --- a/vmm_core/state_unit/src/lib.rs +++ b/vmm_core/state_unit/src/lib.rs @@ -481,8 +481,11 @@ impl StateUnits { let inner = self.inner.lock(); check( op, - inner.units.values().filter_map(|unit| { - (unit.state == State::StartUncertain).then(|| { + inner + .units + .values() + .filter(|unit| unit.state == State::StartUncertain) + .map(|unit| { ( unit.name.clone(), Err::<(), _>(anyhow::anyhow!( @@ -490,8 +493,7 @@ impl StateUnits { unit.state )), ) - }) - }), + }), ) } diff --git a/vmm_core/state_unit/src/start.rs b/vmm_core/state_unit/src/start.rs index ed7b00a3cde..ca25e86e27d 100644 --- a/vmm_core/state_unit/src/start.rs +++ b/vmm_core/state_unit/src/start.rs @@ -85,6 +85,7 @@ impl StateUnits { let mut failures = Vec::new(); let mut started_ids = Vec::new(); + let mut failed_start_ids = Vec::new(); { let mut inner = self.inner.lock(); for (name, id, result) in results { @@ -102,6 +103,7 @@ impl StateUnits { } StartResult::Failed(error) => { unit.state = State::StartUncertain; + failed_start_ids.push(id); failures.push(format!("{name}: {error:#}")); } StartResult::Uncertain(error) => { @@ -116,6 +118,19 @@ impl StateUnits { self.running = true; Ok(()) } else { + if !failed_start_ids.is_empty() { + self.run_op( + "failed_start_cleanup", + Some(&failed_start_ids), + State::StartUncertain, + State::Stopping, + State::StartUncertain, + StateRequest::Stop, + |_, _| Some(()), + |unit| Some(&unit.dependents), + ) + .await; + } if !started_ids.is_empty() { self.run_op( "failed_start_rollback", diff --git a/vmm_core/state_unit/src/tests/start.rs b/vmm_core/state_unit/src/tests/start.rs index db93798328a..8b6cee1258a 100644 --- a/vmm_core/state_unit/src/tests/start.rs +++ b/vmm_core/state_unit/src/tests/start.rs @@ -25,8 +25,8 @@ struct StartResultUnit { impl StateUnit for StartResultUnit { async fn start(&mut self) -> anyhow::Result<()> { - anyhow::ensure!(!self.fail, "intentional start failure"); self.started.store(true, Ordering::Relaxed); + anyhow::ensure!(!self.fail, "intentional start failure"); Ok(()) } @@ -71,13 +71,15 @@ async fn test_start_failure_blocks_dependent(driver: DefaultDriver) { ) }) .unwrap(); + let failed_started = Arc::new(AtomicBool::new(false)); + let failed_stopped = Arc::new(AtomicBool::new(false)); let failed = units .add("failed") .spawn(&driver, |recv| { run_unit( StartResultUnit { - started: Arc::new(AtomicBool::new(false)), - stopped: Arc::new(AtomicBool::new(false)), + started: failed_started.clone(), + stopped: failed_stopped.clone(), fail: true, }, recv, @@ -104,6 +106,8 @@ async fn test_start_failure_blocks_dependent(driver: DefaultDriver) { assert!(error.to_string().contains("intentional start failure")); assert!(independent_started.load(Ordering::Relaxed)); assert!(independent_stopped.load(Ordering::Relaxed)); + assert!(failed_started.load(Ordering::Relaxed)); + assert!(failed_stopped.load(Ordering::Relaxed)); assert!(!dependent_started.load(Ordering::Relaxed)); assert!(!units.is_running()); let retry_error = units.start().await.unwrap_err(); From 55942a68365db0619af8f0b9405b807e870a2fa5 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Mon, 12 Oct 2026 18:58:55 -0300 Subject: [PATCH 05/17] state_unit: preserve cleaned start failures Distinguish failed starts that rollback already stopped from starts whose outcome is uncertain. During a later full stop, skip the former and stop the latter, then restore both terminal markers so no lifecycle operation can silently reuse a partially initialized unit. Extend the hot-add regression test to reject duplicate stop calls and verify that start, reset, save, and restore remain blocked after the full stop. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- vmm_core/state_unit/src/lib.rs | 41 +++++++++++++++++++------- vmm_core/state_unit/src/start.rs | 2 +- vmm_core/state_unit/src/tests/start.rs | 9 +++++- 3 files changed, 40 insertions(+), 12 deletions(-) diff --git a/vmm_core/state_unit/src/lib.rs b/vmm_core/state_unit/src/lib.rs index b1732809111..087f1e04303 100644 --- a/vmm_core/state_unit/src/lib.rs +++ b/vmm_core/state_unit/src/lib.rs @@ -222,6 +222,7 @@ enum State { Saving, Restoring, StartUncertain, + StartFailedStopped, } #[derive(Debug)] @@ -484,7 +485,12 @@ impl StateUnits { inner .units .values() - .filter(|unit| unit.state == State::StartUncertain) + .filter(|unit| { + matches!( + unit.state, + State::StartUncertain | State::StartFailedStopped + ) + }) .map(|unit| { ( unit.name.clone(), @@ -516,16 +522,24 @@ impl StateUnits { /// Stops all the state units. pub async fn stop(&mut self) { assert!(self.running); - { + // Make uncertain starts receive Stop and already-cleaned failures skip + // it, then restore both terminal markers after the ordered operation. + let terminal_states = { let mut inner = self.inner.lock(); - for unit in inner.units.values_mut() { - if unit.state == State::StartUncertain { - // Start may have changed the unit before failing, so it - // must receive a stop request with the running units. - unit.state = State::Running; - } - } - } + inner + .units + .iter_mut() + .filter_map(|(&id, unit)| { + let state = unit.state; + match state { + State::StartUncertain => unit.state = State::Running, + State::StartFailedStopped => unit.state = State::Stopped, + _ => return None, + } + Some((id, state)) + }) + .collect::>() + }; // Stop units in reverse dependency order so that a dependency is not // stopped before its dependant. self.run_op( @@ -539,6 +553,13 @@ impl StateUnits { |unit| Some(&unit.dependents), ) .await; + let mut inner = self.inner.lock(); + for (id, state) in terminal_states { + if let Some(unit) = inner.units.get_mut(&id) { + assert_eq!(unit.state, State::Stopped); + unit.state = state; + } + } self.running = false; } diff --git a/vmm_core/state_unit/src/start.rs b/vmm_core/state_unit/src/start.rs index ca25e86e27d..b474f4074ee 100644 --- a/vmm_core/state_unit/src/start.rs +++ b/vmm_core/state_unit/src/start.rs @@ -124,7 +124,7 @@ impl StateUnits { Some(&failed_start_ids), State::StartUncertain, State::Stopping, - State::StartUncertain, + State::StartFailedStopped, StateRequest::Stop, |_, _| Some(()), |unit| Some(&unit.dependents), diff --git a/vmm_core/state_unit/src/tests/start.rs b/vmm_core/state_unit/src/tests/start.rs index 8b6cee1258a..1b456f2bc0c 100644 --- a/vmm_core/state_unit/src/tests/start.rs +++ b/vmm_core/state_unit/src/tests/start.rs @@ -31,7 +31,7 @@ impl StateUnit for StartResultUnit { } async fn stop(&mut self) { - self.stopped.store(true, Ordering::Relaxed); + assert!(!self.stopped.swap(true, Ordering::Relaxed)); } async fn reset(&mut self) -> anyhow::Result<()> { @@ -152,7 +152,14 @@ async fn test_stop_cleans_up_failed_start_while_running(driver: DefaultDriver) { units.start_stopped_units().await.unwrap_err(); assert!(units.is_running()); + let retry_error = units.start_stopped_units().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); units.stop().await; assert!(failed_stopped.load(Ordering::Relaxed)); assert!(!units.is_running()); + let retry_error = units.start().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); + assert!(units.reset().await.is_err()); + assert!(units.save().await.is_err()); + assert!(units.restore(Vec::new()).await.is_err()); } From cb87ff9364c702e3903c04bb391bf0e35df3121b Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Mon, 12 Oct 2026 19:17:05 -0300 Subject: [PATCH 06/17] state_unit: test uncertain start cleanup Exercise the transport-uncertain start path by dropping a Start RPC response while keeping the unit receiver alive. Verify that retry and stopped lifecycle operations remain rejected and that a later full stop sends exactly one cleanup request. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- vmm_core/state_unit/src/tests/start.rs | 61 ++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/vmm_core/state_unit/src/tests/start.rs b/vmm_core/state_unit/src/tests/start.rs index 1b456f2bc0c..dfeb7402370 100644 --- a/vmm_core/state_unit/src/tests/start.rs +++ b/vmm_core/state_unit/src/tests/start.rs @@ -3,9 +3,11 @@ //! Tests for fallible start. +use crate::StateRequest; use crate::StateUnit; use crate::StateUnits; use crate::run_unit; +use futures::StreamExt; use inspect::InspectMut; use pal_async::DefaultDriver; use pal_async::async_test; @@ -163,3 +165,62 @@ async fn test_stop_cleans_up_failed_start_while_running(driver: DefaultDriver) { assert!(units.save().await.is_err()); assert!(units.restore(Vec::new()).await.is_err()); } + +#[async_test] +async fn test_uncertain_start_is_cleaned_once_and_remains_terminal(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let _running = units + .add("running") + .spawn(&driver, |recv| { + run_unit( + StartResultUnit { + started: Arc::new(AtomicBool::new(false)), + stopped: Arc::new(AtomicBool::new(false)), + fail: false, + }, + recv, + ) + }) + .unwrap(); + units.start().await.unwrap(); + + let uncertain_started = Arc::new(AtomicBool::new(false)); + let uncertain_stopped = Arc::new(AtomicBool::new(false)); + let _uncertain = units + .add("uncertain") + .spawn(&driver, |mut recv| { + let mut unit = StartResultUnit { + started: uncertain_started.clone(), + stopped: uncertain_stopped.clone(), + fail: false, + }; + async move { + while let Some(request) = recv.next().await { + match request { + StateRequest::Start(response) => { + unit.started.store(true, Ordering::Relaxed); + drop(response); + } + request => request.apply(&mut unit).await, + } + } + unit + } + }) + .unwrap(); + + units.start_stopped_units().await.unwrap_err(); + assert!(uncertain_started.load(Ordering::Relaxed)); + assert!(units.is_running()); + let retry_error = units.start_stopped_units().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); + + units.stop().await; + assert!(uncertain_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); + let retry_error = units.start().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); + assert!(units.reset().await.is_err()); + assert!(units.save().await.is_err()); + assert!(units.restore(Vec::new()).await.is_err()); +} From f408c58c7e1d485b4b924b955b57d5062ab201ec Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Mon, 12 Oct 2026 19:17:06 -0300 Subject: [PATCH 07/17] underhill: keep failed diagnostic resume stopped Return a diagnostic resume's state-unit start error through its failable RPC without terminating the worker. The state-unit collection has rolled back and remains stopped, allowing the caller to inspect or tear down the unusable VM without turning the expected RPC failure into an unexpected worker stop. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- openhcl/underhill_core/src/dispatch/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/openhcl/underhill_core/src/dispatch/mod.rs b/openhcl/underhill_core/src/dispatch/mod.rs index 06c6f173e83..352f30baba6 100644 --- a/openhcl/underhill_core/src/dispatch/mod.rs +++ b/openhcl/underhill_core/src/dispatch/mod.rs @@ -422,7 +422,6 @@ impl LoadedVm { rpc.complete(Err(RemoteError::new( error.context("failed to resume VM"), ))); - break None; } else { rpc.complete(Ok(true)); } From abfb5308b3c19c8326ac9ef0ad8a52faa21235ce Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Mon, 12 Oct 2026 19:33:20 -0300 Subject: [PATCH 08/17] state_unit: ignore removed rollback units Filter selected operation IDs against the live unit map when constructing a ready set. This preserves the existing concurrent-removal behavior if a unit handle disappears between failed-start cleanup and dependency rollback. Add a regression test that builds a selected ready set after its unit has been removed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- vmm_core/state_unit/src/lib.rs | 2 +- vmm_core/state_unit/src/tests/start.rs | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/vmm_core/state_unit/src/lib.rs b/vmm_core/state_unit/src/lib.rs index 087f1e04303..0c3bcb52909 100644 --- a/vmm_core/state_unit/src/lib.rs +++ b/vmm_core/state_unit/src/lib.rs @@ -821,7 +821,7 @@ impl Inner { let units = if let Some(unit_ids) = unit_ids { unit_ids .iter() - .map(|id| map(*id, &self.units[id])) + .filter_map(|id| self.units.get(id).map(|unit| map(*id, unit))) .collect() } else { self.units.iter().map(|(id, unit)| map(*id, unit)).collect() diff --git a/vmm_core/state_unit/src/tests/start.rs b/vmm_core/state_unit/src/tests/start.rs index dfeb7402370..d5f471df429 100644 --- a/vmm_core/state_unit/src/tests/start.rs +++ b/vmm_core/state_unit/src/tests/start.rs @@ -224,3 +224,26 @@ async fn test_uncertain_start_is_cleaned_once_and_remains_terminal(driver: Defau assert!(units.save().await.is_err()); assert!(units.restore(Vec::new()).await.is_err()); } + +#[async_test] +async fn test_rollback_ready_set_ignores_removed_unit(driver: DefaultDriver) { + let units = StateUnits::new(); + let unit = units + .add("removed") + .spawn(&driver, |recv| { + run_unit( + StartResultUnit { + started: Arc::new(AtomicBool::new(false)), + stopped: Arc::new(AtomicBool::new(false)), + fail: false, + }, + recv, + ) + }) + .unwrap(); + let id = unit.handle().id.id; + unit.remove().await; + + let ready_set = units.inner.lock().ready_set(Some(&[id])); + assert!(ready_set.0.is_empty()); +} From 9bab03ff5430bd2e8a5c04b28ca5a2b10c903560 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Mon, 12 Oct 2026 19:47:59 -0300 Subject: [PATCH 09/17] underhill: forward diagnostic resume errors Complete the diagnostic Resume RPC with a state-unit start failure and keep the control loop alive. Reserve propagation out of run_control for transport failures, so a request-level resume error neither drops its response nor shuts down the control mesh. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- openhcl/underhill_core/src/lib.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/openhcl/underhill_core/src/lib.rs b/openhcl/underhill_core/src/lib.rs index f36b578149e..69cb928ad5c 100644 --- a/openhcl/underhill_core/src/lib.rs +++ b/openhcl/underhill_core/src/lib.rs @@ -686,11 +686,18 @@ async fn run_control( continue; }; - let was_resumed = workers + let was_resumed = match workers .vm_rpc .call(UhVmRpc::Resume, ()) .await - .context("failed to resume VM")??; + .context("failed to resume VM")? + { + Ok(was_resumed) => was_resumed, + Err(error) => { + rpc.complete(Err(error)); + continue; + } + }; let was_halted = workers .vm_rpc From 1ac5d058d9930c4ffb4b285bf022d11428bf23e8 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Mon, 12 Oct 2026 20:10:49 -0300 Subject: [PATCH 10/17] state_unit: best-effort stop uncertain starts Attempt ordered Stop requests for both explicit and transport-uncertain start failures before rolling back successful dependencies. A confirmed cleanup uses the already-stopped terminal state; a dropped cleanup response preserves StartUncertain so a later full stop can retry without hiding uncertainty. Cover initial uncertain-start cleanup and the retry path where the first Stop response is also dropped. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- vmm_core/state_unit/src/start.rs | 70 ++++++++++++++++++++------ vmm_core/state_unit/src/tests/start.rs | 50 ++++++++++++++++++ 2 files changed, 105 insertions(+), 15 deletions(-) diff --git a/vmm_core/state_unit/src/start.rs b/vmm_core/state_unit/src/start.rs index b474f4074ee..f5d83bcc938 100644 --- a/vmm_core/state_unit/src/start.rs +++ b/vmm_core/state_unit/src/start.rs @@ -85,7 +85,7 @@ impl StateUnits { let mut failures = Vec::new(); let mut started_ids = Vec::new(); - let mut failed_start_ids = Vec::new(); + let mut failed_or_uncertain_ids = Vec::new(); { let mut inner = self.inner.lock(); for (name, id, result) in results { @@ -103,11 +103,12 @@ impl StateUnits { } StartResult::Failed(error) => { unit.state = State::StartUncertain; - failed_start_ids.push(id); + failed_or_uncertain_ids.push(id); failures.push(format!("{name}: {error:#}")); } StartResult::Uncertain(error) => { unit.state = State::StartUncertain; + failed_or_uncertain_ids.push(id); failures.push(format!("{name}: {error:#}")); } } @@ -118,19 +119,7 @@ impl StateUnits { self.running = true; Ok(()) } else { - if !failed_start_ids.is_empty() { - self.run_op( - "failed_start_cleanup", - Some(&failed_start_ids), - State::StartUncertain, - State::Stopping, - State::StartFailedStopped, - StateRequest::Stop, - |_, _| Some(()), - |unit| Some(&unit.dependents), - ) - .await; - } + failures.extend(self.cleanup_failed_starts(&failed_or_uncertain_ids).await); if !started_ids.is_empty() { self.run_op( "failed_start_rollback", @@ -148,4 +137,55 @@ impl StateUnits { anyhow::bail!("state units could not start: {}", failures.join("; ")) } } + + async fn cleanup_failed_starts(&self, unit_ids: &[u64]) -> Vec { + let mut operations = Vec::new(); + let ready_set; + { + let mut inner = self.inner.lock(); + ready_set = inner.ready_set(Some(unit_ids)); + for (&id, unit) in inner + .units + .iter_mut() + .filter(|(id, _)| ready_set.0.contains_key(id)) + { + assert_eq!(unit.state, State::StartUncertain); + let name = unit.name.clone(); + let dependents = unit.dependents.iter().copied().collect::>(); + let ready_set = ready_set.clone(); + let stop = state_change(name.clone(), unit, StateRequest::Stop, Some(())); + operations.push(async move { + ready_set + .wait("failed_start_cleanup", id, &dependents) + .await; + let result = stop.await; + ready_set.done(id, true); + (name, id, result) + }); + unit.state = State::Stopping; + } + } + + let results = join_all(operations).await; + let mut failures = Vec::new(); + let mut inner = self.inner.lock(); + for (name, id, result) in results { + let Some(unit) = inner.units.get_mut(&id) else { + continue; + }; + assert_eq!(unit.state, State::Stopping); + match result { + Ok(Some(())) => unit.state = State::StartFailedStopped, + Ok(None) => { + unit.state = State::StartUncertain; + failures.push(format!("{name}: cleanup stop request was not sent")); + } + Err(error) => { + unit.state = State::StartUncertain; + failures.push(format!("{name}: cleanup stop failed: {error:#}")); + } + } + } + failures + } } diff --git a/vmm_core/state_unit/src/tests/start.rs b/vmm_core/state_unit/src/tests/start.rs index d5f471df429..51df0c6a348 100644 --- a/vmm_core/state_unit/src/tests/start.rs +++ b/vmm_core/state_unit/src/tests/start.rs @@ -13,6 +13,7 @@ use pal_async::DefaultDriver; use pal_async::async_test; use std::sync::Arc; use std::sync::atomic::AtomicBool; +use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; use test_with_tracing::test; use vmcore::save_restore::RestoreError; @@ -186,9 +187,11 @@ async fn test_uncertain_start_is_cleaned_once_and_remains_terminal(driver: Defau let uncertain_started = Arc::new(AtomicBool::new(false)); let uncertain_stopped = Arc::new(AtomicBool::new(false)); + let stop_requests = Arc::new(AtomicUsize::new(0)); let _uncertain = units .add("uncertain") .spawn(&driver, |mut recv| { + let stop_requests = stop_requests.clone(); let mut unit = StartResultUnit { started: uncertain_started.clone(), stopped: uncertain_stopped.clone(), @@ -201,6 +204,11 @@ async fn test_uncertain_start_is_cleaned_once_and_remains_terminal(driver: Defau unit.started.store(true, Ordering::Relaxed); drop(response); } + StateRequest::Stop(response) + if stop_requests.fetch_add(1, Ordering::Relaxed) == 0 => + { + drop(response); + } request => request.apply(&mut unit).await, } } @@ -211,11 +219,53 @@ async fn test_uncertain_start_is_cleaned_once_and_remains_terminal(driver: Defau units.start_stopped_units().await.unwrap_err(); assert!(uncertain_started.load(Ordering::Relaxed)); + assert_eq!(stop_requests.load(Ordering::Relaxed), 1); + assert!(!uncertain_stopped.load(Ordering::Relaxed)); assert!(units.is_running()); let retry_error = units.start_stopped_units().await.unwrap_err(); assert!(retry_error.to_string().contains("terminal state")); units.stop().await; + assert_eq!(stop_requests.load(Ordering::Relaxed), 2); + assert!(uncertain_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); + let retry_error = units.start().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); + assert!(units.reset().await.is_err()); + assert!(units.save().await.is_err()); + assert!(units.restore(Vec::new()).await.is_err()); +} + +#[async_test] +async fn test_initial_uncertain_start_is_cleaned(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let uncertain_started = Arc::new(AtomicBool::new(false)); + let uncertain_stopped = Arc::new(AtomicBool::new(false)); + let _uncertain = units + .add("uncertain") + .spawn(&driver, |mut recv| { + let mut unit = StartResultUnit { + started: uncertain_started.clone(), + stopped: uncertain_stopped.clone(), + fail: false, + }; + async move { + while let Some(request) = recv.next().await { + match request { + StateRequest::Start(response) => { + unit.started.store(true, Ordering::Relaxed); + drop(response); + } + request => request.apply(&mut unit).await, + } + } + unit + } + }) + .unwrap(); + + units.start().await.unwrap_err(); + assert!(uncertain_started.load(Ordering::Relaxed)); assert!(uncertain_stopped.load(Ordering::Relaxed)); assert!(!units.is_running()); let retry_error = units.start().await.unwrap_err(); From 762b8ea6f29eae14773a2bacdcc18d873aae6381 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Mon, 12 Oct 2026 20:24:03 -0300 Subject: [PATCH 11/17] state_unit: handle rollback transport failures Use the best-effort Stop path for successfully started units during failed start rollback as well as for failed or uncertain starters. A dropped Stop response now marks that unit terminal, contributes to the returned error, and does not panic the VM worker. Add a regression where a unit acknowledges Start and exits before sibling failure triggers rollback. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- vmm_core/state_unit/src/start.rs | 44 ++++++++++++++---------- vmm_core/state_unit/src/tests/start.rs | 47 ++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 18 deletions(-) diff --git a/vmm_core/state_unit/src/start.rs b/vmm_core/state_unit/src/start.rs index f5d83bcc938..217802fc89a 100644 --- a/vmm_core/state_unit/src/start.rs +++ b/vmm_core/state_unit/src/start.rs @@ -119,26 +119,36 @@ impl StateUnits { self.running = true; Ok(()) } else { - failures.extend(self.cleanup_failed_starts(&failed_or_uncertain_ids).await); - if !started_ids.is_empty() { - self.run_op( + failures.extend( + self.stop_after_failed_start( + "failed_start_cleanup", + &failed_or_uncertain_ids, + State::StartUncertain, + State::StartFailedStopped, + ) + .await, + ); + failures.extend( + self.stop_after_failed_start( "failed_start_rollback", - Some(&started_ids), + &started_ids, State::Running, - State::Stopping, State::Stopped, - StateRequest::Stop, - |_, _| Some(()), - |unit| Some(&unit.dependents), ) - .await; - } + .await, + ); self.running = was_running; anyhow::bail!("state units could not start: {}", failures.join("; ")) } } - async fn cleanup_failed_starts(&self, unit_ids: &[u64]) -> Vec { + async fn stop_after_failed_start( + &self, + op: &'static str, + unit_ids: &[u64], + old_state: State, + stopped_state: State, + ) -> Vec { let mut operations = Vec::new(); let ready_set; { @@ -149,15 +159,13 @@ impl StateUnits { .iter_mut() .filter(|(id, _)| ready_set.0.contains_key(id)) { - assert_eq!(unit.state, State::StartUncertain); + assert_eq!(unit.state, old_state); let name = unit.name.clone(); let dependents = unit.dependents.iter().copied().collect::>(); let ready_set = ready_set.clone(); let stop = state_change(name.clone(), unit, StateRequest::Stop, Some(())); operations.push(async move { - ready_set - .wait("failed_start_cleanup", id, &dependents) - .await; + ready_set.wait(op, id, &dependents).await; let result = stop.await; ready_set.done(id, true); (name, id, result) @@ -175,14 +183,14 @@ impl StateUnits { }; assert_eq!(unit.state, State::Stopping); match result { - Ok(Some(())) => unit.state = State::StartFailedStopped, + Ok(Some(())) => unit.state = stopped_state, Ok(None) => { unit.state = State::StartUncertain; - failures.push(format!("{name}: cleanup stop request was not sent")); + failures.push(format!("{name}: {op} stop request was not sent")); } Err(error) => { unit.state = State::StartUncertain; - failures.push(format!("{name}: cleanup stop failed: {error:#}")); + failures.push(format!("{name}: {op} stop failed: {error:#}")); } } } diff --git a/vmm_core/state_unit/src/tests/start.rs b/vmm_core/state_unit/src/tests/start.rs index 51df0c6a348..da37724a75a 100644 --- a/vmm_core/state_unit/src/tests/start.rs +++ b/vmm_core/state_unit/src/tests/start.rs @@ -275,6 +275,53 @@ async fn test_initial_uncertain_start_is_cleaned(driver: DefaultDriver) { assert!(units.restore(Vec::new()).await.is_err()); } +#[async_test] +async fn test_started_unit_exit_during_rollback_returns_error(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let _exited = units + .add("exited") + .spawn(&driver, |mut recv| async move { + let unit = StartResultUnit { + started: Arc::new(AtomicBool::new(true)), + stopped: Arc::new(AtomicBool::new(false)), + fail: false, + }; + let request = recv.next().await.unwrap(); + let StateRequest::Start(response) = request else { + panic!("unexpected request"); + }; + response.handle_failable_sync(|()| anyhow::Ok(())); + unit + }) + .unwrap(); + let _failed = units + .add("failed") + .spawn(&driver, |recv| { + run_unit( + StartResultUnit { + started: Arc::new(AtomicBool::new(false)), + stopped: Arc::new(AtomicBool::new(false)), + fail: true, + }, + recv, + ) + }) + .unwrap(); + + let error = units.start().await.unwrap_err(); + assert!( + error + .to_string() + .contains("failed_start_rollback stop failed") + ); + assert!(!units.is_running()); + let retry_error = units.start().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); + assert!(units.reset().await.is_err()); + assert!(units.save().await.is_err()); + assert!(units.restore(Vec::new()).await.is_err()); +} + #[async_test] async fn test_rollback_ready_set_ignores_removed_unit(driver: DefaultDriver) { let units = StateUnits::new(); From 79a67f7958cf13fded7311e748d57e9199c57d50 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Mon, 12 Oct 2026 20:24:03 -0300 Subject: [PATCH 12/17] underhill: retain worker after recovery failures After a failed restart or diagnostic save cannot resume state units, complete the request error but keep the rolled-back worker event loop alive. This lets mesh_worker report RestartFailed and reconnect control instead of following it with an unexpected worker stop. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- openhcl/underhill_core/src/dispatch/mod.rs | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/openhcl/underhill_core/src/dispatch/mod.rs b/openhcl/underhill_core/src/dispatch/mod.rs index 352f30baba6..17072126226 100644 --- a/openhcl/underhill_core/src/dispatch/mod.rs +++ b/openhcl/underhill_core/src/dispatch/mod.rs @@ -326,7 +326,7 @@ impl LoadedVm { Event::WorkerRpc(message) => match message { WorkerRpc::Stop => break None, WorkerRpc::Restart(rpc) => { - let (state, terminate) = async { + let state = async { let running = self.stop().await; match self .save( @@ -337,7 +337,7 @@ impl LoadedVm { ) .await { - Ok(servicing_state) => (Some((rpc, servicing_state)), false), + Ok(servicing_state) => Some((rpc, servicing_state)), Err(err) => { if running { if let Err(start_error) = self.start(None).await { @@ -346,20 +346,17 @@ impl LoadedVm { "worker restart failed ({err:#}) and the VM could not resume" )), ))); - return (None, true); + return None; } } rpc.complete(Err(RemoteError::new(err))); - (None, false) + None } } } .instrument(tracing::info_span!("restart", CVM_ALLOWED)) .await; - if terminate { - break None; - } if let Some((rpc, servicing_state)) = state { break Some(LoadedVmState { restart_rpc: rpc, @@ -441,7 +438,7 @@ impl LoadedVm { rpc.complete(Err(RemoteError::new( error.context("failed to resume VM after save"), ))); - break None; + continue; } rpc.complete(result.map(mesh::payload::encode).map_err(RemoteError::new)); } From de3d1007631a27e052fccfd1b4411eade4ad0f80 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Wed, 21 Oct 2026 11:23:17 -0300 Subject: [PATCH 13/17] underhill: make hardware reseal start fallible Adapt the upstream hardware-reseal state unit to the fallible StateUnit start contract. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- openhcl/underhill_core/src/hardware_reseal.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openhcl/underhill_core/src/hardware_reseal.rs b/openhcl/underhill_core/src/hardware_reseal.rs index 1fbb8dbf98b..51fe81346f7 100644 --- a/openhcl/underhill_core/src/hardware_reseal.rs +++ b/openhcl/underhill_core/src/hardware_reseal.rs @@ -402,7 +402,7 @@ impl inspect::InspectMut for HardwareReseal { } impl StateUnit for HardwareReseal { - async fn start(&mut self) { + async fn start(&mut self) -> anyhow::Result<()> { // Starting or resuming does not create work or reset retry backoff. // Pending notifications and recovery survive a normal stop/start. self.schedule.running = true; @@ -413,6 +413,7 @@ impl StateUnit for HardwareReseal { pending_notification = self.notification.pending.load(Ordering::Acquire), "hardware reseal worker started" ); + Ok(()) } async fn stop(&mut self) { From a52450eacdf0375aca922afc3c9470e6c1b3079a Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Fri, 23 Oct 2026 08:23:51 -0300 Subject: [PATCH 14/17] underhill: handle fallible reseal starts in tests Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- openhcl/underhill_core/src/hardware_reseal/tests.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/openhcl/underhill_core/src/hardware_reseal/tests.rs b/openhcl/underhill_core/src/hardware_reseal/tests.rs index 7081da1a42d..78ba94ca702 100644 --- a/openhcl/underhill_core/src/hardware_reseal/tests.rs +++ b/openhcl/underhill_core/src/hardware_reseal/tests.rs @@ -642,6 +642,7 @@ async fn finish_attempt( if start { drive_until(run.as_mut(), send.call(StateRequest::Start, ())) .await + .unwrap() .unwrap(); } drive_until(run.as_mut(), milestone).await; @@ -837,6 +838,7 @@ async fn stopped_worker_keeps_notification_and_start_honors_not_before(driver: D assert!(notification.pending.load(Ordering::SeqCst)); drive_until(run.as_mut(), send.call(StateRequest::Start, ())) .await + .unwrap() .unwrap(); assert!(!notification.pending.load(Ordering::SeqCst)); drive_until(run.as_mut(), send.call(StateRequest::Stop, ())) @@ -874,6 +876,7 @@ async fn stop_drains_blocked_write_and_final_flush_and_retains_late_event(driver let mut run = pin!(fixture.worker.run(recv)); drive_until(run.as_mut(), send.call(StateRequest::Start, ())) .await + .unwrap() .unwrap(); drive_until(run.as_mut(), fixture.io.blocked()).await; let mut stop = pin!(send.call(StateRequest::Stop, ())); @@ -940,6 +943,7 @@ async fn blocked_hardware_keeps_executor_responsive_and_stop_drains_attempt(driv let mut run = pin!(fixture.worker.run(recv)); drive_until(run.as_mut(), send.call(StateRequest::Start, ())) .await + .unwrap() .unwrap(); drive_until(run.as_mut(), gate.reached()).await; { @@ -1078,7 +1082,7 @@ async fn failed_report_keeps_recovery_pending_and_retries_successfully(driver: D let deadline = fixture.worker.schedule.deadline; let not_before = fixture.worker.schedule.not_before; - fixture.worker.start().await; + fixture.worker.start().await.unwrap(); fixture.worker.stop().await; assert!(fixture.worker.schedule.force_reseal); assert_eq!(fixture.worker.schedule.failures, 1); @@ -1300,6 +1304,7 @@ async fn changed_report_during_pre_or_post_verification_keeps_recovery_pending( let mut run = pin!(fixture.worker.run(recv)); drive_until(run.as_mut(), send.call(StateRequest::Start, ())) .await + .unwrap() .unwrap(); if post_flush { drive_until(run.as_mut(), fixture.io.blocked()).await; @@ -1418,6 +1423,7 @@ async fn migration_during_event_flush_is_detected_without_another_event(driver: let mut run = pin!(fixture.worker.run(recv)); drive_until(run.as_mut(), send.call(StateRequest::Start, ())) .await + .unwrap() .unwrap(); drive_until(run.as_mut(), fixture.io.blocked()).await; fixture.hardware.identity.store(0x73, Ordering::SeqCst); @@ -1500,7 +1506,7 @@ async fn restore_and_reset_preserve_upgrade_and_failed_flush_obligation(driver: assert_eq!(fixture.worker.schedule.deadline, deadline); assert_eq!(fixture.worker.schedule.not_before, not_before); assert!(!fixture.worker.notification.pending.load(Ordering::SeqCst)); - fixture.worker.start().await; + fixture.worker.start().await.unwrap(); fixture.worker.stop().await; assert!(fixture.worker.schedule.force_reseal); assert_eq!(fixture.worker.schedule.failures, 1); @@ -1552,6 +1558,7 @@ async fn failed_final_flush_retries_write_even_when_cached_protector_matches( let mut run = pin!(fixture.worker.run(recv)); drive_until(run.as_mut(), send.call(StateRequest::Start, ())) .await + .unwrap() .unwrap(); drive_until(run.as_mut(), fixture.io.blocked()).await; let stop = send.call(StateRequest::Stop, ()); @@ -1771,7 +1778,7 @@ async fn running_save_and_restore_fail_without_consuming_pending_work(driver: De let mut fixture = Fixture::new(&driver, true).await; fixture.worker.notification.notify(); let saved = fixture.worker.save().await.unwrap().unwrap(); - fixture.worker.start().await; + fixture.worker.start().await.unwrap(); let before = worker_snapshot(&fixture.worker); assert!(fixture.worker.save().await.is_err()); assert!(fixture.worker.restore(saved).await.is_err()); From 78a362e16d782262f348eec1c95cb447cc5235ab Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Fri, 25 Sep 2026 00:01:32 -0700 Subject: [PATCH 15/17] openvmm: make VM resume fallible Resuming a VM starts its state units, which can now fail to start, but `VmRpc::Resume` could only report whether the VM was resumed, so a start failure could not reach the requester. Make `VmRpc::Resume` a failable RPC, and make `LoadedVm::resume` return the error of starting the state units. The worker propagates it when it resumes the VM after a restart, a pulse save/restore, a reset, or a state dump. The REPL logs a failed resume, the ttrpc `ResumeVm` handler and Petri return the error, and launching a VM that is not paused fails if it does not resume. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: ef640993-0003-4e49-b1ed-1b8b4776d0ef --- openvmm/openvmm_core/src/worker/dispatch.rs | 20 +++++++++---------- .../openvmm_core/src/worker/dispatch/dump.rs | 4 +++- openvmm/openvmm_defs/src/rpc.rs | 2 +- openvmm/openvmm_entry/src/lib.rs | 5 ++++- openvmm/openvmm_entry/src/repl.rs | 7 +++++-- openvmm/openvmm_entry/src/ttrpc/mod.rs | 2 +- petri/src/worker.rs | 4 ++-- 7 files changed, 26 insertions(+), 18 deletions(-) diff --git a/openvmm/openvmm_core/src/worker/dispatch.rs b/openvmm/openvmm_core/src/worker/dispatch.rs index 2ae6541952f..97c15bb73f7 100644 --- a/openvmm/openvmm_core/src/worker/dispatch.rs +++ b/openvmm/openvmm_core/src/worker/dispatch.rs @@ -401,7 +401,7 @@ impl Worker for VmWorker { LOADED_VM.store(&vm); if running { - vm.resume().await; + vm.resume().await?; } Ok(Self { vm, @@ -3693,18 +3693,16 @@ impl LoadedVmInner { impl LoadedVm { #[tracing::instrument(name = "resume_vm", target = "openvmm::perf", skip_all)] - async fn resume(&mut self) -> bool { + async fn resume(&mut self) -> anyhow::Result { if self.running { - return false; + return Ok(false); } - // Resume cannot report a failure to its callers, so treat a unit that - // fails to start like any other unrecoverable state unit failure. self.state_units .start() .await - .expect("state units failed to start"); + .context("VM state units failed to start")?; self.running = true; - true + Ok(true) } async fn pause(&mut self) -> bool { @@ -3862,7 +3860,9 @@ impl LoadedVm { rpc.handle(async |()| self.inner.partition_unit.clear_halt().await) .await } - VmRpc::Resume(rpc) => rpc.handle(async |()| self.resume().await).await, + VmRpc::Resume(rpc) => { + rpc.handle_failable(async |()| self.resume().await).await + } VmRpc::Pause(rpc) => rpc.handle(async |()| self.pause().await).await, VmRpc::Save(rpc) => { rpc.handle_failable(async |()| self.save().await.map(ProtobufMessage::new)) @@ -3947,7 +3947,7 @@ impl LoadedVm { self.save_reset_restore().await?; if paused { - self.resume().await; + self.resume().await?; } Ok(()) }) @@ -4435,7 +4435,7 @@ impl LoadedVm { } if resume { - self.resume().await; + self.resume().await?; } Ok(()) } diff --git a/openvmm/openvmm_core/src/worker/dispatch/dump.rs b/openvmm/openvmm_core/src/worker/dispatch/dump.rs index ec47b71126d..a0108c686da 100644 --- a/openvmm/openvmm_core/src/worker/dispatch/dump.rs +++ b/openvmm/openvmm_core/src/worker/dispatch/dump.rs @@ -19,7 +19,9 @@ impl LoadedVm { let was_running = self.pause().await; let result = self.dump_state_inner(file).await; if was_running { - self.resume().await; + self.resume() + .await + .context("failed to resume VM after dump")?; } result } diff --git a/openvmm/openvmm_defs/src/rpc.rs b/openvmm/openvmm_defs/src/rpc.rs index 58af188f5ef..e6d8e702320 100644 --- a/openvmm/openvmm_defs/src/rpc.rs +++ b/openvmm/openvmm_defs/src/rpc.rs @@ -20,7 +20,7 @@ use vm_resource::kind::VmbusDeviceHandleKind; #[derive(MeshPayload)] pub enum VmRpc { Save(FailableRpc<(), ProtobufMessage>), - Resume(Rpc<(), bool>), + Resume(FailableRpc<(), bool>), Pause(Rpc<(), bool>), ClearHalt(Rpc<(), bool>), Reset(FailableRpc<(), ()>), diff --git a/openvmm/openvmm_entry/src/lib.rs b/openvmm/openvmm_entry/src/lib.rs index 6965a1140d1..31133fd579a 100644 --- a/openvmm/openvmm_entry/src/lib.rs +++ b/openvmm/openvmm_entry/src/lib.rs @@ -3019,7 +3019,10 @@ async fn run_control_inner( } if !opt.paused { - vm_rpc.call(VmRpc::Resume, ()).await?; + anyhow::ensure!( + vm_rpc.call_failable(VmRpc::Resume, ()).await?, + "VM failed to start; inspect the worker log for the device startup error" + ); } let paravisor_diag = Arc::new(diag_client::DiagClient::from_dialer( diff --git a/openvmm/openvmm_entry/src/repl.rs b/openvmm/openvmm_entry/src/repl.rs index 75beaa7e162..4460cd84c1e 100644 --- a/openvmm/openvmm_entry/src/repl.rs +++ b/openvmm/openvmm_entry/src/repl.rs @@ -664,7 +664,7 @@ pub(crate) async fn run_repl( enum StateChange { Pause(bool), - Resume(bool), + Resume(Result), Reset(Result<(), RemoteError>), PulseSaveRestore(Result<(), PulseSaveRestoreError>), ServiceVtl2(anyhow::Result), @@ -766,13 +766,16 @@ pub(crate) async fn run_repl( tracing::warn!("already paused"); } } - StateChange::Resume(success) => { + StateChange::Resume(Ok(success)) => { if success { tracing::info!("resumed complete"); } else { tracing::warn!("already running"); } } + StateChange::Resume(Err(err)) => { + tracing::error!(error = &err as &dyn std::error::Error, "resume failed") + } StateChange::Reset(r) => match r { Ok(()) => tracing::info!("reset complete"), Err(err) => tracing::error!( diff --git a/openvmm/openvmm_entry/src/ttrpc/mod.rs b/openvmm/openvmm_entry/src/ttrpc/mod.rs index 406ddfffe10..402d62da512 100644 --- a/openvmm/openvmm_entry/src/ttrpc/mod.rs +++ b/openvmm/openvmm_entry/src/ttrpc/mod.rs @@ -1458,7 +1458,7 @@ impl VmService { async fn resume_vm(&mut self) -> anyhow::Result<()> { let vm = self.vm.clone().context("VM not created yet")?; vm.worker_rpc - .call(VmRpc::Resume, ()) + .call_failable(VmRpc::Resume, ()) .await .map(drop) .context("resume failed")?; diff --git a/petri/src/worker.rs b/petri/src/worker.rs index 5071503305d..5b5ca3440d6 100644 --- a/petri/src/worker.rs +++ b/petri/src/worker.rs @@ -52,8 +52,8 @@ impl Worker { self.rpc.call(VmRpc::Pause, ()).await } - pub(crate) async fn resume(&self) -> Result { - self.rpc.call(VmRpc::Resume, ()).await + pub(crate) async fn resume(&self) -> anyhow::Result { + Ok(self.rpc.call_failable(VmRpc::Resume, ()).await?) } pub(crate) async fn save(&self) -> anyhow::Result { From 4720397aef4e5f002ac7bb52cff89d8545e7e331 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Mon, 12 Oct 2026 20:28:08 -0300 Subject: [PATCH 16/17] openvmm: retain worker after restart recovery failure When a failed worker restart cannot resume the state units, mark the loaded VM stopped after reporting RestartFailed and keep its event loop alive. This avoids following the controller's restart-failure recovery with an unexpected WorkerStopped event while preserving coherent running flags. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- openvmm/openvmm_core/src/worker/dispatch.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openvmm/openvmm_core/src/worker/dispatch.rs b/openvmm/openvmm_core/src/worker/dispatch.rs index 97c15bb73f7..179e2ba5b4b 100644 --- a/openvmm/openvmm_core/src/worker/dispatch.rs +++ b/openvmm/openvmm_core/src/worker/dispatch.rs @@ -3831,10 +3831,11 @@ impl LoadedVm { Err(err) => { if stopped { if let Err(start_error) = self.state_units.start().await { + self.running = false; rpc.complete(Err(RemoteError::new(start_error.context( "worker restart failed and the VM could not resume", )))); - break; + continue; } } rpc.complete(Err(RemoteError::new(err))); From 4fe5dc97c5681d95bd62039868a3e481029433a1 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Mon, 2 Nov 2026 09:10:50 -0800 Subject: [PATCH 17/17] underhill: tear down the worker after unrecoverable recovery failures If a worker restart or a diagnostic save fails and the VM then also fails to resume, the state units have been rolled back and the worker cannot run the VM again. Keeping the worker alive in that state only hides the failure from the host. Complete the pending RPC with the combined error and then leave the worker loop through the common teardown path, like servicing and autostart recovery failures already do. The worker host reports the worker as stopped, which OpenHCL treats as fatal. Failures after which the VM did resume still keep the worker running, and a failed diagnostic Resume stays a request-level error. This reverses the retain-the-worker behavior of 79a67f795. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- openhcl/underhill_core/src/dispatch/mod.rs | 43 ++++++++++++++++------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/openhcl/underhill_core/src/dispatch/mod.rs b/openhcl/underhill_core/src/dispatch/mod.rs index 17072126226..a336eac9e08 100644 --- a/openhcl/underhill_core/src/dispatch/mod.rs +++ b/openhcl/underhill_core/src/dispatch/mod.rs @@ -326,7 +326,17 @@ impl LoadedVm { Event::WorkerRpc(message) => match message { WorkerRpc::Stop => break None, WorkerRpc::Restart(rpc) => { - let state = async { + enum RestartOutcome { + Restart(R, S), + // The restart failed, but the VM is back in its + // previous state, so this worker can keep serving. + Recovered, + // The restart failed and the VM could not be + // resumed, so this worker cannot serve any more. + Unrecoverable, + } + + let outcome = async { let running = self.stop().await; match self .save( @@ -337,7 +347,7 @@ impl LoadedVm { ) .await { - Ok(servicing_state) => Some((rpc, servicing_state)), + Ok(servicing_state) => RestartOutcome::Restart(rpc, servicing_state), Err(err) => { if running { if let Err(start_error) = self.start(None).await { @@ -346,24 +356,31 @@ impl LoadedVm { "worker restart failed ({err:#}) and the VM could not resume" )), ))); - return None; + return RestartOutcome::Unrecoverable; } } rpc.complete(Err(RemoteError::new(err))); - None + RestartOutcome::Recovered } } } .instrument(tracing::info_span!("restart", CVM_ALLOWED)) .await; - if let Some((rpc, servicing_state)) = state { - break Some(LoadedVmState { - restart_rpc: rpc, - servicing_state, - vm_rpc, - control_send: self.control_send.lock().take().unwrap(), - }); + match outcome { + RestartOutcome::Restart(rpc, servicing_state) => { + break Some(LoadedVmState { + restart_rpc: rpc, + servicing_state, + vm_rpc, + control_send: self.control_send.lock().take().unwrap(), + }); + } + RestartOutcome::Recovered => {} + // The error has already completed the RPC. Tear + // down through the common path rather than keep a + // worker whose state units can no longer run. + RestartOutcome::Unrecoverable => break None, } } WorkerRpc::Inspect(deferred) => deferred.respond(|resp| { @@ -438,7 +455,9 @@ impl LoadedVm { rpc.complete(Err(RemoteError::new( error.context("failed to resume VM after save"), ))); - continue; + // The state units rolled back and cannot run + // again, so tear down rather than keep this worker. + break None; } rpc.complete(result.map(mesh::payload::encode).map_err(RemoteError::new)); }