From 78e752de1bbadef1a8b7da72af3345894e753a6d Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Mon, 9 Nov 2026 23:12:13 -0800 Subject: [PATCH 1/2] state_unit: handle a unit in a terminal state instead of panicking A unit whose task does not answer its start request is left in the terminal StartUncertain state, which only a later start refused. Any other state change that reached such a unit panicked the VM worker on run_op's state assertion: a stop of a running set, as after a failed hot-add, with "unit ... in StartUncertain state, should be Running or Stopped", and a reset, save, restore, or time advance after a failed start with "should be Stopped or Stopped". So a lifecycle request that followed a failed start, such as a reset after a failed resume or after a failed rollback of a microVM snapshot capture, took the process down (microsoft/nvx#1148). Those requests now handle the terminal state, as the later revisions of microsoft/openvmm#4872, which proposes this chain's start rollback, do: - A reset, save, restore, or time advance fails, naming each unit in a terminal state, before it sends any unit a request. It checks this before it checks that the units are stopped, so that it fails rather than panics even while such a unit keeps the units running. - A stop also stops the units whose start is uncertain, since they may be running, in the same reverse dependency order as the others, and leaves them in their terminal state, so that a later start still fails. A unit whose task has exited still panics the stop that it does not answer, as any unit's does; microsoft/nvx#1363 tracks that. Tests: a stop of a running set stops a unit whose start is uncertain before the unit that it depends on, and the unit stays terminal; after a start that leaves a unit terminal, and while such a unit keeps the units running, a reset, save, restore, and time advance each fail, naming only that unit, without sending any unit a request. Without the change, the stop and the first reset panic on the state assertion, and the reset of a running set on its running assertion. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- vmm_core/state_unit/src/advance_time.rs | 4 + vmm_core/state_unit/src/lib.rs | 59 +++++- vmm_core/state_unit/src/start.rs | 6 +- vmm_core/state_unit/src/tests/terminal.rs | 242 ++++++++++++++++++++++ 4 files changed, 306 insertions(+), 5 deletions(-) create mode 100644 vmm_core/state_unit/src/tests/terminal.rs diff --git a/vmm_core/state_unit/src/advance_time.rs b/vmm_core/state_unit/src/advance_time.rs index 68a6b21430b..ae2416da05c 100644 --- a/vmm_core/state_unit/src/advance_time.rs +++ b/vmm_core/state_unit/src/advance_time.rs @@ -15,7 +15,11 @@ impl StateUnits { /// Only units added with [`UnitBuilder::advances_time`](crate::UnitBuilder::advances_time) /// receive the request; the others pass through the operation without a /// round trip. + /// + /// Fails without sending a request while a unit is in a terminal state, + /// whether or not the units run. Otherwise panics if running. pub async fn advance_time(&mut self, duration: Duration) -> Result<(), StateTransitionError> { + self.check_no_terminal_units("advance_time")?; assert!(!self.running); let results = self .run_op( diff --git a/vmm_core/state_unit/src/lib.rs b/vmm_core/state_unit/src/lib.rs index df8d2ffaff9..79641acfe22 100644 --- a/vmm_core/state_unit/src/lib.rs +++ b/vmm_core/state_unit/src/lib.rs @@ -553,6 +553,28 @@ impl StateUnits { self.running } + /// Fails `op`, naming each unit in a terminal state, if any unit is in + /// one. + fn check_no_terminal_units(&self, op: &'static str) -> Result<(), StateTransitionError> { + let inner = self.inner.lock(); + check( + op, + inner + .units + .values() + .filter(|unit| matches!(unit.state, State::StartUncertain)) + .map(|unit| { + ( + unit.name.clone(), + Err::<(), _>(anyhow::anyhow!( + "unit is in terminal state {:?}", + unit.state + )), + ) + }), + ) + } + /// Starts any units that are individually stopped, because they were added /// via [`StateUnits::add`] while the VM was running. /// @@ -571,8 +593,25 @@ impl StateUnits { } /// Stops all the state units. + /// + /// This also stops the units whose start is uncertain, since they may be + /// running, and leaves them in that terminal state. pub async fn stop(&mut self) { assert!(self.running); + // Make the units whose start is uncertain receive the stop, then + // restore their terminal state after the ordered operation. + let uncertain_ids = { + let mut inner = self.inner.lock(); + inner + .units + .iter_mut() + .filter(|(_, unit)| unit.state == State::StartUncertain) + .map(|(&id, unit)| { + unit.state = State::Running; + id + }) + .collect::>() + }; // Stop units in reverse dependency order so that a dependency is not // stopped before its dependant. self.run_op( @@ -586,13 +625,22 @@ impl StateUnits { |unit| Some(&unit.dependents), ) .await; + let mut inner = self.inner.lock(); + for id in uncertain_ids { + if let Some(unit) = inner.units.get_mut(&id) { + assert_eq!(unit.state, State::Stopped); + unit.state = State::StartUncertain; + } + } self.running = false; } /// Resets all the state units. /// - /// Panics if running. + /// Fails without sending a request while a unit is in a terminal state, + /// whether or not the units run. Otherwise panics if running. pub async fn reset(&mut self) -> Result<(), StateTransitionError> { + self.check_no_terminal_units("reset")?; assert!(!self.running); self.restore_allowed = false; // Reset in dependency order so that dependants observe their @@ -617,8 +665,10 @@ impl StateUnits { /// Saves all the state units. /// - /// Panics if running. + /// Fails without sending a request while a unit is in a terminal state, + /// whether or not the units run. Otherwise panics if running. pub async fn save(&mut self) -> Result, StateTransitionError> { + self.check_no_terminal_units("save")?; assert!(!self.running); // Save can occur in any order since it will not observably mutate // state. @@ -652,11 +702,13 @@ impl StateUnits { /// unit-name validation errors leave eligibility unchanged. Saving does not /// consume eligibility, and saving restored state before start is supported. /// - /// Panics if running. + /// Fails without sending a request while a unit is in a terminal state, + /// whether or not the units run. Otherwise panics if running. pub async fn restore( &mut self, states: Vec, ) -> Result<(), StateTransitionError> { + self.check_no_terminal_units("restore")?; assert!(!self.running); if !self.restore_allowed { return Err(StateTransitionError { @@ -1145,6 +1197,7 @@ mod tests { mod quiesce; mod slow; mod start; + mod terminal; use super::StateTransitionErrorKind; use super::StateUnit; diff --git a/vmm_core/state_unit/src/start.rs b/vmm_core/state_unit/src/start.rs index dedb26ec200..e6799e1b73d 100644 --- a/vmm_core/state_unit/src/start.rs +++ b/vmm_core/state_unit/src/start.rs @@ -22,8 +22,10 @@ impl StateUnits { /// not start or did not confirm its start. If a unit fails to confirm its /// start, or its task does not answer its start or confirmation request, /// the units that did start, the failed unit among them, are stopped - /// again. A unit whose task does not answer is left in a terminal state - /// that fails later starts; a unit that failed to confirm its start can + /// again. A unit whose task does not answer is left in a terminal state: + /// while it is in that state, later starts, resets, saves, restores, and + /// time advances fail, and a stop stops it, since it may be running, but + /// leaves it in that state. A unit that failed to confirm its start can /// start again. /// /// [`StateUnit::confirm_start`]: crate::StateUnit::confirm_start diff --git a/vmm_core/state_unit/src/tests/terminal.rs b/vmm_core/state_unit/src/tests/terminal.rs new file mode 100644 index 00000000000..04707bc0ff0 --- /dev/null +++ b/vmm_core/state_unit/src/tests/terminal.rs @@ -0,0 +1,242 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Tests for the requests that reach a set with a unit in a terminal state. + +use crate::StateRequest; +use crate::StateTransitionError; +use crate::StateTransitionErrorKind; +use crate::StateUnit; +use crate::StateUnits; +use crate::UnitErrorSet; +use crate::run_unit; +use futures::StreamExt; +use inspect::InspectMut; +use mesh::Receiver; +use pal_async::DefaultDriver; +use pal_async::async_test; +use parking_lot::Mutex; +use std::sync::Arc; +use std::time::Duration; +use test_with_tracing::test; +use vmcore::save_restore::RestoreError; +use vmcore::save_restore::SaveError; +use vmcore::save_restore::SavedStateBlob; + +/// The requests that [`LoggingUnit`]s handled, in order. +#[derive(Clone, Default)] +struct Log(Arc>>); + +impl Log { + fn push(&self, unit: &str, request: &str) { + self.0.lock().push(format!("{unit} {request}")); + } + + fn take(&self) -> Vec { + std::mem::take(&mut *self.0.lock()) + } +} + +/// A unit that logs each state change request that it handles. +struct LoggingUnit { + name: &'static str, + log: Log, +} + +impl LoggingUnit { + fn new(name: &'static str, log: &Log) -> Self { + Self { + name, + log: log.clone(), + } + } +} + +impl StateUnit for LoggingUnit { + async fn start(&mut self) { + self.log.push(self.name, "start"); + } + + async fn stop(&mut self) { + self.log.push(self.name, "stop"); + } + + async fn reset(&mut self) -> anyhow::Result<()> { + self.log.push(self.name, "reset"); + Ok(()) + } + + async fn save(&mut self) -> Result, SaveError> { + self.log.push(self.name, "save"); + Ok(None) + } + + async fn restore(&mut self, _state: SavedStateBlob) -> Result<(), RestoreError> { + self.log.push(self.name, "restore"); + Ok(()) + } + + async fn advance_time(&mut self, _duration: Duration) -> anyhow::Result<()> { + self.log.push(self.name, "advance_time"); + Ok(()) + } +} + +impl InspectMut for LoggingUnit { + fn inspect_mut(&mut self, req: inspect::Request<'_>) { + req.respond(); + } +} + +/// Runs `unit`, but drops each start request after starting the unit, without +/// answering it, so that the start is uncertain while the unit runs and +/// answers later requests. +async fn run_unanswering_start( + mut unit: LoggingUnit, + mut recv: Receiver, +) -> LoggingUnit { + while let Some(request) = recv.next().await { + match request { + StateRequest::Start(response) => { + unit.start().await; + drop(response); + } + request => request.apply(&mut unit).await, + } + } + unit +} + +/// Starts a set with a unit named `dependency`, then adds a unit named +/// `uncertain` that depends on it, as a hot-add does, and whose start is +/// uncertain, and returns the set, which still runs, with the handles of both +/// units. +async fn running_set_with_an_uncertain_unit( + driver: &DefaultDriver, + log: &Log, +) -> (StateUnits, Vec>) { + let mut units = StateUnits::new(); + let dependency = units + .add("dependency") + .spawn(driver, |recv| { + run_unit(LoggingUnit::new("dependency", log), recv) + }) + .unwrap(); + units.start().await.unwrap(); + + let uncertain = units + .add("uncertain") + .depends_on(dependency.handle()) + .spawn(driver, |recv| { + run_unanswering_start(LoggingUnit::new("uncertain", log), recv) + }) + .unwrap(); + let error = units.start_stopped_units().await.unwrap_err(); + assert!(format!("{error:#}").contains("uncertain"), "{error:#}"); + assert!(units.is_running()); + assert_eq!(log.take(), ["dependency start", "uncertain start"]); + (units, vec![dependency, uncertain]) +} + +/// Checks that `error` is the failure of `op` because of the unit named +/// `name` alone. +fn assert_terminal_unit_error(error: StateTransitionError, op: &str, name: &str) { + let StateTransitionErrorKind::Units(UnitErrorSet(errors)) = &error.error else { + panic!("{op} failed for another reason: {error:?}"); + }; + let names = errors + .iter() + .map(|(name, _)| name.as_ref()) + .collect::>(); + assert_eq!( + (error.op, names.as_slice()), + (op, [name].as_slice()), + "{error:?}" + ); +} + +/// Checks that a reset, save, restore, and time advance of `units` each fail +/// because of the unit named `name`, which is in a terminal state, without +/// sending a unit a request. +async fn assert_requests_fail(units: &mut StateUnits, name: &str, log: &Log) { + assert_terminal_unit_error(units.reset().await.unwrap_err(), "reset", name); + let Err(error) = units.save().await else { + panic!("save succeeded"); + }; + assert_terminal_unit_error(error, "save", name); + assert_terminal_unit_error( + units.restore(Vec::new()).await.unwrap_err(), + "restore", + name, + ); + assert_terminal_unit_error( + units + .advance_time(Duration::from_secs(1)) + .await + .unwrap_err(), + "advance_time", + name, + ); + assert_eq!(log.take(), Vec::::new()); +} + +#[async_test] +async fn a_stop_stops_a_unit_whose_start_is_uncertain(driver: DefaultDriver) { + let log = Log::default(); + let (mut units, _handles) = running_set_with_an_uncertain_unit(&driver, &log).await; + + // The unit whose start is uncertain may be running, so the stop stops it, + // before the unit that it depends on. + units.stop().await; + assert!(!units.is_running()); + assert_eq!(log.take(), ["uncertain stop", "dependency stop"]); + + // The unit stays in its terminal state. + let error = units.start().await.unwrap_err(); + assert!(error.to_string().contains("terminal state"), "{error:#}"); + assert_requests_fail(&mut units, "uncertain", &log).await; +} + +#[async_test] +async fn requests_fail_after_a_start_leaves_a_terminal_unit(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let log = Log::default(); + let _healthy = units + .add("healthy") + .advances_time() + .spawn(&driver, |recv| { + run_unit(LoggingUnit::new("healthy", &log), recv) + }) + .unwrap(); + // A unit whose task exits at its start request without answering it, as + // a task that has failed would. + let _lost = units + .add("lost") + .spawn(&driver, |mut recv| async move { + while let Some(request) = recv.next().await { + if matches!(request, StateRequest::Start(_)) { + break; + } + } + }) + .unwrap(); + + let error = units.start().await.unwrap_err(); + assert!(format!("{error:#}").contains("lost"), "{error:#}"); + // The rollback stopped the unit that started. + assert_eq!(log.take(), ["healthy start", "healthy stop"]); + + assert_requests_fail(&mut units, "lost", &log).await; +} + +#[async_test] +async fn requests_fail_while_a_terminal_unit_keeps_the_set_running(driver: DefaultDriver) { + let log = Log::default(); + let (mut units, _handles) = running_set_with_an_uncertain_unit(&driver, &log).await; + + // The terminal unit fails the requests before the set's running state + // matters, so that a caller that considers the VM stopped while such a + // unit keeps the set running gets an error rather than a panic. + assert_requests_fail(&mut units, "uncertain", &log).await; + assert!(units.is_running()); +} From 6ddd6ca505a7212d92e2a8e561a19635f7185d4a Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Mon, 9 Nov 2026 23:12:23 -0800 Subject: [PATCH 2/2] openvmm_core: tear the VM down when a restart cannot resume it A worker restart stops the state units of a running VM before it saves the VM, and starts them again when the save fails. When that start failed too, the worker reported the failure and kept serving the VM, which it still marked running although its state units were stopped, or left in a terminal state by the failed start. The next pause, reset, or pulse save and restore then stopped units that did not run, and so did the worker's teardown, and each panicked on the running assertion of StateUnits::stop (microsoft/nvx#1148). The worker now tears the VM down after such a restart, as the later revisions of microsoft/openvmm#4872 do, since the VM cannot resume. Its teardown stops the state units when they run, rather than when the VM is marked running, so that it does not stop them again. The microVM profile refuses worker restarts, so no NVX microVM reaches this path. No test covers it: no harness of the worker can fail a unit's start after a failed save in a running worker. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- openvmm/openvmm_core/src/worker/dispatch.rs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/openvmm/openvmm_core/src/worker/dispatch.rs b/openvmm/openvmm_core/src/worker/dispatch.rs index 50488e2de2d..a2695e19c5a 100644 --- a/openvmm/openvmm_core/src/worker/dispatch.rs +++ b/openvmm/openvmm_core/src/worker/dispatch.rs @@ -4185,7 +4185,12 @@ impl LoadedVm { rpc.complete(Err(RemoteError::new(start_error.context( "worker restart failed and the VM could not resume", )))); - continue; + // The VM cannot resume, but it is + // still marked running while its + // state units are stopped, so a later + // pause would stop them again. Tear + // the VM down instead. + break; } } rpc.complete(Err(RemoteError::new(err))); @@ -4623,7 +4628,13 @@ impl LoadedVm { } device_removal.drain().await; - self.pause().await; + // Stop the state units whenever they run, not only when the VM does: + // after a failed state change, such as a restart that could not resume + // the VM, the two can disagree. + if self.state_units.is_running() { + self.state_units.stop().await; + } + self.running = false; self.teardown_chipset_devices().await; self.inner.partition_unit.teardown().await; if let Some(vmbus) = self.inner.vmbus_server {