diff --git a/openvmm/openvmm_core/src/worker/dispatch.rs b/openvmm/openvmm_core/src/worker/dispatch.rs index 50488e2de2d..a2695e19c5a 100644 --- a/openvmm/openvmm_core/src/worker/dispatch.rs +++ b/openvmm/openvmm_core/src/worker/dispatch.rs @@ -4185,7 +4185,12 @@ impl LoadedVm { rpc.complete(Err(RemoteError::new(start_error.context( "worker restart failed and the VM could not resume", )))); - continue; + // The VM cannot resume, but it is + // still marked running while its + // state units are stopped, so a later + // pause would stop them again. Tear + // the VM down instead. + break; } } rpc.complete(Err(RemoteError::new(err))); @@ -4623,7 +4628,13 @@ impl LoadedVm { } device_removal.drain().await; - self.pause().await; + // Stop the state units whenever they run, not only when the VM does: + // after a failed state change, such as a restart that could not resume + // the VM, the two can disagree. + if self.state_units.is_running() { + self.state_units.stop().await; + } + self.running = false; self.teardown_chipset_devices().await; self.inner.partition_unit.teardown().await; if let Some(vmbus) = self.inner.vmbus_server { diff --git a/vmm_core/state_unit/src/advance_time.rs b/vmm_core/state_unit/src/advance_time.rs index 68a6b21430b..ae2416da05c 100644 --- a/vmm_core/state_unit/src/advance_time.rs +++ b/vmm_core/state_unit/src/advance_time.rs @@ -15,7 +15,11 @@ impl StateUnits { /// Only units added with [`UnitBuilder::advances_time`](crate::UnitBuilder::advances_time) /// receive the request; the others pass through the operation without a /// round trip. + /// + /// Fails without sending a request while a unit is in a terminal state, + /// whether or not the units run. Otherwise panics if running. pub async fn advance_time(&mut self, duration: Duration) -> Result<(), StateTransitionError> { + self.check_no_terminal_units("advance_time")?; assert!(!self.running); let results = self .run_op( diff --git a/vmm_core/state_unit/src/lib.rs b/vmm_core/state_unit/src/lib.rs index df8d2ffaff9..79641acfe22 100644 --- a/vmm_core/state_unit/src/lib.rs +++ b/vmm_core/state_unit/src/lib.rs @@ -553,6 +553,28 @@ impl StateUnits { self.running } + /// Fails `op`, naming each unit in a terminal state, if any unit is in + /// one. + fn check_no_terminal_units(&self, op: &'static str) -> Result<(), StateTransitionError> { + let inner = self.inner.lock(); + check( + op, + inner + .units + .values() + .filter(|unit| matches!(unit.state, State::StartUncertain)) + .map(|unit| { + ( + unit.name.clone(), + Err::<(), _>(anyhow::anyhow!( + "unit is in terminal state {:?}", + unit.state + )), + ) + }), + ) + } + /// Starts any units that are individually stopped, because they were added /// via [`StateUnits::add`] while the VM was running. /// @@ -571,8 +593,25 @@ impl StateUnits { } /// Stops all the state units. + /// + /// This also stops the units whose start is uncertain, since they may be + /// running, and leaves them in that terminal state. pub async fn stop(&mut self) { assert!(self.running); + // Make the units whose start is uncertain receive the stop, then + // restore their terminal state after the ordered operation. + let uncertain_ids = { + let mut inner = self.inner.lock(); + inner + .units + .iter_mut() + .filter(|(_, unit)| unit.state == State::StartUncertain) + .map(|(&id, unit)| { + unit.state = State::Running; + id + }) + .collect::>() + }; // Stop units in reverse dependency order so that a dependency is not // stopped before its dependant. self.run_op( @@ -586,13 +625,22 @@ impl StateUnits { |unit| Some(&unit.dependents), ) .await; + let mut inner = self.inner.lock(); + for id in uncertain_ids { + if let Some(unit) = inner.units.get_mut(&id) { + assert_eq!(unit.state, State::Stopped); + unit.state = State::StartUncertain; + } + } self.running = false; } /// Resets all the state units. /// - /// Panics if running. + /// Fails without sending a request while a unit is in a terminal state, + /// whether or not the units run. Otherwise panics if running. pub async fn reset(&mut self) -> Result<(), StateTransitionError> { + self.check_no_terminal_units("reset")?; assert!(!self.running); self.restore_allowed = false; // Reset in dependency order so that dependants observe their @@ -617,8 +665,10 @@ impl StateUnits { /// Saves all the state units. /// - /// Panics if running. + /// Fails without sending a request while a unit is in a terminal state, + /// whether or not the units run. Otherwise panics if running. pub async fn save(&mut self) -> Result, StateTransitionError> { + self.check_no_terminal_units("save")?; assert!(!self.running); // Save can occur in any order since it will not observably mutate // state. @@ -652,11 +702,13 @@ impl StateUnits { /// unit-name validation errors leave eligibility unchanged. Saving does not /// consume eligibility, and saving restored state before start is supported. /// - /// Panics if running. + /// Fails without sending a request while a unit is in a terminal state, + /// whether or not the units run. Otherwise panics if running. pub async fn restore( &mut self, states: Vec, ) -> Result<(), StateTransitionError> { + self.check_no_terminal_units("restore")?; assert!(!self.running); if !self.restore_allowed { return Err(StateTransitionError { @@ -1145,6 +1197,7 @@ mod tests { mod quiesce; mod slow; mod start; + mod terminal; use super::StateTransitionErrorKind; use super::StateUnit; diff --git a/vmm_core/state_unit/src/start.rs b/vmm_core/state_unit/src/start.rs index dedb26ec200..e6799e1b73d 100644 --- a/vmm_core/state_unit/src/start.rs +++ b/vmm_core/state_unit/src/start.rs @@ -22,8 +22,10 @@ impl StateUnits { /// not start or did not confirm its start. If a unit fails to confirm its /// start, or its task does not answer its start or confirmation request, /// the units that did start, the failed unit among them, are stopped - /// again. A unit whose task does not answer is left in a terminal state - /// that fails later starts; a unit that failed to confirm its start can + /// again. A unit whose task does not answer is left in a terminal state: + /// while it is in that state, later starts, resets, saves, restores, and + /// time advances fail, and a stop stops it, since it may be running, but + /// leaves it in that state. A unit that failed to confirm its start can /// start again. /// /// [`StateUnit::confirm_start`]: crate::StateUnit::confirm_start diff --git a/vmm_core/state_unit/src/tests/terminal.rs b/vmm_core/state_unit/src/tests/terminal.rs new file mode 100644 index 00000000000..04707bc0ff0 --- /dev/null +++ b/vmm_core/state_unit/src/tests/terminal.rs @@ -0,0 +1,242 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Tests for the requests that reach a set with a unit in a terminal state. + +use crate::StateRequest; +use crate::StateTransitionError; +use crate::StateTransitionErrorKind; +use crate::StateUnit; +use crate::StateUnits; +use crate::UnitErrorSet; +use crate::run_unit; +use futures::StreamExt; +use inspect::InspectMut; +use mesh::Receiver; +use pal_async::DefaultDriver; +use pal_async::async_test; +use parking_lot::Mutex; +use std::sync::Arc; +use std::time::Duration; +use test_with_tracing::test; +use vmcore::save_restore::RestoreError; +use vmcore::save_restore::SaveError; +use vmcore::save_restore::SavedStateBlob; + +/// The requests that [`LoggingUnit`]s handled, in order. +#[derive(Clone, Default)] +struct Log(Arc>>); + +impl Log { + fn push(&self, unit: &str, request: &str) { + self.0.lock().push(format!("{unit} {request}")); + } + + fn take(&self) -> Vec { + std::mem::take(&mut *self.0.lock()) + } +} + +/// A unit that logs each state change request that it handles. +struct LoggingUnit { + name: &'static str, + log: Log, +} + +impl LoggingUnit { + fn new(name: &'static str, log: &Log) -> Self { + Self { + name, + log: log.clone(), + } + } +} + +impl StateUnit for LoggingUnit { + async fn start(&mut self) { + self.log.push(self.name, "start"); + } + + async fn stop(&mut self) { + self.log.push(self.name, "stop"); + } + + async fn reset(&mut self) -> anyhow::Result<()> { + self.log.push(self.name, "reset"); + Ok(()) + } + + async fn save(&mut self) -> Result, SaveError> { + self.log.push(self.name, "save"); + Ok(None) + } + + async fn restore(&mut self, _state: SavedStateBlob) -> Result<(), RestoreError> { + self.log.push(self.name, "restore"); + Ok(()) + } + + async fn advance_time(&mut self, _duration: Duration) -> anyhow::Result<()> { + self.log.push(self.name, "advance_time"); + Ok(()) + } +} + +impl InspectMut for LoggingUnit { + fn inspect_mut(&mut self, req: inspect::Request<'_>) { + req.respond(); + } +} + +/// Runs `unit`, but drops each start request after starting the unit, without +/// answering it, so that the start is uncertain while the unit runs and +/// answers later requests. +async fn run_unanswering_start( + mut unit: LoggingUnit, + mut recv: Receiver, +) -> LoggingUnit { + while let Some(request) = recv.next().await { + match request { + StateRequest::Start(response) => { + unit.start().await; + drop(response); + } + request => request.apply(&mut unit).await, + } + } + unit +} + +/// Starts a set with a unit named `dependency`, then adds a unit named +/// `uncertain` that depends on it, as a hot-add does, and whose start is +/// uncertain, and returns the set, which still runs, with the handles of both +/// units. +async fn running_set_with_an_uncertain_unit( + driver: &DefaultDriver, + log: &Log, +) -> (StateUnits, Vec>) { + let mut units = StateUnits::new(); + let dependency = units + .add("dependency") + .spawn(driver, |recv| { + run_unit(LoggingUnit::new("dependency", log), recv) + }) + .unwrap(); + units.start().await.unwrap(); + + let uncertain = units + .add("uncertain") + .depends_on(dependency.handle()) + .spawn(driver, |recv| { + run_unanswering_start(LoggingUnit::new("uncertain", log), recv) + }) + .unwrap(); + let error = units.start_stopped_units().await.unwrap_err(); + assert!(format!("{error:#}").contains("uncertain"), "{error:#}"); + assert!(units.is_running()); + assert_eq!(log.take(), ["dependency start", "uncertain start"]); + (units, vec![dependency, uncertain]) +} + +/// Checks that `error` is the failure of `op` because of the unit named +/// `name` alone. +fn assert_terminal_unit_error(error: StateTransitionError, op: &str, name: &str) { + let StateTransitionErrorKind::Units(UnitErrorSet(errors)) = &error.error else { + panic!("{op} failed for another reason: {error:?}"); + }; + let names = errors + .iter() + .map(|(name, _)| name.as_ref()) + .collect::>(); + assert_eq!( + (error.op, names.as_slice()), + (op, [name].as_slice()), + "{error:?}" + ); +} + +/// Checks that a reset, save, restore, and time advance of `units` each fail +/// because of the unit named `name`, which is in a terminal state, without +/// sending a unit a request. +async fn assert_requests_fail(units: &mut StateUnits, name: &str, log: &Log) { + assert_terminal_unit_error(units.reset().await.unwrap_err(), "reset", name); + let Err(error) = units.save().await else { + panic!("save succeeded"); + }; + assert_terminal_unit_error(error, "save", name); + assert_terminal_unit_error( + units.restore(Vec::new()).await.unwrap_err(), + "restore", + name, + ); + assert_terminal_unit_error( + units + .advance_time(Duration::from_secs(1)) + .await + .unwrap_err(), + "advance_time", + name, + ); + assert_eq!(log.take(), Vec::::new()); +} + +#[async_test] +async fn a_stop_stops_a_unit_whose_start_is_uncertain(driver: DefaultDriver) { + let log = Log::default(); + let (mut units, _handles) = running_set_with_an_uncertain_unit(&driver, &log).await; + + // The unit whose start is uncertain may be running, so the stop stops it, + // before the unit that it depends on. + units.stop().await; + assert!(!units.is_running()); + assert_eq!(log.take(), ["uncertain stop", "dependency stop"]); + + // The unit stays in its terminal state. + let error = units.start().await.unwrap_err(); + assert!(error.to_string().contains("terminal state"), "{error:#}"); + assert_requests_fail(&mut units, "uncertain", &log).await; +} + +#[async_test] +async fn requests_fail_after_a_start_leaves_a_terminal_unit(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let log = Log::default(); + let _healthy = units + .add("healthy") + .advances_time() + .spawn(&driver, |recv| { + run_unit(LoggingUnit::new("healthy", &log), recv) + }) + .unwrap(); + // A unit whose task exits at its start request without answering it, as + // a task that has failed would. + let _lost = units + .add("lost") + .spawn(&driver, |mut recv| async move { + while let Some(request) = recv.next().await { + if matches!(request, StateRequest::Start(_)) { + break; + } + } + }) + .unwrap(); + + let error = units.start().await.unwrap_err(); + assert!(format!("{error:#}").contains("lost"), "{error:#}"); + // The rollback stopped the unit that started. + assert_eq!(log.take(), ["healthy start", "healthy stop"]); + + assert_requests_fail(&mut units, "lost", &log).await; +} + +#[async_test] +async fn requests_fail_while_a_terminal_unit_keeps_the_set_running(driver: DefaultDriver) { + let log = Log::default(); + let (mut units, _handles) = running_set_with_an_uncertain_unit(&driver, &log).await; + + // The terminal unit fails the requests before the set's running state + // matters, so that a caller that considers the VM stopped while such a + // unit keeps the set running gets an error rather than a panic. + assert_requests_fail(&mut units, "uncertain", &log).await; + assert!(units.is_running()); +}