From 2fcd1573d0ae8448e7d8e52d3308647bf5a6e342 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Tue, 10 Nov 2026 04:55:05 -0800 Subject: [PATCH 1/3] state_unit: roll back a partial start without panicking on a lost unit When a start fails, its rollback stops the units that did start with run_op, which panics when one of them does not answer, as when its task exited after it started (microsoft/nvx#1363). The rollback also stops a started unit while a unit that depends on it may still run, such as a unit whose start is uncertain, or a unit that ran before a hot-add. Stop the started units in reverse dependency order, as the later revisions of microsoft/openvmm#4872 do: a started unit is not sent a stop request while a unit that depends on it is not stopped, unless that unit was removed, and a started unit whose stop is blocked or not answered is left in the terminal StartUncertain state. Such a unit may still run, so the units then stay running, so that StateUnits::stop can stop it. A unit that failed to confirm its start is still stopped with the units that started, and can start again. Tests: a dependency of a unit whose start is uncertain, a started unit whose task exits before the rollback stops it, a new dependency of a running unit, and a unit that drops the rollback's stop request each fail the start and stay running, and a later stop stops them. Before the change, two of them panic in run_op with UnitRecvError, and the other two stop the dependency. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/worker/dispatch/microvm.rs | 3 +- vmm_core/state_unit/src/start.rs | 116 ++++++- vmm_core/state_unit/src/tests/start.rs | 288 +++++++++++++++++- 3 files changed, 391 insertions(+), 16 deletions(-) diff --git a/openvmm/openvmm_core/src/worker/dispatch/microvm.rs b/openvmm/openvmm_core/src/worker/dispatch/microvm.rs index 8c104360819..64bab0f81b9 100644 --- a/openvmm/openvmm_core/src/worker/dispatch/microvm.rs +++ b/openvmm/openvmm_core/src/worker/dispatch/microvm.rs @@ -423,7 +423,8 @@ impl LoadedVm { /// that runs a quiesced VM again. /// /// A unit whose task does not answer leaves the units that did start - /// stopped again, and fails the rollback. + /// stopped again, and fails the rollback. A unit that such a unit depends + /// on, or that does not answer its stop, stays running instead. pub(super) async fn resume_after_failed_snapshot(&mut self) -> anyhow::Result<()> { self.state_units.start().await?; self.running = true; diff --git a/vmm_core/state_unit/src/start.rs b/vmm_core/state_unit/src/start.rs index e6799e1b73d..25c8468d582 100644 --- a/vmm_core/state_unit/src/start.rs +++ b/vmm_core/state_unit/src/start.rs @@ -11,7 +11,9 @@ use super::StateRequest; use super::StateUnits; use super::state_change; use futures::future::join_all; +use std::collections::BTreeSet; use std::time::Instant; +use tracing::Instrument; impl StateUnits { /// Starts all stopped units in dependency order. @@ -28,6 +30,14 @@ impl StateUnits { /// leaves it in that state. A unit that failed to confirm its start can /// start again. /// + /// The units that did start are stopped again in reverse dependency order. + /// A started unit that a unit that is not stopped depends on, such as a + /// unit whose start is uncertain, is not stopped, since that unit may + /// still use it, and a started unit whose task does not answer its stop + /// request may still run. Either is left in the terminal state too, and + /// the units then stay running, so that [`StateUnits::stop`] can stop + /// them. + /// /// [`StateUnit::confirm_start`]: crate::StateUnit::confirm_start pub(super) async fn start_with_rollback(&mut self) -> anyhow::Result<()> { enum StartResult { @@ -136,21 +146,99 @@ impl StateUnits { self.running = true; Ok(()) } else { - if !started_ids.is_empty() { - self.run_op( - "failed_start_rollback", - Some(&started_ids), - State::Running, - State::Stopping, - State::Stopped, - StateRequest::Stop, - |_, _| Some(()), - |unit| Some(&unit.dependents), - ) - .await; - } - self.running = was_running; + let rollback_failures = self.stop_after_failed_start(&started_ids).await; + // A unit that the rollback did not stop may still run. + self.running = was_running || !rollback_failures.is_empty(); + failures.extend(rollback_failures); anyhow::bail!("state units could not start: {}", failures.join("; ")) } } + + /// Stops the units in `started_ids` again in reverse dependency order. + /// + /// Every other unit that is not stopped, such as one that was already + /// running or one whose start is uncertain, blocks the stop of its + /// dependencies without receiving a stop request. A unit whose stop is + /// blocked or not answered is left in a terminal state. Returns the + /// failures. + async fn stop_after_failed_start(&self, started_ids: &[u64]) -> Vec { + const OP: &str = "failed_start_rollback"; + let started_ids = started_ids.iter().copied().collect::>(); + let mut operations = Vec::new(); + { + let mut inner = self.inner.lock(); + let ready_set = inner.ready_set(None); + for (&id, unit) in &inner.units { + if !started_ids.contains(&id) { + ready_set.done(id, unit.state == State::Stopped); + } + } + for (&id, unit) in inner + .units + .iter_mut() + .filter(|(id, _)| started_ids.contains(id)) + { + assert_eq!(unit.state, State::Running); + let name = unit.name.clone(); + let dependents = unit.dependents.iter().copied().collect::>(); + let ready_set = ready_set.clone(); + let units = self.inner.clone(); + let stop = state_change(name.clone(), unit, StateRequest::Stop, Some(())); + operations.push(async move { + if !ready_set.wait(OP, id, &dependents).await { + // A unit removed while it waits no longer uses its + // dependencies, so they can stop. + ready_set.done(id, !units.lock().units.contains_key(&id)); + return (name, id, None); + } + let result = stop.await; + // Removing a unit ends its task, which then cannot answer. + // A removed unit no longer uses its dependencies, so they + // can stop. + let stopped = + matches!(result, Ok(Some(()))) || !units.lock().units.contains_key(&id); + ready_set.done(id, stopped); + (name, id, Some(result)) + }); + unit.state = State::Stopping; + } + } + let results = async { + let start = Instant::now(); + let results = join_all(operations).await; + tracing::info!(duration = ?Instant::now() - start, "state change complete"); + results + } + .instrument(tracing::info_span!("state_change", operation = OP)) + .await; + + let mut failures = Vec::new(); + let mut inner = self.inner.lock(); + for (name, id, result) in results { + let Some(unit) = inner.units.get_mut(&id) else { + continue; + }; + assert_eq!(unit.state, State::Stopping); + match result { + Some(Ok(Some(()))) => unit.state = State::Stopped, + Some(Ok(None)) => { + unit.state = State::StartUncertain; + failures.push(format!( + "{name}: {OP} stop failed: stop request was not sent" + )); + } + Some(Err(error)) => { + unit.state = State::StartUncertain; + failures.push(format!("{name}: {OP} stop failed: {error:#}")); + } + None => { + unit.state = State::StartUncertain; + failures.push(format!( + "{name}: {OP} stop skipped because a dependent was not stopped" + )); + } + } + } + failures + } } diff --git a/vmm_core/state_unit/src/tests/start.rs b/vmm_core/state_unit/src/tests/start.rs index d41d23f3bc9..b60ae8cec57 100644 --- a/vmm_core/state_unit/src/tests/start.rs +++ b/vmm_core/state_unit/src/tests/start.rs @@ -3,17 +3,20 @@ //! Tests for start with rollback. +use crate::State; use crate::StateRequest; use crate::StateUnit; use crate::StateUnits; use crate::run_unit; use futures::StreamExt; use inspect::InspectMut; +use mesh::Receiver; use pal_async::DefaultDriver; use pal_async::async_test; use parking_lot::Mutex; use std::sync::Arc; use std::sync::atomic::AtomicBool; +use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; use std::time::Duration; use test_with_tracing::test; @@ -21,7 +24,7 @@ use vmcore::save_restore::RestoreError; use vmcore::save_restore::SaveError; use vmcore::save_restore::SavedStateBlob; -struct RecordingUnit { +pub(super) struct RecordingUnit { started: Arc, stopped: Arc, } @@ -54,6 +57,46 @@ impl InspectMut for RecordingUnit { } } +pub(super) fn recording_unit( + started: &Arc, + stopped: &Arc, +) -> RecordingUnit { + RecordingUnit { + started: started.clone(), + stopped: stopped.clone(), + } +} + +pub(super) fn unit_state(units: &StateUnits, name: &str) -> State { + units + .inner + .lock() + .units + .values() + .find(|unit| unit.name.as_ref() == name) + .unwrap() + .state +} + +/// Runs `unit`, but drops each start request after starting the unit, without +/// answering it, so that the start is uncertain while the unit still answers +/// later requests. +async fn run_unanswering_start( + mut unit: RecordingUnit, + mut recv: Receiver, +) -> RecordingUnit { + while let Some(request) = recv.next().await { + match request { + StateRequest::Start(response) => { + unit.start().await; + drop(response); + } + request => request.apply(&mut unit).await, + } + } + unit +} + #[async_test] async fn an_unanswered_start_rolls_back_and_blocks_dependents(driver: DefaultDriver) { let mut units = StateUnits::new(); @@ -323,3 +366,246 @@ async fn an_unanswered_confirmation_leaves_a_terminal_unit(driver: DefaultDriver let retry_error = units.start().await.unwrap_err(); assert!(retry_error.to_string().contains("terminal state")); } + +#[async_test] +async fn test_uncertain_start_blocks_dependency_rollback(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let dependency_started = Arc::new(AtomicBool::new(false)); + let dependency_stopped = Arc::new(AtomicBool::new(false)); + let dependency = units + .add("dependency") + .spawn(&driver, |recv| { + run_unit( + recording_unit(&dependency_started, &dependency_stopped), + recv, + ) + }) + .unwrap(); + let unanswered_started = Arc::new(AtomicBool::new(false)); + let unanswered_stopped = Arc::new(AtomicBool::new(false)); + let _unanswered = units + .add("unanswered") + .depends_on(dependency.handle()) + .spawn(&driver, |recv| { + run_unanswering_start( + recording_unit(&unanswered_started, &unanswered_stopped), + recv, + ) + }) + .unwrap(); + + let error = units.start().await.unwrap_err().to_string(); + assert!( + error.contains("dependency: failed_start_rollback stop skipped"), + "{error}" + ); + assert!(dependency_started.load(Ordering::Relaxed)); + assert!(unanswered_started.load(Ordering::Relaxed)); + // The unit whose start is uncertain may use its dependency, so the + // rollback leaves the dependency running, and the units stay running. + assert!(!dependency_stopped.load(Ordering::Relaxed)); + assert!(units.is_running()); + assert_eq!(unit_state(&units, "dependency"), State::StartUncertain); + assert_eq!(unit_state(&units, "unanswered"), State::StartUncertain); + + // A later stop stops both, the dependency last, and both stay terminal. + units.stop().await; + assert!(unanswered_stopped.load(Ordering::Relaxed)); + assert!(dependency_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); + assert_eq!(unit_state(&units, "dependency"), State::StartUncertain); + assert_eq!(unit_state(&units, "unanswered"), State::StartUncertain); + let retry_error = units.start().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); +} + +#[async_test] +async fn test_started_unit_exit_during_rollback_returns_error(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let dependency_started = Arc::new(AtomicBool::new(false)); + let dependency_stopped = Arc::new(AtomicBool::new(false)); + let dependency = units + .add("dependency") + .spawn(&driver, |recv| { + run_unit( + recording_unit(&dependency_started, &dependency_stopped), + recv, + ) + }) + .unwrap(); + // A unit whose task answers its start and the confirmation of that start, + // and then exits, as a task that has failed would. + let exited = units + .add("exited") + .depends_on(dependency.handle()) + .spawn(&driver, |mut recv| async move { + while let Some(request) = recv.next().await { + match request { + StateRequest::Start(response) => response.complete(()), + StateRequest::ConfirmStart(response) => { + response.complete(Ok(())); + break; + } + request => panic!("unexpected request {request:?}"), + } + } + }) + .unwrap(); + let unanswered_stopped = Arc::new(AtomicBool::new(false)); + let _unanswered = units + .add("unanswered") + .spawn(&driver, |recv| { + run_unanswering_start( + recording_unit(&Arc::new(AtomicBool::new(false)), &unanswered_stopped), + recv, + ) + }) + .unwrap(); + + let error = units.start().await.unwrap_err().to_string(); + assert!( + error.contains("exited: failed_start_rollback stop failed"), + "{error}" + ); + assert!( + error.contains("dependency: failed_start_rollback stop skipped"), + "{error}" + ); + assert!(dependency_started.load(Ordering::Relaxed)); + assert!(!dependency_stopped.load(Ordering::Relaxed)); + // The dependency is still running, so a later stop must be able to stop it. + assert!(units.is_running()); + assert_eq!(unit_state(&units, "exited"), State::StartUncertain); + assert_eq!(unit_state(&units, "dependency"), State::StartUncertain); + + // Once the exited unit is removed, a stop stops the others. + exited.remove().await; + units.stop().await; + assert!(dependency_stopped.load(Ordering::Relaxed)); + assert!(unanswered_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); + let retry_error = units.start().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); +} + +#[async_test] +async fn test_failed_hot_add_does_not_stop_dependency_of_running_unit(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let running_dependent_started = Arc::new(AtomicBool::new(false)); + let running_dependent_stopped = Arc::new(AtomicBool::new(false)); + let running_dependent = units + .add("running-dependent") + .spawn(&driver, |recv| { + run_unit( + recording_unit(&running_dependent_started, &running_dependent_stopped), + recv, + ) + }) + .unwrap(); + units.start().await.unwrap(); + + let dependency_started = Arc::new(AtomicBool::new(false)); + let dependency_stopped = Arc::new(AtomicBool::new(false)); + let _dependency = units + .add("new-dependency") + .dependency_of(running_dependent.handle()) + .spawn(&driver, |recv| { + run_unit( + recording_unit(&dependency_started, &dependency_stopped), + recv, + ) + }) + .unwrap(); + let unanswered_started = Arc::new(AtomicBool::new(false)); + let unanswered_stopped = Arc::new(AtomicBool::new(false)); + let _unanswered = units + .add("unanswered") + .spawn(&driver, |recv| { + run_unanswering_start( + recording_unit(&unanswered_started, &unanswered_stopped), + recv, + ) + }) + .unwrap(); + + let error = units.start_stopped_units().await.unwrap_err().to_string(); + assert!( + error.contains("new-dependency: failed_start_rollback stop skipped"), + "{error}" + ); + // A running unit depends on the new unit, so the rollback leaves it + // running. + assert!(dependency_started.load(Ordering::Relaxed)); + assert!(!dependency_stopped.load(Ordering::Relaxed)); + assert!(!running_dependent_stopped.load(Ordering::Relaxed)); + assert!(unanswered_started.load(Ordering::Relaxed)); + assert!(!unanswered_stopped.load(Ordering::Relaxed)); + assert!(units.is_running()); + assert_eq!(unit_state(&units, "new-dependency"), State::StartUncertain); + assert_eq!(unit_state(&units, "unanswered"), State::StartUncertain); + + units.stop().await; + assert!(dependency_stopped.load(Ordering::Relaxed)); + assert!(running_dependent_stopped.load(Ordering::Relaxed)); + assert!(unanswered_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); + assert_eq!(unit_state(&units, "new-dependency"), State::StartUncertain); + let retry_error = units.start().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); +} + +#[async_test] +async fn test_unanswered_rollback_stop_can_be_retried(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let started = Arc::new(AtomicBool::new(false)); + let stopped = Arc::new(AtomicBool::new(false)); + let stop_requests = Arc::new(AtomicUsize::new(0)); + // A unit that drops its first stop request without stopping or + // answering. + let _unit = units + .add("rollback-retry") + .spawn(&driver, |mut recv| { + let mut unit = recording_unit(&started, &stopped); + let stop_requests = stop_requests.clone(); + async move { + while let Some(request) = recv.next().await { + match request { + StateRequest::Stop(response) + if stop_requests.fetch_add(1, Ordering::Relaxed) == 0 => + { + drop(response); + } + request => request.apply(&mut unit).await, + } + } + unit + } + }) + .unwrap(); + let unanswered_stopped = Arc::new(AtomicBool::new(false)); + let _unanswered = units + .add("unanswered") + .spawn(&driver, |recv| { + run_unanswering_start( + recording_unit(&Arc::new(AtomicBool::new(false)), &unanswered_stopped), + recv, + ) + }) + .unwrap(); + + let error = units.start().await.unwrap_err().to_string(); + assert!( + error.contains("rollback-retry: failed_start_rollback stop failed"), + "{error}" + ); + assert!(started.load(Ordering::Relaxed)); + assert!(!stopped.load(Ordering::Relaxed)); + assert_eq!(stop_requests.load(Ordering::Relaxed), 1); + assert!(units.is_running()); + + units.stop().await; + assert!(stopped.load(Ordering::Relaxed)); + assert_eq!(stop_requests.load(Ordering::Relaxed), 2); + assert!(unanswered_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); +} From 5a750a8625c59d8ea5de70fa28d4236dabc74bf2 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Tue, 10 Nov 2026 05:08:43 -0800 Subject: [PATCH 2/3] state_unit: fail a stop instead of panicking when a unit is lost A unit whose task has exited, or that drops its stop request, makes StateUnits::stop panic the VM worker with "UnitRecvError { name: .., source: Channel(Closed) }": run_op treats an unanswered request as an unrecoverable unit failure (microsoft/nvx#1363). OpenVMM's profiles abort on panic, so the process ends without reporting the failure or tearing the VM down. A microVM stops its units when it quiesces for a snapshot, and whenever it pauses or tears down. Stop the units in reverse dependency order without panicking, with the ordered stop of the start rollback, which moves to a module that both share, as microsoft/openvmm#4937 proposes upstream. A unit whose stop request is not answered is left in a new terminal state, StopUncertain, and the units that it depends on are not sent a stop request, since it may still use them. StateUnits::stop then returns the failures, and the units stay running, so that a later stop can stop them, as once that unit is removed. A unit removed during a stop, even while it waits for a unit that depends on it, is not a failure, and no longer keeps its dependencies running. A stop also stops a unit in a terminal state, which keeps its marker, and while a unit is in one, a later start, reset, save, restore, or time advance fails. The callers handle the error: - LoadedVm::pause fails, leaving the VM paused and state units running. A reset, pulse save and restore, VMRS dump, or pause reports the failure, and the worker then tears the VM down, since those units can never start again: it does so whenever state units run while the VM does not, as a failed stop, or a start whose rollback did not stop every unit, leaves them. - VmRpc::Pause becomes failable, as VmRpc::Resume is in this chain, so that the REPL, the ttrpc PauseVm handler, the VM controller's disk snapshot, and Petri report a failed pause. During a microVM snapshot boundary it fails, as a resume does, rather than being dropped. - A microVM snapshot quiesce whose units do not all stop fails with the new SnapshotQuiesceError::Unrecoverable, and the controller terminates the source, which can neither roll back nor resume, with nothing published. - A worker restart whose VM cannot stop reports the failure and tears the VM down, and PCI resource assignment returns the error. - The teardown, a failed snapshot boundary, a failed restore readiness event, and an expired restore gate log a failed stop, since the VM ends anyway. - OpenHCL reports a failed stop to the requester of a pause, which UhVmRpc::Pause now allows, a diagnostic save, a worker restart, or a servicing request, and keeps serving, as after a failed start. Tests: a lost unit fails the stop, keeps its dependency running, and fails a later start, reset, save, restore, or time advance, while the units run and after another stop; a unit whose stop is unanswered stops when asked again and stays terminal; a unit removed during a stop does not keep its dependency running, even while it waits behind a unit whose stop is unanswered. At the previous commit, the first two scenarios, and the removal behind an unanswered stop, panic in run_op with UnitRecvError. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- openhcl/underhill_core/src/dispatch/mod.rs | 29 +- openhcl/underhill_core/src/lib.rs | 8 +- openvmm/openvmm_core/src/worker/dispatch.rs | 69 ++++- .../openvmm_core/src/worker/dispatch/dump.rs | 2 +- .../src/worker/dispatch/microvm.rs | 27 +- .../src/worker/dispatch/restore.rs | 17 +- .../src/worker/dispatch/snapshot_rpc.rs | 11 +- openvmm/openvmm_defs/src/rpc.rs | 10 +- openvmm/openvmm_entry/src/repl.rs | 17 +- openvmm/openvmm_entry/src/ttrpc/mod.rs | 2 +- openvmm/openvmm_entry/src/vm_controller.rs | 2 +- .../src/vm_controller/microvm.rs | 7 + petri/src/worker.rs | 4 +- vmm_core/state_unit/src/lib.rs | 73 ++--- vmm_core/state_unit/src/start.rs | 63 +--- vmm_core/state_unit/src/stop.rs | 156 ++++++++++ vmm_core/state_unit/src/tests/slow.rs | 2 +- vmm_core/state_unit/src/tests/start.rs | 8 +- vmm_core/state_unit/src/tests/stop.rs | 287 ++++++++++++++++++ vmm_core/state_unit/src/tests/terminal.rs | 2 +- 20 files changed, 632 insertions(+), 164 deletions(-) create mode 100644 vmm_core/state_unit/src/stop.rs create mode 100644 vmm_core/state_unit/src/tests/stop.rs diff --git a/openhcl/underhill_core/src/dispatch/mod.rs b/openhcl/underhill_core/src/dispatch/mod.rs index 346c8a1d874..ca71ef98f8d 100644 --- a/openhcl/underhill_core/src/dispatch/mod.rs +++ b/openhcl/underhill_core/src/dispatch/mod.rs @@ -79,7 +79,7 @@ use vtl2_settings_worker::handle_vtl2_config_rpc; #[derive(MeshPayload)] pub enum UhVmRpc { - Pause(Rpc<(), bool>), + Pause(FailableRpc<(), bool>), Resume(Rpc<(), bool>), Save(FailableRpc<(), Vec>), ClearHalt(Rpc<(), bool>), // TODO: remove this, and use DebugRequest::Resume @@ -326,7 +326,15 @@ impl LoadedVm { WorkerRpc::Stop => break None, WorkerRpc::Restart(rpc) => { let state = async { - let running = self.stop().await; + let running = match self.stop().await { + Ok(running) => running, + Err(err) => { + rpc.complete(Err(RemoteError::new( + err.context("failed to stop VM for worker restart"), + ))); + return None; + } + }; match self .save( SaveReason::WorkerRestart, @@ -416,10 +424,10 @@ impl LoadedVm { }) .await } - UhVmRpc::Pause(rpc) => rpc.handle(async |()| self.stop().await).await, + UhVmRpc::Pause(rpc) => rpc.handle_failable(async |()| self.stop().await).await, UhVmRpc::Save(rpc) => { rpc.handle_failable(async |()| { - let running = self.stop().await; + let running = self.stop().await?; let r = self .save( SaveReason::Diagnostic, @@ -725,7 +733,7 @@ impl LoadedVm { // Do everything before the log flush under a span. let r = async { - if !self.stop().await { + if !self.stop().await? { // This should only occur if you tried to initiate a // servicing operation after manually pausing underhill // via `ohcldiag-dev`. @@ -921,14 +929,17 @@ impl LoadedVm { } /// Returns true if the VM was previously running. - async fn stop(&mut self) -> bool { + /// + /// Fails if a state unit does not stop. The state units then stay + /// running, and can never start again. + async fn stop(&mut self) -> anyhow::Result { if self.state_units.is_running() { self.last_state_unit_stop = Some(ReferenceTime::new(self.partition.reference_time())); tracing::info!(CVM_ALLOWED, "stopping VM"); - self.state_units.stop().await; - true + self.state_units.stop().await?; + Ok(true) } else { - false + Ok(false) } } diff --git a/openhcl/underhill_core/src/lib.rs b/openhcl/underhill_core/src/lib.rs index 329d1c119f8..2f53d4b1670 100644 --- a/openhcl/underhill_core/src/lib.rs +++ b/openhcl/underhill_core/src/lib.rs @@ -637,7 +637,13 @@ async fn run_control( // FUTURE: consider supporting cancellation driver .spawn("diag-pause", async move { - let was_paused = req.await.expect("failed to pause VM"); + let was_paused = match req.await.expect("failed to pause VM") { + Ok(was_paused) => was_paused, + Err(error) => { + rpc.complete(Err(error)); + return; + } + }; rpc.handle_failable_sync(|_| { if !was_paused { Err(anyhow::anyhow!("VM is already paused")) diff --git a/openvmm/openvmm_core/src/worker/dispatch.rs b/openvmm/openvmm_core/src/worker/dispatch.rs index a2695e19c5a..c878d9ab879 100644 --- a/openvmm/openvmm_core/src/worker/dispatch.rs +++ b/openvmm/openvmm_core/src/worker/dispatch.rs @@ -4018,13 +4018,24 @@ impl LoadedVm { self.state_units.start().await } - async fn pause(&mut self) -> bool { + /// Pauses the VM, returning true if it was running. + /// + /// Fails if a state unit does not stop. The VM no longer runs then, but + /// state units still may, which [`Self::units_left_running`] reports. + async fn pause(&mut self) -> anyhow::Result { if !self.running { - return false; + return Ok(false); } - self.state_units.stop().await; self.running = false; - true + self.state_units.stop().await?; + Ok(true) + } + + /// Returns true if state units run while the VM does not, as a failed + /// stop, or a start whose rollback did not stop every unit, leaves them. + /// They can never start again, so the VM must be torn down. + fn units_left_running(&self) -> bool { + self.state_units.is_running() && !self.running } /// Assign PCI bus numbers and BAR addresses for all boot modes. @@ -4067,10 +4078,16 @@ impl LoadedVm { .context("PCI resource assignment failed"); // Stop state units again; the caller will resume normally. - self.state_units.stop().await; + let stopped = self + .state_units + .stop() + .await + .context("failed to stop devices after PCI resource assignment"); drop(stop_guard); - result + // Devices that did not stop can never start again, which outweighs a + // failed assignment. + stopped.and(result) } pub async fn run( @@ -4111,6 +4128,13 @@ impl LoadedVm { let mut deferred = VecDeque::new(); loop { + // A failed stop, or a start whose rollback did not stop every + // unit, leaves state units running while the VM is paused. They + // can never start again, so tear the VM down. + if self.units_left_running() { + tracing::error!("state units failed to stop; tearing down the VM"); + break; + } let queued = if !device_removal.is_pending() { deferred.pop_front() } else { @@ -4157,6 +4181,7 @@ impl LoadedVm { WorkerRpc::Stop => break, WorkerRpc::Restart(rpc) => { let mut stopped = false; + let mut stop_failed = false; // First run the non-destructive operations. let r = async { let shared_memory = self.inner.memory_manager.shared_memory_backing(); @@ -4164,7 +4189,10 @@ impl LoadedVm { anyhow::bail!("restart is not supported with --private-memory"); } if self.running { - self.state_units.stop().await; + if let Err(error) = self.state_units.stop().await { + stop_failed = true; + return Err(error); + } stopped = true; } let saved_state = self.save().await?; @@ -4180,6 +4208,14 @@ impl LoadedVm { return; } Err(err) => { + if stop_failed { + // The state units that did not stop can + // never start again, so tear the VM down. + rpc.complete(Err(RemoteError::new(err.context( + "worker restart failed and the VM could not stop", + )))); + break; + } if stopped { if let Err(start_error) = self.start_units().await { rpc.complete(Err(RemoteError::new(start_error.context( @@ -4220,7 +4256,9 @@ impl LoadedVm { VmRpc::Resume(rpc) => { rpc.handle_failable(async |()| self.resume().await).await } - VmRpc::Pause(rpc) => rpc.handle(async |()| self.pause().await).await, + VmRpc::Pause(rpc) => { + rpc.handle_failable(async |()| self.pause().await).await + } VmRpc::Save(rpc) => { rpc.handle_failable(async |()| self.save().await.map(ProtobufMessage::new)) .await @@ -4324,7 +4362,7 @@ impl LoadedVm { if !self.inner.partition.supports_reset() { return Err(PulseSaveRestoreError::ResetNotSupported); } - let paused = self.pause().await; + let paused = self.pause().await?; self.save_reset_restore().await?; if paused { @@ -4630,9 +4668,16 @@ impl LoadedVm { device_removal.drain().await; // Stop the state units whenever they run, not only when the VM does: // after a failed state change, such as a restart that could not resume - // the VM, the two can disagree. + // the VM, or a stop that a unit did not answer, the two can disagree. + // The VM is torn down anyway, so a unit that does not stop only leaves + // its dependencies running. if self.state_units.is_running() { - self.state_units.stop().await; + if let Err(error) = self.state_units.stop().await { + tracing::error!( + error = error.as_ref() as &dyn std::error::Error, + "failed to stop the state units before teardown" + ); + } } self.running = false; self.teardown_chipset_devices().await; @@ -4840,7 +4885,7 @@ impl LoadedVm { } async fn reset(&mut self, reload_firmware: bool) -> anyhow::Result<()> { - let resume = self.pause().await; + let resume = self.pause().await?; self.state_units.reset().await?; // TODO: _vmnic diff --git a/openvmm/openvmm_core/src/worker/dispatch/dump.rs b/openvmm/openvmm_core/src/worker/dispatch/dump.rs index a0108c686da..dc095f2d66f 100644 --- a/openvmm/openvmm_core/src/worker/dispatch/dump.rs +++ b/openvmm/openvmm_core/src/worker/dispatch/dump.rs @@ -16,7 +16,7 @@ impl LoadedVm { /// Pauses the VM if running, collects VP state and streams memory, /// then restores the prior running state. pub(super) async fn dump_state(&mut self, file: File) -> anyhow::Result<()> { - let was_running = self.pause().await; + let was_running = self.pause().await?; let result = self.dump_state_inner(file).await; if was_running { self.resume() diff --git a/openvmm/openvmm_core/src/worker/dispatch/microvm.rs b/openvmm/openvmm_core/src/worker/dispatch/microvm.rs index 64bab0f81b9..4953997f31d 100644 --- a/openvmm/openvmm_core/src/worker/dispatch/microvm.rs +++ b/openvmm/openvmm_core/src/worker/dispatch/microvm.rs @@ -319,9 +319,11 @@ impl LoadedVm { /// /// The quiesce stops every state unit and has no deadline: a unit that /// never stops stalls the capture until the process that drives the VM - /// ends it, with nothing published and the source never resumed. Each - /// error after the quiesce leaves every unit stopped, so the VM can roll - /// back with [`Self::resume_after_failed_snapshot`]. + /// ends it, with nothing published and the source never resumed. A unit + /// whose task does not answer fails the quiesce instead: it can never + /// start again, so the VM can neither roll back nor run, and the worker + /// tears it down. Each error after the quiesce leaves every unit stopped, + /// so the VM can roll back with [`Self::resume_after_failed_snapshot`]. pub(super) async fn quiesce_for_snapshot( &mut self, ) -> Result @@ -349,9 +351,13 @@ impl LoadedVm { } let quiesce = openvmm_defs::profile::ProfileSpan::start(); - self.state_units.stop().await; - quiesce.complete("capture", "quiesce", Default::default()); + // The VM no longer runs even if a unit does not stop, which leaves + // the units running for the worker to tear down. self.running = false; + self.state_units.stop().await.map_err(|error| { + openvmm_defs::rpc::SnapshotQuiesceError::Unrecoverable(RemoteError::new(error)) + })?; + quiesce.complete("capture", "quiesce", Default::default()); // The time ABI's capture checks and records: the LAPIC timers, then // the capture anchor and records. The PIT checks itself when saved. @@ -424,7 +430,8 @@ impl LoadedVm { /// /// A unit whose task does not answer leaves the units that did start /// stopped again, and fails the rollback. A unit that such a unit depends - /// on, or that does not answer its stop, stays running instead. + /// on, or that does not answer its stop, stays running instead, and the + /// worker then tears the VM down. pub(super) async fn resume_after_failed_snapshot(&mut self) -> anyhow::Result<()> { self.state_units.start().await?; self.running = true; @@ -438,9 +445,11 @@ impl LoadedVm { match event { Ok(request) => { if !self.establish_snapshot_boundary(request).await { - if self.running { - self.state_units.stop().await; - self.running = false; + if let Err(error) = self.pause().await { + tracing::error!( + error = error.as_ref() as &dyn std::error::Error, + "failed to stop the VM after its snapshot boundary failed" + ); } return false; } diff --git a/openvmm/openvmm_core/src/worker/dispatch/restore.rs b/openvmm/openvmm_core/src/worker/dispatch/restore.rs index 2d9c29fc500..6fd607ac36a 100644 --- a/openvmm/openvmm_core/src/worker/dispatch/restore.rs +++ b/openvmm/openvmm_core/src/worker/dispatch/restore.rs @@ -348,7 +348,14 @@ impl LoadedVm { } }); if let Err(error) = signal_result { - self.state_units.stop().await; + // A unit that does not stop stays running while the VM is + // paused, and the worker then tears the VM down. + if let Err(stop_error) = self.state_units.stop().await { + tracing::error!( + error = stop_error.as_ref() as &dyn std::error::Error, + "failed to stop the state units after the restore readiness event failed" + ); + } return Err(error).context("failed to publish restore readiness event"); } } @@ -367,9 +374,11 @@ impl LoadedVm { /// input gate in time. pub(super) async fn handle_restore_gate_timeout(&mut self) { tracing::error!("post-restore input gate acknowledgement timed out"); - if self.running { - self.state_units.stop().await; - self.running = false; + if let Err(error) = self.pause().await { + tracing::error!( + error = error.as_ref() as &dyn std::error::Error, + "failed to stop the VM after the post-restore input gate timed out" + ); } } } diff --git a/openvmm/openvmm_core/src/worker/dispatch/snapshot_rpc.rs b/openvmm/openvmm_core/src/worker/dispatch/snapshot_rpc.rs index a69b4fb00ca..a37088bf3ad 100644 --- a/openvmm/openvmm_core/src/worker/dispatch/snapshot_rpc.rs +++ b/openvmm/openvmm_core/src/worker/dispatch/snapshot_rpc.rs @@ -52,10 +52,11 @@ pub(super) fn filter(message: VmRpc, boundary_active: bool) -> Option { VmRpc::AddVpciDevice(rpc) => rpc.fail(SnapshotBoundaryActive), VmRpc::RemoveVpciDevice(rpc) => rpc.fail(SnapshotBoundaryActive), VmRpc::DumpState(rpc) => rpc.fail(SnapshotBoundaryActive), + VmRpc::Pause(rpc) => rpc.fail(SnapshotBoundaryActive), VmRpc::PulseSaveRestore(rpc) => rpc.complete(Err(PulseSaveRestoreError::Other( RemoteError::new(SnapshotBoundaryActive), ))), - VmRpc::Pause(rpc) | VmRpc::ClearHalt(rpc) => drop(rpc), + VmRpc::ClearHalt(rpc) => drop(rpc), VmRpc::Nmi(rpc) => drop(rpc), } tracelimit::warn_ratelimited!( @@ -190,9 +191,10 @@ mod tests { let write = send.call(VmRpc::WriteMemory, (0, vec![1])); let reset = send.call(VmRpc::Reset, ()); let resume = send.call(VmRpc::Resume, ()); + let pause = send.call(VmRpc::Pause, ()); let remove = send.call(VmRpc::RemovePcieDevice, "port".to_owned()); - for _ in 0..4 { + for _ in 0..5 { assert!(filter(recv.try_recv().unwrap(), true).is_none()); } for result in [block_on(write), block_on(reset), block_on(remove)] { @@ -200,6 +202,7 @@ mod tests { assert!(error.to_string().contains("snapshot boundary is active")); } assert!(block_on(resume).unwrap().is_err()); + assert!(block_on(pause).unwrap().is_err()); let release = send.call(VmRpc::ReleaseSnapshotBoundary, ()); let Some(VmRpc::ReleaseSnapshotBoundary(rpc)) = filter(recv.try_recv().unwrap(), true) @@ -213,13 +216,11 @@ mod tests { #[test] fn infallible_mutations_report_channel_error() { let (send, mut recv) = mesh::channel(); - let pause = send.call(VmRpc::Pause, ()); let clear_halt = send.call(VmRpc::ClearHalt, ()); let nmi = send.call(VmRpc::Nmi, 0); - for _ in 0..3 { + for _ in 0..2 { assert!(filter(recv.try_recv().unwrap(), true).is_none()); } - assert!(block_on(pause).is_err()); assert!(block_on(clear_halt).is_err()); assert!(block_on(nmi).is_err()); } diff --git a/openvmm/openvmm_defs/src/rpc.rs b/openvmm/openvmm_defs/src/rpc.rs index e235ef36c61..5586e9e930c 100644 --- a/openvmm/openvmm_defs/src/rpc.rs +++ b/openvmm/openvmm_defs/src/rpc.rs @@ -31,7 +31,11 @@ pub enum VmRpc { /// Release a post-OUT boundary without starting a snapshot transaction. ReleaseSnapshotBoundary(FailableRpc<(), ()>), Resume(FailableRpc<(), bool>), - Pause(Rpc<(), bool>), + /// Pause the VM, returning whether it was running. + /// + /// Fails if a state unit does not stop; the worker then tears the VM + /// down, since that unit can never start again. + Pause(FailableRpc<(), bool>), ClearHalt(Rpc<(), bool>), Reset(FailableRpc<(), ()>), Nmi(Rpc), @@ -87,6 +91,10 @@ pub enum SnapshotQuiesceError { /// The VM quiesced completely; the controller may request rollback. #[error("snapshot failed after the VM quiesced")] RollbackSafe(#[source] RemoteError), + /// A state unit did not stop, so the VM can neither roll back nor run + /// again; the controller terminates the source. + #[error("snapshot quiesce left the VM unable to run again")] + Unrecoverable(#[source] RemoteError), } #[derive(Debug, MeshPayload, thiserror::Error)] diff --git a/openvmm/openvmm_entry/src/repl.rs b/openvmm/openvmm_entry/src/repl.rs index abab2d71c75..6ed0f51791b 100644 --- a/openvmm/openvmm_entry/src/repl.rs +++ b/openvmm/openvmm_entry/src/repl.rs @@ -667,7 +667,7 @@ pub(crate) async fn run_repl( let mut snapshot_saved = false; enum StateChange { - Pause(bool), + Pause(Result), Resume(Result), Reset(Result<(), RemoteError>), PulseSaveRestore(Result<(), PulseSaveRestoreError>), @@ -767,13 +767,14 @@ pub(crate) async fn run_repl( Event::StateChange(r) => { match r { Ok(sc) => match sc { - StateChange::Pause(success) => { - if success { - tracing::info!("pause complete"); - } else { - tracing::warn!("already paused"); - } - } + StateChange::Pause(r) => match r { + Ok(true) => tracing::info!("pause complete"), + Ok(false) => tracing::warn!("already paused"), + Err(err) => tracing::error!( + error = &err as &dyn std::error::Error, + "pause failed" + ), + }, StateChange::Resume(Ok(success)) => { if success { launch.restore_ready_pending = false; diff --git a/openvmm/openvmm_entry/src/ttrpc/mod.rs b/openvmm/openvmm_entry/src/ttrpc/mod.rs index 487ac4e36c3..8789a82ffc5 100644 --- a/openvmm/openvmm_entry/src/ttrpc/mod.rs +++ b/openvmm/openvmm_entry/src/ttrpc/mod.rs @@ -1701,7 +1701,7 @@ impl VmService { async fn pause_vm(&mut self) -> anyhow::Result<()> { let vm = self.vm.clone().context("VM not created yet")?; vm.worker_rpc - .call(VmRpc::Pause, ()) + .call_failable(VmRpc::Pause, ()) .await .map(drop) .context("pause failed")?; diff --git a/openvmm/openvmm_entry/src/vm_controller.rs b/openvmm/openvmm_entry/src/vm_controller.rs index 61ec2f93502..816232e21c5 100644 --- a/openvmm/openvmm_entry/src/vm_controller.rs +++ b/openvmm/openvmm_entry/src/vm_controller.rs @@ -518,7 +518,7 @@ impl VmController { // Pause the VM. self.vm_rpc - .call(VmRpc::Pause, ()) + .call_failable(VmRpc::Pause, ()) .await .context("failed to pause VM")?; diff --git a/openvmm/openvmm_entry/src/vm_controller/microvm.rs b/openvmm/openvmm_entry/src/vm_controller/microvm.rs index f6dfcf5f12b..350b70c4211 100644 --- a/openvmm/openvmm_entry/src/vm_controller/microvm.rs +++ b/openvmm/openvmm_entry/src/vm_controller/microvm.rs @@ -247,6 +247,13 @@ impl VmController { Ok(Err(SnapshotQuiesceError::RollbackSafe(error))) => { return self.rollback_failed_guest_snapshot(error.into()).await; } + Ok(Err(SnapshotQuiesceError::Unrecoverable(error))) => { + tracing::error!( + error = &error as &dyn std::error::Error, + "microVM snapshot quiesce left the VM unable to run again; terminating source" + ); + return GuestSnapshotAction::Terminate { exit_code: 1 }; + } Err(error) => { tracing::error!( error = &error as &dyn std::error::Error, diff --git a/petri/src/worker.rs b/petri/src/worker.rs index bd9905ec12d..fe17ce04ba5 100644 --- a/petri/src/worker.rs +++ b/petri/src/worker.rs @@ -65,8 +65,8 @@ impl Worker { )) } - pub(crate) async fn pause(&self) -> Result { - self.rpc.call(VmRpc::Pause, ()).await + pub(crate) async fn pause(&self) -> anyhow::Result { + Ok(self.rpc.call_failable(VmRpc::Pause, ()).await?) } /// The VM worker's management RPC channel. diff --git a/vmm_core/state_unit/src/lib.rs b/vmm_core/state_unit/src/lib.rs index 79641acfe22..8b5a6929a7a 100644 --- a/vmm_core/state_unit/src/lib.rs +++ b/vmm_core/state_unit/src/lib.rs @@ -34,6 +34,7 @@ mod advance_time; mod inventory; mod quiesce; mod start; +mod stop; use futures::FutureExt; use futures::StreamExt; @@ -287,6 +288,15 @@ enum State { Restoring, AdvancingTime, StartUncertain, + StopUncertain, +} + +impl State { + /// Whether a unit in this state never changes state again: a request to + /// it was not answered, so its state is unknown. + fn is_terminal(self) -> bool { + matches!(self, State::StartUncertain | State::StopUncertain) + } } #[derive(Debug)] @@ -562,7 +572,7 @@ impl StateUnits { inner .units .values() - .filter(|unit| matches!(unit.state, State::StartUncertain)) + .filter(|unit| unit.state.is_terminal()) .map(|unit| { ( unit.name.clone(), @@ -594,45 +604,15 @@ impl StateUnits { /// Stops all the state units. /// - /// This also stops the units whose start is uncertain, since they may be - /// running, and leaves them in that terminal state. - pub async fn stop(&mut self) { + /// Stops the units in reverse dependency order. A unit in a terminal state + /// may be running, so it is sent a stop request too, and keeps its terminal + /// marker. If a unit's task does not answer its stop request, the stop + /// fails, the unit is left in a terminal state, and the units that it + /// depends on keep running, since it may still use them. The units then + /// stay running, so that a later stop can stop them. + pub async fn stop(&mut self) -> anyhow::Result<()> { assert!(self.running); - // Make the units whose start is uncertain receive the stop, then - // restore their terminal state after the ordered operation. - let uncertain_ids = { - let mut inner = self.inner.lock(); - inner - .units - .iter_mut() - .filter(|(_, unit)| unit.state == State::StartUncertain) - .map(|(&id, unit)| { - unit.state = State::Running; - id - }) - .collect::>() - }; - // Stop units in reverse dependency order so that a dependency is not - // stopped before its dependant. - self.run_op( - "stop", - None, - State::Running, - State::Stopping, - State::Stopped, - StateRequest::Stop, - |_, _| Some(()), - |unit| Some(&unit.dependents), - ) - .await; - let mut inner = self.inner.lock(); - for id in uncertain_ids { - if let Some(unit) = inner.units.get_mut(&id) { - assert_eq!(unit.state, State::Stopped); - unit.state = State::StartUncertain; - } - } - self.running = false; + self.stop_in_order().await } /// Resets all the state units. @@ -1197,6 +1177,7 @@ mod tests { mod quiesce; mod slow; mod start; + mod stop; mod terminal; use super::StateTransitionErrorKind; @@ -1372,7 +1353,7 @@ mod tests { units.restore(state).await.unwrap(); assert_eq!(calls.load(Ordering::Relaxed), 2); units.start().await.unwrap(); - units.stop().await; + units.stop().await.unwrap(); let state = units.save().await.unwrap(); assert!(units.restore(state).await.is_err()); assert_eq!(calls.load(Ordering::Relaxed), 2); @@ -1476,7 +1457,7 @@ mod tests { // Both units are stopped again and can start. units.start().await.unwrap(); - units.stop().await; + units.stop().await.unwrap(); } #[async_test] @@ -1507,10 +1488,10 @@ mod tests { let _c = units .add("c") .spawn(&driver, |recv| run_unit(TestUnit::default(), recv)); - units.stop().await; + units.stop().await.unwrap(); units.start().await.unwrap(); - units.stop().await; + units.stop().await.unwrap(); let state = units.save().await.unwrap(); @@ -1557,7 +1538,7 @@ mod tests { }) .unwrap(); units.start().await.unwrap(); - units.stop().await; + units.stop().await.unwrap(); let state = units.save().await.unwrap(); @@ -1667,7 +1648,7 @@ mod tests { } units.start().await.unwrap(); - units.stop().await; + units.stop().await.unwrap(); a.remove().await; { @@ -1677,7 +1658,7 @@ mod tests { } units.start().await.unwrap(); - units.stop().await; + units.stop().await.unwrap(); // Removing the remaining unit leaves the set empty. c.remove().await; diff --git a/vmm_core/state_unit/src/start.rs b/vmm_core/state_unit/src/start.rs index 25c8468d582..495645183c1 100644 --- a/vmm_core/state_unit/src/start.rs +++ b/vmm_core/state_unit/src/start.rs @@ -10,10 +10,10 @@ use super::State; use super::StateRequest; use super::StateUnits; use super::state_change; +use super::stop::StopResult; use futures::future::join_all; use std::collections::BTreeSet; use std::time::Instant; -use tracing::Instrument; impl StateUnits { /// Starts all stopped units in dependency order. @@ -164,54 +164,7 @@ impl StateUnits { async fn stop_after_failed_start(&self, started_ids: &[u64]) -> Vec { const OP: &str = "failed_start_rollback"; let started_ids = started_ids.iter().copied().collect::>(); - let mut operations = Vec::new(); - { - let mut inner = self.inner.lock(); - let ready_set = inner.ready_set(None); - for (&id, unit) in &inner.units { - if !started_ids.contains(&id) { - ready_set.done(id, unit.state == State::Stopped); - } - } - for (&id, unit) in inner - .units - .iter_mut() - .filter(|(id, _)| started_ids.contains(id)) - { - assert_eq!(unit.state, State::Running); - let name = unit.name.clone(); - let dependents = unit.dependents.iter().copied().collect::>(); - let ready_set = ready_set.clone(); - let units = self.inner.clone(); - let stop = state_change(name.clone(), unit, StateRequest::Stop, Some(())); - operations.push(async move { - if !ready_set.wait(OP, id, &dependents).await { - // A unit removed while it waits no longer uses its - // dependencies, so they can stop. - ready_set.done(id, !units.lock().units.contains_key(&id)); - return (name, id, None); - } - let result = stop.await; - // Removing a unit ends its task, which then cannot answer. - // A removed unit no longer uses its dependencies, so they - // can stop. - let stopped = - matches!(result, Ok(Some(()))) || !units.lock().units.contains_key(&id); - ready_set.done(id, stopped); - (name, id, Some(result)) - }); - unit.state = State::Stopping; - } - } - let results = async { - let start = Instant::now(); - let results = join_all(operations).await; - tracing::info!(duration = ?Instant::now() - start, "state change complete"); - results - } - .instrument(tracing::info_span!("state_change", operation = OP)) - .await; - + let results = self.stop_units(OP, &started_ids).await; let mut failures = Vec::new(); let mut inner = self.inner.lock(); for (name, id, result) in results { @@ -220,18 +173,12 @@ impl StateUnits { }; assert_eq!(unit.state, State::Stopping); match result { - Some(Ok(Some(()))) => unit.state = State::Stopped, - Some(Ok(None)) => { - unit.state = State::StartUncertain; - failures.push(format!( - "{name}: {OP} stop failed: stop request was not sent" - )); - } - Some(Err(error)) => { + StopResult::Stopped => unit.state = State::Stopped, + StopResult::Uncertain(error) => { unit.state = State::StartUncertain; failures.push(format!("{name}: {OP} stop failed: {error:#}")); } - None => { + StopResult::Blocked => { unit.state = State::StartUncertain; failures.push(format!( "{name}: {OP} stop skipped because a dependent was not stopped" diff --git a/vmm_core/state_unit/src/stop.rs b/vmm_core/state_unit/src/stop.rs new file mode 100644 index 00000000000..ef4ca1fdf92 --- /dev/null +++ b/vmm_core/state_unit/src/stop.rs @@ -0,0 +1,156 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Ordered stop. A unit's stop cannot fail, but its task can stop answering: +//! then the units that it depends on are not stopped, since it may still use +//! them, and the stop fails, rather than the whole process panicking. + +use super::State; +use super::StateRequest; +use super::StateUnits; +use super::state_change; +use futures::future::join_all; +use std::collections::BTreeMap; +use std::collections::BTreeSet; +use std::sync::Arc; +use std::time::Instant; +use tracing::Instrument; + +/// The outcome of a unit's stop. +pub(super) enum StopResult { + /// The unit answered its stop request. + Stopped, + /// The unit was not sent a stop request, because a unit that depends on + /// it was not stopped. + Blocked, + /// The unit's task did not answer its stop request, so the unit may still + /// be running. + Uncertain(anyhow::Error), +} + +/// The outcome of a unit's stop, with the unit's name and ID. +pub(super) type UnitStopResult = (Arc, u64, StopResult); + +impl StateUnits { + /// Stops the units that are not stopped in reverse dependency order. + /// + /// A unit in a terminal state that may be running is sent a stop request + /// too, and keeps its terminal marker. A unit whose task does not answer + /// its stop request is left in a terminal state, and the units that it + /// depends on keep their states. The units stay running until every unit + /// has stopped. + pub(super) async fn stop_in_order(&mut self) -> anyhow::Result<()> { + const OP: &str = "stop"; + // Make the units in a terminal state receive a stop request, then + // restore their terminal markers. + let mut ids = BTreeSet::new(); + let mut terminal_states = BTreeMap::new(); + for (&id, unit) in &mut self.inner.lock().units { + match unit.state { + State::Stopped => continue, + State::Running => {} + State::StartUncertain | State::StopUncertain => { + terminal_states.insert(id, unit.state); + unit.state = State::Running; + } + state => panic!("unit {} in {state:?} state before stop", unit.name), + } + ids.insert(id); + } + + let results = self.stop_units(OP, &ids).await; + + let mut failures = Vec::new(); + let mut inner = self.inner.lock(); + for (name, id, result) in results { + let Some(unit) = inner.units.get_mut(&id) else { + continue; + }; + assert_eq!(unit.state, State::Stopping); + let terminal_state = terminal_states.get(&id).copied(); + unit.state = match result { + StopResult::Stopped => terminal_state.unwrap_or(State::Stopped), + StopResult::Uncertain(error) => { + failures.push(format!("{name}: {error:#}")); + State::StopUncertain + } + StopResult::Blocked => { + failures.push(format!( + "{name}: stop skipped because a dependent was not stopped" + )); + terminal_state.unwrap_or(State::Running) + } + }; + } + if failures.is_empty() { + self.running = false; + Ok(()) + } else { + anyhow::bail!("state units could not stop: {}", failures.join("; ")) + } + } + + /// Sends a stop request to each running unit in `ids` in reverse + /// dependency order, and returns the outcome of each unit's stop. + /// + /// A unit is sent its request once every unit that depends on it has + /// stopped. A unit that is not in `ids` has stopped only if it is in the + /// stopped state, and a unit in `ids` once it answers its request or is + /// removed. The units in `ids` are left stopping, for the caller to move + /// each to the state that the outcome of its stop leaves it in. + pub(super) async fn stop_units( + &self, + op: &'static str, + ids: &BTreeSet, + ) -> Vec { + let mut operations = Vec::new(); + { + let mut inner = self.inner.lock(); + let ready_set = inner.ready_set(None); + for (&id, unit) in &inner.units { + if !ids.contains(&id) { + ready_set.done(id, unit.state == State::Stopped); + } + } + for (&id, unit) in inner.units.iter_mut().filter(|(id, _)| ids.contains(id)) { + assert_eq!(unit.state, State::Running); + let name = unit.name.clone(); + let dependents = unit.dependents.iter().copied().collect::>(); + let ready_set = ready_set.clone(); + let units = self.inner.clone(); + let stop = state_change(name.clone(), unit, StateRequest::Stop, Some(())); + operations.push(async move { + if !ready_set.wait(op, id, &dependents).await { + // A unit removed while it waits no longer uses its + // dependencies, so they can stop. + ready_set.done(id, !units.lock().units.contains_key(&id)); + return (name, id, StopResult::Blocked); + } + let result = match stop.await { + Ok(Some(())) => StopResult::Stopped, + Ok(None) => { + StopResult::Uncertain(anyhow::anyhow!("stop request was not sent")) + } + Err(error) => StopResult::Uncertain(error.into()), + }; + // Removing a unit ends its task, which then cannot answer. + // A removed unit no longer uses its dependencies, so they + // can stop. + let stopped = matches!(result, StopResult::Stopped) + || !units.lock().units.contains_key(&id); + ready_set.done(id, stopped); + (name, id, result) + }); + unit.state = State::Stopping; + } + } + async { + let start = Instant::now(); + let results = join_all(operations).await; + tracing::info!(duration = ?Instant::now() - start, "state change complete"); + results + } + .instrument(tracing::info_span!("state_change", operation = op)) + .await + } +} diff --git a/vmm_core/state_unit/src/tests/slow.rs b/vmm_core/state_unit/src/tests/slow.rs index 3b9d0133d4a..78516f96599 100644 --- a/vmm_core/state_unit/src/tests/slow.rs +++ b/vmm_core/state_unit/src/tests/slow.rs @@ -110,7 +110,7 @@ async fn a_slow_stop_is_awaited_and_names_its_unit(driver: DefaultDriver) { units.start().await.unwrap(); assert!(log.unanswered().is_empty()); - units.stop().await; + units.stop().await.unwrap(); assert!(stopped.load(Ordering::Relaxed)); assert!(!units.is_running()); diff --git a/vmm_core/state_unit/src/tests/start.rs b/vmm_core/state_unit/src/tests/start.rs index b60ae8cec57..0dec7ba6837 100644 --- a/vmm_core/state_unit/src/tests/start.rs +++ b/vmm_core/state_unit/src/tests/start.rs @@ -409,7 +409,7 @@ async fn test_uncertain_start_blocks_dependency_rollback(driver: DefaultDriver) assert_eq!(unit_state(&units, "unanswered"), State::StartUncertain); // A later stop stops both, the dependency last, and both stay terminal. - units.stop().await; + units.stop().await.unwrap(); assert!(unanswered_stopped.load(Ordering::Relaxed)); assert!(dependency_stopped.load(Ordering::Relaxed)); assert!(!units.is_running()); @@ -480,7 +480,7 @@ async fn test_started_unit_exit_during_rollback_returns_error(driver: DefaultDri // Once the exited unit is removed, a stop stops the others. exited.remove().await; - units.stop().await; + units.stop().await.unwrap(); assert!(dependency_stopped.load(Ordering::Relaxed)); assert!(unanswered_stopped.load(Ordering::Relaxed)); assert!(!units.is_running()); @@ -544,7 +544,7 @@ async fn test_failed_hot_add_does_not_stop_dependency_of_running_unit(driver: De assert_eq!(unit_state(&units, "new-dependency"), State::StartUncertain); assert_eq!(unit_state(&units, "unanswered"), State::StartUncertain); - units.stop().await; + units.stop().await.unwrap(); assert!(dependency_stopped.load(Ordering::Relaxed)); assert!(running_dependent_stopped.load(Ordering::Relaxed)); assert!(unanswered_stopped.load(Ordering::Relaxed)); @@ -603,7 +603,7 @@ async fn test_unanswered_rollback_stop_can_be_retried(driver: DefaultDriver) { assert_eq!(stop_requests.load(Ordering::Relaxed), 1); assert!(units.is_running()); - units.stop().await; + units.stop().await.unwrap(); assert!(stopped.load(Ordering::Relaxed)); assert_eq!(stop_requests.load(Ordering::Relaxed), 2); assert!(unanswered_stopped.load(Ordering::Relaxed)); diff --git a/vmm_core/state_unit/src/tests/stop.rs b/vmm_core/state_unit/src/tests/stop.rs new file mode 100644 index 00000000000..6dce50b39b2 --- /dev/null +++ b/vmm_core/state_unit/src/tests/stop.rs @@ -0,0 +1,287 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Tests for a stop that fails instead of panicking. + +use super::start::recording_unit; +use super::start::unit_state; +use crate::State; +use crate::StateRequest; +use crate::StateUnits; +use crate::run_unit; +use futures::StreamExt; +use mesh::Receiver; +use pal_async::DefaultDriver; +use pal_async::async_test; +use std::sync::Arc; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; +use std::time::Duration; +use test_with_tracing::test; + +fn flag() -> Arc { + Arc::new(AtomicBool::new(false)) +} + +/// Runs a unit whose task answers its start and the confirmation of that +/// start, and then exits, as a task that has failed would. +async fn run_lost_after_start(mut recv: Receiver) { + while let Some(request) = recv.next().await { + match request { + StateRequest::Start(response) => response.complete(()), + StateRequest::ConfirmStart(response) => { + response.complete(Ok(())); + break; + } + request => panic!("unexpected request {request:?}"), + } + } +} + +#[async_test] +async fn a_lost_unit_fails_the_stop_and_keeps_its_dependencies_running(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let dependency_stopped = flag(); + let dependency = units + .add("dependency") + .spawn(&driver, |recv| { + run_unit(recording_unit(&flag(), &dependency_stopped), recv) + }) + .unwrap(); + let lost = units + .add("lost") + .depends_on(dependency.handle()) + .spawn(&driver, run_lost_after_start) + .unwrap(); + let dependent_stopped = flag(); + let _dependent = units + .add("dependent") + .depends_on(lost.handle()) + .spawn(&driver, |recv| { + run_unit(recording_unit(&flag(), &dependent_stopped), recv) + }) + .unwrap(); + let independent_stopped = flag(); + let _independent = units + .add("independent") + .spawn(&driver, |recv| { + run_unit(recording_unit(&flag(), &independent_stopped), recv) + }) + .unwrap(); + units.start().await.unwrap(); + + let error = units.stop().await.unwrap_err().to_string(); + assert!(error.contains("lost"), "{error}"); + assert!( + error.contains("dependency: stop skipped because a dependent was not stopped"), + "{error}" + ); + assert!(dependent_stopped.load(Ordering::Relaxed)); + assert!(independent_stopped.load(Ordering::Relaxed)); + // The lost unit may still use its dependency, so the dependency runs. + assert!(!dependency_stopped.load(Ordering::Relaxed)); + assert!(units.is_running()); + assert_eq!(unit_state(&units, "lost"), State::StopUncertain); + assert_eq!(unit_state(&units, "dependency"), State::Running); + assert_eq!(unit_state(&units, "dependent"), State::Stopped); + assert_eq!(unit_state(&units, "independent"), State::Stopped); + + // The units can never start again, and the requests that need stopped + // units fail, rather than panic, while the units stay running. + let start_error = units.start().await.unwrap_err(); + assert!(start_error.to_string().contains("terminal state")); + assert_eq!(unit_state(&units, "independent"), State::Stopped); + let reset_error = format!( + "{:#}", + anyhow::Error::from(units.reset().await.unwrap_err()) + ); + assert!( + reset_error.contains("lost: unit is in terminal state"), + "{reset_error}" + ); + assert!(units.save().await.is_err()); + assert!(units.restore(Vec::new()).await.is_err()); + assert!(units.advance_time(Duration::from_secs(1)).await.is_err()); + + // A later stop fails the same way while the lost unit remains. + assert!(units.stop().await.is_err()); + assert!(!dependency_stopped.load(Ordering::Relaxed)); + assert!(units.is_running()); + + // Once the lost unit is removed, the stop can stop its dependency, and + // the remaining units, none of which is in a terminal state, can start. + lost.remove().await; + units.stop().await.unwrap(); + assert!(dependency_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); + units.start().await.unwrap(); + units.stop().await.unwrap(); +} + +#[async_test] +async fn a_unit_whose_stop_is_unanswered_can_be_stopped_again(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let dependency_stopped = flag(); + let dependency = units + .add("dependency") + .spawn(&driver, |recv| { + run_unit(recording_unit(&flag(), &dependency_stopped), recv) + }) + .unwrap(); + let stopped = flag(); + let stop_requests = Arc::new(AtomicUsize::new(0)); + // A unit that drops its first stop request without stopping or answering. + let _unanswered = units + .add("unanswered") + .depends_on(dependency.handle()) + .spawn(&driver, |mut recv| { + let mut unit = recording_unit(&flag(), &stopped); + let stop_requests = stop_requests.clone(); + async move { + while let Some(request) = recv.next().await { + match request { + StateRequest::Stop(response) + if stop_requests.fetch_add(1, Ordering::Relaxed) == 0 => + { + drop(response); + } + request => request.apply(&mut unit).await, + } + } + unit + } + }) + .unwrap(); + units.start().await.unwrap(); + + let error = units.stop().await.unwrap_err().to_string(); + assert!(error.contains("unanswered"), "{error}"); + assert!(!stopped.load(Ordering::Relaxed)); + assert!(!dependency_stopped.load(Ordering::Relaxed)); + assert!(units.is_running()); + assert_eq!(unit_state(&units, "unanswered"), State::StopUncertain); + assert_eq!(unit_state(&units, "dependency"), State::Running); + + // The unit may be running, so a later stop sends it another request, and + // the unit keeps its terminal marker once it stops. + units.stop().await.unwrap(); + assert_eq!(stop_requests.load(Ordering::Relaxed), 2); + assert!(stopped.load(Ordering::Relaxed)); + assert!(dependency_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); + assert_eq!(unit_state(&units, "unanswered"), State::StopUncertain); + assert_eq!(unit_state(&units, "dependency"), State::Stopped); + let start_error = units.start().await.unwrap_err(); + assert!(start_error.to_string().contains("terminal state")); + assert!(units.reset().await.is_err()); + assert!(units.save().await.is_err()); + assert!(units.restore(Vec::new()).await.is_err()); + assert!(units.advance_time(Duration::from_secs(1)).await.is_err()); +} + +#[async_test] +async fn a_unit_removed_during_a_stop_does_not_block_its_dependencies(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let dependency_stopped = flag(); + let dependency = units + .add("dependency") + .spawn(&driver, |recv| { + run_unit(recording_unit(&flag(), &dependency_stopped), recv) + }) + .unwrap(); + let (stop_received_send, stop_received) = mesh::oneshot(); + // A unit that holds its stop request without answering it until the unit + // is removed, which ends its task. + let removed = units + .add("removed") + .depends_on(dependency.handle()) + .spawn(&driver, |mut recv| { + let mut unit = recording_unit(&flag(), &flag()); + async move { + let mut stop_received_send = Some(stop_received_send); + while let Some(request) = recv.next().await { + match request { + StateRequest::Stop(_response) => { + stop_received_send.take().unwrap().send(()); + std::future::pending::<()>().await; + } + request => request.apply(&mut unit).await, + } + } + } + }) + .unwrap(); + units.start().await.unwrap(); + + let remove = async { + stop_received.await.unwrap(); + drop(removed); + }; + let (result, ()) = futures::join!(units.stop(), remove); + result.unwrap(); + assert!(dependency_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); + assert_eq!(unit_state(&units, "dependency"), State::Stopped); +} + +#[async_test] +async fn a_unit_removed_while_blocked_does_not_block_its_dependencies(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let dependency_stopped = flag(); + let dependency = units + .add("dependency") + .spawn(&driver, |recv| { + run_unit(recording_unit(&flag(), &dependency_stopped), recv) + }) + .unwrap(); + let removed = units + .add("removed") + .depends_on(dependency.handle()) + .spawn(&driver, |recv| { + run_unit(recording_unit(&flag(), &flag()), recv) + }) + .unwrap(); + let (stop_received_send, stop_received) = mesh::oneshot(); + let (release_send, release) = mesh::oneshot::<()>(); + // A unit that depends on `removed` and holds its stop request until + // `removed` is removed, and then exits without answering it, so that + // `removed` waits for it and is blocked. + let _unanswered = units + .add("unanswered") + .depends_on(removed.handle()) + .spawn(&driver, |mut recv| { + let mut unit = recording_unit(&flag(), &flag()); + async move { + let mut stop_received_send = Some(stop_received_send); + let mut release = Some(release); + while let Some(request) = recv.next().await { + match request { + StateRequest::Stop(response) => { + stop_received_send.take().unwrap().send(()); + release.take().unwrap().await.unwrap(); + drop(response); + break; + } + request => request.apply(&mut unit).await, + } + } + } + }) + .unwrap(); + units.start().await.unwrap(); + + let remove = async { + stop_received.await.unwrap(); + drop(removed); + release_send.send(()); + }; + let (result, ()) = futures::join!(units.stop(), remove); + let error = result.unwrap_err().to_string(); + assert!(error.contains("unanswered"), "{error}"); + assert!(!error.contains("dependency"), "{error}"); + // The removed unit no longer uses its dependency, so it stops. + assert!(dependency_stopped.load(Ordering::Relaxed)); + assert_eq!(unit_state(&units, "dependency"), State::Stopped); + assert_eq!(unit_state(&units, "unanswered"), State::StopUncertain); +} diff --git a/vmm_core/state_unit/src/tests/terminal.rs b/vmm_core/state_unit/src/tests/terminal.rs index 04707bc0ff0..e699c497321 100644 --- a/vmm_core/state_unit/src/tests/terminal.rs +++ b/vmm_core/state_unit/src/tests/terminal.rs @@ -187,7 +187,7 @@ async fn a_stop_stops_a_unit_whose_start_is_uncertain(driver: DefaultDriver) { // The unit whose start is uncertain may be running, so the stop stops it, // before the unit that it depends on. - units.stop().await; + units.stop().await.unwrap(); assert!(!units.is_running()); assert_eq!(log.take(), ["uncertain stop", "dependency stop"]); From a8a0e35178f61c0e82cbe465b7ebf70e0f0bb543 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Mon, 9 Nov 2026 23:28:43 -0800 Subject: [PATCH 3/3] state_unit: fail a reset, save, restore, or time advance on a lost unit StateUnits::reset, save, restore, and advance_time panic the VM worker when a unit does not answer their request, as when its task has exited (microsoft/nvx#1363): run_op treats that as an unrecoverable unit failure. A microVM saves its state units for a snapshot, and restores them and advances their time when it restores one. Have run_op fail the operation instead, naming the unit in the operation's StateTransitionError, and leave the unit in the operation's terminal state, ResetUncertain, SaveUncertain, RestoreUncertain, or AdvanceTimeUncertain, since its state is unknown, so that a later start, reset, save, restore, or time advance fails too. A unit removed during the operation still does not fail it. run_op now serves only these failable requests, so it returns each unit's result as an error. Tests: a unit built with UnitBuilder::build whose receiver is dropped fails a save, a reset, a restore, and a time advance, naming the unit, while the other units complete it, and stays terminal; a unit removed during a save does not fail it. At the previous commit, the first four panic in run_op with UnitRecvError { name: "lost", source: Channel(Closed) }. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- vmm_core/state_unit/src/advance_time.rs | 12 +- vmm_core/state_unit/src/lib.rs | 80 ++++++--- vmm_core/state_unit/src/tests/lost.rs | 229 ++++++++++++++++++++++++ 3 files changed, 287 insertions(+), 34 deletions(-) create mode 100644 vmm_core/state_unit/src/tests/lost.rs diff --git a/vmm_core/state_unit/src/advance_time.rs b/vmm_core/state_unit/src/advance_time.rs index ae2416da05c..81533cf79e8 100644 --- a/vmm_core/state_unit/src/advance_time.rs +++ b/vmm_core/state_unit/src/advance_time.rs @@ -17,7 +17,9 @@ impl StateUnits { /// round trip. /// /// Fails without sending a request while a unit is in a terminal state, - /// whether or not the units run. Otherwise panics if running. + /// whether or not the units run. Otherwise panics if running. A unit whose + /// task does not answer its request fails the time advance, and is left in + /// a terminal state. pub async fn advance_time(&mut self, duration: Duration) -> Result<(), StateTransitionError> { self.check_no_terminal_units("advance_time")?; assert!(!self.running); @@ -28,17 +30,13 @@ impl StateUnits { State::Stopped, State::AdvancingTime, State::Stopped, + State::AdvanceTimeUncertain, StateRequest::AdvanceTime, |_, unit| unit.advances_time.then_some(duration), |unit| Some(&unit.dependencies), ) .await; - check( - "advance_time", - results - .into_iter() - .map(|(name, result)| (name, result.map_err(anyhow::Error::from))), - )?; + check("advance_time", results)?; Ok(()) } } diff --git a/vmm_core/state_unit/src/lib.rs b/vmm_core/state_unit/src/lib.rs index 8b5a6929a7a..1a4b5d86a73 100644 --- a/vmm_core/state_unit/src/lib.rs +++ b/vmm_core/state_unit/src/lib.rs @@ -289,13 +289,25 @@ enum State { AdvancingTime, StartUncertain, StopUncertain, + ResetUncertain, + SaveUncertain, + RestoreUncertain, + AdvanceTimeUncertain, } impl State { /// Whether a unit in this state never changes state again: a request to /// it was not answered, so its state is unknown. fn is_terminal(self) -> bool { - matches!(self, State::StartUncertain | State::StopUncertain) + matches!( + self, + State::StartUncertain + | State::StopUncertain + | State::ResetUncertain + | State::SaveUncertain + | State::RestoreUncertain + | State::AdvanceTimeUncertain + ) } } @@ -618,7 +630,9 @@ impl StateUnits { /// Resets all the state units. /// /// Fails without sending a request while a unit is in a terminal state, - /// whether or not the units run. Otherwise panics if running. + /// whether or not the units run. Otherwise panics if running. A unit whose + /// task does not answer its reset request fails the reset, and is left in + /// a terminal state. pub async fn reset(&mut self) -> Result<(), StateTransitionError> { self.check_no_terminal_units("reset")?; assert!(!self.running); @@ -632,6 +646,7 @@ impl StateUnits { State::Stopped, State::Resetting, State::Stopped, + State::ResetUncertain, StateRequest::Reset, |_, _| Some(()), |unit| Some(&unit.dependencies), @@ -646,7 +661,9 @@ impl StateUnits { /// Saves all the state units. /// /// Fails without sending a request while a unit is in a terminal state, - /// whether or not the units run. Otherwise panics if running. + /// whether or not the units run. Otherwise panics if running. A unit whose + /// task does not answer its save request fails the save, and is left in a + /// terminal state. pub async fn save(&mut self) -> Result, StateTransitionError> { self.check_no_terminal_units("save")?; assert!(!self.running); @@ -659,6 +676,7 @@ impl StateUnits { State::Stopped, State::Saving, State::Stopped, + State::SaveUncertain, StateRequest::Save, |_, _| Some(()), |_| None, @@ -683,7 +701,9 @@ impl StateUnits { /// consume eligibility, and saving restored state before start is supported. /// /// Fails without sending a request while a unit is in a terminal state, - /// whether or not the units run. Otherwise panics if running. + /// whether or not the units run. Otherwise panics if running. A unit whose + /// task does not answer its restore request fails the restore, and is left + /// in a terminal state. pub async fn restore( &mut self, states: Vec, @@ -736,6 +756,7 @@ impl StateUnits { State::Stopped, State::Restoring, State::Stopped, + State::RestoreUncertain, StateRequest::Restore, |id, _| states_by_id.remove(&id).map(|(_, blob)| blob), |unit| Some(&unit.dependencies), @@ -778,17 +799,23 @@ impl StateUnits { /// then communication with the unit is skipped, but the unit still /// transitions through the interim and into the new state, and its /// dependencies are still waited on by its dependents. - async fn run_op( + /// + /// A unit whose task does not answer its request, as when the task has + /// exited, fails the operation and is left in `uncertain_state`, a terminal + /// state, since its state is unknown. A unit that is removed during the + /// operation does not fail it. + async fn run_op( &self, op: &str, unit_ids: Option<&[u64]>, old_state: State, interim_state: State, new_state: State, - request: impl Copy + FnOnce(Rpc) -> StateRequest, + uncertain_state: State, + request: impl Copy + FnOnce(FailableRpc) -> StateRequest, mut input: impl FnMut(u64, &Unit) -> Option, mut deps: impl FnMut(&Unit) -> Option<&BTreeSet>, - ) -> Vec<(Arc, R)> { + ) -> Vec<(Arc, anyhow::Result)> { let mut done = Vec::new(); let ready_set; { @@ -838,25 +865,20 @@ impl StateUnits { .await; let mut inner = self.inner.lock(); - let r = results - .into_iter() - .filter_map(|(name, id, r)| { - match r { - Ok(Some(r)) => Some((name, r)), - Ok(None) => None, - Err(err) => { - // If the unit was removed during the operation, then - // ignore the failure. Otherwise, panic because unit - // failure is not recoverable. FUTURE: reconsider this - // position. - if inner.units.contains_key(&id) { - panic!("{:?}", err); - } - None + let mut r = Vec::new(); + for (name, id, result) in results { + match result { + Ok(Some(result)) => r.push((name, result.map_err(anyhow::Error::from))), + Ok(None) => {} + Err(err) => { + // A unit removed during the operation does not fail it. + if let Some(unit) = inner.units.get_mut(&id) { + unit.state = uncertain_state; + r.push((name, Err(err.into()))); } } - }) - .collect(); + } + } for (_, unit) in inner .units .iter_mut() @@ -865,10 +887,13 @@ impl StateUnits { if unit.state == interim_state { unit.state = new_state; } else { - assert_eq!( - unit.state, new_state, + assert!( + unit.state == new_state || unit.state == uncertain_state, "unit {} in {:?} state, should be {:?} or {:?}", - unit.name, unit.state, interim_state, new_state + unit.name, + unit.state, + interim_state, + new_state ); } } @@ -1174,6 +1199,7 @@ impl Ready { #[cfg(test)] mod tests { mod inventory; + mod lost; mod quiesce; mod slow; mod start; diff --git a/vmm_core/state_unit/src/tests/lost.rs b/vmm_core/state_unit/src/tests/lost.rs new file mode 100644 index 00000000000..ced7732606f --- /dev/null +++ b/vmm_core/state_unit/src/tests/lost.rs @@ -0,0 +1,229 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Tests for a reset, save, restore, or time advance that a unit whose task +//! has exited does not answer. + +use super::TestUnit; +use super::start::unit_state; +use crate::SavedStateUnit; +use crate::State; +use crate::StateRequest; +use crate::StateTransitionError; +use crate::StateTransitionErrorKind; +use crate::StateUnits; +use crate::UnitErrorSet; +use crate::UnitHandle; +use crate::run_unit; +use futures::StreamExt; +use pal_async::DefaultDriver; +use pal_async::async_test; +use std::time::Duration; +use test_with_tracing::test; +use vmcore::save_restore::NoSavedState; +use vmcore::save_restore::SavedStateBlob; + +/// Adds a unit named `name` whose task has exited: its request receiver is +/// dropped. +fn lost_unit(units: &StateUnits, name: &str) -> UnitHandle { + let (send, recv) = mesh::channel(); + drop(recv); + units.add(name).build(send).unwrap() +} + +/// Checks that `error` is the failure of `op` because the unit named `name`, +/// alone, did not answer its request. +fn assert_unanswered(error: StateTransitionError, op: &str, name: &str) { + let StateTransitionErrorKind::Units(UnitErrorSet(errors)) = &error.error else { + panic!("{op} failed for another reason: {error:?}"); + }; + let names = errors + .iter() + .map(|(name, _)| name.as_ref()) + .collect::>(); + assert_eq!( + (error.op, names.as_slice()), + (op, [name].as_slice()), + "{error:?}" + ); + let message = format!("{:#}", errors[0].1); + assert!( + message.contains("critical unit communication failure"), + "{message}" + ); +} + +/// Checks that a start, reset, save, restore, and time advance of `units` +/// each fail because the unit named `name` is in a terminal state. +async fn assert_terminal(units: &mut StateUnits, name: &str) { + let error = format!("{:#}", units.start().await.unwrap_err()); + assert!( + error.contains(&format!("'{name}' cannot be started from terminal state")), + "{error}" + ); + for error in [ + units.reset().await.unwrap_err(), + units.save().await.err().unwrap(), + units.restore(Vec::new()).await.unwrap_err(), + units + .advance_time(Duration::from_secs(1)) + .await + .unwrap_err(), + ] { + let error = format!("{:#}", anyhow::Error::from(error)); + assert!( + error.contains(&format!("{name}: unit is in terminal state")), + "{error}" + ); + } +} + +/// The saved state of the unit named `name`. +fn saved_state(name: &str) -> SavedStateUnit { + SavedStateUnit { + name: name.to_owned(), + state: SavedStateBlob::new(NoSavedState), + } +} + +#[async_test] +async fn a_lost_unit_fails_a_save(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let _healthy = units + .add("healthy") + .spawn(&driver, |recv| run_unit(TestUnit::default(), recv)) + .unwrap(); + let _lost = lost_unit(&units, "lost"); + + let Err(error) = units.save().await else { + panic!("save succeeded"); + }; + assert_unanswered(error, "save", "lost"); + assert_eq!(unit_state(&units, "lost"), State::SaveUncertain); + assert_eq!(unit_state(&units, "healthy"), State::Stopped); + assert_terminal(&mut units, "lost").await; +} + +#[async_test] +async fn a_lost_unit_fails_a_reset(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let lost = lost_unit(&units, "lost"); + // A unit that depends on the lost unit still resets after it. + let _dependent = units + .add("dependent") + .depends_on(&lost) + .spawn(&driver, |recv| run_unit(TestUnit::default(), recv)) + .unwrap(); + + assert_unanswered(units.reset().await.unwrap_err(), "reset", "lost"); + assert_eq!(unit_state(&units, "lost"), State::ResetUncertain); + assert_eq!(unit_state(&units, "dependent"), State::Stopped); + assert_terminal(&mut units, "lost").await; +} + +#[async_test] +async fn a_lost_unit_fails_a_restore(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let _healthy = units + .add("healthy") + .spawn(&driver, |recv| { + run_unit( + TestUnit { + support_saved_state: true, + ..Default::default() + }, + recv, + ) + }) + .unwrap(); + let _lost = lost_unit(&units, "lost"); + let healthy_state = SavedStateUnit { + name: "healthy".to_owned(), + state: SavedStateBlob::new(super::SavedState(true)), + }; + + assert_unanswered( + units + .restore(vec![healthy_state, saved_state("lost")]) + .await + .unwrap_err(), + "restore", + "lost", + ); + assert_eq!(unit_state(&units, "lost"), State::RestoreUncertain); + assert_eq!(unit_state(&units, "healthy"), State::Stopped); + assert_terminal(&mut units, "lost").await; +} + +#[async_test] +async fn a_lost_unit_fails_a_time_advance(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let _healthy = units + .add("healthy") + .advances_time() + .spawn(&driver, |recv| run_unit(TestUnit::default(), recv)) + .unwrap(); + let (send, recv) = mesh::channel(); + drop(recv); + let _lost = units.add("lost").advances_time().build(send).unwrap(); + + assert_unanswered( + units + .advance_time(Duration::from_secs(1)) + .await + .unwrap_err(), + "advance_time", + "lost", + ); + assert_eq!(unit_state(&units, "lost"), State::AdvanceTimeUncertain); + assert_eq!(unit_state(&units, "healthy"), State::Stopped); + assert_terminal(&mut units, "lost").await; +} + +#[async_test] +async fn a_unit_removed_during_a_save_does_not_fail_it(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let _healthy = units + .add("healthy") + .spawn(&driver, |recv| { + run_unit( + TestUnit { + support_saved_state: true, + ..Default::default() + }, + recv, + ) + }) + .unwrap(); + let (save_received_send, save_received) = mesh::oneshot(); + // A unit that holds its save request without answering it until the unit + // is removed, which ends its task. + let removed = units + .add("removed") + .spawn(&driver, |mut recv| async move { + let mut save_received_send = Some(save_received_send); + while let Some(request) = recv.next().await { + if let StateRequest::Save(_response) = request { + save_received_send.take().unwrap().send(()); + std::future::pending::<()>().await; + } + } + }) + .unwrap(); + + let remove = async { + save_received.await.unwrap(); + drop(removed); + }; + let (result, ()) = futures::join!(units.save(), remove); + let Ok(states) = result else { + panic!("save failed"); + }; + let names = states + .iter() + .map(|state| state.name.as_str()) + .collect::>(); + assert_eq!(names, ["healthy"]); + assert_eq!(unit_state(&units, "healthy"), State::Stopped); + units.start().await.unwrap(); +}