diff --git a/openhcl/underhill_core/src/dispatch/mod.rs b/openhcl/underhill_core/src/dispatch/mod.rs index 346c8a1d874..ca71ef98f8d 100644 --- a/openhcl/underhill_core/src/dispatch/mod.rs +++ b/openhcl/underhill_core/src/dispatch/mod.rs @@ -79,7 +79,7 @@ use vtl2_settings_worker::handle_vtl2_config_rpc; #[derive(MeshPayload)] pub enum UhVmRpc { - Pause(Rpc<(), bool>), + Pause(FailableRpc<(), bool>), Resume(Rpc<(), bool>), Save(FailableRpc<(), Vec>), ClearHalt(Rpc<(), bool>), // TODO: remove this, and use DebugRequest::Resume @@ -326,7 +326,15 @@ impl LoadedVm { WorkerRpc::Stop => break None, WorkerRpc::Restart(rpc) => { let state = async { - let running = self.stop().await; + let running = match self.stop().await { + Ok(running) => running, + Err(err) => { + rpc.complete(Err(RemoteError::new( + err.context("failed to stop VM for worker restart"), + ))); + return None; + } + }; match self .save( SaveReason::WorkerRestart, @@ -416,10 +424,10 @@ impl LoadedVm { }) .await } - UhVmRpc::Pause(rpc) => rpc.handle(async |()| self.stop().await).await, + UhVmRpc::Pause(rpc) => rpc.handle_failable(async |()| self.stop().await).await, UhVmRpc::Save(rpc) => { rpc.handle_failable(async |()| { - let running = self.stop().await; + let running = self.stop().await?; let r = self .save( SaveReason::Diagnostic, @@ -725,7 +733,7 @@ impl LoadedVm { // Do everything before the log flush under a span. let r = async { - if !self.stop().await { + if !self.stop().await? { // This should only occur if you tried to initiate a // servicing operation after manually pausing underhill // via `ohcldiag-dev`. @@ -921,14 +929,17 @@ impl LoadedVm { } /// Returns true if the VM was previously running. - async fn stop(&mut self) -> bool { + /// + /// Fails if a state unit does not stop. The state units then stay + /// running, and can never start again. + async fn stop(&mut self) -> anyhow::Result { if self.state_units.is_running() { self.last_state_unit_stop = Some(ReferenceTime::new(self.partition.reference_time())); tracing::info!(CVM_ALLOWED, "stopping VM"); - self.state_units.stop().await; - true + self.state_units.stop().await?; + Ok(true) } else { - false + Ok(false) } } diff --git a/openhcl/underhill_core/src/lib.rs b/openhcl/underhill_core/src/lib.rs index 329d1c119f8..2f53d4b1670 100644 --- a/openhcl/underhill_core/src/lib.rs +++ b/openhcl/underhill_core/src/lib.rs @@ -637,7 +637,13 @@ async fn run_control( // FUTURE: consider supporting cancellation driver .spawn("diag-pause", async move { - let was_paused = req.await.expect("failed to pause VM"); + let was_paused = match req.await.expect("failed to pause VM") { + Ok(was_paused) => was_paused, + Err(error) => { + rpc.complete(Err(error)); + return; + } + }; rpc.handle_failable_sync(|_| { if !was_paused { Err(anyhow::anyhow!("VM is already paused")) diff --git a/openvmm/openvmm_core/src/worker/dispatch.rs b/openvmm/openvmm_core/src/worker/dispatch.rs index a2695e19c5a..c878d9ab879 100644 --- a/openvmm/openvmm_core/src/worker/dispatch.rs +++ b/openvmm/openvmm_core/src/worker/dispatch.rs @@ -4018,13 +4018,24 @@ impl LoadedVm { self.state_units.start().await } - async fn pause(&mut self) -> bool { + /// Pauses the VM, returning true if it was running. + /// + /// Fails if a state unit does not stop. The VM no longer runs then, but + /// state units still may, which [`Self::units_left_running`] reports. + async fn pause(&mut self) -> anyhow::Result { if !self.running { - return false; + return Ok(false); } - self.state_units.stop().await; self.running = false; - true + self.state_units.stop().await?; + Ok(true) + } + + /// Returns true if state units run while the VM does not, as a failed + /// stop, or a start whose rollback did not stop every unit, leaves them. + /// They can never start again, so the VM must be torn down. + fn units_left_running(&self) -> bool { + self.state_units.is_running() && !self.running } /// Assign PCI bus numbers and BAR addresses for all boot modes. @@ -4067,10 +4078,16 @@ impl LoadedVm { .context("PCI resource assignment failed"); // Stop state units again; the caller will resume normally. - self.state_units.stop().await; + let stopped = self + .state_units + .stop() + .await + .context("failed to stop devices after PCI resource assignment"); drop(stop_guard); - result + // Devices that did not stop can never start again, which outweighs a + // failed assignment. + stopped.and(result) } pub async fn run( @@ -4111,6 +4128,13 @@ impl LoadedVm { let mut deferred = VecDeque::new(); loop { + // A failed stop, or a start whose rollback did not stop every + // unit, leaves state units running while the VM is paused. They + // can never start again, so tear the VM down. + if self.units_left_running() { + tracing::error!("state units failed to stop; tearing down the VM"); + break; + } let queued = if !device_removal.is_pending() { deferred.pop_front() } else { @@ -4157,6 +4181,7 @@ impl LoadedVm { WorkerRpc::Stop => break, WorkerRpc::Restart(rpc) => { let mut stopped = false; + let mut stop_failed = false; // First run the non-destructive operations. let r = async { let shared_memory = self.inner.memory_manager.shared_memory_backing(); @@ -4164,7 +4189,10 @@ impl LoadedVm { anyhow::bail!("restart is not supported with --private-memory"); } if self.running { - self.state_units.stop().await; + if let Err(error) = self.state_units.stop().await { + stop_failed = true; + return Err(error); + } stopped = true; } let saved_state = self.save().await?; @@ -4180,6 +4208,14 @@ impl LoadedVm { return; } Err(err) => { + if stop_failed { + // The state units that did not stop can + // never start again, so tear the VM down. + rpc.complete(Err(RemoteError::new(err.context( + "worker restart failed and the VM could not stop", + )))); + break; + } if stopped { if let Err(start_error) = self.start_units().await { rpc.complete(Err(RemoteError::new(start_error.context( @@ -4220,7 +4256,9 @@ impl LoadedVm { VmRpc::Resume(rpc) => { rpc.handle_failable(async |()| self.resume().await).await } - VmRpc::Pause(rpc) => rpc.handle(async |()| self.pause().await).await, + VmRpc::Pause(rpc) => { + rpc.handle_failable(async |()| self.pause().await).await + } VmRpc::Save(rpc) => { rpc.handle_failable(async |()| self.save().await.map(ProtobufMessage::new)) .await @@ -4324,7 +4362,7 @@ impl LoadedVm { if !self.inner.partition.supports_reset() { return Err(PulseSaveRestoreError::ResetNotSupported); } - let paused = self.pause().await; + let paused = self.pause().await?; self.save_reset_restore().await?; if paused { @@ -4630,9 +4668,16 @@ impl LoadedVm { device_removal.drain().await; // Stop the state units whenever they run, not only when the VM does: // after a failed state change, such as a restart that could not resume - // the VM, the two can disagree. + // the VM, or a stop that a unit did not answer, the two can disagree. + // The VM is torn down anyway, so a unit that does not stop only leaves + // its dependencies running. if self.state_units.is_running() { - self.state_units.stop().await; + if let Err(error) = self.state_units.stop().await { + tracing::error!( + error = error.as_ref() as &dyn std::error::Error, + "failed to stop the state units before teardown" + ); + } } self.running = false; self.teardown_chipset_devices().await; @@ -4840,7 +4885,7 @@ impl LoadedVm { } async fn reset(&mut self, reload_firmware: bool) -> anyhow::Result<()> { - let resume = self.pause().await; + let resume = self.pause().await?; self.state_units.reset().await?; // TODO: _vmnic diff --git a/openvmm/openvmm_core/src/worker/dispatch/dump.rs b/openvmm/openvmm_core/src/worker/dispatch/dump.rs index a0108c686da..dc095f2d66f 100644 --- a/openvmm/openvmm_core/src/worker/dispatch/dump.rs +++ b/openvmm/openvmm_core/src/worker/dispatch/dump.rs @@ -16,7 +16,7 @@ impl LoadedVm { /// Pauses the VM if running, collects VP state and streams memory, /// then restores the prior running state. pub(super) async fn dump_state(&mut self, file: File) -> anyhow::Result<()> { - let was_running = self.pause().await; + let was_running = self.pause().await?; let result = self.dump_state_inner(file).await; if was_running { self.resume() diff --git a/openvmm/openvmm_core/src/worker/dispatch/microvm.rs b/openvmm/openvmm_core/src/worker/dispatch/microvm.rs index 8c104360819..4953997f31d 100644 --- a/openvmm/openvmm_core/src/worker/dispatch/microvm.rs +++ b/openvmm/openvmm_core/src/worker/dispatch/microvm.rs @@ -319,9 +319,11 @@ impl LoadedVm { /// /// The quiesce stops every state unit and has no deadline: a unit that /// never stops stalls the capture until the process that drives the VM - /// ends it, with nothing published and the source never resumed. Each - /// error after the quiesce leaves every unit stopped, so the VM can roll - /// back with [`Self::resume_after_failed_snapshot`]. + /// ends it, with nothing published and the source never resumed. A unit + /// whose task does not answer fails the quiesce instead: it can never + /// start again, so the VM can neither roll back nor run, and the worker + /// tears it down. Each error after the quiesce leaves every unit stopped, + /// so the VM can roll back with [`Self::resume_after_failed_snapshot`]. pub(super) async fn quiesce_for_snapshot( &mut self, ) -> Result @@ -349,9 +351,13 @@ impl LoadedVm { } let quiesce = openvmm_defs::profile::ProfileSpan::start(); - self.state_units.stop().await; - quiesce.complete("capture", "quiesce", Default::default()); + // The VM no longer runs even if a unit does not stop, which leaves + // the units running for the worker to tear down. self.running = false; + self.state_units.stop().await.map_err(|error| { + openvmm_defs::rpc::SnapshotQuiesceError::Unrecoverable(RemoteError::new(error)) + })?; + quiesce.complete("capture", "quiesce", Default::default()); // The time ABI's capture checks and records: the LAPIC timers, then // the capture anchor and records. The PIT checks itself when saved. @@ -423,7 +429,9 @@ impl LoadedVm { /// that runs a quiesced VM again. /// /// A unit whose task does not answer leaves the units that did start - /// stopped again, and fails the rollback. + /// stopped again, and fails the rollback. A unit that such a unit depends + /// on, or that does not answer its stop, stays running instead, and the + /// worker then tears the VM down. pub(super) async fn resume_after_failed_snapshot(&mut self) -> anyhow::Result<()> { self.state_units.start().await?; self.running = true; @@ -437,9 +445,11 @@ impl LoadedVm { match event { Ok(request) => { if !self.establish_snapshot_boundary(request).await { - if self.running { - self.state_units.stop().await; - self.running = false; + if let Err(error) = self.pause().await { + tracing::error!( + error = error.as_ref() as &dyn std::error::Error, + "failed to stop the VM after its snapshot boundary failed" + ); } return false; } diff --git a/openvmm/openvmm_core/src/worker/dispatch/restore.rs b/openvmm/openvmm_core/src/worker/dispatch/restore.rs index 2d9c29fc500..6fd607ac36a 100644 --- a/openvmm/openvmm_core/src/worker/dispatch/restore.rs +++ b/openvmm/openvmm_core/src/worker/dispatch/restore.rs @@ -348,7 +348,14 @@ impl LoadedVm { } }); if let Err(error) = signal_result { - self.state_units.stop().await; + // A unit that does not stop stays running while the VM is + // paused, and the worker then tears the VM down. + if let Err(stop_error) = self.state_units.stop().await { + tracing::error!( + error = stop_error.as_ref() as &dyn std::error::Error, + "failed to stop the state units after the restore readiness event failed" + ); + } return Err(error).context("failed to publish restore readiness event"); } } @@ -367,9 +374,11 @@ impl LoadedVm { /// input gate in time. pub(super) async fn handle_restore_gate_timeout(&mut self) { tracing::error!("post-restore input gate acknowledgement timed out"); - if self.running { - self.state_units.stop().await; - self.running = false; + if let Err(error) = self.pause().await { + tracing::error!( + error = error.as_ref() as &dyn std::error::Error, + "failed to stop the VM after the post-restore input gate timed out" + ); } } } diff --git a/openvmm/openvmm_core/src/worker/dispatch/snapshot_rpc.rs b/openvmm/openvmm_core/src/worker/dispatch/snapshot_rpc.rs index a69b4fb00ca..a37088bf3ad 100644 --- a/openvmm/openvmm_core/src/worker/dispatch/snapshot_rpc.rs +++ b/openvmm/openvmm_core/src/worker/dispatch/snapshot_rpc.rs @@ -52,10 +52,11 @@ pub(super) fn filter(message: VmRpc, boundary_active: bool) -> Option { VmRpc::AddVpciDevice(rpc) => rpc.fail(SnapshotBoundaryActive), VmRpc::RemoveVpciDevice(rpc) => rpc.fail(SnapshotBoundaryActive), VmRpc::DumpState(rpc) => rpc.fail(SnapshotBoundaryActive), + VmRpc::Pause(rpc) => rpc.fail(SnapshotBoundaryActive), VmRpc::PulseSaveRestore(rpc) => rpc.complete(Err(PulseSaveRestoreError::Other( RemoteError::new(SnapshotBoundaryActive), ))), - VmRpc::Pause(rpc) | VmRpc::ClearHalt(rpc) => drop(rpc), + VmRpc::ClearHalt(rpc) => drop(rpc), VmRpc::Nmi(rpc) => drop(rpc), } tracelimit::warn_ratelimited!( @@ -190,9 +191,10 @@ mod tests { let write = send.call(VmRpc::WriteMemory, (0, vec![1])); let reset = send.call(VmRpc::Reset, ()); let resume = send.call(VmRpc::Resume, ()); + let pause = send.call(VmRpc::Pause, ()); let remove = send.call(VmRpc::RemovePcieDevice, "port".to_owned()); - for _ in 0..4 { + for _ in 0..5 { assert!(filter(recv.try_recv().unwrap(), true).is_none()); } for result in [block_on(write), block_on(reset), block_on(remove)] { @@ -200,6 +202,7 @@ mod tests { assert!(error.to_string().contains("snapshot boundary is active")); } assert!(block_on(resume).unwrap().is_err()); + assert!(block_on(pause).unwrap().is_err()); let release = send.call(VmRpc::ReleaseSnapshotBoundary, ()); let Some(VmRpc::ReleaseSnapshotBoundary(rpc)) = filter(recv.try_recv().unwrap(), true) @@ -213,13 +216,11 @@ mod tests { #[test] fn infallible_mutations_report_channel_error() { let (send, mut recv) = mesh::channel(); - let pause = send.call(VmRpc::Pause, ()); let clear_halt = send.call(VmRpc::ClearHalt, ()); let nmi = send.call(VmRpc::Nmi, 0); - for _ in 0..3 { + for _ in 0..2 { assert!(filter(recv.try_recv().unwrap(), true).is_none()); } - assert!(block_on(pause).is_err()); assert!(block_on(clear_halt).is_err()); assert!(block_on(nmi).is_err()); } diff --git a/openvmm/openvmm_defs/src/rpc.rs b/openvmm/openvmm_defs/src/rpc.rs index e235ef36c61..5586e9e930c 100644 --- a/openvmm/openvmm_defs/src/rpc.rs +++ b/openvmm/openvmm_defs/src/rpc.rs @@ -31,7 +31,11 @@ pub enum VmRpc { /// Release a post-OUT boundary without starting a snapshot transaction. ReleaseSnapshotBoundary(FailableRpc<(), ()>), Resume(FailableRpc<(), bool>), - Pause(Rpc<(), bool>), + /// Pause the VM, returning whether it was running. + /// + /// Fails if a state unit does not stop; the worker then tears the VM + /// down, since that unit can never start again. + Pause(FailableRpc<(), bool>), ClearHalt(Rpc<(), bool>), Reset(FailableRpc<(), ()>), Nmi(Rpc), @@ -87,6 +91,10 @@ pub enum SnapshotQuiesceError { /// The VM quiesced completely; the controller may request rollback. #[error("snapshot failed after the VM quiesced")] RollbackSafe(#[source] RemoteError), + /// A state unit did not stop, so the VM can neither roll back nor run + /// again; the controller terminates the source. + #[error("snapshot quiesce left the VM unable to run again")] + Unrecoverable(#[source] RemoteError), } #[derive(Debug, MeshPayload, thiserror::Error)] diff --git a/openvmm/openvmm_entry/src/repl.rs b/openvmm/openvmm_entry/src/repl.rs index abab2d71c75..6ed0f51791b 100644 --- a/openvmm/openvmm_entry/src/repl.rs +++ b/openvmm/openvmm_entry/src/repl.rs @@ -667,7 +667,7 @@ pub(crate) async fn run_repl( let mut snapshot_saved = false; enum StateChange { - Pause(bool), + Pause(Result), Resume(Result), Reset(Result<(), RemoteError>), PulseSaveRestore(Result<(), PulseSaveRestoreError>), @@ -767,13 +767,14 @@ pub(crate) async fn run_repl( Event::StateChange(r) => { match r { Ok(sc) => match sc { - StateChange::Pause(success) => { - if success { - tracing::info!("pause complete"); - } else { - tracing::warn!("already paused"); - } - } + StateChange::Pause(r) => match r { + Ok(true) => tracing::info!("pause complete"), + Ok(false) => tracing::warn!("already paused"), + Err(err) => tracing::error!( + error = &err as &dyn std::error::Error, + "pause failed" + ), + }, StateChange::Resume(Ok(success)) => { if success { launch.restore_ready_pending = false; diff --git a/openvmm/openvmm_entry/src/ttrpc/mod.rs b/openvmm/openvmm_entry/src/ttrpc/mod.rs index 487ac4e36c3..8789a82ffc5 100644 --- a/openvmm/openvmm_entry/src/ttrpc/mod.rs +++ b/openvmm/openvmm_entry/src/ttrpc/mod.rs @@ -1701,7 +1701,7 @@ impl VmService { async fn pause_vm(&mut self) -> anyhow::Result<()> { let vm = self.vm.clone().context("VM not created yet")?; vm.worker_rpc - .call(VmRpc::Pause, ()) + .call_failable(VmRpc::Pause, ()) .await .map(drop) .context("pause failed")?; diff --git a/openvmm/openvmm_entry/src/vm_controller.rs b/openvmm/openvmm_entry/src/vm_controller.rs index 61ec2f93502..816232e21c5 100644 --- a/openvmm/openvmm_entry/src/vm_controller.rs +++ b/openvmm/openvmm_entry/src/vm_controller.rs @@ -518,7 +518,7 @@ impl VmController { // Pause the VM. self.vm_rpc - .call(VmRpc::Pause, ()) + .call_failable(VmRpc::Pause, ()) .await .context("failed to pause VM")?; diff --git a/openvmm/openvmm_entry/src/vm_controller/microvm.rs b/openvmm/openvmm_entry/src/vm_controller/microvm.rs index f6dfcf5f12b..350b70c4211 100644 --- a/openvmm/openvmm_entry/src/vm_controller/microvm.rs +++ b/openvmm/openvmm_entry/src/vm_controller/microvm.rs @@ -247,6 +247,13 @@ impl VmController { Ok(Err(SnapshotQuiesceError::RollbackSafe(error))) => { return self.rollback_failed_guest_snapshot(error.into()).await; } + Ok(Err(SnapshotQuiesceError::Unrecoverable(error))) => { + tracing::error!( + error = &error as &dyn std::error::Error, + "microVM snapshot quiesce left the VM unable to run again; terminating source" + ); + return GuestSnapshotAction::Terminate { exit_code: 1 }; + } Err(error) => { tracing::error!( error = &error as &dyn std::error::Error, diff --git a/petri/src/worker.rs b/petri/src/worker.rs index bd9905ec12d..fe17ce04ba5 100644 --- a/petri/src/worker.rs +++ b/petri/src/worker.rs @@ -65,8 +65,8 @@ impl Worker { )) } - pub(crate) async fn pause(&self) -> Result { - self.rpc.call(VmRpc::Pause, ()).await + pub(crate) async fn pause(&self) -> anyhow::Result { + Ok(self.rpc.call_failable(VmRpc::Pause, ()).await?) } /// The VM worker's management RPC channel. diff --git a/vmm_core/state_unit/src/advance_time.rs b/vmm_core/state_unit/src/advance_time.rs index ae2416da05c..81533cf79e8 100644 --- a/vmm_core/state_unit/src/advance_time.rs +++ b/vmm_core/state_unit/src/advance_time.rs @@ -17,7 +17,9 @@ impl StateUnits { /// round trip. /// /// Fails without sending a request while a unit is in a terminal state, - /// whether or not the units run. Otherwise panics if running. + /// whether or not the units run. Otherwise panics if running. A unit whose + /// task does not answer its request fails the time advance, and is left in + /// a terminal state. pub async fn advance_time(&mut self, duration: Duration) -> Result<(), StateTransitionError> { self.check_no_terminal_units("advance_time")?; assert!(!self.running); @@ -28,17 +30,13 @@ impl StateUnits { State::Stopped, State::AdvancingTime, State::Stopped, + State::AdvanceTimeUncertain, StateRequest::AdvanceTime, |_, unit| unit.advances_time.then_some(duration), |unit| Some(&unit.dependencies), ) .await; - check( - "advance_time", - results - .into_iter() - .map(|(name, result)| (name, result.map_err(anyhow::Error::from))), - )?; + check("advance_time", results)?; Ok(()) } } diff --git a/vmm_core/state_unit/src/lib.rs b/vmm_core/state_unit/src/lib.rs index 79641acfe22..1a4b5d86a73 100644 --- a/vmm_core/state_unit/src/lib.rs +++ b/vmm_core/state_unit/src/lib.rs @@ -34,6 +34,7 @@ mod advance_time; mod inventory; mod quiesce; mod start; +mod stop; use futures::FutureExt; use futures::StreamExt; @@ -287,6 +288,27 @@ enum State { Restoring, AdvancingTime, StartUncertain, + StopUncertain, + ResetUncertain, + SaveUncertain, + RestoreUncertain, + AdvanceTimeUncertain, +} + +impl State { + /// Whether a unit in this state never changes state again: a request to + /// it was not answered, so its state is unknown. + fn is_terminal(self) -> bool { + matches!( + self, + State::StartUncertain + | State::StopUncertain + | State::ResetUncertain + | State::SaveUncertain + | State::RestoreUncertain + | State::AdvanceTimeUncertain + ) + } } #[derive(Debug)] @@ -562,7 +584,7 @@ impl StateUnits { inner .units .values() - .filter(|unit| matches!(unit.state, State::StartUncertain)) + .filter(|unit| unit.state.is_terminal()) .map(|unit| { ( unit.name.clone(), @@ -594,51 +616,23 @@ impl StateUnits { /// Stops all the state units. /// - /// This also stops the units whose start is uncertain, since they may be - /// running, and leaves them in that terminal state. - pub async fn stop(&mut self) { + /// Stops the units in reverse dependency order. A unit in a terminal state + /// may be running, so it is sent a stop request too, and keeps its terminal + /// marker. If a unit's task does not answer its stop request, the stop + /// fails, the unit is left in a terminal state, and the units that it + /// depends on keep running, since it may still use them. The units then + /// stay running, so that a later stop can stop them. + pub async fn stop(&mut self) -> anyhow::Result<()> { assert!(self.running); - // Make the units whose start is uncertain receive the stop, then - // restore their terminal state after the ordered operation. - let uncertain_ids = { - let mut inner = self.inner.lock(); - inner - .units - .iter_mut() - .filter(|(_, unit)| unit.state == State::StartUncertain) - .map(|(&id, unit)| { - unit.state = State::Running; - id - }) - .collect::>() - }; - // Stop units in reverse dependency order so that a dependency is not - // stopped before its dependant. - self.run_op( - "stop", - None, - State::Running, - State::Stopping, - State::Stopped, - StateRequest::Stop, - |_, _| Some(()), - |unit| Some(&unit.dependents), - ) - .await; - let mut inner = self.inner.lock(); - for id in uncertain_ids { - if let Some(unit) = inner.units.get_mut(&id) { - assert_eq!(unit.state, State::Stopped); - unit.state = State::StartUncertain; - } - } - self.running = false; + self.stop_in_order().await } /// Resets all the state units. /// /// Fails without sending a request while a unit is in a terminal state, - /// whether or not the units run. Otherwise panics if running. + /// whether or not the units run. Otherwise panics if running. A unit whose + /// task does not answer its reset request fails the reset, and is left in + /// a terminal state. pub async fn reset(&mut self) -> Result<(), StateTransitionError> { self.check_no_terminal_units("reset")?; assert!(!self.running); @@ -652,6 +646,7 @@ impl StateUnits { State::Stopped, State::Resetting, State::Stopped, + State::ResetUncertain, StateRequest::Reset, |_, _| Some(()), |unit| Some(&unit.dependencies), @@ -666,7 +661,9 @@ impl StateUnits { /// Saves all the state units. /// /// Fails without sending a request while a unit is in a terminal state, - /// whether or not the units run. Otherwise panics if running. + /// whether or not the units run. Otherwise panics if running. A unit whose + /// task does not answer its save request fails the save, and is left in a + /// terminal state. pub async fn save(&mut self) -> Result, StateTransitionError> { self.check_no_terminal_units("save")?; assert!(!self.running); @@ -679,6 +676,7 @@ impl StateUnits { State::Stopped, State::Saving, State::Stopped, + State::SaveUncertain, StateRequest::Save, |_, _| Some(()), |_| None, @@ -703,7 +701,9 @@ impl StateUnits { /// consume eligibility, and saving restored state before start is supported. /// /// Fails without sending a request while a unit is in a terminal state, - /// whether or not the units run. Otherwise panics if running. + /// whether or not the units run. Otherwise panics if running. A unit whose + /// task does not answer its restore request fails the restore, and is left + /// in a terminal state. pub async fn restore( &mut self, states: Vec, @@ -756,6 +756,7 @@ impl StateUnits { State::Stopped, State::Restoring, State::Stopped, + State::RestoreUncertain, StateRequest::Restore, |id, _| states_by_id.remove(&id).map(|(_, blob)| blob), |unit| Some(&unit.dependencies), @@ -798,17 +799,23 @@ impl StateUnits { /// then communication with the unit is skipped, but the unit still /// transitions through the interim and into the new state, and its /// dependencies are still waited on by its dependents. - async fn run_op( + /// + /// A unit whose task does not answer its request, as when the task has + /// exited, fails the operation and is left in `uncertain_state`, a terminal + /// state, since its state is unknown. A unit that is removed during the + /// operation does not fail it. + async fn run_op( &self, op: &str, unit_ids: Option<&[u64]>, old_state: State, interim_state: State, new_state: State, - request: impl Copy + FnOnce(Rpc) -> StateRequest, + uncertain_state: State, + request: impl Copy + FnOnce(FailableRpc) -> StateRequest, mut input: impl FnMut(u64, &Unit) -> Option, mut deps: impl FnMut(&Unit) -> Option<&BTreeSet>, - ) -> Vec<(Arc, R)> { + ) -> Vec<(Arc, anyhow::Result)> { let mut done = Vec::new(); let ready_set; { @@ -858,25 +865,20 @@ impl StateUnits { .await; let mut inner = self.inner.lock(); - let r = results - .into_iter() - .filter_map(|(name, id, r)| { - match r { - Ok(Some(r)) => Some((name, r)), - Ok(None) => None, - Err(err) => { - // If the unit was removed during the operation, then - // ignore the failure. Otherwise, panic because unit - // failure is not recoverable. FUTURE: reconsider this - // position. - if inner.units.contains_key(&id) { - panic!("{:?}", err); - } - None + let mut r = Vec::new(); + for (name, id, result) in results { + match result { + Ok(Some(result)) => r.push((name, result.map_err(anyhow::Error::from))), + Ok(None) => {} + Err(err) => { + // A unit removed during the operation does not fail it. + if let Some(unit) = inner.units.get_mut(&id) { + unit.state = uncertain_state; + r.push((name, Err(err.into()))); } } - }) - .collect(); + } + } for (_, unit) in inner .units .iter_mut() @@ -885,10 +887,13 @@ impl StateUnits { if unit.state == interim_state { unit.state = new_state; } else { - assert_eq!( - unit.state, new_state, + assert!( + unit.state == new_state || unit.state == uncertain_state, "unit {} in {:?} state, should be {:?} or {:?}", - unit.name, unit.state, interim_state, new_state + unit.name, + unit.state, + interim_state, + new_state ); } } @@ -1194,9 +1199,11 @@ impl Ready { #[cfg(test)] mod tests { mod inventory; + mod lost; mod quiesce; mod slow; mod start; + mod stop; mod terminal; use super::StateTransitionErrorKind; @@ -1372,7 +1379,7 @@ mod tests { units.restore(state).await.unwrap(); assert_eq!(calls.load(Ordering::Relaxed), 2); units.start().await.unwrap(); - units.stop().await; + units.stop().await.unwrap(); let state = units.save().await.unwrap(); assert!(units.restore(state).await.is_err()); assert_eq!(calls.load(Ordering::Relaxed), 2); @@ -1476,7 +1483,7 @@ mod tests { // Both units are stopped again and can start. units.start().await.unwrap(); - units.stop().await; + units.stop().await.unwrap(); } #[async_test] @@ -1507,10 +1514,10 @@ mod tests { let _c = units .add("c") .spawn(&driver, |recv| run_unit(TestUnit::default(), recv)); - units.stop().await; + units.stop().await.unwrap(); units.start().await.unwrap(); - units.stop().await; + units.stop().await.unwrap(); let state = units.save().await.unwrap(); @@ -1557,7 +1564,7 @@ mod tests { }) .unwrap(); units.start().await.unwrap(); - units.stop().await; + units.stop().await.unwrap(); let state = units.save().await.unwrap(); @@ -1667,7 +1674,7 @@ mod tests { } units.start().await.unwrap(); - units.stop().await; + units.stop().await.unwrap(); a.remove().await; { @@ -1677,7 +1684,7 @@ mod tests { } units.start().await.unwrap(); - units.stop().await; + units.stop().await.unwrap(); // Removing the remaining unit leaves the set empty. c.remove().await; diff --git a/vmm_core/state_unit/src/start.rs b/vmm_core/state_unit/src/start.rs index e6799e1b73d..495645183c1 100644 --- a/vmm_core/state_unit/src/start.rs +++ b/vmm_core/state_unit/src/start.rs @@ -10,7 +10,9 @@ use super::State; use super::StateRequest; use super::StateUnits; use super::state_change; +use super::stop::StopResult; use futures::future::join_all; +use std::collections::BTreeSet; use std::time::Instant; impl StateUnits { @@ -28,6 +30,14 @@ impl StateUnits { /// leaves it in that state. A unit that failed to confirm its start can /// start again. /// + /// The units that did start are stopped again in reverse dependency order. + /// A started unit that a unit that is not stopped depends on, such as a + /// unit whose start is uncertain, is not stopped, since that unit may + /// still use it, and a started unit whose task does not answer its stop + /// request may still run. Either is left in the terminal state too, and + /// the units then stay running, so that [`StateUnits::stop`] can stop + /// them. + /// /// [`StateUnit::confirm_start`]: crate::StateUnit::confirm_start pub(super) async fn start_with_rollback(&mut self) -> anyhow::Result<()> { enum StartResult { @@ -136,21 +146,46 @@ impl StateUnits { self.running = true; Ok(()) } else { - if !started_ids.is_empty() { - self.run_op( - "failed_start_rollback", - Some(&started_ids), - State::Running, - State::Stopping, - State::Stopped, - StateRequest::Stop, - |_, _| Some(()), - |unit| Some(&unit.dependents), - ) - .await; - } - self.running = was_running; + let rollback_failures = self.stop_after_failed_start(&started_ids).await; + // A unit that the rollback did not stop may still run. + self.running = was_running || !rollback_failures.is_empty(); + failures.extend(rollback_failures); anyhow::bail!("state units could not start: {}", failures.join("; ")) } } + + /// Stops the units in `started_ids` again in reverse dependency order. + /// + /// Every other unit that is not stopped, such as one that was already + /// running or one whose start is uncertain, blocks the stop of its + /// dependencies without receiving a stop request. A unit whose stop is + /// blocked or not answered is left in a terminal state. Returns the + /// failures. + async fn stop_after_failed_start(&self, started_ids: &[u64]) -> Vec { + const OP: &str = "failed_start_rollback"; + let started_ids = started_ids.iter().copied().collect::>(); + let results = self.stop_units(OP, &started_ids).await; + let mut failures = Vec::new(); + let mut inner = self.inner.lock(); + for (name, id, result) in results { + let Some(unit) = inner.units.get_mut(&id) else { + continue; + }; + assert_eq!(unit.state, State::Stopping); + match result { + StopResult::Stopped => unit.state = State::Stopped, + StopResult::Uncertain(error) => { + unit.state = State::StartUncertain; + failures.push(format!("{name}: {OP} stop failed: {error:#}")); + } + StopResult::Blocked => { + unit.state = State::StartUncertain; + failures.push(format!( + "{name}: {OP} stop skipped because a dependent was not stopped" + )); + } + } + } + failures + } } diff --git a/vmm_core/state_unit/src/stop.rs b/vmm_core/state_unit/src/stop.rs new file mode 100644 index 00000000000..ef4ca1fdf92 --- /dev/null +++ b/vmm_core/state_unit/src/stop.rs @@ -0,0 +1,156 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Ordered stop. A unit's stop cannot fail, but its task can stop answering: +//! then the units that it depends on are not stopped, since it may still use +//! them, and the stop fails, rather than the whole process panicking. + +use super::State; +use super::StateRequest; +use super::StateUnits; +use super::state_change; +use futures::future::join_all; +use std::collections::BTreeMap; +use std::collections::BTreeSet; +use std::sync::Arc; +use std::time::Instant; +use tracing::Instrument; + +/// The outcome of a unit's stop. +pub(super) enum StopResult { + /// The unit answered its stop request. + Stopped, + /// The unit was not sent a stop request, because a unit that depends on + /// it was not stopped. + Blocked, + /// The unit's task did not answer its stop request, so the unit may still + /// be running. + Uncertain(anyhow::Error), +} + +/// The outcome of a unit's stop, with the unit's name and ID. +pub(super) type UnitStopResult = (Arc, u64, StopResult); + +impl StateUnits { + /// Stops the units that are not stopped in reverse dependency order. + /// + /// A unit in a terminal state that may be running is sent a stop request + /// too, and keeps its terminal marker. A unit whose task does not answer + /// its stop request is left in a terminal state, and the units that it + /// depends on keep their states. The units stay running until every unit + /// has stopped. + pub(super) async fn stop_in_order(&mut self) -> anyhow::Result<()> { + const OP: &str = "stop"; + // Make the units in a terminal state receive a stop request, then + // restore their terminal markers. + let mut ids = BTreeSet::new(); + let mut terminal_states = BTreeMap::new(); + for (&id, unit) in &mut self.inner.lock().units { + match unit.state { + State::Stopped => continue, + State::Running => {} + State::StartUncertain | State::StopUncertain => { + terminal_states.insert(id, unit.state); + unit.state = State::Running; + } + state => panic!("unit {} in {state:?} state before stop", unit.name), + } + ids.insert(id); + } + + let results = self.stop_units(OP, &ids).await; + + let mut failures = Vec::new(); + let mut inner = self.inner.lock(); + for (name, id, result) in results { + let Some(unit) = inner.units.get_mut(&id) else { + continue; + }; + assert_eq!(unit.state, State::Stopping); + let terminal_state = terminal_states.get(&id).copied(); + unit.state = match result { + StopResult::Stopped => terminal_state.unwrap_or(State::Stopped), + StopResult::Uncertain(error) => { + failures.push(format!("{name}: {error:#}")); + State::StopUncertain + } + StopResult::Blocked => { + failures.push(format!( + "{name}: stop skipped because a dependent was not stopped" + )); + terminal_state.unwrap_or(State::Running) + } + }; + } + if failures.is_empty() { + self.running = false; + Ok(()) + } else { + anyhow::bail!("state units could not stop: {}", failures.join("; ")) + } + } + + /// Sends a stop request to each running unit in `ids` in reverse + /// dependency order, and returns the outcome of each unit's stop. + /// + /// A unit is sent its request once every unit that depends on it has + /// stopped. A unit that is not in `ids` has stopped only if it is in the + /// stopped state, and a unit in `ids` once it answers its request or is + /// removed. The units in `ids` are left stopping, for the caller to move + /// each to the state that the outcome of its stop leaves it in. + pub(super) async fn stop_units( + &self, + op: &'static str, + ids: &BTreeSet, + ) -> Vec { + let mut operations = Vec::new(); + { + let mut inner = self.inner.lock(); + let ready_set = inner.ready_set(None); + for (&id, unit) in &inner.units { + if !ids.contains(&id) { + ready_set.done(id, unit.state == State::Stopped); + } + } + for (&id, unit) in inner.units.iter_mut().filter(|(id, _)| ids.contains(id)) { + assert_eq!(unit.state, State::Running); + let name = unit.name.clone(); + let dependents = unit.dependents.iter().copied().collect::>(); + let ready_set = ready_set.clone(); + let units = self.inner.clone(); + let stop = state_change(name.clone(), unit, StateRequest::Stop, Some(())); + operations.push(async move { + if !ready_set.wait(op, id, &dependents).await { + // A unit removed while it waits no longer uses its + // dependencies, so they can stop. + ready_set.done(id, !units.lock().units.contains_key(&id)); + return (name, id, StopResult::Blocked); + } + let result = match stop.await { + Ok(Some(())) => StopResult::Stopped, + Ok(None) => { + StopResult::Uncertain(anyhow::anyhow!("stop request was not sent")) + } + Err(error) => StopResult::Uncertain(error.into()), + }; + // Removing a unit ends its task, which then cannot answer. + // A removed unit no longer uses its dependencies, so they + // can stop. + let stopped = matches!(result, StopResult::Stopped) + || !units.lock().units.contains_key(&id); + ready_set.done(id, stopped); + (name, id, result) + }); + unit.state = State::Stopping; + } + } + async { + let start = Instant::now(); + let results = join_all(operations).await; + tracing::info!(duration = ?Instant::now() - start, "state change complete"); + results + } + .instrument(tracing::info_span!("state_change", operation = op)) + .await + } +} diff --git a/vmm_core/state_unit/src/tests/lost.rs b/vmm_core/state_unit/src/tests/lost.rs new file mode 100644 index 00000000000..ced7732606f --- /dev/null +++ b/vmm_core/state_unit/src/tests/lost.rs @@ -0,0 +1,229 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Tests for a reset, save, restore, or time advance that a unit whose task +//! has exited does not answer. + +use super::TestUnit; +use super::start::unit_state; +use crate::SavedStateUnit; +use crate::State; +use crate::StateRequest; +use crate::StateTransitionError; +use crate::StateTransitionErrorKind; +use crate::StateUnits; +use crate::UnitErrorSet; +use crate::UnitHandle; +use crate::run_unit; +use futures::StreamExt; +use pal_async::DefaultDriver; +use pal_async::async_test; +use std::time::Duration; +use test_with_tracing::test; +use vmcore::save_restore::NoSavedState; +use vmcore::save_restore::SavedStateBlob; + +/// Adds a unit named `name` whose task has exited: its request receiver is +/// dropped. +fn lost_unit(units: &StateUnits, name: &str) -> UnitHandle { + let (send, recv) = mesh::channel(); + drop(recv); + units.add(name).build(send).unwrap() +} + +/// Checks that `error` is the failure of `op` because the unit named `name`, +/// alone, did not answer its request. +fn assert_unanswered(error: StateTransitionError, op: &str, name: &str) { + let StateTransitionErrorKind::Units(UnitErrorSet(errors)) = &error.error else { + panic!("{op} failed for another reason: {error:?}"); + }; + let names = errors + .iter() + .map(|(name, _)| name.as_ref()) + .collect::>(); + assert_eq!( + (error.op, names.as_slice()), + (op, [name].as_slice()), + "{error:?}" + ); + let message = format!("{:#}", errors[0].1); + assert!( + message.contains("critical unit communication failure"), + "{message}" + ); +} + +/// Checks that a start, reset, save, restore, and time advance of `units` +/// each fail because the unit named `name` is in a terminal state. +async fn assert_terminal(units: &mut StateUnits, name: &str) { + let error = format!("{:#}", units.start().await.unwrap_err()); + assert!( + error.contains(&format!("'{name}' cannot be started from terminal state")), + "{error}" + ); + for error in [ + units.reset().await.unwrap_err(), + units.save().await.err().unwrap(), + units.restore(Vec::new()).await.unwrap_err(), + units + .advance_time(Duration::from_secs(1)) + .await + .unwrap_err(), + ] { + let error = format!("{:#}", anyhow::Error::from(error)); + assert!( + error.contains(&format!("{name}: unit is in terminal state")), + "{error}" + ); + } +} + +/// The saved state of the unit named `name`. +fn saved_state(name: &str) -> SavedStateUnit { + SavedStateUnit { + name: name.to_owned(), + state: SavedStateBlob::new(NoSavedState), + } +} + +#[async_test] +async fn a_lost_unit_fails_a_save(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let _healthy = units + .add("healthy") + .spawn(&driver, |recv| run_unit(TestUnit::default(), recv)) + .unwrap(); + let _lost = lost_unit(&units, "lost"); + + let Err(error) = units.save().await else { + panic!("save succeeded"); + }; + assert_unanswered(error, "save", "lost"); + assert_eq!(unit_state(&units, "lost"), State::SaveUncertain); + assert_eq!(unit_state(&units, "healthy"), State::Stopped); + assert_terminal(&mut units, "lost").await; +} + +#[async_test] +async fn a_lost_unit_fails_a_reset(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let lost = lost_unit(&units, "lost"); + // A unit that depends on the lost unit still resets after it. + let _dependent = units + .add("dependent") + .depends_on(&lost) + .spawn(&driver, |recv| run_unit(TestUnit::default(), recv)) + .unwrap(); + + assert_unanswered(units.reset().await.unwrap_err(), "reset", "lost"); + assert_eq!(unit_state(&units, "lost"), State::ResetUncertain); + assert_eq!(unit_state(&units, "dependent"), State::Stopped); + assert_terminal(&mut units, "lost").await; +} + +#[async_test] +async fn a_lost_unit_fails_a_restore(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let _healthy = units + .add("healthy") + .spawn(&driver, |recv| { + run_unit( + TestUnit { + support_saved_state: true, + ..Default::default() + }, + recv, + ) + }) + .unwrap(); + let _lost = lost_unit(&units, "lost"); + let healthy_state = SavedStateUnit { + name: "healthy".to_owned(), + state: SavedStateBlob::new(super::SavedState(true)), + }; + + assert_unanswered( + units + .restore(vec![healthy_state, saved_state("lost")]) + .await + .unwrap_err(), + "restore", + "lost", + ); + assert_eq!(unit_state(&units, "lost"), State::RestoreUncertain); + assert_eq!(unit_state(&units, "healthy"), State::Stopped); + assert_terminal(&mut units, "lost").await; +} + +#[async_test] +async fn a_lost_unit_fails_a_time_advance(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let _healthy = units + .add("healthy") + .advances_time() + .spawn(&driver, |recv| run_unit(TestUnit::default(), recv)) + .unwrap(); + let (send, recv) = mesh::channel(); + drop(recv); + let _lost = units.add("lost").advances_time().build(send).unwrap(); + + assert_unanswered( + units + .advance_time(Duration::from_secs(1)) + .await + .unwrap_err(), + "advance_time", + "lost", + ); + assert_eq!(unit_state(&units, "lost"), State::AdvanceTimeUncertain); + assert_eq!(unit_state(&units, "healthy"), State::Stopped); + assert_terminal(&mut units, "lost").await; +} + +#[async_test] +async fn a_unit_removed_during_a_save_does_not_fail_it(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let _healthy = units + .add("healthy") + .spawn(&driver, |recv| { + run_unit( + TestUnit { + support_saved_state: true, + ..Default::default() + }, + recv, + ) + }) + .unwrap(); + let (save_received_send, save_received) = mesh::oneshot(); + // A unit that holds its save request without answering it until the unit + // is removed, which ends its task. + let removed = units + .add("removed") + .spawn(&driver, |mut recv| async move { + let mut save_received_send = Some(save_received_send); + while let Some(request) = recv.next().await { + if let StateRequest::Save(_response) = request { + save_received_send.take().unwrap().send(()); + std::future::pending::<()>().await; + } + } + }) + .unwrap(); + + let remove = async { + save_received.await.unwrap(); + drop(removed); + }; + let (result, ()) = futures::join!(units.save(), remove); + let Ok(states) = result else { + panic!("save failed"); + }; + let names = states + .iter() + .map(|state| state.name.as_str()) + .collect::>(); + assert_eq!(names, ["healthy"]); + assert_eq!(unit_state(&units, "healthy"), State::Stopped); + units.start().await.unwrap(); +} diff --git a/vmm_core/state_unit/src/tests/slow.rs b/vmm_core/state_unit/src/tests/slow.rs index 3b9d0133d4a..78516f96599 100644 --- a/vmm_core/state_unit/src/tests/slow.rs +++ b/vmm_core/state_unit/src/tests/slow.rs @@ -110,7 +110,7 @@ async fn a_slow_stop_is_awaited_and_names_its_unit(driver: DefaultDriver) { units.start().await.unwrap(); assert!(log.unanswered().is_empty()); - units.stop().await; + units.stop().await.unwrap(); assert!(stopped.load(Ordering::Relaxed)); assert!(!units.is_running()); diff --git a/vmm_core/state_unit/src/tests/start.rs b/vmm_core/state_unit/src/tests/start.rs index d41d23f3bc9..0dec7ba6837 100644 --- a/vmm_core/state_unit/src/tests/start.rs +++ b/vmm_core/state_unit/src/tests/start.rs @@ -3,17 +3,20 @@ //! Tests for start with rollback. +use crate::State; use crate::StateRequest; use crate::StateUnit; use crate::StateUnits; use crate::run_unit; use futures::StreamExt; use inspect::InspectMut; +use mesh::Receiver; use pal_async::DefaultDriver; use pal_async::async_test; use parking_lot::Mutex; use std::sync::Arc; use std::sync::atomic::AtomicBool; +use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; use std::time::Duration; use test_with_tracing::test; @@ -21,7 +24,7 @@ use vmcore::save_restore::RestoreError; use vmcore::save_restore::SaveError; use vmcore::save_restore::SavedStateBlob; -struct RecordingUnit { +pub(super) struct RecordingUnit { started: Arc, stopped: Arc, } @@ -54,6 +57,46 @@ impl InspectMut for RecordingUnit { } } +pub(super) fn recording_unit( + started: &Arc, + stopped: &Arc, +) -> RecordingUnit { + RecordingUnit { + started: started.clone(), + stopped: stopped.clone(), + } +} + +pub(super) fn unit_state(units: &StateUnits, name: &str) -> State { + units + .inner + .lock() + .units + .values() + .find(|unit| unit.name.as_ref() == name) + .unwrap() + .state +} + +/// Runs `unit`, but drops each start request after starting the unit, without +/// answering it, so that the start is uncertain while the unit still answers +/// later requests. +async fn run_unanswering_start( + mut unit: RecordingUnit, + mut recv: Receiver, +) -> RecordingUnit { + while let Some(request) = recv.next().await { + match request { + StateRequest::Start(response) => { + unit.start().await; + drop(response); + } + request => request.apply(&mut unit).await, + } + } + unit +} + #[async_test] async fn an_unanswered_start_rolls_back_and_blocks_dependents(driver: DefaultDriver) { let mut units = StateUnits::new(); @@ -323,3 +366,246 @@ async fn an_unanswered_confirmation_leaves_a_terminal_unit(driver: DefaultDriver let retry_error = units.start().await.unwrap_err(); assert!(retry_error.to_string().contains("terminal state")); } + +#[async_test] +async fn test_uncertain_start_blocks_dependency_rollback(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let dependency_started = Arc::new(AtomicBool::new(false)); + let dependency_stopped = Arc::new(AtomicBool::new(false)); + let dependency = units + .add("dependency") + .spawn(&driver, |recv| { + run_unit( + recording_unit(&dependency_started, &dependency_stopped), + recv, + ) + }) + .unwrap(); + let unanswered_started = Arc::new(AtomicBool::new(false)); + let unanswered_stopped = Arc::new(AtomicBool::new(false)); + let _unanswered = units + .add("unanswered") + .depends_on(dependency.handle()) + .spawn(&driver, |recv| { + run_unanswering_start( + recording_unit(&unanswered_started, &unanswered_stopped), + recv, + ) + }) + .unwrap(); + + let error = units.start().await.unwrap_err().to_string(); + assert!( + error.contains("dependency: failed_start_rollback stop skipped"), + "{error}" + ); + assert!(dependency_started.load(Ordering::Relaxed)); + assert!(unanswered_started.load(Ordering::Relaxed)); + // The unit whose start is uncertain may use its dependency, so the + // rollback leaves the dependency running, and the units stay running. + assert!(!dependency_stopped.load(Ordering::Relaxed)); + assert!(units.is_running()); + assert_eq!(unit_state(&units, "dependency"), State::StartUncertain); + assert_eq!(unit_state(&units, "unanswered"), State::StartUncertain); + + // A later stop stops both, the dependency last, and both stay terminal. + units.stop().await.unwrap(); + assert!(unanswered_stopped.load(Ordering::Relaxed)); + assert!(dependency_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); + assert_eq!(unit_state(&units, "dependency"), State::StartUncertain); + assert_eq!(unit_state(&units, "unanswered"), State::StartUncertain); + let retry_error = units.start().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); +} + +#[async_test] +async fn test_started_unit_exit_during_rollback_returns_error(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let dependency_started = Arc::new(AtomicBool::new(false)); + let dependency_stopped = Arc::new(AtomicBool::new(false)); + let dependency = units + .add("dependency") + .spawn(&driver, |recv| { + run_unit( + recording_unit(&dependency_started, &dependency_stopped), + recv, + ) + }) + .unwrap(); + // A unit whose task answers its start and the confirmation of that start, + // and then exits, as a task that has failed would. + let exited = units + .add("exited") + .depends_on(dependency.handle()) + .spawn(&driver, |mut recv| async move { + while let Some(request) = recv.next().await { + match request { + StateRequest::Start(response) => response.complete(()), + StateRequest::ConfirmStart(response) => { + response.complete(Ok(())); + break; + } + request => panic!("unexpected request {request:?}"), + } + } + }) + .unwrap(); + let unanswered_stopped = Arc::new(AtomicBool::new(false)); + let _unanswered = units + .add("unanswered") + .spawn(&driver, |recv| { + run_unanswering_start( + recording_unit(&Arc::new(AtomicBool::new(false)), &unanswered_stopped), + recv, + ) + }) + .unwrap(); + + let error = units.start().await.unwrap_err().to_string(); + assert!( + error.contains("exited: failed_start_rollback stop failed"), + "{error}" + ); + assert!( + error.contains("dependency: failed_start_rollback stop skipped"), + "{error}" + ); + assert!(dependency_started.load(Ordering::Relaxed)); + assert!(!dependency_stopped.load(Ordering::Relaxed)); + // The dependency is still running, so a later stop must be able to stop it. + assert!(units.is_running()); + assert_eq!(unit_state(&units, "exited"), State::StartUncertain); + assert_eq!(unit_state(&units, "dependency"), State::StartUncertain); + + // Once the exited unit is removed, a stop stops the others. + exited.remove().await; + units.stop().await.unwrap(); + assert!(dependency_stopped.load(Ordering::Relaxed)); + assert!(unanswered_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); + let retry_error = units.start().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); +} + +#[async_test] +async fn test_failed_hot_add_does_not_stop_dependency_of_running_unit(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let running_dependent_started = Arc::new(AtomicBool::new(false)); + let running_dependent_stopped = Arc::new(AtomicBool::new(false)); + let running_dependent = units + .add("running-dependent") + .spawn(&driver, |recv| { + run_unit( + recording_unit(&running_dependent_started, &running_dependent_stopped), + recv, + ) + }) + .unwrap(); + units.start().await.unwrap(); + + let dependency_started = Arc::new(AtomicBool::new(false)); + let dependency_stopped = Arc::new(AtomicBool::new(false)); + let _dependency = units + .add("new-dependency") + .dependency_of(running_dependent.handle()) + .spawn(&driver, |recv| { + run_unit( + recording_unit(&dependency_started, &dependency_stopped), + recv, + ) + }) + .unwrap(); + let unanswered_started = Arc::new(AtomicBool::new(false)); + let unanswered_stopped = Arc::new(AtomicBool::new(false)); + let _unanswered = units + .add("unanswered") + .spawn(&driver, |recv| { + run_unanswering_start( + recording_unit(&unanswered_started, &unanswered_stopped), + recv, + ) + }) + .unwrap(); + + let error = units.start_stopped_units().await.unwrap_err().to_string(); + assert!( + error.contains("new-dependency: failed_start_rollback stop skipped"), + "{error}" + ); + // A running unit depends on the new unit, so the rollback leaves it + // running. + assert!(dependency_started.load(Ordering::Relaxed)); + assert!(!dependency_stopped.load(Ordering::Relaxed)); + assert!(!running_dependent_stopped.load(Ordering::Relaxed)); + assert!(unanswered_started.load(Ordering::Relaxed)); + assert!(!unanswered_stopped.load(Ordering::Relaxed)); + assert!(units.is_running()); + assert_eq!(unit_state(&units, "new-dependency"), State::StartUncertain); + assert_eq!(unit_state(&units, "unanswered"), State::StartUncertain); + + units.stop().await.unwrap(); + assert!(dependency_stopped.load(Ordering::Relaxed)); + assert!(running_dependent_stopped.load(Ordering::Relaxed)); + assert!(unanswered_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); + assert_eq!(unit_state(&units, "new-dependency"), State::StartUncertain); + let retry_error = units.start().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); +} + +#[async_test] +async fn test_unanswered_rollback_stop_can_be_retried(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let started = Arc::new(AtomicBool::new(false)); + let stopped = Arc::new(AtomicBool::new(false)); + let stop_requests = Arc::new(AtomicUsize::new(0)); + // A unit that drops its first stop request without stopping or + // answering. + let _unit = units + .add("rollback-retry") + .spawn(&driver, |mut recv| { + let mut unit = recording_unit(&started, &stopped); + let stop_requests = stop_requests.clone(); + async move { + while let Some(request) = recv.next().await { + match request { + StateRequest::Stop(response) + if stop_requests.fetch_add(1, Ordering::Relaxed) == 0 => + { + drop(response); + } + request => request.apply(&mut unit).await, + } + } + unit + } + }) + .unwrap(); + let unanswered_stopped = Arc::new(AtomicBool::new(false)); + let _unanswered = units + .add("unanswered") + .spawn(&driver, |recv| { + run_unanswering_start( + recording_unit(&Arc::new(AtomicBool::new(false)), &unanswered_stopped), + recv, + ) + }) + .unwrap(); + + let error = units.start().await.unwrap_err().to_string(); + assert!( + error.contains("rollback-retry: failed_start_rollback stop failed"), + "{error}" + ); + assert!(started.load(Ordering::Relaxed)); + assert!(!stopped.load(Ordering::Relaxed)); + assert_eq!(stop_requests.load(Ordering::Relaxed), 1); + assert!(units.is_running()); + + units.stop().await.unwrap(); + assert!(stopped.load(Ordering::Relaxed)); + assert_eq!(stop_requests.load(Ordering::Relaxed), 2); + assert!(unanswered_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); +} diff --git a/vmm_core/state_unit/src/tests/stop.rs b/vmm_core/state_unit/src/tests/stop.rs new file mode 100644 index 00000000000..6dce50b39b2 --- /dev/null +++ b/vmm_core/state_unit/src/tests/stop.rs @@ -0,0 +1,287 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Tests for a stop that fails instead of panicking. + +use super::start::recording_unit; +use super::start::unit_state; +use crate::State; +use crate::StateRequest; +use crate::StateUnits; +use crate::run_unit; +use futures::StreamExt; +use mesh::Receiver; +use pal_async::DefaultDriver; +use pal_async::async_test; +use std::sync::Arc; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; +use std::time::Duration; +use test_with_tracing::test; + +fn flag() -> Arc { + Arc::new(AtomicBool::new(false)) +} + +/// Runs a unit whose task answers its start and the confirmation of that +/// start, and then exits, as a task that has failed would. +async fn run_lost_after_start(mut recv: Receiver) { + while let Some(request) = recv.next().await { + match request { + StateRequest::Start(response) => response.complete(()), + StateRequest::ConfirmStart(response) => { + response.complete(Ok(())); + break; + } + request => panic!("unexpected request {request:?}"), + } + } +} + +#[async_test] +async fn a_lost_unit_fails_the_stop_and_keeps_its_dependencies_running(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let dependency_stopped = flag(); + let dependency = units + .add("dependency") + .spawn(&driver, |recv| { + run_unit(recording_unit(&flag(), &dependency_stopped), recv) + }) + .unwrap(); + let lost = units + .add("lost") + .depends_on(dependency.handle()) + .spawn(&driver, run_lost_after_start) + .unwrap(); + let dependent_stopped = flag(); + let _dependent = units + .add("dependent") + .depends_on(lost.handle()) + .spawn(&driver, |recv| { + run_unit(recording_unit(&flag(), &dependent_stopped), recv) + }) + .unwrap(); + let independent_stopped = flag(); + let _independent = units + .add("independent") + .spawn(&driver, |recv| { + run_unit(recording_unit(&flag(), &independent_stopped), recv) + }) + .unwrap(); + units.start().await.unwrap(); + + let error = units.stop().await.unwrap_err().to_string(); + assert!(error.contains("lost"), "{error}"); + assert!( + error.contains("dependency: stop skipped because a dependent was not stopped"), + "{error}" + ); + assert!(dependent_stopped.load(Ordering::Relaxed)); + assert!(independent_stopped.load(Ordering::Relaxed)); + // The lost unit may still use its dependency, so the dependency runs. + assert!(!dependency_stopped.load(Ordering::Relaxed)); + assert!(units.is_running()); + assert_eq!(unit_state(&units, "lost"), State::StopUncertain); + assert_eq!(unit_state(&units, "dependency"), State::Running); + assert_eq!(unit_state(&units, "dependent"), State::Stopped); + assert_eq!(unit_state(&units, "independent"), State::Stopped); + + // The units can never start again, and the requests that need stopped + // units fail, rather than panic, while the units stay running. + let start_error = units.start().await.unwrap_err(); + assert!(start_error.to_string().contains("terminal state")); + assert_eq!(unit_state(&units, "independent"), State::Stopped); + let reset_error = format!( + "{:#}", + anyhow::Error::from(units.reset().await.unwrap_err()) + ); + assert!( + reset_error.contains("lost: unit is in terminal state"), + "{reset_error}" + ); + assert!(units.save().await.is_err()); + assert!(units.restore(Vec::new()).await.is_err()); + assert!(units.advance_time(Duration::from_secs(1)).await.is_err()); + + // A later stop fails the same way while the lost unit remains. + assert!(units.stop().await.is_err()); + assert!(!dependency_stopped.load(Ordering::Relaxed)); + assert!(units.is_running()); + + // Once the lost unit is removed, the stop can stop its dependency, and + // the remaining units, none of which is in a terminal state, can start. + lost.remove().await; + units.stop().await.unwrap(); + assert!(dependency_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); + units.start().await.unwrap(); + units.stop().await.unwrap(); +} + +#[async_test] +async fn a_unit_whose_stop_is_unanswered_can_be_stopped_again(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let dependency_stopped = flag(); + let dependency = units + .add("dependency") + .spawn(&driver, |recv| { + run_unit(recording_unit(&flag(), &dependency_stopped), recv) + }) + .unwrap(); + let stopped = flag(); + let stop_requests = Arc::new(AtomicUsize::new(0)); + // A unit that drops its first stop request without stopping or answering. + let _unanswered = units + .add("unanswered") + .depends_on(dependency.handle()) + .spawn(&driver, |mut recv| { + let mut unit = recording_unit(&flag(), &stopped); + let stop_requests = stop_requests.clone(); + async move { + while let Some(request) = recv.next().await { + match request { + StateRequest::Stop(response) + if stop_requests.fetch_add(1, Ordering::Relaxed) == 0 => + { + drop(response); + } + request => request.apply(&mut unit).await, + } + } + unit + } + }) + .unwrap(); + units.start().await.unwrap(); + + let error = units.stop().await.unwrap_err().to_string(); + assert!(error.contains("unanswered"), "{error}"); + assert!(!stopped.load(Ordering::Relaxed)); + assert!(!dependency_stopped.load(Ordering::Relaxed)); + assert!(units.is_running()); + assert_eq!(unit_state(&units, "unanswered"), State::StopUncertain); + assert_eq!(unit_state(&units, "dependency"), State::Running); + + // The unit may be running, so a later stop sends it another request, and + // the unit keeps its terminal marker once it stops. + units.stop().await.unwrap(); + assert_eq!(stop_requests.load(Ordering::Relaxed), 2); + assert!(stopped.load(Ordering::Relaxed)); + assert!(dependency_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); + assert_eq!(unit_state(&units, "unanswered"), State::StopUncertain); + assert_eq!(unit_state(&units, "dependency"), State::Stopped); + let start_error = units.start().await.unwrap_err(); + assert!(start_error.to_string().contains("terminal state")); + assert!(units.reset().await.is_err()); + assert!(units.save().await.is_err()); + assert!(units.restore(Vec::new()).await.is_err()); + assert!(units.advance_time(Duration::from_secs(1)).await.is_err()); +} + +#[async_test] +async fn a_unit_removed_during_a_stop_does_not_block_its_dependencies(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let dependency_stopped = flag(); + let dependency = units + .add("dependency") + .spawn(&driver, |recv| { + run_unit(recording_unit(&flag(), &dependency_stopped), recv) + }) + .unwrap(); + let (stop_received_send, stop_received) = mesh::oneshot(); + // A unit that holds its stop request without answering it until the unit + // is removed, which ends its task. + let removed = units + .add("removed") + .depends_on(dependency.handle()) + .spawn(&driver, |mut recv| { + let mut unit = recording_unit(&flag(), &flag()); + async move { + let mut stop_received_send = Some(stop_received_send); + while let Some(request) = recv.next().await { + match request { + StateRequest::Stop(_response) => { + stop_received_send.take().unwrap().send(()); + std::future::pending::<()>().await; + } + request => request.apply(&mut unit).await, + } + } + } + }) + .unwrap(); + units.start().await.unwrap(); + + let remove = async { + stop_received.await.unwrap(); + drop(removed); + }; + let (result, ()) = futures::join!(units.stop(), remove); + result.unwrap(); + assert!(dependency_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); + assert_eq!(unit_state(&units, "dependency"), State::Stopped); +} + +#[async_test] +async fn a_unit_removed_while_blocked_does_not_block_its_dependencies(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let dependency_stopped = flag(); + let dependency = units + .add("dependency") + .spawn(&driver, |recv| { + run_unit(recording_unit(&flag(), &dependency_stopped), recv) + }) + .unwrap(); + let removed = units + .add("removed") + .depends_on(dependency.handle()) + .spawn(&driver, |recv| { + run_unit(recording_unit(&flag(), &flag()), recv) + }) + .unwrap(); + let (stop_received_send, stop_received) = mesh::oneshot(); + let (release_send, release) = mesh::oneshot::<()>(); + // A unit that depends on `removed` and holds its stop request until + // `removed` is removed, and then exits without answering it, so that + // `removed` waits for it and is blocked. + let _unanswered = units + .add("unanswered") + .depends_on(removed.handle()) + .spawn(&driver, |mut recv| { + let mut unit = recording_unit(&flag(), &flag()); + async move { + let mut stop_received_send = Some(stop_received_send); + let mut release = Some(release); + while let Some(request) = recv.next().await { + match request { + StateRequest::Stop(response) => { + stop_received_send.take().unwrap().send(()); + release.take().unwrap().await.unwrap(); + drop(response); + break; + } + request => request.apply(&mut unit).await, + } + } + } + }) + .unwrap(); + units.start().await.unwrap(); + + let remove = async { + stop_received.await.unwrap(); + drop(removed); + release_send.send(()); + }; + let (result, ()) = futures::join!(units.stop(), remove); + let error = result.unwrap_err().to_string(); + assert!(error.contains("unanswered"), "{error}"); + assert!(!error.contains("dependency"), "{error}"); + // The removed unit no longer uses its dependency, so it stops. + assert!(dependency_stopped.load(Ordering::Relaxed)); + assert_eq!(unit_state(&units, "dependency"), State::Stopped); + assert_eq!(unit_state(&units, "unanswered"), State::StopUncertain); +} diff --git a/vmm_core/state_unit/src/tests/terminal.rs b/vmm_core/state_unit/src/tests/terminal.rs index 04707bc0ff0..e699c497321 100644 --- a/vmm_core/state_unit/src/tests/terminal.rs +++ b/vmm_core/state_unit/src/tests/terminal.rs @@ -187,7 +187,7 @@ async fn a_stop_stops_a_unit_whose_start_is_uncertain(driver: DefaultDriver) { // The unit whose start is uncertain may be running, so the stop stops it, // before the unit that it depends on. - units.stop().await; + units.stop().await.unwrap(); assert!(!units.is_running()); assert_eq!(log.take(), ["uncertain stop", "dependency stop"]);