diff --git a/openhcl/underhill_core/src/dispatch/mod.rs b/openhcl/underhill_core/src/dispatch/mod.rs index 20eebc55c06..a336eac9e08 100644 --- a/openhcl/underhill_core/src/dispatch/mod.rs +++ b/openhcl/underhill_core/src/dispatch/mod.rs @@ -79,7 +79,7 @@ use vtl2_settings_worker::handle_vtl2_config_rpc; #[derive(MeshPayload)] pub enum UhVmRpc { Pause(Rpc<(), bool>), - Resume(Rpc<(), bool>), + Resume(FailableRpc<(), bool>), Save(FailableRpc<(), Vec>), ClearHalt(Rpc<(), bool>), // TODO: remove this, and use DebugRequest::Resume PacketCapture(FailableRpc, PacketCaptureParams>), @@ -233,9 +233,12 @@ impl LoadedVm { correlation_id: Option, mut vm_rpc: mesh::Receiver, mut worker_rpc: mesh::Receiver>, - ) -> Option> { + ) -> anyhow::Result>> { if autostart_vps { - self.start(correlation_id).await; + if let Err(error) = self.start(correlation_id).await { + self.teardown().await; + return Err(error.context("failed to autostart VM")); + } } // VTL2 settings services @@ -323,7 +326,17 @@ impl LoadedVm { Event::WorkerRpc(message) => match message { WorkerRpc::Stop => break None, WorkerRpc::Restart(rpc) => { - let state = async { + enum RestartOutcome { + Restart(R, S), + // The restart failed, but the VM is back in its + // previous state, so this worker can keep serving. + Recovered, + // The restart failed and the VM could not be + // resumed, so this worker cannot serve any more. + Unrecoverable, + } + + let outcome = async { let running = self.stop().await; match self .save( @@ -334,26 +347,40 @@ impl LoadedVm { ) .await { - Ok(servicing_state) => Some((rpc, servicing_state)), + Ok(servicing_state) => RestartOutcome::Restart(rpc, servicing_state), Err(err) => { if running { - self.start(None).await; + if let Err(start_error) = self.start(None).await { + rpc.complete(Err(RemoteError::new( + start_error.context(format!( + "worker restart failed ({err:#}) and the VM could not resume" + )), + ))); + return RestartOutcome::Unrecoverable; + } } rpc.complete(Err(RemoteError::new(err))); - None + RestartOutcome::Recovered } } } .instrument(tracing::info_span!("restart", CVM_ALLOWED)) .await; - if let Some((rpc, servicing_state)) = state { - break Some(LoadedVmState { - restart_rpc: rpc, - servicing_state, - vm_rpc, - control_send: self.control_send.lock().take().unwrap(), - }); + match outcome { + RestartOutcome::Restart(rpc, servicing_state) => { + break Some(LoadedVmState { + restart_rpc: rpc, + servicing_state, + vm_rpc, + control_send: self.control_send.lock().take().unwrap(), + }); + } + RestartOutcome::Recovered => {} + // The error has already completed the RPC. Tear + // down through the common path rather than keep a + // worker whose state units can no longer run. + RestartOutcome::Unrecoverable => break None, } } WorkerRpc::Inspect(deferred) => deferred.respond(|resp| { @@ -403,34 +430,36 @@ impl LoadedVm { } Event::UhVmRpc(msg) => match msg { UhVmRpc::Resume(rpc) => { - rpc.handle(async |()| { - if !self.state_units.is_running() { - self.start(None).await; - true - } else { - false - } - }) - .await + if self.state_units.is_running() { + rpc.complete(Ok(false)); + } else if let Err(error) = self.start(None).await { + rpc.complete(Err(RemoteError::new( + error.context("failed to resume VM"), + ))); + } else { + rpc.complete(Ok(true)); + } } UhVmRpc::Pause(rpc) => rpc.handle(async |()| self.stop().await).await, UhVmRpc::Save(rpc) => { - rpc.handle_failable(async |()| { - let running = self.stop().await; - let r = self - .save( - SaveReason::Diagnostic, - None, - KeepAliveConfig::Disabled, - KeepAliveConfig::Disabled, - ) - .await; - if running { - self.start(None).await; - } - r.map(mesh::payload::encode) - }) - .await + let running = self.stop().await; + let result = self + .save( + SaveReason::Diagnostic, + None, + KeepAliveConfig::Disabled, + KeepAliveConfig::Disabled, + ) + .await; + if running && let Err(error) = self.start(None).await { + rpc.complete(Err(RemoteError::new( + error.context("failed to resume VM after save"), + ))); + // The state units rolled back and cannot run + // again, so tear down rather than keep this worker. + break None; + } + rpc.complete(result.map(mesh::payload::encode).map_err(RemoteError::new)); } UhVmRpc::ClearHalt(rpc) => { rpc.handle(async |()| self.partition_unit.clear_halt().await) @@ -582,6 +611,12 @@ impl LoadedVm { } }; + self.teardown().await; + + Ok(state) + } + + async fn teardown(mut self) { // Drain resealing before tearing down any storage transport. Removing // the unit closes its request stream; in-flight I/O finishes first. if let Some(resealer) = self.hardware_reseal.take() { @@ -599,8 +634,6 @@ impl LoadedVm { if let Some(vmbus) = self.vmbus_server { vmbus.remove().await.shutdown().await; } - - state } /// Handles a servicing request from the host. @@ -651,7 +684,9 @@ impl LoadedVm { .context("failed to notify host of servicing-while-paused failure")?; if running { - self.start(Some(correlation_id)).await; + self.start(Some(correlation_id)) + .await + .context("failed to resume VM after servicing failure")?; } false } @@ -881,8 +916,8 @@ impl LoadedVm { } } - async fn start(&mut self, correlation_id: Option) { - self.state_units.start().await; + async fn start(&mut self, correlation_id: Option) -> anyhow::Result<()> { + self.state_units.start().await?; // Log the boot/blackout time. let reference_time = ReferenceTime::new(self.partition.reference_time()); @@ -908,6 +943,8 @@ impl LoadedVm { "starting VM" ) } + + Ok(()) } /// Returns true if the VM was previously running. @@ -1134,7 +1171,20 @@ impl LoadedVm { ) .await?; - self.state_units.start_stopped_units().await; + if let Err(error) = self.state_units.start_stopped_units().await { + if let Err(remove_error) = self + .network_settings + .as_mut() + .unwrap() + .remove_network(instance_id) + .await + { + return Err(error.context(format!( + "failed to remove network after start failure: {remove_error:#}" + ))); + } + return Err(error); + } self.emuplat_servicing.netvsp_state.push(save_state); Ok(()) diff --git a/openhcl/underhill_core/src/hardware_reseal.rs b/openhcl/underhill_core/src/hardware_reseal.rs index 1fbb8dbf98b..51fe81346f7 100644 --- a/openhcl/underhill_core/src/hardware_reseal.rs +++ b/openhcl/underhill_core/src/hardware_reseal.rs @@ -402,7 +402,7 @@ impl inspect::InspectMut for HardwareReseal { } impl StateUnit for HardwareReseal { - async fn start(&mut self) { + async fn start(&mut self) -> anyhow::Result<()> { // Starting or resuming does not create work or reset retry backoff. // Pending notifications and recovery survive a normal stop/start. self.schedule.running = true; @@ -413,6 +413,7 @@ impl StateUnit for HardwareReseal { pending_notification = self.notification.pending.load(Ordering::Acquire), "hardware reseal worker started" ); + Ok(()) } async fn stop(&mut self) { diff --git a/openhcl/underhill_core/src/hardware_reseal/tests.rs b/openhcl/underhill_core/src/hardware_reseal/tests.rs index 7081da1a42d..78ba94ca702 100644 --- a/openhcl/underhill_core/src/hardware_reseal/tests.rs +++ b/openhcl/underhill_core/src/hardware_reseal/tests.rs @@ -642,6 +642,7 @@ async fn finish_attempt( if start { drive_until(run.as_mut(), send.call(StateRequest::Start, ())) .await + .unwrap() .unwrap(); } drive_until(run.as_mut(), milestone).await; @@ -837,6 +838,7 @@ async fn stopped_worker_keeps_notification_and_start_honors_not_before(driver: D assert!(notification.pending.load(Ordering::SeqCst)); drive_until(run.as_mut(), send.call(StateRequest::Start, ())) .await + .unwrap() .unwrap(); assert!(!notification.pending.load(Ordering::SeqCst)); drive_until(run.as_mut(), send.call(StateRequest::Stop, ())) @@ -874,6 +876,7 @@ async fn stop_drains_blocked_write_and_final_flush_and_retains_late_event(driver let mut run = pin!(fixture.worker.run(recv)); drive_until(run.as_mut(), send.call(StateRequest::Start, ())) .await + .unwrap() .unwrap(); drive_until(run.as_mut(), fixture.io.blocked()).await; let mut stop = pin!(send.call(StateRequest::Stop, ())); @@ -940,6 +943,7 @@ async fn blocked_hardware_keeps_executor_responsive_and_stop_drains_attempt(driv let mut run = pin!(fixture.worker.run(recv)); drive_until(run.as_mut(), send.call(StateRequest::Start, ())) .await + .unwrap() .unwrap(); drive_until(run.as_mut(), gate.reached()).await; { @@ -1078,7 +1082,7 @@ async fn failed_report_keeps_recovery_pending_and_retries_successfully(driver: D let deadline = fixture.worker.schedule.deadline; let not_before = fixture.worker.schedule.not_before; - fixture.worker.start().await; + fixture.worker.start().await.unwrap(); fixture.worker.stop().await; assert!(fixture.worker.schedule.force_reseal); assert_eq!(fixture.worker.schedule.failures, 1); @@ -1300,6 +1304,7 @@ async fn changed_report_during_pre_or_post_verification_keeps_recovery_pending( let mut run = pin!(fixture.worker.run(recv)); drive_until(run.as_mut(), send.call(StateRequest::Start, ())) .await + .unwrap() .unwrap(); if post_flush { drive_until(run.as_mut(), fixture.io.blocked()).await; @@ -1418,6 +1423,7 @@ async fn migration_during_event_flush_is_detected_without_another_event(driver: let mut run = pin!(fixture.worker.run(recv)); drive_until(run.as_mut(), send.call(StateRequest::Start, ())) .await + .unwrap() .unwrap(); drive_until(run.as_mut(), fixture.io.blocked()).await; fixture.hardware.identity.store(0x73, Ordering::SeqCst); @@ -1500,7 +1506,7 @@ async fn restore_and_reset_preserve_upgrade_and_failed_flush_obligation(driver: assert_eq!(fixture.worker.schedule.deadline, deadline); assert_eq!(fixture.worker.schedule.not_before, not_before); assert!(!fixture.worker.notification.pending.load(Ordering::SeqCst)); - fixture.worker.start().await; + fixture.worker.start().await.unwrap(); fixture.worker.stop().await; assert!(fixture.worker.schedule.force_reseal); assert_eq!(fixture.worker.schedule.failures, 1); @@ -1552,6 +1558,7 @@ async fn failed_final_flush_retries_write_even_when_cached_protector_matches( let mut run = pin!(fixture.worker.run(recv)); drive_until(run.as_mut(), send.call(StateRequest::Start, ())) .await + .unwrap() .unwrap(); drive_until(run.as_mut(), fixture.io.blocked()).await; let stop = send.call(StateRequest::Stop, ()); @@ -1771,7 +1778,7 @@ async fn running_save_and_restore_fail_without_consuming_pending_work(driver: De let mut fixture = Fixture::new(&driver, true).await; fixture.worker.notification.notify(); let saved = fixture.worker.save().await.unwrap().unwrap(); - fixture.worker.start().await; + fixture.worker.start().await.unwrap(); let before = worker_snapshot(&fixture.worker); assert!(fixture.worker.save().await.is_err()); assert!(fixture.worker.restore(saved).await.is_err()); diff --git a/openhcl/underhill_core/src/lib.rs b/openhcl/underhill_core/src/lib.rs index 329d1c119f8..69cb928ad5c 100644 --- a/openhcl/underhill_core/src/lib.rs +++ b/openhcl/underhill_core/src/lib.rs @@ -686,11 +686,18 @@ async fn run_control( continue; }; - let was_resumed = workers + let was_resumed = match workers .vm_rpc .call(UhVmRpc::Resume, ()) .await - .context("failed to resumed VM")?; + .context("failed to resume VM")? + { + Ok(was_resumed) => was_resumed, + Err(error) => { + rpc.complete(Err(error)); + continue; + } + }; let was_halted = workers .vm_rpc diff --git a/openhcl/underhill_core/src/vmbus_relay_unit.rs b/openhcl/underhill_core/src/vmbus_relay_unit.rs index bce72bc9b17..21a172e45d9 100644 --- a/openhcl/underhill_core/src/vmbus_relay_unit.rs +++ b/openhcl/underhill_core/src/vmbus_relay_unit.rs @@ -43,8 +43,9 @@ impl VmbusRelayHandle { struct VmbusRelayUnit(HostVmbusTransport); impl StateUnit for &'_ VmbusRelayUnit { - async fn start(&mut self) { + async fn start(&mut self) -> anyhow::Result<()> { self.0.start(); + Ok(()) } async fn stop(&mut self) { diff --git a/openhcl/underhill_core/src/worker.rs b/openhcl/underhill_core/src/worker.rs index 702894bd524..694653db74e 100644 --- a/openhcl/underhill_core/src/worker.rs +++ b/openhcl/underhill_core/src/worker.rs @@ -465,6 +465,7 @@ impl Worker for UnderhillVmWorker { )); tracing::info!(CVM_ALLOWED, "terminating worker"); self.get_thread.join().unwrap(); + let state = state?; if let Some(state) = state { let params = UnderhillWorkerParameters { env_cfg: self.env_cfg, diff --git a/openvmm/openvmm_core/src/worker/dispatch.rs b/openvmm/openvmm_core/src/worker/dispatch.rs index ce8e17f1d79..179e2ba5b4b 100644 --- a/openvmm/openvmm_core/src/worker/dispatch.rs +++ b/openvmm/openvmm_core/src/worker/dispatch.rs @@ -401,7 +401,7 @@ impl Worker for VmWorker { LOADED_VM.store(&vm); if running { - vm.resume().await; + vm.resume().await?; } Ok(Self { vm, @@ -3693,13 +3693,16 @@ impl LoadedVmInner { impl LoadedVm { #[tracing::instrument(name = "resume_vm", target = "openvmm::perf", skip_all)] - async fn resume(&mut self) -> bool { + async fn resume(&mut self) -> anyhow::Result { if self.running { - return false; + return Ok(false); } - self.state_units.start().await; + self.state_units + .start() + .await + .context("VM state units failed to start")?; self.running = true; - true + Ok(true) } async fn pause(&mut self) -> bool { @@ -3738,7 +3741,10 @@ impl LoadedVm { let stop_guard = self.inner.partition_unit.temporarily_stop_vps().await; // Start state units so device config space is accessible. - self.state_units.start().await; + self.state_units + .start() + .await + .context("failed to start devices for PCI resource assignment")?; let result = ecam_config_access::assign_pci_resources_for_root_complexes( &self.inner.chipset, @@ -3824,7 +3830,13 @@ impl LoadedVm { } Err(err) => { if stopped { - self.state_units.start().await; + if let Err(start_error) = self.state_units.start().await { + self.running = false; + rpc.complete(Err(RemoteError::new(start_error.context( + "worker restart failed and the VM could not resume", + )))); + continue; + } } rpc.complete(Err(RemoteError::new(err))); } @@ -3849,7 +3861,9 @@ impl LoadedVm { rpc.handle(async |()| self.inner.partition_unit.clear_halt().await) .await } - VmRpc::Resume(rpc) => rpc.handle(async |()| self.resume().await).await, + VmRpc::Resume(rpc) => { + rpc.handle_failable(async |()| self.resume().await).await + } VmRpc::Pause(rpc) => rpc.handle(async |()| self.pause().await).await, VmRpc::Save(rpc) => { rpc.handle_failable(async |()| self.save().await.map(ProtobufMessage::new)) @@ -3897,7 +3911,15 @@ impl LoadedVm { ) .await?; self.inner.vmbus_devices.push(device); - self.state_units.start_stopped_units().await; + if let Err(error) = self.state_units.start_stopped_units().await { + let device = self + .inner + .vmbus_devices + .pop() + .expect("device was just pushed"); + drop(device.remove().await); + return Err(error); + } anyhow::Ok(()) }) .await @@ -3926,7 +3948,7 @@ impl LoadedVm { self.save_reset_restore().await?; if paused { - self.resume().await; + self.resume().await?; } Ok(()) }) @@ -4043,7 +4065,15 @@ impl LoadedVm { // MSI. The guest may begin probing config space // immediately after receiving the interrupt, so // the device must be ready first. - self.state_units.start_stopped_units().await; + if let Err(error) = self.state_units.start_stopped_units().await { + let (_, unit, _) = self + .inner + .pcie_hotplug_devices + .pop() + .expect("device was just pushed"); + unit.remove().await; + return Err(error); + } // Now attach the device and notify the guest. if let Err(e) = rc.lock().hotplug_add_device( @@ -4406,7 +4436,7 @@ impl LoadedVm { } if resume { - self.resume().await; + self.resume().await?; } Ok(()) } diff --git a/openvmm/openvmm_core/src/worker/dispatch/dump.rs b/openvmm/openvmm_core/src/worker/dispatch/dump.rs index ec47b71126d..a0108c686da 100644 --- a/openvmm/openvmm_core/src/worker/dispatch/dump.rs +++ b/openvmm/openvmm_core/src/worker/dispatch/dump.rs @@ -19,7 +19,9 @@ impl LoadedVm { let was_running = self.pause().await; let result = self.dump_state_inner(file).await; if was_running { - self.resume().await; + self.resume() + .await + .context("failed to resume VM after dump")?; } result } diff --git a/openvmm/openvmm_defs/src/rpc.rs b/openvmm/openvmm_defs/src/rpc.rs index 58af188f5ef..e6d8e702320 100644 --- a/openvmm/openvmm_defs/src/rpc.rs +++ b/openvmm/openvmm_defs/src/rpc.rs @@ -20,7 +20,7 @@ use vm_resource::kind::VmbusDeviceHandleKind; #[derive(MeshPayload)] pub enum VmRpc { Save(FailableRpc<(), ProtobufMessage>), - Resume(Rpc<(), bool>), + Resume(FailableRpc<(), bool>), Pause(Rpc<(), bool>), ClearHalt(Rpc<(), bool>), Reset(FailableRpc<(), ()>), diff --git a/openvmm/openvmm_entry/src/lib.rs b/openvmm/openvmm_entry/src/lib.rs index 6965a1140d1..31133fd579a 100644 --- a/openvmm/openvmm_entry/src/lib.rs +++ b/openvmm/openvmm_entry/src/lib.rs @@ -3019,7 +3019,10 @@ async fn run_control_inner( } if !opt.paused { - vm_rpc.call(VmRpc::Resume, ()).await?; + anyhow::ensure!( + vm_rpc.call_failable(VmRpc::Resume, ()).await?, + "VM failed to start; inspect the worker log for the device startup error" + ); } let paravisor_diag = Arc::new(diag_client::DiagClient::from_dialer( diff --git a/openvmm/openvmm_entry/src/repl.rs b/openvmm/openvmm_entry/src/repl.rs index 75beaa7e162..4460cd84c1e 100644 --- a/openvmm/openvmm_entry/src/repl.rs +++ b/openvmm/openvmm_entry/src/repl.rs @@ -664,7 +664,7 @@ pub(crate) async fn run_repl( enum StateChange { Pause(bool), - Resume(bool), + Resume(Result), Reset(Result<(), RemoteError>), PulseSaveRestore(Result<(), PulseSaveRestoreError>), ServiceVtl2(anyhow::Result), @@ -766,13 +766,16 @@ pub(crate) async fn run_repl( tracing::warn!("already paused"); } } - StateChange::Resume(success) => { + StateChange::Resume(Ok(success)) => { if success { tracing::info!("resumed complete"); } else { tracing::warn!("already running"); } } + StateChange::Resume(Err(err)) => { + tracing::error!(error = &err as &dyn std::error::Error, "resume failed") + } StateChange::Reset(r) => match r { Ok(()) => tracing::info!("reset complete"), Err(err) => tracing::error!( diff --git a/openvmm/openvmm_entry/src/ttrpc/mod.rs b/openvmm/openvmm_entry/src/ttrpc/mod.rs index b67512a24c4..3770cd338c6 100644 --- a/openvmm/openvmm_entry/src/ttrpc/mod.rs +++ b/openvmm/openvmm_entry/src/ttrpc/mod.rs @@ -1543,7 +1543,7 @@ impl VmService { async fn resume_vm(&mut self) -> anyhow::Result<()> { let vm = self.vm.clone().context("VM not created yet")?; vm.worker_rpc - .call(VmRpc::Resume, ()) + .call_failable(VmRpc::Resume, ()) .await .map(drop) .context("resume failed")?; diff --git a/petri/src/worker.rs b/petri/src/worker.rs index 5071503305d..5b5ca3440d6 100644 --- a/petri/src/worker.rs +++ b/petri/src/worker.rs @@ -52,8 +52,8 @@ impl Worker { self.rpc.call(VmRpc::Pause, ()).await } - pub(crate) async fn resume(&self) -> Result { - self.rpc.call(VmRpc::Resume, ()).await + pub(crate) async fn resume(&self) -> anyhow::Result { + Ok(self.rpc.call_failable(VmRpc::Resume, ()).await?) } pub(crate) async fn save(&self) -> anyhow::Result { diff --git a/vm/devices/pci/vpci_relay/src/lib.rs b/vm/devices/pci/vpci_relay/src/lib.rs index 22e971f1f90..76bae759c18 100644 --- a/vm/devices/pci/vpci_relay/src/lib.rs +++ b/vm/devices/pci/vpci_relay/src/lib.rs @@ -490,7 +490,7 @@ impl VpciRelay { .await? }; - entry.insert(RelayedDevice { + let device = RelayedDevice { bus_instance_id: instance_id, bus_client: vpci_client, vpci_device: vpci_device.clone(), @@ -498,9 +498,13 @@ impl VpciRelay { bus_unit, device_unit, ready_to_remove: false, - }); + }; - state_units.start_stopped_units().await; + if let Err(error) = state_units.start_stopped_units().await { + device.remove().await; + return Err(error); + } + entry.insert(device); Ok(()) } } diff --git a/vmm_core/src/device_builder.rs b/vmm_core/src/device_builder.rs index 089ac1444c9..440d98a9dc4 100644 --- a/vmm_core/src/device_builder.rs +++ b/vmm_core/src/device_builder.rs @@ -144,7 +144,11 @@ pub async fn build_dynamic_vpci_device( }; let pending_offer = pending_offer.context("missing deferred VPCI channel offer")?; - state_units.start_stopped_units().await; + if let Err(error) = state_units.start_stopped_units().await { + vpci_unit.remove().await; + pci_unit.remove().await; + return Err(error); + } if let Err(error) = pending_offer .offer_registered(&vpci_bus, driver_source, vmbus, state_units.is_running()) .await diff --git a/vmm_core/src/input_distributor.rs b/vmm_core/src/input_distributor.rs index 6aafd50d85d..7d6aef61e1c 100644 --- a/vmm_core/src/input_distributor.rs +++ b/vmm_core/src/input_distributor.rs @@ -195,8 +195,9 @@ struct Inner { } impl StateUnit for Inner { - async fn start(&mut self) { + async fn start(&mut self) -> anyhow::Result<()> { self.running = true; + Ok(()) } async fn stop(&mut self) { diff --git a/vmm_core/src/partition_unit.rs b/vmm_core/src/partition_unit.rs index 3dd1ea5f802..962d9db6dab 100644 --- a/vmm_core/src/partition_unit.rs +++ b/vmm_core/src/partition_unit.rs @@ -594,10 +594,11 @@ impl Drop for StopGuard { } impl StateUnit for PartitionUnitRunner { - async fn start(&mut self) { + async fn start(&mut self) -> anyhow::Result<()> { self.partition.thaw_time(); self.unit_started = true; self.try_start(); + Ok(()) } async fn stop(&mut self) { diff --git a/vmm_core/src/partition_unit/tests.rs b/vmm_core/src/partition_unit/tests.rs index 2c62663243c..29e0690ec37 100644 --- a/vmm_core/src/partition_unit/tests.rs +++ b/vmm_core/src/partition_unit/tests.rs @@ -131,7 +131,7 @@ fn time_stays_frozen_across_initial_restore_and_reset() { StateUnit::reset(&mut runner).await.unwrap(); assert!(clock.lock().frozen); assert!(clock.lock().transitions.is_empty()); - StateUnit::start(&mut runner).await; + StateUnit::start(&mut runner).await.unwrap(); assert_eq!(clock.lock().transitions, [false]); }); } @@ -141,7 +141,7 @@ fn full_stop_freezes_time_until_resume() { block_on(async { let (mut runner, clock, _) = new_runner(); for _ in 0..2 { - StateUnit::start(&mut runner).await; + StateUnit::start(&mut runner).await.unwrap(); assert!(!clock.lock().frozen); StateUnit::stop(&mut runner).await; let state = StateUnit::save(&mut runner).await.unwrap().unwrap(); @@ -157,7 +157,7 @@ fn full_stop_freezes_time_until_resume() { fn temporary_stops_and_debugger_halts_leave_time_running() { block_on(async { let (mut runner, clock, halt) = new_runner(); - StateUnit::start(&mut runner).await; + StateUnit::start(&mut runner).await.unwrap(); runner.stop_vps().await; runner.stop_vps().await; assert!(!clock.lock().frozen); @@ -178,14 +178,14 @@ fn temporary_stops_and_debugger_halts_leave_time_running() { fn temporary_stop_guards_do_not_override_full_stop() { block_on(async { let (mut runner, clock, _) = new_runner(); - StateUnit::start(&mut runner).await; + StateUnit::start(&mut runner).await.unwrap(); runner.stop_vps().await; runner.stop_vps().await; StateUnit::stop(&mut runner).await; runner.resume_vps(); assert!(clock.lock().frozen); - StateUnit::start(&mut runner).await; + StateUnit::start(&mut runner).await.unwrap(); assert!(!clock.lock().frozen); runner.resume_vps(); assert_eq!(clock.lock().transitions, [false, true, false]); @@ -196,7 +196,7 @@ fn temporary_stop_guards_do_not_override_full_stop() { fn temporary_scrub_thaws_time_before_restarting_vps() { block_on(async { let (mut runner, clock, _) = new_runner(); - StateUnit::start(&mut runner).await; + StateUnit::start(&mut runner).await.unwrap(); runner.stop_vps().await; runner.partition.scrub_vtl(Vtl::Vtl2).unwrap(); assert!(clock.lock().frozen); diff --git a/vmm_core/src/vmbus_unit.rs b/vmm_core/src/vmbus_unit.rs index 5d6d9b0f398..76024683b0d 100644 --- a/vmm_core/src/vmbus_unit.rs +++ b/vmm_core/src/vmbus_unit.rs @@ -78,8 +78,9 @@ impl VmbusServerHandle { struct VmbusServerUnit(VmbusServer); impl StateUnit for &'_ VmbusServerUnit { - async fn start(&mut self) { + async fn start(&mut self) -> anyhow::Result<()> { self.0.start(); + Ok(()) } async fn stop(&mut self) { @@ -134,8 +135,9 @@ impl ChannelUnit { } impl StateUnit for &'_ ChannelUnit { - async fn start(&mut self) { + async fn start(&mut self) -> anyhow::Result<()> { self.0.start(); + Ok(()) } async fn stop(&mut self) { @@ -205,8 +207,9 @@ impl Inspect for SimpleChannelUnit { } impl StateUnit for &'_ SimpleChannelUnit { - async fn start(&mut self) { + async fn start(&mut self) -> anyhow::Result<()> { self.0.start(); + Ok(()) } async fn stop(&mut self) { diff --git a/vmm_core/src/vmtime_unit.rs b/vmm_core/src/vmtime_unit.rs index a53688cb527..9fa55a3c138 100644 --- a/vmm_core/src/vmtime_unit.rs +++ b/vmm_core/src/vmtime_unit.rs @@ -17,8 +17,9 @@ use vmcore::vmtime::VmTimeKeeper; struct KeeperUnit<'a>(#[inspect(mut)] &'a mut VmTimeKeeper); impl StateUnit for KeeperUnit<'_> { - async fn start(&mut self) { + async fn start(&mut self) -> anyhow::Result<()> { self.0.start().await; + Ok(()) } async fn stop(&mut self) { diff --git a/vmm_core/state_unit/src/lib.rs b/vmm_core/state_unit/src/lib.rs index 87a9525e73d..928730e6a39 100644 --- a/vmm_core/state_unit/src/lib.rs +++ b/vmm_core/state_unit/src/lib.rs @@ -30,6 +30,8 @@ #![forbid(unsafe_code)] +mod start; + use futures::FutureExt; use futures::StreamExt; use futures::future::join_all; @@ -70,7 +72,7 @@ use vmcore::save_restore::SavedStateBlob; #[derive(Debug, MeshPayload)] pub enum StateRequest { /// Start asynchronous operations. - Start(Rpc<(), ()>), + Start(FailableRpc<(), ()>), /// Stop asynchronous operations. Stop(Rpc<(), ()>), @@ -96,7 +98,7 @@ pub enum StateRequest { #[expect(async_fn_in_trait)] // Don't need Send bounds pub trait StateUnit: InspectMut { /// Start asynchronous processing. - async fn start(&mut self); + async fn start(&mut self) -> anyhow::Result<()>; /// Stop asynchronous processing. async fn stop(&mut self); @@ -192,7 +194,7 @@ impl StateRequest { /// Runs this state request against `unit`. pub async fn apply(self, unit: &mut impl StateUnit) { match self { - StateRequest::Start(rpc) => rpc.handle(async |()| unit.start().await).await, + StateRequest::Start(rpc) => rpc.handle_failable(async |()| unit.start().await).await, StateRequest::Stop(rpc) => rpc.handle(async |()| unit.stop().await).await, StateRequest::Reset(rpc) => rpc.handle_failable(async |()| unit.reset().await).await, StateRequest::Save(rpc) => rpc.handle_failable(async |()| unit.save().await).await, @@ -222,6 +224,8 @@ enum State { Resetting, Saving, Restoring, + StartUncertain, + StartFailedStopped, } #[derive(Debug)] @@ -486,36 +490,69 @@ impl StateUnits { self.running } + fn check_no_terminal_units(&self, op: &'static str) -> Result<(), StateTransitionError> { + let inner = self.inner.lock(); + check( + op, + inner + .units + .values() + .filter(|unit| { + matches!( + unit.state, + State::StartUncertain | State::StartFailedStopped + ) + }) + .map(|unit| { + ( + unit.name.clone(), + Err::<(), _>(anyhow::anyhow!( + "unit is in terminal state {:?}", + unit.state + )), + ) + }), + ) + } + /// Starts any units that are individually stopped, because they were added /// via [`StateUnits::add`] while the VM was running. /// /// Does nothing if all units are stopped, via [`StateUnits::stop`]. - pub async fn start_stopped_units(&mut self) { + pub async fn start_stopped_units(&mut self) -> anyhow::Result<()> { if self.is_running() { - self.start().await; + self.start().await?; } + Ok(()) } /// Starts all the state units. - pub async fn start(&mut self) { + pub async fn start(&mut self) -> anyhow::Result<()> { self.restore_allowed = false; - self.run_op( - "start", - None, - State::Stopped, - State::Starting, - State::Running, - StateRequest::Start, - |_, _| Some(()), - |unit| Some(&unit.dependencies), - ) - .await; - self.running = true; + self.start_with_rollback().await } /// Stops all the state units. pub async fn stop(&mut self) { assert!(self.running); + // Make uncertain starts receive Stop and already-cleaned failures skip + // it, then restore both terminal markers after the ordered operation. + let terminal_states = { + let mut inner = self.inner.lock(); + inner + .units + .iter_mut() + .filter_map(|(&id, unit)| { + let state = unit.state; + match state { + State::StartUncertain => unit.state = State::Running, + State::StartFailedStopped => unit.state = State::Stopped, + _ => return None, + } + Some((id, state)) + }) + .collect::>() + }; // Stop units in reverse dependency order so that a dependency is not // stopped before its dependant. self.run_op( @@ -529,6 +566,13 @@ impl StateUnits { |unit| Some(&unit.dependents), ) .await; + let mut inner = self.inner.lock(); + for (id, state) in terminal_states { + if let Some(unit) = inner.units.get_mut(&id) { + assert_eq!(unit.state, State::Stopped); + unit.state = state; + } + } self.running = false; } @@ -537,6 +581,7 @@ impl StateUnits { /// Panics if running. pub async fn reset(&mut self) -> Result<(), StateTransitionError> { assert!(!self.running); + self.check_no_terminal_units("reset")?; self.restore_allowed = false; // Reset in dependency order so that dependants observe their // dependencies' reset state. @@ -563,6 +608,7 @@ impl StateUnits { /// Panics if running. pub async fn save(&mut self) -> Result, StateTransitionError> { assert!(!self.running); + self.check_no_terminal_units("save")?; // Save can occur in any order since it will not observably mutate // state. let r = self @@ -601,6 +647,7 @@ impl StateUnits { states: Vec, ) -> Result<(), StateTransitionError> { assert!(!self.running); + self.check_no_terminal_units("restore")?; if !self.restore_allowed { return Err(StateTransitionError { op: "restore", @@ -801,7 +848,7 @@ impl Inner { let units = if let Some(unit_ids) = unit_ids { unit_ids .iter() - .map(|id| map(*id, &self.units[id])) + .filter_map(|id| self.units.get(id).map(|unit| map(*id, unit))) .collect() } else { self.units.iter().map(|(id, unit)| map(*id, unit)).collect() @@ -1047,6 +1094,8 @@ impl Ready { #[cfg(test)] mod tests { + mod start; + use super::StateTransitionErrorKind; use super::StateUnit; use super::StateUnits; @@ -1079,7 +1128,9 @@ mod tests { struct SavedState(bool); impl StateUnit for TestUnit { - async fn start(&mut self) {} + async fn start(&mut self) -> anyhow::Result<()> { + Ok(()) + } async fn stop(&mut self) {} @@ -1121,7 +1172,9 @@ mod tests { } impl StateUnit for TestUnitSetDep { - async fn start(&mut self) {} + async fn start(&mut self) -> anyhow::Result<()> { + Ok(()) + } async fn stop(&mut self) {} @@ -1161,7 +1214,9 @@ mod tests { } impl StateUnit for FailingUnit { - async fn start(&mut self) {} + async fn start(&mut self) -> anyhow::Result<()> { + Ok(()) + } async fn stop(&mut self) {} async fn reset(&mut self) -> anyhow::Result<()> { @@ -1219,7 +1274,7 @@ mod tests { let state = units.save().await.unwrap(); units.restore(state).await.unwrap(); assert_eq!(calls.load(Ordering::Relaxed), 2); - units.start().await; + units.start().await.unwrap(); units.stop().await; let state = units.save().await.unwrap(); assert!(units.restore(state).await.is_err()); @@ -1293,13 +1348,13 @@ mod tests { .add("b") .spawn(&driver, |recv| run_unit(TestUnit::default(), recv)) .unwrap(); - units.start().await; + units.start().await.unwrap(); let _c = units .add("c") .spawn(&driver, |recv| run_unit(TestUnit::default(), recv)); units.stop().await; - units.start().await; + units.start().await.unwrap(); units.stop().await; @@ -1347,7 +1402,7 @@ mod tests { ) }) .unwrap(); - units.start().await; + units.start().await.unwrap(); units.stop().await; let state = units.save().await.unwrap(); @@ -1457,7 +1512,7 @@ mod tests { ); } - units.start().await; + units.start().await.unwrap(); units.stop().await; a.remove().await; @@ -1467,7 +1522,7 @@ mod tests { assert!(inner.units[&c.handle().id.id].dependencies.is_empty()); } - units.start().await; + units.start().await.unwrap(); units.stop().await; // Removing the remaining unit leaves the set empty. diff --git a/vmm_core/state_unit/src/start.rs b/vmm_core/state_unit/src/start.rs new file mode 100644 index 00000000000..217802fc89a --- /dev/null +++ b/vmm_core/state_unit/src/start.rs @@ -0,0 +1,199 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Fallible start with rollback. If a unit fails to start, its dependents do +//! not start, and the units that did start are stopped again. + +use super::State; +use super::StateRequest; +use super::StateUnits; +use super::state_change; +use futures::future::join_all; +use std::time::Instant; + +impl StateUnits { + /// Starts all stopped units in dependency order. + /// + /// A unit does not start if one of its dependencies fails to start. On any + /// failure, the units that did start are stopped again, and a unit whose + /// start failed or is uncertain is left in a terminal state that fails + /// later starts. + pub(super) async fn start_with_rollback(&mut self) -> anyhow::Result<()> { + enum StartResult { + Started, + DependencyFailed, + Failed(anyhow::Error), + Uncertain(anyhow::Error), + } + + let was_running = self.running; + let mut operations = Vec::new(); + let ready_set; + { + let mut inner = self.inner.lock(); + if let Some(unit) = inner + .units + .values() + .find(|unit| !matches!(unit.state, State::Stopped | State::Running)) + { + anyhow::bail!( + "state unit '{}' cannot be started from terminal state {:?}", + unit.name, + unit.state + ); + } + ready_set = inner.ready_set(None); + for (&id, unit) in &mut inner.units { + if unit.state == State::Running { + ready_set.done(id, true); + continue; + } + assert_eq!( + unit.state, + State::Stopped, + "unit {} is not stopped before start", + unit.name, + ); + let name = unit.name.clone(); + let dependencies = unit.dependencies.iter().copied().collect::>(); + let ready_set = ready_set.clone(); + let start = state_change(name.clone(), unit, StateRequest::Start, Some(())); + operations.push(async move { + if !ready_set.wait("start", id, &dependencies).await { + ready_set.done(id, false); + return (name, id, StartResult::DependencyFailed); + } + + let result = match start.await { + Ok(Some(Ok(()))) => StartResult::Started, + Ok(Some(Err(error))) => StartResult::Failed(error.into()), + Ok(None) => { + StartResult::Uncertain(anyhow::anyhow!("start request was not sent")) + } + Err(error) => StartResult::Uncertain(error.into()), + }; + ready_set.done(id, matches!(result, StartResult::Started)); + (name, id, result) + }); + unit.state = State::Starting; + } + } + + let start = Instant::now(); + let results = join_all(operations).await; + tracing::info!(duration = ?Instant::now() - start, "state-unit start complete"); + + let mut failures = Vec::new(); + let mut started_ids = Vec::new(); + let mut failed_or_uncertain_ids = Vec::new(); + { + let mut inner = self.inner.lock(); + for (name, id, result) in results { + let Some(unit) = inner.units.get_mut(&id) else { + continue; + }; + match result { + StartResult::Started => { + unit.state = State::Running; + started_ids.push(id); + } + StartResult::DependencyFailed => { + unit.state = State::Stopped; + failures.push(format!("{name}: a dependency did not start")); + } + StartResult::Failed(error) => { + unit.state = State::StartUncertain; + failed_or_uncertain_ids.push(id); + failures.push(format!("{name}: {error:#}")); + } + StartResult::Uncertain(error) => { + unit.state = State::StartUncertain; + failed_or_uncertain_ids.push(id); + failures.push(format!("{name}: {error:#}")); + } + } + } + } + + if failures.is_empty() { + self.running = true; + Ok(()) + } else { + failures.extend( + self.stop_after_failed_start( + "failed_start_cleanup", + &failed_or_uncertain_ids, + State::StartUncertain, + State::StartFailedStopped, + ) + .await, + ); + failures.extend( + self.stop_after_failed_start( + "failed_start_rollback", + &started_ids, + State::Running, + State::Stopped, + ) + .await, + ); + self.running = was_running; + anyhow::bail!("state units could not start: {}", failures.join("; ")) + } + } + + async fn stop_after_failed_start( + &self, + op: &'static str, + unit_ids: &[u64], + old_state: State, + stopped_state: State, + ) -> Vec { + let mut operations = Vec::new(); + let ready_set; + { + let mut inner = self.inner.lock(); + ready_set = inner.ready_set(Some(unit_ids)); + for (&id, unit) in inner + .units + .iter_mut() + .filter(|(id, _)| ready_set.0.contains_key(id)) + { + assert_eq!(unit.state, old_state); + let name = unit.name.clone(); + let dependents = unit.dependents.iter().copied().collect::>(); + let ready_set = ready_set.clone(); + let stop = state_change(name.clone(), unit, StateRequest::Stop, Some(())); + operations.push(async move { + ready_set.wait(op, id, &dependents).await; + let result = stop.await; + ready_set.done(id, true); + (name, id, result) + }); + unit.state = State::Stopping; + } + } + + let results = join_all(operations).await; + let mut failures = Vec::new(); + let mut inner = self.inner.lock(); + for (name, id, result) in results { + let Some(unit) = inner.units.get_mut(&id) else { + continue; + }; + assert_eq!(unit.state, State::Stopping); + match result { + Ok(Some(())) => unit.state = stopped_state, + Ok(None) => { + unit.state = State::StartUncertain; + failures.push(format!("{name}: {op} stop request was not sent")); + } + Err(error) => { + unit.state = State::StartUncertain; + failures.push(format!("{name}: {op} stop failed: {error:#}")); + } + } + } + failures + } +} diff --git a/vmm_core/state_unit/src/tests/start.rs b/vmm_core/state_unit/src/tests/start.rs new file mode 100644 index 00000000000..da37724a75a --- /dev/null +++ b/vmm_core/state_unit/src/tests/start.rs @@ -0,0 +1,346 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Tests for fallible start. + +use crate::StateRequest; +use crate::StateUnit; +use crate::StateUnits; +use crate::run_unit; +use futures::StreamExt; +use inspect::InspectMut; +use pal_async::DefaultDriver; +use pal_async::async_test; +use std::sync::Arc; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; +use test_with_tracing::test; +use vmcore::save_restore::RestoreError; +use vmcore::save_restore::SaveError; +use vmcore::save_restore::SavedStateBlob; + +struct StartResultUnit { + started: Arc, + stopped: Arc, + fail: bool, +} + +impl StateUnit for StartResultUnit { + async fn start(&mut self) -> anyhow::Result<()> { + self.started.store(true, Ordering::Relaxed); + anyhow::ensure!(!self.fail, "intentional start failure"); + Ok(()) + } + + async fn stop(&mut self) { + assert!(!self.stopped.swap(true, Ordering::Relaxed)); + } + + async fn reset(&mut self) -> anyhow::Result<()> { + Ok(()) + } + + async fn save(&mut self) -> Result, SaveError> { + Ok(None) + } + + async fn restore(&mut self, _state: SavedStateBlob) -> Result<(), RestoreError> { + Err(RestoreError::SavedStateNotSupported) + } +} + +impl InspectMut for StartResultUnit { + fn inspect_mut(&mut self, req: inspect::Request<'_>) { + req.respond(); + } +} + +#[async_test] +async fn test_start_failure_blocks_dependent(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let independent_started = Arc::new(AtomicBool::new(false)); + let independent_stopped = Arc::new(AtomicBool::new(false)); + let _independent = units + .add("independent") + .spawn(&driver, |recv| { + run_unit( + StartResultUnit { + started: independent_started.clone(), + stopped: independent_stopped.clone(), + fail: false, + }, + recv, + ) + }) + .unwrap(); + let failed_started = Arc::new(AtomicBool::new(false)); + let failed_stopped = Arc::new(AtomicBool::new(false)); + let failed = units + .add("failed") + .spawn(&driver, |recv| { + run_unit( + StartResultUnit { + started: failed_started.clone(), + stopped: failed_stopped.clone(), + fail: true, + }, + recv, + ) + }) + .unwrap(); + let dependent_started = Arc::new(AtomicBool::new(false)); + let _dependent = units + .add("dependent") + .depends_on(failed.handle()) + .spawn(&driver, |recv| { + run_unit( + StartResultUnit { + started: dependent_started.clone(), + stopped: Arc::new(AtomicBool::new(false)), + fail: false, + }, + recv, + ) + }) + .unwrap(); + + let error = units.start().await.unwrap_err(); + assert!(error.to_string().contains("intentional start failure")); + assert!(independent_started.load(Ordering::Relaxed)); + assert!(independent_stopped.load(Ordering::Relaxed)); + assert!(failed_started.load(Ordering::Relaxed)); + assert!(failed_stopped.load(Ordering::Relaxed)); + assert!(!dependent_started.load(Ordering::Relaxed)); + assert!(!units.is_running()); + let retry_error = units.start().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); + assert!(units.reset().await.is_err()); + assert!(units.save().await.is_err()); + assert!(units.restore(Vec::new()).await.is_err()); +} + +#[async_test] +async fn test_stop_cleans_up_failed_start_while_running(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let _running = units + .add("running") + .spawn(&driver, |recv| { + run_unit( + StartResultUnit { + started: Arc::new(AtomicBool::new(false)), + stopped: Arc::new(AtomicBool::new(false)), + fail: false, + }, + recv, + ) + }) + .unwrap(); + units.start().await.unwrap(); + + let failed_stopped = Arc::new(AtomicBool::new(false)); + let _failed = units + .add("failed") + .spawn(&driver, |recv| { + run_unit( + StartResultUnit { + started: Arc::new(AtomicBool::new(false)), + stopped: failed_stopped.clone(), + fail: true, + }, + recv, + ) + }) + .unwrap(); + + units.start_stopped_units().await.unwrap_err(); + assert!(units.is_running()); + let retry_error = units.start_stopped_units().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); + units.stop().await; + assert!(failed_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); + let retry_error = units.start().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); + assert!(units.reset().await.is_err()); + assert!(units.save().await.is_err()); + assert!(units.restore(Vec::new()).await.is_err()); +} + +#[async_test] +async fn test_uncertain_start_is_cleaned_once_and_remains_terminal(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let _running = units + .add("running") + .spawn(&driver, |recv| { + run_unit( + StartResultUnit { + started: Arc::new(AtomicBool::new(false)), + stopped: Arc::new(AtomicBool::new(false)), + fail: false, + }, + recv, + ) + }) + .unwrap(); + units.start().await.unwrap(); + + let uncertain_started = Arc::new(AtomicBool::new(false)); + let uncertain_stopped = Arc::new(AtomicBool::new(false)); + let stop_requests = Arc::new(AtomicUsize::new(0)); + let _uncertain = units + .add("uncertain") + .spawn(&driver, |mut recv| { + let stop_requests = stop_requests.clone(); + let mut unit = StartResultUnit { + started: uncertain_started.clone(), + stopped: uncertain_stopped.clone(), + fail: false, + }; + async move { + while let Some(request) = recv.next().await { + match request { + StateRequest::Start(response) => { + unit.started.store(true, Ordering::Relaxed); + drop(response); + } + StateRequest::Stop(response) + if stop_requests.fetch_add(1, Ordering::Relaxed) == 0 => + { + drop(response); + } + request => request.apply(&mut unit).await, + } + } + unit + } + }) + .unwrap(); + + units.start_stopped_units().await.unwrap_err(); + assert!(uncertain_started.load(Ordering::Relaxed)); + assert_eq!(stop_requests.load(Ordering::Relaxed), 1); + assert!(!uncertain_stopped.load(Ordering::Relaxed)); + assert!(units.is_running()); + let retry_error = units.start_stopped_units().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); + + units.stop().await; + assert_eq!(stop_requests.load(Ordering::Relaxed), 2); + assert!(uncertain_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); + let retry_error = units.start().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); + assert!(units.reset().await.is_err()); + assert!(units.save().await.is_err()); + assert!(units.restore(Vec::new()).await.is_err()); +} + +#[async_test] +async fn test_initial_uncertain_start_is_cleaned(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let uncertain_started = Arc::new(AtomicBool::new(false)); + let uncertain_stopped = Arc::new(AtomicBool::new(false)); + let _uncertain = units + .add("uncertain") + .spawn(&driver, |mut recv| { + let mut unit = StartResultUnit { + started: uncertain_started.clone(), + stopped: uncertain_stopped.clone(), + fail: false, + }; + async move { + while let Some(request) = recv.next().await { + match request { + StateRequest::Start(response) => { + unit.started.store(true, Ordering::Relaxed); + drop(response); + } + request => request.apply(&mut unit).await, + } + } + unit + } + }) + .unwrap(); + + units.start().await.unwrap_err(); + assert!(uncertain_started.load(Ordering::Relaxed)); + assert!(uncertain_stopped.load(Ordering::Relaxed)); + assert!(!units.is_running()); + let retry_error = units.start().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); + assert!(units.reset().await.is_err()); + assert!(units.save().await.is_err()); + assert!(units.restore(Vec::new()).await.is_err()); +} + +#[async_test] +async fn test_started_unit_exit_during_rollback_returns_error(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let _exited = units + .add("exited") + .spawn(&driver, |mut recv| async move { + let unit = StartResultUnit { + started: Arc::new(AtomicBool::new(true)), + stopped: Arc::new(AtomicBool::new(false)), + fail: false, + }; + let request = recv.next().await.unwrap(); + let StateRequest::Start(response) = request else { + panic!("unexpected request"); + }; + response.handle_failable_sync(|()| anyhow::Ok(())); + unit + }) + .unwrap(); + let _failed = units + .add("failed") + .spawn(&driver, |recv| { + run_unit( + StartResultUnit { + started: Arc::new(AtomicBool::new(false)), + stopped: Arc::new(AtomicBool::new(false)), + fail: true, + }, + recv, + ) + }) + .unwrap(); + + let error = units.start().await.unwrap_err(); + assert!( + error + .to_string() + .contains("failed_start_rollback stop failed") + ); + assert!(!units.is_running()); + let retry_error = units.start().await.unwrap_err(); + assert!(retry_error.to_string().contains("terminal state")); + assert!(units.reset().await.is_err()); + assert!(units.save().await.is_err()); + assert!(units.restore(Vec::new()).await.is_err()); +} + +#[async_test] +async fn test_rollback_ready_set_ignores_removed_unit(driver: DefaultDriver) { + let units = StateUnits::new(); + let unit = units + .add("removed") + .spawn(&driver, |recv| { + run_unit( + StartResultUnit { + started: Arc::new(AtomicBool::new(false)), + stopped: Arc::new(AtomicBool::new(false)), + fail: false, + }, + recv, + ) + }) + .unwrap(); + let id = unit.handle().id.id; + unit.remove().await; + + let ready_set = units.inner.lock().ready_set(Some(&[id])); + assert!(ready_set.0.is_empty()); +} diff --git a/vmm_core/vmotherboard/src/chipset/backing/arc_mutex/state_unit.rs b/vmm_core/vmotherboard/src/chipset/backing/arc_mutex/state_unit.rs index 5f296e7e28f..2c9550410fd 100644 --- a/vmm_core/vmotherboard/src/chipset/backing/arc_mutex/state_unit.rs +++ b/vmm_core/vmotherboard/src/chipset/backing/arc_mutex/state_unit.rs @@ -139,13 +139,14 @@ impl ArcMutexChipsetDeviceUnit { } impl StateUnit for ArcMutexChipsetDeviceUnit { - async fn start(&mut self) { + async fn start(&mut self) -> anyhow::Result<()> { self.running = true; // Poll the device at least once. let mut device = self.device.lock(); device.start(); device.poll_device(&mut Context::from_waker(&waker_ref(&self.poll_event))); + Ok(()) } async fn stop(&mut self) { diff --git a/vmm_core/vmotherboard/src/chipset/builder/mod.rs b/vmm_core/vmotherboard/src/chipset/builder/mod.rs index 788e80f98b1..a8e47c0fae0 100644 --- a/vmm_core/vmotherboard/src/chipset/builder/mod.rs +++ b/vmm_core/vmotherboard/src/chipset/builder/mod.rs @@ -379,7 +379,9 @@ mod chipset_unit { pub struct ChipsetUnit<'a>(pub &'a Chipset); impl StateUnit for ChipsetUnit<'_> { - async fn start(&mut self) {} + async fn start(&mut self) -> anyhow::Result<()> { + Ok(()) + } async fn stop(&mut self) {} diff --git a/vmm_core/vmotherboard/src/chipset/line_sets.rs b/vmm_core/vmotherboard/src/chipset/line_sets.rs index dd6d73f7d7b..9d8517a8176 100644 --- a/vmm_core/vmotherboard/src/chipset/line_sets.rs +++ b/vmm_core/vmotherboard/src/chipset/line_sets.rs @@ -60,7 +60,9 @@ impl LineSets { struct LineSetUnit<'a>(&'a LineSet); impl StateUnit for LineSetUnit<'_> { - async fn start(&mut self) {} + async fn start(&mut self) -> anyhow::Result<()> { + Ok(()) + } async fn stop(&mut self) {}