diff --git a/Guide/src/user_guide/openvmm/snapshots.md b/Guide/src/user_guide/openvmm/snapshots.md index c1ca637e07e..6d4faf3c527 100644 --- a/Guide/src/user_guide/openvmm/snapshots.md +++ b/Guide/src/user_guide/openvmm/snapshots.md @@ -136,6 +136,11 @@ requests report a reply-channel error because their result types cannot carry an application error. Retry a rejected operation after the boundary is released; ordinary paused VMs are not subject to this restriction. +The snapshot lifecycle RPCs must follow the capture's order. The worker +rejects a snapshot quiesce outside a guest-requested boundary, without +stopping the VM, and refuses to release the boundary of a VM that it has +quiesced, which only the rollback of a failed capture runs again. + ## Restoring a snapshot To restore, pass the snapshot directory with `--restore-snapshot`: diff --git a/openvmm/openvmm_core/src/worker/dispatch/microvm.rs b/openvmm/openvmm_core/src/worker/dispatch/microvm.rs index 3a523b7b9b5..8c104360819 100644 --- a/openvmm/openvmm_core/src/worker/dispatch/microvm.rs +++ b/openvmm/openvmm_core/src/worker/dispatch/microvm.rs @@ -268,12 +268,24 @@ impl LoadedVm { } } + /// Releases the guest-requested snapshot boundary of a running VM: resumes + /// host input, completes the guest's snapshot transaction, and lets the + /// vCPUs run. + /// + /// The boundary of a quiesced VM, whose snapshot the controller holds, is + /// released only by [`Self::resume_after_failed_snapshot`], which starts + /// the VM's state units first: the boundary keeps refusing an ordinary + /// resume until then. pub(super) async fn release_snapshot_boundary(&mut self) -> anyhow::Result<()> { // Resume host input only while a boundary holds it. anyhow::ensure!( self.snapshot_boundary.transaction_complete.is_some(), "no active microVM snapshot boundary" ); + anyhow::ensure!( + self.running, + "the snapshot boundary of a quiesced microVM is released only by its rollback" + ); self.state_units.resume_input_after_save().await?; let transaction_complete = self .snapshot_boundary @@ -300,6 +312,11 @@ impl LoadedVm { /// Quiesces the VM for a guest-requested snapshot and saves its state, /// leaving it stopped. /// + /// The VM must be running at a snapshot boundary that its guest requested, + /// which holds the stopped VM until the snapshot's rollback: outside one, + /// the request is rejected and changes nothing, since an ordinary resume + /// would run the VM again after its state was saved. + /// /// The quiesce stops every state unit and has no deadline: a unit that /// never stops stalls the capture until the process that drives the VM /// ends it, with nothing published and the source never resumed. Each @@ -323,6 +340,13 @@ impl LoadedVm { RemoteError::new(anyhow::anyhow!("VM is already stopped")), )); } + if self.snapshot_boundary.stop_guard.is_none() { + return Err(openvmm_defs::rpc::SnapshotQuiesceError::Rejected( + RemoteError::new(anyhow::anyhow!( + "guest-requested snapshot quiesce requires an active snapshot boundary" + )), + )); + } let quiesce = openvmm_defs::profile::ProfileSpan::start(); self.state_units.stop().await; @@ -395,7 +419,8 @@ impl LoadedVm { } /// Resumes the VM after a snapshot failure that followed a complete - /// quiesce, and releases the snapshot boundary. + /// quiesce, and releases the snapshot boundary. This is the only request + /// that runs a quiesced VM again. /// /// A unit whose task does not answer leaves the units that did start /// stopped again, and fails the rollback. diff --git a/petri/src/vm/openvmm/microvm.rs b/petri/src/vm/openvmm/microvm.rs index 22e3c678c72..28dfcb29ef9 100644 --- a/petri/src/vm/openvmm/microvm.rs +++ b/petri/src/vm/openvmm/microvm.rs @@ -3,6 +3,8 @@ //! OpenVMM-specific microVM construction and runtime support. +use super::PetriVmConfigOpenVmm; +use super::PetriVmResourcesOpenVmm; use super::construct::SerialData; use super::runtime::PetriVmInner; use crate::Firmware; @@ -11,15 +13,21 @@ use anyhow::Context; use chipset_resources::microvm::MicrovmPortbHandle; use chipset_resources::microvm::MicrovmPortbTimeAbi; use chipset_resources::microvm::MicrovmShutdownHandle; +use chipset_resources::microvm::MicrovmSnapshotBoundaryRequest; use chipset_resources::microvm::MicrovmSnapshotRequestHandle; +use chipset_resources::microvm::MicrovmSnapshotScratchPolicy; use chipset_resources::pit::PitDeviceHandle; use cpu_profile::HostCpuSignature; +use mesh::rpc::RpcSend; use openvmm_defs::config::Config; use openvmm_defs::config::LinuxDirectBootMode; use openvmm_defs::config::LinuxIsolationConfig; use openvmm_defs::config::LoadMode; use openvmm_defs::microvm::MachineProfile; use openvmm_defs::microvm::MicrovmConfig; +use openvmm_defs::rpc::SnapshotQuiesceError; +use openvmm_defs::rpc::SnapshotSaveResponse; +use openvmm_defs::rpc::VmRpc; use openvmm_defs::time_abi::TimeAbiParameters; use pal_async::DefaultDriver; use pal_async::socket::PolledSocket; @@ -39,6 +47,65 @@ pub(super) struct RuntimeResources { output: mesh::Receiver>, /// The output that followed the marker of the latest wait. unread: Vec, + /// The guest-requested snapshot boundaries that the VM worker reports, + /// when the test is the VM's snapshot controller. + snapshot_boundaries: Option>, + /// The VM worker's ends of the snapshot boundary channels, until the + /// worker launches. + worker_snapshot_channels: Option, +} + +/// The VM worker's ends of the channels of guest-requested snapshot +/// boundaries. +pub(crate) struct SnapshotBoundaryChannels { + /// The guest's snapshot requests, from the snapshot-request port. + pub(crate) requests: mesh::Receiver, + /// Reports each boundary that the worker establishes to the snapshot + /// controller. + pub(crate) ready: mesh::Sender, +} + +impl PetriVmConfigOpenVmm { + /// Makes the test the snapshot controller of a microVM: the VM worker + /// holds each guest snapshot request at a snapshot boundary, with the + /// vCPUs stopped, and reports it, which + /// [`PetriVmOpenVmm::wait_for_microvm_snapshot_boundary`](super::PetriVmOpenVmm::wait_for_microvm_snapshot_boundary) + /// waits for. The test then sends the snapshot requests that a controller + /// sends, such as + /// [`PetriVmOpenVmm::release_microvm_snapshot_boundary`](super::PetriVmOpenVmm::release_microvm_snapshot_boundary). + /// Otherwise the snapshot-request port lets each guest request continue at + /// once, as it does when the VM has no snapshot destination. + pub fn with_microvm_snapshot_boundary(mut self) -> Self { + let resources = self + .resources + .microvm + .as_mut() + .expect("snapshot boundaries require a microVM with its portb console"); + let port = self + .config + .chipset_devices + .iter_mut() + .find(|device| device.name == MicrovmSnapshotRequestHandle::ID) + .expect("a microVM has a snapshot-request port"); + let (notify, requests) = mesh::channel(); + let (ready, boundaries) = mesh::channel(); + port.resource = MicrovmSnapshotRequestHandle { + notify: Some(notify), + } + .into_resource(); + resources.snapshot_boundaries = Some(boundaries); + resources.worker_snapshot_channels = Some(SnapshotBoundaryChannels { requests, ready }); + self + } +} + +/// Takes the VM worker's ends of the snapshot boundary channels of +/// [`PetriVmConfigOpenVmm::with_microvm_snapshot_boundary`], for the worker's +/// launch. +pub(super) fn take_worker_snapshot_channels( + resources: &mut PetriVmResourcesOpenVmm, +) -> Option { + resources.microvm.as_mut()?.worker_snapshot_channels.take() } pub(super) struct PortbSerial { @@ -369,6 +436,8 @@ pub(super) fn configure_serial( runtime: RuntimeResources { output, unread: Vec::new(), + snapshot_boundaries: None, + worker_snapshot_channels: None, }, }), }) @@ -474,7 +543,7 @@ impl PetriVmInner { marker: &[u8], ) -> anyhow::Result<()> { anyhow::ensure!(!marker.is_empty(), "microVM portb marker cannot be empty"); - let RuntimeResources { output, unread } = self + let RuntimeResources { output, unread, .. } = self .resources .microvm .as_mut() @@ -503,6 +572,55 @@ impl PetriVmInner { .await .map_err(anyhow::Error::from) } + + /// Waits for the VM worker to hold a guest snapshot request at its + /// snapshot boundary, and returns the request's scratch policy. + pub(super) async fn wait_for_microvm_snapshot_boundary( + &mut self, + ) -> anyhow::Result { + let boundaries = self + .resources + .microvm + .as_mut() + .and_then(|resources| resources.snapshot_boundaries.as_mut()) + .context("the test is not the microVM's snapshot controller")?; + boundaries + .recv() + .await + .context("the VM worker closed its snapshot boundary channel") + } + + /// Quiesces the VM for the snapshot of its boundary, as a snapshot + /// controller does, and returns the VM worker's response. + pub(super) async fn quiesce_microvm_for_snapshot( + &self, + ) -> anyhow::Result> { + Ok(self + .worker + .rpc() + .call(VmRpc::QuiesceForSnapshot, ()) + .await?) + } + + /// Releases the VM's snapshot boundary, as a snapshot controller does when + /// it takes no snapshot. + pub(super) async fn release_microvm_snapshot_boundary(&self) -> anyhow::Result<()> { + Ok(self + .worker + .rpc() + .call_failable(VmRpc::ReleaseSnapshotBoundary, ()) + .await?) + } + + /// Rolls the VM's snapshot back, as a snapshot controller does when a + /// snapshot fails after its quiesce: the VM runs again from its boundary. + pub(super) async fn resume_microvm_after_failed_snapshot(&self) -> anyhow::Result<()> { + Ok(self + .worker + .rpc() + .call_failable(VmRpc::ResumeAfterFailedSnapshot, ()) + .await?) + } } #[cfg(test)] diff --git a/petri/src/vm/openvmm/runtime.rs b/petri/src/vm/openvmm/runtime.rs index 2bfc51eced9..80d1dc7ddf0 100644 --- a/petri/src/vm/openvmm/runtime.rs +++ b/petri/src/vm/openvmm/runtime.rs @@ -17,6 +17,7 @@ use crate::worker::Worker; use anyhow::Context; use async_trait::async_trait; use chipset_resources::acpi_power_button::PowerButtonRpc; +use chipset_resources::microvm::MicrovmSnapshotScratchPolicy; use framebuffer::View; use futures::FutureExt; use futures_concurrency::future::Race; @@ -30,6 +31,8 @@ use mesh::rpc::RpcError; use mesh::rpc::RpcSend; use mesh_process::Mesh; use openvmm_defs::rpc::PulseSaveRestoreError; +use openvmm_defs::rpc::SnapshotQuiesceError; +use openvmm_defs::rpc::SnapshotSaveResponse; use pal_async::socket::PolledSocket; use petri_artifacts_core::ResolvedArtifact; #[cfg(target_os = "linux")] @@ -390,6 +393,32 @@ impl PetriVmOpenVmm { /// Perform one pulse save/restore operation. pub async fn pulse_save_restore(&mut self) -> anyhow::Result<()> ); + petri_vm_fn!( + /// Wait for the VM worker to hold a guest snapshot request of a microVM + /// at its snapshot boundary, with the vCPUs stopped, and return the + /// request's scratch policy. The test must be the VM's snapshot + /// controller, as + /// [`PetriVmConfigOpenVmm::with_microvm_snapshot_boundary`](super::PetriVmConfigOpenVmm::with_microvm_snapshot_boundary) + /// makes it. + pub async fn wait_for_microvm_snapshot_boundary(&mut self) -> anyhow::Result + ); + petri_vm_fn!( + /// Quiesce a microVM for the snapshot of its boundary, as a snapshot + /// controller does, and return the VM worker's response: the saved + /// state, or why the worker did not save it. + pub async fn quiesce_microvm_for_snapshot(&mut self) -> anyhow::Result> + ); + petri_vm_fn!( + /// Release a microVM's snapshot boundary, as a snapshot controller does + /// when it takes no snapshot. + pub async fn release_microvm_snapshot_boundary(&mut self) -> anyhow::Result<()> + ); + petri_vm_fn!( + /// Roll a microVM's snapshot back, as a snapshot controller does when a + /// snapshot fails after its quiesce: the VM runs again from its + /// boundary. + pub async fn resume_microvm_after_failed_snapshot(&mut self) -> anyhow::Result<()> + ); /// Wrap the provided future in a race with the worker process's halt /// notification channel. This is useful for preventing a future from diff --git a/petri/src/vm/openvmm/start.rs b/petri/src/vm/openvmm/start.rs index f14b1f389a1..25e000c4f2d 100644 --- a/petri/src/vm/openvmm/start.rs +++ b/petri/src/vm/openvmm/start.rs @@ -177,9 +177,12 @@ impl PetriVmConfigOpenVmm { ); } - let (worker, halt_notif) = Worker::launch(&host, config, shared_memory) - .await - .context("failed to launch vm worker")?; + let microvm_snapshot_boundary = + super::microvm::take_worker_snapshot_channels(&mut resources); + let (worker, halt_notif) = + Worker::launch(&host, config, shared_memory, microvm_snapshot_boundary) + .await + .context("failed to launch vm worker")?; let worker = Arc::new(worker); diff --git a/petri/src/worker.rs b/petri/src/worker.rs index 297722662a7..bd9905ec12d 100644 --- a/petri/src/worker.rs +++ b/petri/src/worker.rs @@ -25,6 +25,7 @@ impl Worker { host: &WorkerHost, cfg: Config, shared_memory: Option, + microvm_snapshot_boundary: Option, ) -> anyhow::Result<(Self, mesh::Receiver)> { let (vm_rpc, rpc_recv) = mesh::channel(); let (notify_send, notify_recv) = mesh::channel(); @@ -34,6 +35,9 @@ impl Worker { openvmm_helpers::hypervisor::choose_hypervisor, openvmm_helpers::hypervisor::microvm::choose_microvm_hypervisor, )?; + let (snapshot_boundary_requests, snapshot_ready) = microvm_snapshot_boundary + .map(|channels| (channels.requests, channels.ready)) + .unzip(); let params = VmWorkerParameters { hypervisor, cfg, @@ -41,8 +45,8 @@ impl Worker { shared_memory, shared_memory_copy_on_write: false, snapshot_restore_guards: None, - snapshot_boundary_requests: None, - snapshot_ready: None, + snapshot_boundary_requests, + snapshot_ready, restore_time: None, restore_ready_sink: None, restore_gate_timeout: None, @@ -65,6 +69,11 @@ impl Worker { self.rpc.call(VmRpc::Pause, ()).await } + /// The VM worker's management RPC channel. + pub(crate) fn rpc(&self) -> &mesh::Sender { + &self.rpc + } + pub(crate) async fn resume(&self) -> anyhow::Result { Ok(self.rpc.call_failable(VmRpc::Resume, ()).await?) } diff --git a/vmm_tests/vmm_tests/tests/tests/x86_64/microvm.rs b/vmm_tests/vmm_tests/tests/tests/x86_64/microvm.rs index d0f88429e8e..661dca3b6f3 100644 --- a/vmm_tests/vmm_tests/tests/tests/x86_64/microvm.rs +++ b/vmm_tests/vmm_tests/tests/tests/x86_64/microvm.rs @@ -5,12 +5,26 @@ use anyhow::Context; use mesh::CancelContext; +use openvmm_defs::rpc::SnapshotQuiesceError; use petri::PetriHaltReason; +use petri::PetriVm; use petri::PetriVmBuilder; use petri::openvmm::OpenVmmPetriBackend; use std::time::Duration; use vmm_test_macros::openvmm_test_no_agent; +/// Time for a microVM guest to boot and reach its snapshot request. +const BOOT_TIMEOUT: Duration = Duration::from_secs(120); +/// Time for a running microVM guest to take its next step. +const GUEST_TIMEOUT: Duration = Duration::from_secs(60); +/// The power-off status of the guests of the snapshot controller tests. +const GUEST_STATUS: u8 = 7; +/// The guest's request for a snapshot with fresh scratch, a write of 0 to the +/// snapshot-request port, 0x605. It returns once the snapshot boundary that +/// holds it is released. +const SNAPSHOT_REQUEST: &str = + "printf '\\x00' | dd of=/dev/port bs=1 seek=1541 count=1 conv=notrunc 2>/dev/null\n"; + fn portb_output(bytes: &[u8]) -> anyhow::Result { use std::fmt::Write as _; @@ -115,3 +129,232 @@ async fn phase_1_lifecycle(config: PetriVmBuilder) -> anyho vm.teardown().await } + +/// Returns the guest's command that powers the microVM off with `status`, a +/// write to the shutdown port, 0x604. +fn power_off(status: u8) -> String { + format!( + "printf '\\{status:03o}' | dd of=/dev/port bs=1 seek=1540 count=1 conv=notrunc 2>/dev/null\n" + ) +} + +/// Boots a 1-vCPU microVM whose init runs `script` and then idles, with the +/// test as its snapshot controller. +async fn run_snapshot_controller_guest( + config: PetriVmBuilder, + script: &str, +) -> anyhow::Result> { + let script = format!( + "#!/bin/busybox sh\n\ + mount -t devtmpfs devtmpfs /dev 2>/dev/null || true\n\ + {script}\ + while :; do sleep 1; done\n" + ); + let config = config.with_microvm_machine(1); + let initrd = config.prepare_initrd_with_file("microvm-test", script.as_bytes(), 0o100755)?; + config + .with_prebuilt_initrd(initrd) + .modify_backend(|config| config.with_microvm_snapshot_boundary()) + .run_without_agent() + .await +} + +/// Waits for the VM worker to hold the guest's snapshot request at a snapshot +/// boundary. +async fn wait_for_snapshot_boundary(vm: &mut PetriVm) -> anyhow::Result<()> { + CancelContext::new() + .with_timeout(BOOT_TIMEOUT) + .until_cancelled(vm.backend().wait_for_microvm_snapshot_boundary()) + .await + .context("the guest's snapshot request reached no snapshot boundary")??; + Ok(()) +} + +/// Waits for the guest to write `marker` to portb, failing with `failure`. +async fn wait_for_guest_output( + vm: &mut PetriVm, + marker: &[u8], + failure: &'static str, +) -> anyhow::Result<()> { + CancelContext::new() + .with_timeout(GUEST_TIMEOUT) + .until_cancelled(vm.backend().wait_for_microvm_portb_bytes(marker)) + .await + .context(failure)? +} + +/// Waits for the guest to power the microVM off with `status`. +async fn wait_for_power_off( + vm: &mut PetriVm, + status: u8, +) -> anyhow::Result<()> { + let halt = CancelContext::new() + .with_timeout(GUEST_TIMEOUT) + .until_cancelled(vm.wait_for_halt()) + .await + .context("timed out waiting for the guest to power off")??; + assert_eq!(halt.reason, PetriHaltReason::PowerOff); + assert!( + halt.detail.contains(&format!("code: {status}")), + "expected microVM shutdown status {status}, got {}", + halt.detail + ); + Ok(()) +} + +/// Returns the state of the VM worker's partition state unit, which runs the +/// vCPUs: `running`, or `stopped` once the VM quiesced. +async fn partition_unit_state(vm: &PetriVm) -> anyhow::Result { + let node = vm.inspect_vmm("partition/unit_state").await?; + let state: serde_json::Value = serde_json::from_str(&node.json().to_string()) + .context("failed to parse the partition's unit state")?; + state + .as_str() + .map(str::to_owned) + .context("the partition's unit state is not a string") +} + +/// The test, as the microVM's snapshot controller, takes each of the +/// controller's ways out of a guest-requested snapshot boundary: it releases +/// the first boundary without a snapshot, and quiesces the VM for the second +/// one's snapshot, then rolls the snapshot back. Each time the guest +/// continues after its snapshot request, and an ordinary resume cannot pass +/// the boundary. +#[openvmm_test_no_agent(linux_direct_x64)] +async fn snapshot_boundary_release_and_rollback( + config: PetriVmBuilder, +) -> anyhow::Result<()> { + const RELEASED: &[u8] = b"SNAPSHOT-RELEASED\n"; + const ROLLED_BACK: &[u8] = b"SNAPSHOT-ROLLED-BACK\n"; + + let mut vm = run_snapshot_controller_guest( + config, + &format!( + "{SNAPSHOT_REQUEST}{}{SNAPSHOT_REQUEST}{}{}", + portb_output(RELEASED)?, + portb_output(ROLLED_BACK)?, + power_off(GUEST_STATUS), + ), + ) + .await?; + + wait_for_snapshot_boundary(&mut vm).await?; + vm.backend() + .resume() + .await + .expect_err("an ordinary resume passed a snapshot boundary"); + vm.backend().release_microvm_snapshot_boundary().await?; + wait_for_guest_output( + &mut vm, + RELEASED, + "the guest did not continue after its snapshot boundary's release", + ) + .await?; + + wait_for_snapshot_boundary(&mut vm).await?; + let snapshot = vm + .backend() + .quiesce_microvm_for_snapshot() + .await? + .context("the snapshot quiesce failed")?; + assert!(!snapshot.state_unit_names.is_empty()); + vm.backend() + .resume() + .await + .expect_err("an ordinary resume passed a snapshot boundary"); + vm.backend().resume_microvm_after_failed_snapshot().await?; + wait_for_guest_output( + &mut vm, + ROLLED_BACK, + "the guest did not continue after its snapshot's rollback", + ) + .await?; + + wait_for_power_off(&mut vm, GUEST_STATUS).await?; + vm.teardown().await +} + +/// Once the VM worker has quiesced a microVM for the snapshot of its +/// snapshot boundary, only the snapshot's rollback runs the VM again: the +/// boundary's release, which a snapshot controller sends only when it takes +/// no snapshot, is refused, and so is an ordinary resume, so the VM stays +/// stopped until the rollback. +#[openvmm_test_no_agent(linux_direct_x64)] +async fn snapshot_quiesce_resumes_only_on_rollback( + config: PetriVmBuilder, +) -> anyhow::Result<()> { + const CONTINUED: &[u8] = b"SNAPSHOT-CONTINUED\n"; + /// How long the quiesced VM is watched for running again. + const STOPPED_WINDOW: Duration = Duration::from_secs(5); + + let mut vm = run_snapshot_controller_guest( + config, + &format!( + "{SNAPSHOT_REQUEST}{}{}", + portb_output(CONTINUED)?, + power_off(GUEST_STATUS), + ), + ) + .await?; + + wait_for_snapshot_boundary(&mut vm).await?; + let snapshot = vm + .backend() + .quiesce_microvm_for_snapshot() + .await? + .context("the snapshot quiesce failed")?; + assert!(!snapshot.state_unit_names.is_empty()); + + let release = vm.backend().release_microvm_snapshot_boundary().await; + let resume = vm.backend().resume().await; + let output = CancelContext::new() + .with_timeout(STOPPED_WINDOW) + .until_cancelled(vm.backend().wait_for_microvm_portb_bytes(CONTINUED)) + .await; + let state = partition_unit_state(&vm).await?; + assert!( + release.is_err() && resume.is_err() && output.is_err() && state == "stopped", + "the quiesced VM ran again without the snapshot's rollback: the boundary's \ + release returned {release:?}, the ordinary resume returned {resume:?}, the \ + guest's continuation within {STOPPED_WINDOW:?} was {output:?}, and the \ + partition is {state}" + ); + + vm.backend().resume_microvm_after_failed_snapshot().await?; + wait_for_guest_output( + &mut vm, + CONTINUED, + "the guest did not continue after its snapshot's rollback", + ) + .await?; + wait_for_power_off(&mut vm, GUEST_STATUS).await?; + vm.teardown().await +} + +/// The VM worker quiesces a microVM for a snapshot only at a snapshot +/// boundary that its guest requested: outside one, the quiesce is rejected +/// and leaves the VM running, since no boundary would then keep an ordinary +/// resume from running the VM again after its state was saved. +#[openvmm_test_no_agent(linux_direct_x64)] +async fn snapshot_quiesce_requires_a_boundary( + config: PetriVmBuilder, +) -> anyhow::Result<()> { + const READY: &[u8] = b"SNAPSHOT-GUEST-READY\n"; + + let mut vm = run_snapshot_controller_guest(config, &portb_output(READY)?).await?; + + CancelContext::new() + .with_timeout(BOOT_TIMEOUT) + .until_cancelled(vm.backend().wait_for_microvm_portb_bytes(READY)) + .await + .context("timed out waiting for the microVM guest to boot")??; + let quiesce = vm.backend().quiesce_microvm_for_snapshot().await?; + let state = partition_unit_state(&vm).await?; + assert!( + matches!(quiesce, Err(SnapshotQuiesceError::Rejected(_))) && state == "running", + "a snapshot quiesce outside a snapshot boundary returned {:?}, and left the \ + partition {state}", + quiesce.map(|snapshot| snapshot.state_unit_names) + ); + vm.teardown().await +}