From b5d20bf84dbd279b766994e1b86ce9b0bd6d30b4 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Mon, 9 Nov 2026 23:14:42 -0800 Subject: [PATCH 1/2] petri: let a microVM test be its snapshot controller A microVM's guest requests a snapshot by writing to the snapshot-request port, and the VM worker then holds it at a snapshot boundary, with host input gated and the vCPUs stopped, until its snapshot controller, the VM controller of openvmm_entry, releases the boundary without a snapshot, or quiesces the VM for the snapshot and then ends it or rolls the snapshot back. Petri launched the VM worker without the boundary channels, so the port let each request continue at once, as it does without a snapshot destination, and no VMM test could take the controller's part, let alone send the controller's requests in an order that the controller does not. PetriVmConfigOpenVmm::with_microvm_snapshot_boundary gives the port and the VM worker the boundary channels, and keeps the worker's reports of its boundaries for the test. PetriVmOpenVmm then waits for a boundary with wait_for_microvm_snapshot_boundary, and sends the controller's requests with quiesce_microvm_for_snapshot, release_microvm_snapshot_boundary, and resume_microvm_after_failed_snapshot. A VM launched without the option is launched as before. Test: as a microVM's snapshot controller, the test releases the guest's first snapshot boundary without a snapshot, and quiesces the VM for the second one's snapshot and then rolls the snapshot back. Each time, an ordinary resume does not pass the boundary, and the guest continues after its request once the test lets it. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- petri/src/vm/openvmm/microvm.rs | 120 ++++++++++++++- petri/src/vm/openvmm/runtime.rs | 29 ++++ petri/src/vm/openvmm/start.rs | 9 +- petri/src/worker.rs | 13 +- .../vmm_tests/tests/tests/x86_64/microvm.rs | 145 ++++++++++++++++++ 5 files changed, 310 insertions(+), 6 deletions(-) diff --git a/petri/src/vm/openvmm/microvm.rs b/petri/src/vm/openvmm/microvm.rs index 22e3c678c72..28dfcb29ef9 100644 --- a/petri/src/vm/openvmm/microvm.rs +++ b/petri/src/vm/openvmm/microvm.rs @@ -3,6 +3,8 @@ //! OpenVMM-specific microVM construction and runtime support. +use super::PetriVmConfigOpenVmm; +use super::PetriVmResourcesOpenVmm; use super::construct::SerialData; use super::runtime::PetriVmInner; use crate::Firmware; @@ -11,15 +13,21 @@ use anyhow::Context; use chipset_resources::microvm::MicrovmPortbHandle; use chipset_resources::microvm::MicrovmPortbTimeAbi; use chipset_resources::microvm::MicrovmShutdownHandle; +use chipset_resources::microvm::MicrovmSnapshotBoundaryRequest; use chipset_resources::microvm::MicrovmSnapshotRequestHandle; +use chipset_resources::microvm::MicrovmSnapshotScratchPolicy; use chipset_resources::pit::PitDeviceHandle; use cpu_profile::HostCpuSignature; +use mesh::rpc::RpcSend; use openvmm_defs::config::Config; use openvmm_defs::config::LinuxDirectBootMode; use openvmm_defs::config::LinuxIsolationConfig; use openvmm_defs::config::LoadMode; use openvmm_defs::microvm::MachineProfile; use openvmm_defs::microvm::MicrovmConfig; +use openvmm_defs::rpc::SnapshotQuiesceError; +use openvmm_defs::rpc::SnapshotSaveResponse; +use openvmm_defs::rpc::VmRpc; use openvmm_defs::time_abi::TimeAbiParameters; use pal_async::DefaultDriver; use pal_async::socket::PolledSocket; @@ -39,6 +47,65 @@ pub(super) struct RuntimeResources { output: mesh::Receiver>, /// The output that followed the marker of the latest wait. unread: Vec, + /// The guest-requested snapshot boundaries that the VM worker reports, + /// when the test is the VM's snapshot controller. + snapshot_boundaries: Option>, + /// The VM worker's ends of the snapshot boundary channels, until the + /// worker launches. + worker_snapshot_channels: Option, +} + +/// The VM worker's ends of the channels of guest-requested snapshot +/// boundaries. +pub(crate) struct SnapshotBoundaryChannels { + /// The guest's snapshot requests, from the snapshot-request port. + pub(crate) requests: mesh::Receiver, + /// Reports each boundary that the worker establishes to the snapshot + /// controller. + pub(crate) ready: mesh::Sender, +} + +impl PetriVmConfigOpenVmm { + /// Makes the test the snapshot controller of a microVM: the VM worker + /// holds each guest snapshot request at a snapshot boundary, with the + /// vCPUs stopped, and reports it, which + /// [`PetriVmOpenVmm::wait_for_microvm_snapshot_boundary`](super::PetriVmOpenVmm::wait_for_microvm_snapshot_boundary) + /// waits for. The test then sends the snapshot requests that a controller + /// sends, such as + /// [`PetriVmOpenVmm::release_microvm_snapshot_boundary`](super::PetriVmOpenVmm::release_microvm_snapshot_boundary). + /// Otherwise the snapshot-request port lets each guest request continue at + /// once, as it does when the VM has no snapshot destination. + pub fn with_microvm_snapshot_boundary(mut self) -> Self { + let resources = self + .resources + .microvm + .as_mut() + .expect("snapshot boundaries require a microVM with its portb console"); + let port = self + .config + .chipset_devices + .iter_mut() + .find(|device| device.name == MicrovmSnapshotRequestHandle::ID) + .expect("a microVM has a snapshot-request port"); + let (notify, requests) = mesh::channel(); + let (ready, boundaries) = mesh::channel(); + port.resource = MicrovmSnapshotRequestHandle { + notify: Some(notify), + } + .into_resource(); + resources.snapshot_boundaries = Some(boundaries); + resources.worker_snapshot_channels = Some(SnapshotBoundaryChannels { requests, ready }); + self + } +} + +/// Takes the VM worker's ends of the snapshot boundary channels of +/// [`PetriVmConfigOpenVmm::with_microvm_snapshot_boundary`], for the worker's +/// launch. +pub(super) fn take_worker_snapshot_channels( + resources: &mut PetriVmResourcesOpenVmm, +) -> Option { + resources.microvm.as_mut()?.worker_snapshot_channels.take() } pub(super) struct PortbSerial { @@ -369,6 +436,8 @@ pub(super) fn configure_serial( runtime: RuntimeResources { output, unread: Vec::new(), + snapshot_boundaries: None, + worker_snapshot_channels: None, }, }), }) @@ -474,7 +543,7 @@ impl PetriVmInner { marker: &[u8], ) -> anyhow::Result<()> { anyhow::ensure!(!marker.is_empty(), "microVM portb marker cannot be empty"); - let RuntimeResources { output, unread } = self + let RuntimeResources { output, unread, .. } = self .resources .microvm .as_mut() @@ -503,6 +572,55 @@ impl PetriVmInner { .await .map_err(anyhow::Error::from) } + + /// Waits for the VM worker to hold a guest snapshot request at its + /// snapshot boundary, and returns the request's scratch policy. + pub(super) async fn wait_for_microvm_snapshot_boundary( + &mut self, + ) -> anyhow::Result { + let boundaries = self + .resources + .microvm + .as_mut() + .and_then(|resources| resources.snapshot_boundaries.as_mut()) + .context("the test is not the microVM's snapshot controller")?; + boundaries + .recv() + .await + .context("the VM worker closed its snapshot boundary channel") + } + + /// Quiesces the VM for the snapshot of its boundary, as a snapshot + /// controller does, and returns the VM worker's response. + pub(super) async fn quiesce_microvm_for_snapshot( + &self, + ) -> anyhow::Result> { + Ok(self + .worker + .rpc() + .call(VmRpc::QuiesceForSnapshot, ()) + .await?) + } + + /// Releases the VM's snapshot boundary, as a snapshot controller does when + /// it takes no snapshot. + pub(super) async fn release_microvm_snapshot_boundary(&self) -> anyhow::Result<()> { + Ok(self + .worker + .rpc() + .call_failable(VmRpc::ReleaseSnapshotBoundary, ()) + .await?) + } + + /// Rolls the VM's snapshot back, as a snapshot controller does when a + /// snapshot fails after its quiesce: the VM runs again from its boundary. + pub(super) async fn resume_microvm_after_failed_snapshot(&self) -> anyhow::Result<()> { + Ok(self + .worker + .rpc() + .call_failable(VmRpc::ResumeAfterFailedSnapshot, ()) + .await?) + } } #[cfg(test)] diff --git a/petri/src/vm/openvmm/runtime.rs b/petri/src/vm/openvmm/runtime.rs index 2bfc51eced9..80d1dc7ddf0 100644 --- a/petri/src/vm/openvmm/runtime.rs +++ b/petri/src/vm/openvmm/runtime.rs @@ -17,6 +17,7 @@ use crate::worker::Worker; use anyhow::Context; use async_trait::async_trait; use chipset_resources::acpi_power_button::PowerButtonRpc; +use chipset_resources::microvm::MicrovmSnapshotScratchPolicy; use framebuffer::View; use futures::FutureExt; use futures_concurrency::future::Race; @@ -30,6 +31,8 @@ use mesh::rpc::RpcError; use mesh::rpc::RpcSend; use mesh_process::Mesh; use openvmm_defs::rpc::PulseSaveRestoreError; +use openvmm_defs::rpc::SnapshotQuiesceError; +use openvmm_defs::rpc::SnapshotSaveResponse; use pal_async::socket::PolledSocket; use petri_artifacts_core::ResolvedArtifact; #[cfg(target_os = "linux")] @@ -390,6 +393,32 @@ impl PetriVmOpenVmm { /// Perform one pulse save/restore operation. pub async fn pulse_save_restore(&mut self) -> anyhow::Result<()> ); + petri_vm_fn!( + /// Wait for the VM worker to hold a guest snapshot request of a microVM + /// at its snapshot boundary, with the vCPUs stopped, and return the + /// request's scratch policy. The test must be the VM's snapshot + /// controller, as + /// [`PetriVmConfigOpenVmm::with_microvm_snapshot_boundary`](super::PetriVmConfigOpenVmm::with_microvm_snapshot_boundary) + /// makes it. + pub async fn wait_for_microvm_snapshot_boundary(&mut self) -> anyhow::Result + ); + petri_vm_fn!( + /// Quiesce a microVM for the snapshot of its boundary, as a snapshot + /// controller does, and return the VM worker's response: the saved + /// state, or why the worker did not save it. + pub async fn quiesce_microvm_for_snapshot(&mut self) -> anyhow::Result> + ); + petri_vm_fn!( + /// Release a microVM's snapshot boundary, as a snapshot controller does + /// when it takes no snapshot. + pub async fn release_microvm_snapshot_boundary(&mut self) -> anyhow::Result<()> + ); + petri_vm_fn!( + /// Roll a microVM's snapshot back, as a snapshot controller does when a + /// snapshot fails after its quiesce: the VM runs again from its + /// boundary. + pub async fn resume_microvm_after_failed_snapshot(&mut self) -> anyhow::Result<()> + ); /// Wrap the provided future in a race with the worker process's halt /// notification channel. This is useful for preventing a future from diff --git a/petri/src/vm/openvmm/start.rs b/petri/src/vm/openvmm/start.rs index f14b1f389a1..25e000c4f2d 100644 --- a/petri/src/vm/openvmm/start.rs +++ b/petri/src/vm/openvmm/start.rs @@ -177,9 +177,12 @@ impl PetriVmConfigOpenVmm { ); } - let (worker, halt_notif) = Worker::launch(&host, config, shared_memory) - .await - .context("failed to launch vm worker")?; + let microvm_snapshot_boundary = + super::microvm::take_worker_snapshot_channels(&mut resources); + let (worker, halt_notif) = + Worker::launch(&host, config, shared_memory, microvm_snapshot_boundary) + .await + .context("failed to launch vm worker")?; let worker = Arc::new(worker); diff --git a/petri/src/worker.rs b/petri/src/worker.rs index 297722662a7..bd9905ec12d 100644 --- a/petri/src/worker.rs +++ b/petri/src/worker.rs @@ -25,6 +25,7 @@ impl Worker { host: &WorkerHost, cfg: Config, shared_memory: Option, + microvm_snapshot_boundary: Option, ) -> anyhow::Result<(Self, mesh::Receiver)> { let (vm_rpc, rpc_recv) = mesh::channel(); let (notify_send, notify_recv) = mesh::channel(); @@ -34,6 +35,9 @@ impl Worker { openvmm_helpers::hypervisor::choose_hypervisor, openvmm_helpers::hypervisor::microvm::choose_microvm_hypervisor, )?; + let (snapshot_boundary_requests, snapshot_ready) = microvm_snapshot_boundary + .map(|channels| (channels.requests, channels.ready)) + .unzip(); let params = VmWorkerParameters { hypervisor, cfg, @@ -41,8 +45,8 @@ impl Worker { shared_memory, shared_memory_copy_on_write: false, snapshot_restore_guards: None, - snapshot_boundary_requests: None, - snapshot_ready: None, + snapshot_boundary_requests, + snapshot_ready, restore_time: None, restore_ready_sink: None, restore_gate_timeout: None, @@ -65,6 +69,11 @@ impl Worker { self.rpc.call(VmRpc::Pause, ()).await } + /// The VM worker's management RPC channel. + pub(crate) fn rpc(&self) -> &mesh::Sender { + &self.rpc + } + pub(crate) async fn resume(&self) -> anyhow::Result { Ok(self.rpc.call_failable(VmRpc::Resume, ()).await?) } diff --git a/vmm_tests/vmm_tests/tests/tests/x86_64/microvm.rs b/vmm_tests/vmm_tests/tests/tests/x86_64/microvm.rs index d0f88429e8e..ff055cae015 100644 --- a/vmm_tests/vmm_tests/tests/tests/x86_64/microvm.rs +++ b/vmm_tests/vmm_tests/tests/tests/x86_64/microvm.rs @@ -6,11 +6,24 @@ use anyhow::Context; use mesh::CancelContext; use petri::PetriHaltReason; +use petri::PetriVm; use petri::PetriVmBuilder; use petri::openvmm::OpenVmmPetriBackend; use std::time::Duration; use vmm_test_macros::openvmm_test_no_agent; +/// Time for a microVM guest to boot and reach its snapshot request. +const BOOT_TIMEOUT: Duration = Duration::from_secs(120); +/// Time for a running microVM guest to take its next step. +const GUEST_TIMEOUT: Duration = Duration::from_secs(60); +/// The power-off status of the guests of the snapshot controller tests. +const GUEST_STATUS: u8 = 7; +/// The guest's request for a snapshot with fresh scratch, a write of 0 to the +/// snapshot-request port, 0x605. It returns once the snapshot boundary that +/// holds it is released. +const SNAPSHOT_REQUEST: &str = + "printf '\\x00' | dd of=/dev/port bs=1 seek=1541 count=1 conv=notrunc 2>/dev/null\n"; + fn portb_output(bytes: &[u8]) -> anyhow::Result { use std::fmt::Write as _; @@ -115,3 +128,135 @@ async fn phase_1_lifecycle(config: PetriVmBuilder) -> anyho vm.teardown().await } + +/// Returns the guest's command that powers the microVM off with `status`, a +/// write to the shutdown port, 0x604. +fn power_off(status: u8) -> String { + format!( + "printf '\\{status:03o}' | dd of=/dev/port bs=1 seek=1540 count=1 conv=notrunc 2>/dev/null\n" + ) +} + +/// Boots a 1-vCPU microVM whose init runs `script` and then idles, with the +/// test as its snapshot controller. +async fn run_snapshot_controller_guest( + config: PetriVmBuilder, + script: &str, +) -> anyhow::Result> { + let script = format!( + "#!/bin/busybox sh\n\ + mount -t devtmpfs devtmpfs /dev 2>/dev/null || true\n\ + {script}\ + while :; do sleep 1; done\n" + ); + let config = config.with_microvm_machine(1); + let initrd = config.prepare_initrd_with_file("microvm-test", script.as_bytes(), 0o100755)?; + config + .with_prebuilt_initrd(initrd) + .modify_backend(|config| config.with_microvm_snapshot_boundary()) + .run_without_agent() + .await +} + +/// Waits for the VM worker to hold the guest's snapshot request at a snapshot +/// boundary. +async fn wait_for_snapshot_boundary(vm: &mut PetriVm) -> anyhow::Result<()> { + CancelContext::new() + .with_timeout(BOOT_TIMEOUT) + .until_cancelled(vm.backend().wait_for_microvm_snapshot_boundary()) + .await + .context("the guest's snapshot request reached no snapshot boundary")??; + Ok(()) +} + +/// Waits for the guest to write `marker` to portb, failing with `failure`. +async fn wait_for_guest_output( + vm: &mut PetriVm, + marker: &[u8], + failure: &'static str, +) -> anyhow::Result<()> { + CancelContext::new() + .with_timeout(GUEST_TIMEOUT) + .until_cancelled(vm.backend().wait_for_microvm_portb_bytes(marker)) + .await + .context(failure)? +} + +/// Waits for the guest to power the microVM off with `status`. +async fn wait_for_power_off( + vm: &mut PetriVm, + status: u8, +) -> anyhow::Result<()> { + let halt = CancelContext::new() + .with_timeout(GUEST_TIMEOUT) + .until_cancelled(vm.wait_for_halt()) + .await + .context("timed out waiting for the guest to power off")??; + assert_eq!(halt.reason, PetriHaltReason::PowerOff); + assert!( + halt.detail.contains(&format!("code: {status}")), + "expected microVM shutdown status {status}, got {}", + halt.detail + ); + Ok(()) +} + +/// The test, as the microVM's snapshot controller, takes each of the +/// controller's ways out of a guest-requested snapshot boundary: it releases +/// the first boundary without a snapshot, and quiesces the VM for the second +/// one's snapshot, then rolls the snapshot back. Each time the guest +/// continues after its snapshot request, and an ordinary resume cannot pass +/// the boundary. +#[openvmm_test_no_agent(linux_direct_x64)] +async fn snapshot_boundary_release_and_rollback( + config: PetriVmBuilder, +) -> anyhow::Result<()> { + const RELEASED: &[u8] = b"SNAPSHOT-RELEASED\n"; + const ROLLED_BACK: &[u8] = b"SNAPSHOT-ROLLED-BACK\n"; + + let mut vm = run_snapshot_controller_guest( + config, + &format!( + "{SNAPSHOT_REQUEST}{}{SNAPSHOT_REQUEST}{}{}", + portb_output(RELEASED)?, + portb_output(ROLLED_BACK)?, + power_off(GUEST_STATUS), + ), + ) + .await?; + + wait_for_snapshot_boundary(&mut vm).await?; + vm.backend() + .resume() + .await + .expect_err("an ordinary resume passed a snapshot boundary"); + vm.backend().release_microvm_snapshot_boundary().await?; + wait_for_guest_output( + &mut vm, + RELEASED, + "the guest did not continue after its snapshot boundary's release", + ) + .await?; + + wait_for_snapshot_boundary(&mut vm).await?; + let snapshot = vm + .backend() + .quiesce_microvm_for_snapshot() + .await? + .context("the snapshot quiesce failed")?; + assert!(!snapshot.state_unit_names.is_empty()); + vm.backend() + .resume() + .await + .expect_err("an ordinary resume passed a snapshot boundary"); + vm.backend().resume_microvm_after_failed_snapshot().await?; + wait_for_guest_output( + &mut vm, + ROLLED_BACK, + "the guest did not continue after its snapshot's rollback", + ) + .await?; + + wait_for_power_off(&mut vm, GUEST_STATUS).await?; + vm.teardown().await +} From 4c2697aef28d7fa69f788ce4e65b6554c1cfb935 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Mon, 9 Nov 2026 23:21:23 -0800 Subject: [PATCH 2/2] openvmm_core: run a quiesced microVM again only through its rollback Once the VM worker had quiesced a microVM for a snapshot and handed its state to the snapshot controller, an ordinary Resume could run the VM again without the snapshot's rollback, in two ways (microsoft/nvx#1170): - ReleaseSnapshotBoundary released the boundary of a quiesced VM: it resumed host input, completed the guest's snapshot transaction, and dropped the vCPU stop guard of the stopped VM. With the boundary gone, nothing refused a Resume, which started the VM again. - QuiesceForSnapshot stopped and saved a running VM that held no snapshot boundary, so no boundary refused the Resume that followed. The controller sends neither order: it quiesces only at a boundary that the worker reported, and after a quiesce it ends the source or rolls the snapshot back with ResumeAfterFailedSnapshot, which starts the state units before it releases the boundary. But the worker relied on that order for the capture's guarantee that a source whose snapshot may be published runs again only through the rollback. The worker now holds it itself. release_snapshot_boundary refuses the boundary of a quiesced VM, before it resumes input or changes the boundary, so only the rollback, which runs the VM first, releases it, and the boundary keeps refusing an ordinary Resume until then. quiesce_for_snapshot rejects a VM that holds no snapshot boundary with SnapshotQuiesceError::Rejected, without stopping it. The controller's own orders are unchanged: a release without a snapshot, and the rollback's release, both find the VM running. The OpenVMM Guide says that the worker refuses the other orders. Tests, with the test as the microVM's snapshot controller: after a quiesce at the guest's boundary, the boundary's release and an ordinary resume both fail, the partition's state unit stays stopped, and the guest does not continue for 5 s, and then the rollback lets it continue and power off; and a quiesce of a running VM that requested no snapshot is rejected and leaves the partition's state unit running. Without the fix, the release and the resume succeed, the partition runs, and the guest continues, and the quiesce outside a boundary returns the VM's saved state and leaves the partition stopped. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Guide/src/user_guide/openvmm/snapshots.md | 5 + .../src/worker/dispatch/microvm.rs | 27 ++++- .../vmm_tests/tests/tests/x86_64/microvm.rs | 98 +++++++++++++++++++ 3 files changed, 129 insertions(+), 1 deletion(-) diff --git a/Guide/src/user_guide/openvmm/snapshots.md b/Guide/src/user_guide/openvmm/snapshots.md index c1ca637e07e..6d4faf3c527 100644 --- a/Guide/src/user_guide/openvmm/snapshots.md +++ b/Guide/src/user_guide/openvmm/snapshots.md @@ -136,6 +136,11 @@ requests report a reply-channel error because their result types cannot carry an application error. Retry a rejected operation after the boundary is released; ordinary paused VMs are not subject to this restriction. +The snapshot lifecycle RPCs must follow the capture's order. The worker +rejects a snapshot quiesce outside a guest-requested boundary, without +stopping the VM, and refuses to release the boundary of a VM that it has +quiesced, which only the rollback of a failed capture runs again. + ## Restoring a snapshot To restore, pass the snapshot directory with `--restore-snapshot`: diff --git a/openvmm/openvmm_core/src/worker/dispatch/microvm.rs b/openvmm/openvmm_core/src/worker/dispatch/microvm.rs index 3a523b7b9b5..8c104360819 100644 --- a/openvmm/openvmm_core/src/worker/dispatch/microvm.rs +++ b/openvmm/openvmm_core/src/worker/dispatch/microvm.rs @@ -268,12 +268,24 @@ impl LoadedVm { } } + /// Releases the guest-requested snapshot boundary of a running VM: resumes + /// host input, completes the guest's snapshot transaction, and lets the + /// vCPUs run. + /// + /// The boundary of a quiesced VM, whose snapshot the controller holds, is + /// released only by [`Self::resume_after_failed_snapshot`], which starts + /// the VM's state units first: the boundary keeps refusing an ordinary + /// resume until then. pub(super) async fn release_snapshot_boundary(&mut self) -> anyhow::Result<()> { // Resume host input only while a boundary holds it. anyhow::ensure!( self.snapshot_boundary.transaction_complete.is_some(), "no active microVM snapshot boundary" ); + anyhow::ensure!( + self.running, + "the snapshot boundary of a quiesced microVM is released only by its rollback" + ); self.state_units.resume_input_after_save().await?; let transaction_complete = self .snapshot_boundary @@ -300,6 +312,11 @@ impl LoadedVm { /// Quiesces the VM for a guest-requested snapshot and saves its state, /// leaving it stopped. /// + /// The VM must be running at a snapshot boundary that its guest requested, + /// which holds the stopped VM until the snapshot's rollback: outside one, + /// the request is rejected and changes nothing, since an ordinary resume + /// would run the VM again after its state was saved. + /// /// The quiesce stops every state unit and has no deadline: a unit that /// never stops stalls the capture until the process that drives the VM /// ends it, with nothing published and the source never resumed. Each @@ -323,6 +340,13 @@ impl LoadedVm { RemoteError::new(anyhow::anyhow!("VM is already stopped")), )); } + if self.snapshot_boundary.stop_guard.is_none() { + return Err(openvmm_defs::rpc::SnapshotQuiesceError::Rejected( + RemoteError::new(anyhow::anyhow!( + "guest-requested snapshot quiesce requires an active snapshot boundary" + )), + )); + } let quiesce = openvmm_defs::profile::ProfileSpan::start(); self.state_units.stop().await; @@ -395,7 +419,8 @@ impl LoadedVm { } /// Resumes the VM after a snapshot failure that followed a complete - /// quiesce, and releases the snapshot boundary. + /// quiesce, and releases the snapshot boundary. This is the only request + /// that runs a quiesced VM again. /// /// A unit whose task does not answer leaves the units that did start /// stopped again, and fails the rollback. diff --git a/vmm_tests/vmm_tests/tests/tests/x86_64/microvm.rs b/vmm_tests/vmm_tests/tests/tests/x86_64/microvm.rs index ff055cae015..661dca3b6f3 100644 --- a/vmm_tests/vmm_tests/tests/tests/x86_64/microvm.rs +++ b/vmm_tests/vmm_tests/tests/tests/x86_64/microvm.rs @@ -5,6 +5,7 @@ use anyhow::Context; use mesh::CancelContext; +use openvmm_defs::rpc::SnapshotQuiesceError; use petri::PetriHaltReason; use petri::PetriVm; use petri::PetriVmBuilder; @@ -201,6 +202,18 @@ async fn wait_for_power_off( Ok(()) } +/// Returns the state of the VM worker's partition state unit, which runs the +/// vCPUs: `running`, or `stopped` once the VM quiesced. +async fn partition_unit_state(vm: &PetriVm) -> anyhow::Result { + let node = vm.inspect_vmm("partition/unit_state").await?; + let state: serde_json::Value = serde_json::from_str(&node.json().to_string()) + .context("failed to parse the partition's unit state")?; + state + .as_str() + .map(str::to_owned) + .context("the partition's unit state is not a string") +} + /// The test, as the microVM's snapshot controller, takes each of the /// controller's ways out of a guest-requested snapshot boundary: it releases /// the first boundary without a snapshot, and quiesces the VM for the second @@ -260,3 +273,88 @@ async fn snapshot_boundary_release_and_rollback( wait_for_power_off(&mut vm, GUEST_STATUS).await?; vm.teardown().await } + +/// Once the VM worker has quiesced a microVM for the snapshot of its +/// snapshot boundary, only the snapshot's rollback runs the VM again: the +/// boundary's release, which a snapshot controller sends only when it takes +/// no snapshot, is refused, and so is an ordinary resume, so the VM stays +/// stopped until the rollback. +#[openvmm_test_no_agent(linux_direct_x64)] +async fn snapshot_quiesce_resumes_only_on_rollback( + config: PetriVmBuilder, +) -> anyhow::Result<()> { + const CONTINUED: &[u8] = b"SNAPSHOT-CONTINUED\n"; + /// How long the quiesced VM is watched for running again. + const STOPPED_WINDOW: Duration = Duration::from_secs(5); + + let mut vm = run_snapshot_controller_guest( + config, + &format!( + "{SNAPSHOT_REQUEST}{}{}", + portb_output(CONTINUED)?, + power_off(GUEST_STATUS), + ), + ) + .await?; + + wait_for_snapshot_boundary(&mut vm).await?; + let snapshot = vm + .backend() + .quiesce_microvm_for_snapshot() + .await? + .context("the snapshot quiesce failed")?; + assert!(!snapshot.state_unit_names.is_empty()); + + let release = vm.backend().release_microvm_snapshot_boundary().await; + let resume = vm.backend().resume().await; + let output = CancelContext::new() + .with_timeout(STOPPED_WINDOW) + .until_cancelled(vm.backend().wait_for_microvm_portb_bytes(CONTINUED)) + .await; + let state = partition_unit_state(&vm).await?; + assert!( + release.is_err() && resume.is_err() && output.is_err() && state == "stopped", + "the quiesced VM ran again without the snapshot's rollback: the boundary's \ + release returned {release:?}, the ordinary resume returned {resume:?}, the \ + guest's continuation within {STOPPED_WINDOW:?} was {output:?}, and the \ + partition is {state}" + ); + + vm.backend().resume_microvm_after_failed_snapshot().await?; + wait_for_guest_output( + &mut vm, + CONTINUED, + "the guest did not continue after its snapshot's rollback", + ) + .await?; + wait_for_power_off(&mut vm, GUEST_STATUS).await?; + vm.teardown().await +} + +/// The VM worker quiesces a microVM for a snapshot only at a snapshot +/// boundary that its guest requested: outside one, the quiesce is rejected +/// and leaves the VM running, since no boundary would then keep an ordinary +/// resume from running the VM again after its state was saved. +#[openvmm_test_no_agent(linux_direct_x64)] +async fn snapshot_quiesce_requires_a_boundary( + config: PetriVmBuilder, +) -> anyhow::Result<()> { + const READY: &[u8] = b"SNAPSHOT-GUEST-READY\n"; + + let mut vm = run_snapshot_controller_guest(config, &portb_output(READY)?).await?; + + CancelContext::new() + .with_timeout(BOOT_TIMEOUT) + .until_cancelled(vm.backend().wait_for_microvm_portb_bytes(READY)) + .await + .context("timed out waiting for the microVM guest to boot")??; + let quiesce = vm.backend().quiesce_microvm_for_snapshot().await?; + let state = partition_unit_state(&vm).await?; + assert!( + matches!(quiesce, Err(SnapshotQuiesceError::Rejected(_))) && state == "running", + "a snapshot quiesce outside a snapshot boundary returned {:?}, and left the \ + partition {state}", + quiesce.map(|snapshot| snapshot.state_unit_names) + ); + vm.teardown().await +}