Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions Guide/src/user_guide/openvmm/snapshots.md
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,11 @@ requests report a reply-channel error because their result types cannot carry
an application error. Retry a rejected operation after the boundary is
released; ordinary paused VMs are not subject to this restriction.

The snapshot lifecycle RPCs must follow the capture's order. The worker
rejects a snapshot quiesce outside a guest-requested boundary, without
stopping the VM, and refuses to release the boundary of a VM that it has
quiesced, which only the rollback of a failed capture runs again.

## Restoring a snapshot

To restore, pass the snapshot directory with `--restore-snapshot`:
Expand Down
27 changes: 26 additions & 1 deletion openvmm/openvmm_core/src/worker/dispatch/microvm.rs
Original file line number Diff line number Diff line change
Expand Up @@ -268,12 +268,24 @@ impl LoadedVm {
}
}

/// Releases the guest-requested snapshot boundary of a running VM: resumes
/// host input, completes the guest's snapshot transaction, and lets the
/// vCPUs run.
///
/// The boundary of a quiesced VM, whose snapshot the controller holds, is
/// released only by [`Self::resume_after_failed_snapshot`], which starts
/// the VM's state units first: the boundary keeps refusing an ordinary
/// resume until then.
pub(super) async fn release_snapshot_boundary(&mut self) -> anyhow::Result<()> {
// Resume host input only while a boundary holds it.
anyhow::ensure!(
self.snapshot_boundary.transaction_complete.is_some(),
"no active microVM snapshot boundary"
);
anyhow::ensure!(
self.running,
"the snapshot boundary of a quiesced microVM is released only by its rollback"
);
self.state_units.resume_input_after_save().await?;
let transaction_complete = self
.snapshot_boundary
Expand All @@ -300,6 +312,11 @@ impl LoadedVm {
/// Quiesces the VM for a guest-requested snapshot and saves its state,
/// leaving it stopped.
///
/// The VM must be running at a snapshot boundary that its guest requested,
/// which holds the stopped VM until the snapshot's rollback: outside one,
/// the request is rejected and changes nothing, since an ordinary resume
/// would run the VM again after its state was saved.
///
/// The quiesce stops every state unit and has no deadline: a unit that
/// never stops stalls the capture until the process that drives the VM
/// ends it, with nothing published and the source never resumed. Each
Expand All @@ -323,6 +340,13 @@ impl LoadedVm {
RemoteError::new(anyhow::anyhow!("VM is already stopped")),
));
}
if self.snapshot_boundary.stop_guard.is_none() {
return Err(openvmm_defs::rpc::SnapshotQuiesceError::Rejected(
RemoteError::new(anyhow::anyhow!(
"guest-requested snapshot quiesce requires an active snapshot boundary"
)),
));
}

let quiesce = openvmm_defs::profile::ProfileSpan::start();
self.state_units.stop().await;
Expand Down Expand Up @@ -395,7 +419,8 @@ impl LoadedVm {
}

/// Resumes the VM after a snapshot failure that followed a complete
/// quiesce, and releases the snapshot boundary.
/// quiesce, and releases the snapshot boundary. This is the only request
/// that runs a quiesced VM again.
///
/// A unit whose task does not answer leaves the units that did start
/// stopped again, and fails the rollback.
Expand Down
120 changes: 119 additions & 1 deletion petri/src/vm/openvmm/microvm.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@

//! OpenVMM-specific microVM construction and runtime support.

use super::PetriVmConfigOpenVmm;
use super::PetriVmResourcesOpenVmm;
use super::construct::SerialData;
use super::runtime::PetriVmInner;
use crate::Firmware;
Expand All @@ -11,15 +13,21 @@ use anyhow::Context;
use chipset_resources::microvm::MicrovmPortbHandle;
use chipset_resources::microvm::MicrovmPortbTimeAbi;
use chipset_resources::microvm::MicrovmShutdownHandle;
use chipset_resources::microvm::MicrovmSnapshotBoundaryRequest;
use chipset_resources::microvm::MicrovmSnapshotRequestHandle;
use chipset_resources::microvm::MicrovmSnapshotScratchPolicy;
use chipset_resources::pit::PitDeviceHandle;
use cpu_profile::HostCpuSignature;
use mesh::rpc::RpcSend;
use openvmm_defs::config::Config;
use openvmm_defs::config::LinuxDirectBootMode;
use openvmm_defs::config::LinuxIsolationConfig;
use openvmm_defs::config::LoadMode;
use openvmm_defs::microvm::MachineProfile;
use openvmm_defs::microvm::MicrovmConfig;
use openvmm_defs::rpc::SnapshotQuiesceError;
use openvmm_defs::rpc::SnapshotSaveResponse;
use openvmm_defs::rpc::VmRpc;
use openvmm_defs::time_abi::TimeAbiParameters;
use pal_async::DefaultDriver;
use pal_async::socket::PolledSocket;
Expand All @@ -39,6 +47,65 @@ pub(super) struct RuntimeResources {
output: mesh::Receiver<Vec<u8>>,
/// The output that followed the marker of the latest wait.
unread: Vec<u8>,
/// The guest-requested snapshot boundaries that the VM worker reports,
/// when the test is the VM's snapshot controller.
snapshot_boundaries: Option<mesh::Receiver<MicrovmSnapshotScratchPolicy>>,
/// The VM worker's ends of the snapshot boundary channels, until the
/// worker launches.
worker_snapshot_channels: Option<SnapshotBoundaryChannels>,
}

/// The VM worker's ends of the channels of guest-requested snapshot
/// boundaries.
pub(crate) struct SnapshotBoundaryChannels {
/// The guest's snapshot requests, from the snapshot-request port.
pub(crate) requests: mesh::Receiver<MicrovmSnapshotBoundaryRequest>,
/// Reports each boundary that the worker establishes to the snapshot
/// controller.
pub(crate) ready: mesh::Sender<MicrovmSnapshotScratchPolicy>,
}

impl PetriVmConfigOpenVmm {
/// Makes the test the snapshot controller of a microVM: the VM worker
/// holds each guest snapshot request at a snapshot boundary, with the
/// vCPUs stopped, and reports it, which
/// [`PetriVmOpenVmm::wait_for_microvm_snapshot_boundary`](super::PetriVmOpenVmm::wait_for_microvm_snapshot_boundary)
/// waits for. The test then sends the snapshot requests that a controller
/// sends, such as
/// [`PetriVmOpenVmm::release_microvm_snapshot_boundary`](super::PetriVmOpenVmm::release_microvm_snapshot_boundary).
/// Otherwise the snapshot-request port lets each guest request continue at
/// once, as it does when the VM has no snapshot destination.
pub fn with_microvm_snapshot_boundary(mut self) -> Self {
let resources = self
.resources
.microvm
.as_mut()
.expect("snapshot boundaries require a microVM with its portb console");
let port = self
.config
.chipset_devices
.iter_mut()
.find(|device| device.name == MicrovmSnapshotRequestHandle::ID)
.expect("a microVM has a snapshot-request port");
let (notify, requests) = mesh::channel();
let (ready, boundaries) = mesh::channel();
port.resource = MicrovmSnapshotRequestHandle {
notify: Some(notify),
}
.into_resource();
resources.snapshot_boundaries = Some(boundaries);
resources.worker_snapshot_channels = Some(SnapshotBoundaryChannels { requests, ready });
self
}
}

/// Takes the VM worker's ends of the snapshot boundary channels of
/// [`PetriVmConfigOpenVmm::with_microvm_snapshot_boundary`], for the worker's
/// launch.
pub(super) fn take_worker_snapshot_channels(
resources: &mut PetriVmResourcesOpenVmm,
) -> Option<SnapshotBoundaryChannels> {
resources.microvm.as_mut()?.worker_snapshot_channels.take()
}

pub(super) struct PortbSerial {
Expand Down Expand Up @@ -369,6 +436,8 @@ pub(super) fn configure_serial(
runtime: RuntimeResources {
output,
unread: Vec::new(),
snapshot_boundaries: None,
worker_snapshot_channels: None,
},
}),
})
Expand Down Expand Up @@ -474,7 +543,7 @@ impl PetriVmInner {
marker: &[u8],
) -> anyhow::Result<()> {
anyhow::ensure!(!marker.is_empty(), "microVM portb marker cannot be empty");
let RuntimeResources { output, unread } = self
let RuntimeResources { output, unread, .. } = self
.resources
.microvm
.as_mut()
Expand Down Expand Up @@ -503,6 +572,55 @@ impl PetriVmInner {
.await
.map_err(anyhow::Error::from)
}

/// Waits for the VM worker to hold a guest snapshot request at its
/// snapshot boundary, and returns the request's scratch policy.
pub(super) async fn wait_for_microvm_snapshot_boundary(
&mut self,
) -> anyhow::Result<MicrovmSnapshotScratchPolicy> {
let boundaries = self
.resources
.microvm
.as_mut()
.and_then(|resources| resources.snapshot_boundaries.as_mut())
.context("the test is not the microVM's snapshot controller")?;
boundaries
.recv()
.await
.context("the VM worker closed its snapshot boundary channel")
}

/// Quiesces the VM for the snapshot of its boundary, as a snapshot
/// controller does, and returns the VM worker's response.
pub(super) async fn quiesce_microvm_for_snapshot(
&self,
) -> anyhow::Result<Result<SnapshotSaveResponse, SnapshotQuiesceError>> {
Ok(self
.worker
.rpc()
.call(VmRpc::QuiesceForSnapshot, ())
.await?)
}

/// Releases the VM's snapshot boundary, as a snapshot controller does when
/// it takes no snapshot.
pub(super) async fn release_microvm_snapshot_boundary(&self) -> anyhow::Result<()> {
Ok(self
.worker
.rpc()
.call_failable(VmRpc::ReleaseSnapshotBoundary, ())
.await?)
}

/// Rolls the VM's snapshot back, as a snapshot controller does when a
/// snapshot fails after its quiesce: the VM runs again from its boundary.
pub(super) async fn resume_microvm_after_failed_snapshot(&self) -> anyhow::Result<()> {
Ok(self
.worker
.rpc()
.call_failable(VmRpc::ResumeAfterFailedSnapshot, ())
.await?)
}
}

#[cfg(test)]
Expand Down
29 changes: 29 additions & 0 deletions petri/src/vm/openvmm/runtime.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ use crate::worker::Worker;
use anyhow::Context;
use async_trait::async_trait;
use chipset_resources::acpi_power_button::PowerButtonRpc;
use chipset_resources::microvm::MicrovmSnapshotScratchPolicy;
use framebuffer::View;
use futures::FutureExt;
use futures_concurrency::future::Race;
Expand All @@ -30,6 +31,8 @@ use mesh::rpc::RpcError;
use mesh::rpc::RpcSend;
use mesh_process::Mesh;
use openvmm_defs::rpc::PulseSaveRestoreError;
use openvmm_defs::rpc::SnapshotQuiesceError;
use openvmm_defs::rpc::SnapshotSaveResponse;
use pal_async::socket::PolledSocket;
use petri_artifacts_core::ResolvedArtifact;
#[cfg(target_os = "linux")]
Expand Down Expand Up @@ -390,6 +393,32 @@ impl PetriVmOpenVmm {
/// Perform one pulse save/restore operation.
pub async fn pulse_save_restore(&mut self) -> anyhow::Result<()>
);
petri_vm_fn!(
/// Wait for the VM worker to hold a guest snapshot request of a microVM
/// at its snapshot boundary, with the vCPUs stopped, and return the
/// request's scratch policy. The test must be the VM's snapshot
/// controller, as
/// [`PetriVmConfigOpenVmm::with_microvm_snapshot_boundary`](super::PetriVmConfigOpenVmm::with_microvm_snapshot_boundary)
/// makes it.
pub async fn wait_for_microvm_snapshot_boundary(&mut self) -> anyhow::Result<MicrovmSnapshotScratchPolicy>
);
petri_vm_fn!(
/// Quiesce a microVM for the snapshot of its boundary, as a snapshot
/// controller does, and return the VM worker's response: the saved
/// state, or why the worker did not save it.
pub async fn quiesce_microvm_for_snapshot(&mut self) -> anyhow::Result<Result<SnapshotSaveResponse, SnapshotQuiesceError>>
);
petri_vm_fn!(
/// Release a microVM's snapshot boundary, as a snapshot controller does
/// when it takes no snapshot.
pub async fn release_microvm_snapshot_boundary(&mut self) -> anyhow::Result<()>
);
petri_vm_fn!(
/// Roll a microVM's snapshot back, as a snapshot controller does when a
/// snapshot fails after its quiesce: the VM runs again from its
/// boundary.
pub async fn resume_microvm_after_failed_snapshot(&mut self) -> anyhow::Result<()>
);

/// Wrap the provided future in a race with the worker process's halt
/// notification channel. This is useful for preventing a future from
Expand Down
9 changes: 6 additions & 3 deletions petri/src/vm/openvmm/start.rs
Original file line number Diff line number Diff line change
Expand Up @@ -177,9 +177,12 @@ impl PetriVmConfigOpenVmm {
);
}

let (worker, halt_notif) = Worker::launch(&host, config, shared_memory)
.await
.context("failed to launch vm worker")?;
let microvm_snapshot_boundary =
super::microvm::take_worker_snapshot_channels(&mut resources);
let (worker, halt_notif) =
Worker::launch(&host, config, shared_memory, microvm_snapshot_boundary)
.await
.context("failed to launch vm worker")?;

let worker = Arc::new(worker);

Expand Down
13 changes: 11 additions & 2 deletions petri/src/worker.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ impl Worker {
host: &WorkerHost,
cfg: Config,
shared_memory: Option<openvmm_defs::worker::SharedMemoryFd>,
microvm_snapshot_boundary: Option<crate::openvmm::microvm::SnapshotBoundaryChannels>,
) -> anyhow::Result<(Self, mesh::Receiver<HaltReason>)> {
let (vm_rpc, rpc_recv) = mesh::channel();
let (notify_send, notify_recv) = mesh::channel();
Expand All @@ -34,15 +35,18 @@ impl Worker {
openvmm_helpers::hypervisor::choose_hypervisor,
openvmm_helpers::hypervisor::microvm::choose_microvm_hypervisor,
)?;
let (snapshot_boundary_requests, snapshot_ready) = microvm_snapshot_boundary
.map(|channels| (channels.requests, channels.ready))
.unzip();
let params = VmWorkerParameters {
hypervisor,
cfg,
saved_state: None,
shared_memory,
shared_memory_copy_on_write: false,
snapshot_restore_guards: None,
snapshot_boundary_requests: None,
snapshot_ready: None,
snapshot_boundary_requests,
snapshot_ready,
restore_time: None,
restore_ready_sink: None,
restore_gate_timeout: None,
Expand All @@ -65,6 +69,11 @@ impl Worker {
self.rpc.call(VmRpc::Pause, ()).await
}

/// The VM worker's management RPC channel.
pub(crate) fn rpc(&self) -> &mesh::Sender<VmRpc> {
&self.rpc
}

pub(crate) async fn resume(&self) -> anyhow::Result<bool> {
Ok(self.rpc.call_failable(VmRpc::Resume, ()).await?)
}
Expand Down
Loading
Loading