From 8a994544afdec2d9cb61c8176b9119dbc0fdd6e0 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Sun, 8 Nov 2026 17:59:23 -0800 Subject: [PATCH 1/2] openvmm_helpers: refuse earlier snapshot formats before decoding them Restoring a snapshot of an earlier OpenVMM fails with a decoding error, not with E_SNAPSHOT_VERSION and its request to recapture the snapshot. Restore decodes the whole manifest, and only then checks its version and magic, but the earlier formats give some field numbers other types than manifest version 2 does: - Version 6, which OpenVMM wrote until the v4.0 microVM chain restarted the manifest at version 2, keeps its machine contract, a message, in field 11 and its saved-state schema version, a varint, in field 13, where version 2 keeps the schema version and the capture tier, a string: failed to decode snapshot manifest: decoding failed: expected a varint - The first version 2, which predates the format magic, keeps the SHA-256 digests of state.bin and memory.bin in fields 9 and 10, where version 2 now keeps its machine contract and its magic: failed to decode snapshot manifest: decoding failed: unknown wire type 3 The Guide says that restore rejects any other version or magic with E_SNAPSHOT_VERSION, and so does NVX's guide to snapshots of an earlier OpenVMM (microsoft/nvx#1210). On KVM and MSHV, on Azure Linux 3 virtual machines, and on WHP, on a Windows virtual machine, the OpenVMM of NVX release v0.1.0-dev.304f8e1686b4 captured a microVM snapshot of manifest version 6, which it restores, and which feature/microvm/dev 99aac19b3 refuses with the first error. Decode the version alone, which every manifest format, upstream's included, keeps in field 1, and then the magic alone, which version 2 keeps in field 10, and refuse any other version or magic with E_SNAPSHOT_VERSION before decoding the rest of the manifest. With this change, those restores fail with [E_SNAPSHOT_VERSION] snapshot manifest version 6 is not supported (expected 2); recapture the snapshot A malformed manifest still fails to decode, and a manifest of version 2 still gets every check that it got before. Tests: read_snapshot_refuses_a_version_6_manifest_by_its_version and read_snapshot_refuses_an_early_version_2_manifest_by_its_magic, in openvmm_helpers' restore tests, each of which fails at feature/microvm/dev 99aac19b3 with the error above. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Guide/src/dev_guide/snapshot_format.md | 14 +- .../openvmm_helpers/src/snapshot/format.rs | 69 +++++++- .../openvmm_helpers/src/snapshot/restore.rs | 153 +++++++++++++++++- 3 files changed, 221 insertions(+), 15 deletions(-) diff --git a/Guide/src/dev_guide/snapshot_format.md b/Guide/src/dev_guide/snapshot_format.md index e9b71531c85..23cc19ac1ae 100644 --- a/Guide/src/dev_guide/snapshot_format.md +++ b/Guide/src/dev_guide/snapshot_format.md @@ -25,11 +25,15 @@ crate's protobuf encoding. Every snapshot uses manifest version 2 with format magic `OPENVMM_SNAPSHOT_V2\0`. Restore rejects any other version or magic with -`E_SNAPSHOT_VERSION`, which asks for a recapture. A microVM snapshot's machine -contract carries the NVX time ABI records (the time contract and the CPU -profile record), and a host-driven standard-machine snapshot has no machine -contract. Tiered microVM snapshots record capture tier, clone/resume policy, -and consumed configuration sections. +`E_SNAPSHOT_VERSION`, which asks for a recapture. It decodes the version, +field 1, and then the magic, field 10, each alone before the rest of the +manifest, since earlier formats give some fields other types: version 6, the +manifest of earlier releases, keeps its machine contract in field 11, and the +first version 2, which has no magic, keeps artifact digests in fields 9 and 10. +A microVM snapshot's machine contract carries the NVX time ABI records (the +time contract and the CPU profile record), and a host-driven standard-machine +snapshot has no machine contract. Tiered microVM snapshots record capture +tier, clone/resume policy, and consumed configuration sections. The format is a local machine-state contract, not an authenticated container. Every snapshot receives the same regular-file, no-follow/no-reparse, bounded diff --git a/openvmm/openvmm_helpers/src/snapshot/format.rs b/openvmm/openvmm_helpers/src/snapshot/format.rs index a61fe749afe..412dbc34054 100644 --- a/openvmm/openvmm_helpers/src/snapshot/format.rs +++ b/openvmm/openvmm_helpers/src/snapshot/format.rs @@ -8,6 +8,8 @@ use super::MANIFEST_VERSION; use super::SnapshotManifest; use super::microvm; +use anyhow::Context; +use mesh::payload::Protobuf; use mesh::payload::Timestamp; use sha2::Digest; use virt::time_abi::TimeAbiCode; @@ -101,27 +103,80 @@ pub(super) fn verify_digest( Ok(()) } -/// Validates the manifest's format identity: [`MANIFEST_VERSION`], the -/// matching magic, and the saved-state schema. Any other version or magic is +/// The version of a manifest, which every manifest format, upstream's +/// included, keeps in field 1. +#[derive(Protobuf)] +#[mesh(package = "openvmm.snapshot")] +struct ManifestVersion { + #[mesh(1)] + version: u32, +} + +/// The format magic of a manifest of [`MANIFEST_VERSION`], which it keeps in +/// field 10. +#[derive(Protobuf)] +#[mesh(package = "openvmm.snapshot")] +struct ManifestFormatMagic { + #[mesh(10)] + format_magic: Vec, +} + +/// Decodes a manifest of this format, one that [`validate_manifest_header`] +/// accepts. +/// +/// Earlier formats give some field numbers other types, so decoding one of +/// their manifests whole fails as if it were malformed. This decodes the +/// version alone, which every format keeps in field 1, then the magic alone, +/// and refuses any other version or magic with `E_SNAPSHOT_VERSION` before it +/// decodes the rest. The magic comes second, since only the formats of this +/// version agree on the type of field 10. +pub(super) fn decode_manifest(bytes: &[u8]) -> anyhow::Result { + let ManifestVersion { version } = + mesh::payload::decode(bytes).context("failed to decode snapshot manifest")?; + validate_manifest_version(version)?; + let ManifestFormatMagic { format_magic } = + mesh::payload::decode(bytes).context("failed to decode snapshot manifest")?; + validate_format_magic(&format_magic)?; + let manifest: SnapshotManifest = + mesh::payload::decode(bytes).context("failed to decode snapshot manifest")?; + validate_manifest_header(&manifest)?; + Ok(manifest) +} + +/// Refuses any other manifest version than [`MANIFEST_VERSION`] with /// `E_SNAPSHOT_VERSION`, which asks for a recapture. -pub(super) fn validate_manifest_header(manifest: &SnapshotManifest) -> anyhow::Result<()> { - if manifest.version != MANIFEST_VERSION { +fn validate_manifest_version(version: u32) -> anyhow::Result<()> { + if version != MANIFEST_VERSION { return Err(TimeAbiError::new( TimeAbiCode::SnapshotVersion, format!( - "snapshot manifest version {} is not supported (expected {MANIFEST_VERSION}); recapture the snapshot", - manifest.version + "snapshot manifest version {version} is not supported (expected {MANIFEST_VERSION}); recapture the snapshot" ), ) .into()); } - if manifest.format_magic != SNAPSHOT_FORMAT_MAGIC { + Ok(()) +} + +/// Refuses any other format magic than [`SNAPSHOT_FORMAT_MAGIC`] with +/// `E_SNAPSHOT_VERSION`, which asks for a recapture. +fn validate_format_magic(format_magic: &[u8]) -> anyhow::Result<()> { + if format_magic != SNAPSHOT_FORMAT_MAGIC { return Err(TimeAbiError::new( TimeAbiCode::SnapshotVersion, "snapshot format magic is invalid; recapture the snapshot", ) .into()); } + Ok(()) +} + +/// Validates the manifest's format identity: [`MANIFEST_VERSION`], the +/// matching magic, and the saved-state schema. Any other version or magic is +/// `E_SNAPSHOT_VERSION`, which asks for a recapture. +pub(super) fn validate_manifest_header(manifest: &SnapshotManifest) -> anyhow::Result<()> { + validate_manifest_version(manifest.version)?; + validate_format_magic(&manifest.format_magic)?; anyhow::ensure!( manifest.saved_state_schema_version == SAVED_STATE_SCHEMA_VERSION, "snapshot saved-state schema version {} is unsupported", diff --git a/openvmm/openvmm_helpers/src/snapshot/restore.rs b/openvmm/openvmm_helpers/src/snapshot/restore.rs index 56d7a720e39..d2497693b9b 100644 --- a/openvmm/openvmm_helpers/src/snapshot/restore.rs +++ b/openvmm/openvmm_helpers/src/snapshot/restore.rs @@ -14,6 +14,7 @@ use super::format::RESUME_CLAIM_FILE_NAME; use super::format::SCRATCH_FILE_NAME; use super::format::SNAPSHOT_RESTORE_POLICY_RESUME; use super::format::STATE_FILE_NAME; +use super::format::decode_manifest; use super::format::validate_manifest_contents; use super::format::validate_manifest_header; use super::fs::OpenedFileGeneration; @@ -193,9 +194,7 @@ fn read_snapshot_manifest_from_directory( fn decode_snapshot_manifest(manifest_file: &std::fs::File) -> anyhow::Result { let manifest_bytes = read_bounded_open_file(manifest_file, MAX_MANIFEST_SIZE_BYTES, "snapshot manifest")?; - let manifest: SnapshotManifest = - mesh::payload::decode(&manifest_bytes).context("failed to decode snapshot manifest")?; - validate_manifest_header(&manifest)?; + let manifest = decode_manifest(&manifest_bytes)?; validate_manifest_contents(&manifest)?; if let Some(contract) = &manifest.machine_contract { microvm::validate_machine_contract_shape( @@ -413,8 +412,11 @@ mod tests { use super::super::publish::write_snapshot_from_owned_memory_and_scratch_files; use super::super::tests::test_manifest; use super::*; + use mesh::payload::Protobuf; + use mesh::payload::Timestamp; use openvmm_defs::microvm::MICROVM_COMMAND_LINE_MAX_SIZE; use openvmm_defs::microvm::check_microvm_command_line; + use sha2::Digest; use std::io::Read; #[cfg(any(target_os = "linux", windows))] use std::io::Seek; @@ -612,6 +614,151 @@ mod tests { assert!(error.contains("recapture the snapshot"), "{error}"); } + /// Returns the errors of reading a snapshot whole and its manifest alone, + /// once `manifest` replaces the manifest of a snapshot of this format. + fn read_errors_with_manifest(manifest: &[u8]) -> Vec { + let dir = tempfile::tempdir().unwrap(); + let snap_dir = dir.path().join("snap"); + let mem_path = dir.path().join("memory.bin"); + std::fs::write(&mem_path, vec![0_u8; 1024]).unwrap(); + write_snapshot(&snap_dir, &test_manifest(), b"state", &mem_path).unwrap(); + std::fs::write(snap_dir.join(MANIFEST_FILE_NAME), manifest).unwrap(); + [ + read_snapshot(&snap_dir, 1024).err().unwrap(), + read_snapshot_manifest(&snap_dir).err().unwrap(), + ] + .iter() + .map(|error| format!("{error:#}")) + .collect() + } + + #[test] + fn read_snapshot_refuses_a_version_6_manifest_by_its_version() { + // The manifest of earlier OpenVMM releases, version 6, gives fields 11 + // and 13 other types than this format does: its machine contract, a + // message, is where this format keeps the saved-state schema version, + // a varint, and its schema version where this format keeps the capture + // tier, a string. + #[derive(Protobuf)] + #[mesh(package = "openvmm.snapshot")] + struct ManifestV6 { + #[mesh(1)] + version: u32, + #[mesh(2)] + created_at: Timestamp, + #[mesh(3)] + openvmm_version: String, + #[mesh(4)] + memory_size_bytes: u64, + #[mesh(5)] + vp_count: u32, + #[mesh(6)] + page_size: u32, + #[mesh(7)] + architecture: String, + #[mesh(8)] + state_size_bytes: u64, + #[mesh(11)] + machine_contract: Option, + #[mesh(12)] + format_magic: Vec, + #[mesh(13)] + saved_state_schema_version: u32, + #[mesh(14)] + saved_state_root_type: String, + #[mesh(15)] + snapshot_tier: String, + #[mesh(16)] + restore_policy: String, + #[mesh(17)] + consumed_config_sections: u32, + } + + let manifest = mesh::payload::encode(ManifestV6 { + version: 6, + created_at: Timestamp { + seconds: 1234567890, + nanos: 0, + }, + openvmm_version: "test-0.1.0".to_owned(), + memory_size_bytes: 1024, + vp_count: 2, + page_size: 4096, + architecture: "x86_64".to_owned(), + state_size_bytes: 5, + machine_contract: Some(test_machine_contract()), + format_magic: b"OPENVMM_SNAPSHOT_V6\0".to_vec(), + saved_state_schema_version: 1, + saved_state_root_type: "openvmm.SavedState".to_owned(), + snapshot_tier: "platform".to_owned(), + restore_policy: "clone".to_owned(), + consumed_config_sections: 1, + }); + for error in read_errors_with_manifest(&manifest) { + assert!(error.contains("[E_SNAPSHOT_VERSION]"), "{error}"); + assert!( + error.contains("snapshot manifest version 6 is not supported"), + "{error}" + ); + assert!(error.contains("recapture the snapshot"), "{error}"); + } + } + + #[test] + fn read_snapshot_refuses_an_early_version_2_manifest_by_its_magic() { + // The first manifest version 2, which predates the format magic, kept + // the SHA-256 digests of `state.bin` and `memory.bin` in fields 9 and + // 10, where this format keeps its machine contract and its magic. + #[derive(Protobuf)] + #[mesh(package = "openvmm.snapshot")] + struct ManifestV2WithDigests { + #[mesh(1)] + version: u32, + #[mesh(2)] + created_at: Timestamp, + #[mesh(3)] + openvmm_version: String, + #[mesh(4)] + memory_size_bytes: u64, + #[mesh(5)] + vp_count: u32, + #[mesh(6)] + page_size: u32, + #[mesh(7)] + architecture: String, + #[mesh(8)] + state_size_bytes: u64, + #[mesh(9)] + state_sha256: Vec, + #[mesh(10)] + memory_sha256: Vec, + } + + let manifest = mesh::payload::encode(ManifestV2WithDigests { + version: 2, + created_at: Timestamp { + seconds: 1234567890, + nanos: 0, + }, + openvmm_version: "test-0.1.0".to_owned(), + memory_size_bytes: 1024, + vp_count: 2, + page_size: 4096, + architecture: "x86_64".to_owned(), + state_size_bytes: 5, + state_sha256: sha2::Sha256::digest(b"state").to_vec(), + memory_sha256: sha2::Sha256::digest([0_u8; 1024]).to_vec(), + }); + for error in read_errors_with_manifest(&manifest) { + assert!(error.contains("[E_SNAPSHOT_VERSION]"), "{error}"); + assert!( + error.contains("snapshot format magic is invalid"), + "{error}" + ); + assert!(error.contains("recapture the snapshot"), "{error}"); + } + } + #[test] fn read_snapshot_rejects_a_command_line_over_the_limit_without_rewriting_it() { let dir = tempfile::tempdir().unwrap(); From 98e5b3ca504ea18ffdcc7181e27516f7d7e5a6d7 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Sun, 8 Nov 2026 18:00:19 -0800 Subject: [PATCH 2/2] openvmm_helpers: reserve the manifest fields that upstream assigns Upstream's SaveVM and RestoreVM (microsoft/openvmm#4553) record a snapshot ID in the manifest. It first took field 8, which this manifest gives to the length of state.bin, a varint that a GUID cannot decode. Upstream has since moved `snapshot_id` to field 18 instead, so that neither side renumbers a field, and reserves fields 19 and 20 for `compatibility_bundle_digest` and `memory_artifacts`, as its pull request records (microsoft/nvx#1210). This manifest's fields end at 15, so the field that it adds next after 16 and 17 would take 18. Record the reservation where the fields are declared, and in the Guide's rule for extending the format, so that no field of this manifest takes 18, 19, or 20. No field changes, so the format stays version 2, and every snapshot stays restorable. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Guide/src/dev_guide/snapshot_format.md | 4 +++- openvmm/openvmm_helpers/src/snapshot.rs | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Guide/src/dev_guide/snapshot_format.md b/Guide/src/dev_guide/snapshot_format.md index 23cc19ac1ae..5cfeab29c1a 100644 --- a/Guide/src/dev_guide/snapshot_format.md +++ b/Guide/src/dev_guide/snapshot_format.md @@ -213,7 +213,9 @@ Key unsupported categories: ## Extending the format When adding new fields to `SnapshotManifest`, use the next available mesh -tag number. +tag number other than 18, 19, and 20: upstream OpenVMM assigns field 18 to +the `snapshot_id` that its SaveVM API records, and reserves fields 19 and 20 +for `compatibility_bundle_digest` and `memory_artifacts`. The protobuf encoding is forward-compatible: older readers will ignore unknown fields. However, removing or reordering existing fields is a breaking change. See [Save State](contrib/save-state.md) for the full set of diff --git a/openvmm/openvmm_helpers/src/snapshot.rs b/openvmm/openvmm_helpers/src/snapshot.rs index 4f95b1435ae..6539ae8b6e9 100644 --- a/openvmm/openvmm_helpers/src/snapshot.rs +++ b/openvmm/openvmm_helpers/src/snapshot.rs @@ -70,6 +70,10 @@ pub struct SnapshotManifest { /// Bitmask of configuration sections consumed before capture. #[mesh(15)] pub consumed_config_sections: u32, + // Upstream OpenVMM assigns field 18 to the `snapshot_id` that its SaveVM + // API records, and reserves fields 19 and 20 for + // `compatibility_bundle_digest` and `memory_artifacts`, so no other field + // of this manifest takes them. } /// Validate that a snapshot manifest is compatible with the running VM config.