From c40dff64a5ae93cda9bff61f8f5dcb5e68293637 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Fri, 30 Oct 2026 18:18:13 -0700 Subject: [PATCH 1/2] microVM: let an untiered restore install a new policy on a NIC template --restore-network-policy installs the policy of a restore's options in place of the complete deny-all policy of a NIC template, but only at a platform restore: validate_network_policy_restore refused it for every other tier, including the empty tier of a snapshot without sandbox blocks, which the non-sandbox mode captures and restores (microsoft/nvx#942). Such a snapshot could only keep the policy that it was captured with, and never take ingress, explicit host loopback, or forwards. Admit the option for a snapshot without a tier too. Everything else stays as at a platform restore: the template check, which reads the NIC's network record and not the tier; the restore contract, which compares the whole record against the template's deny-all; and the device, which installs the new policy, with its forwards bound, before any vCPU runs. The workload-start and instance-checkpoint tiers keep refusing the option. The one difference from a platform snapshot, which is captured before any tenant input, is that a snapshot without a tier may hold a workload that has already run. Under the complete deny-all, that workload had no connectivity: its endpoint held no flows, listeners, or forwards, and its guest learned no DNS server, so nothing in the snapshot depends on the policy, and the restore grants the workload connectivity that it never had. The option's documentation records this decision. The tier check's test now admits both tiers and still refuses the later ones, and the test that restores one NIC template with eight policies passes the tier check for both, since no other restore-time check reads the tier. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Guide/src/dev_guide/snapshot_format.md | 12 ++--- Guide/src/reference/openvmm/management/cli.md | 48 +++++++++++-------- Guide/src/user_guide/openvmm/snapshots.md | 20 ++++---- openvmm/openvmm_entry/src/cli_args/microvm.rs | 3 +- openvmm/openvmm_entry/src/microvm/network.rs | 13 ++++- openvmm/openvmm_entry/src/microvm/restore.rs | 40 ++++++++++------ 6 files changed, 83 insertions(+), 53 deletions(-) diff --git a/Guide/src/dev_guide/snapshot_format.md b/Guide/src/dev_guide/snapshot_format.md index 10917ca110f..7b52988a2df 100644 --- a/Guide/src/dev_guide/snapshot_format.md +++ b/Guide/src/dev_guide/snapshot_format.md @@ -50,12 +50,12 @@ bound egress policy. Encoding version 2 includes the static prefix, guest MAC, and canonical ARP next-hop set in addition to the policy rules. An absent encoding-version field denotes version 1, allowing network snapshots produced before endpoint next-hop binding to retain their original digest semantics. -A platform snapshot whose digest is that of the complete deny-all policy bound -to its identity, in encoding version 3, or version 4 with IPv6, is a NIC -template, whose restore may install another policy with -`--restore-network-policy`. Earlier encodings do not record the host-loopback -action or the proxy endpoint, so they never identify a template, and the -format records nothing else for it. +A platform snapshot, or a snapshot without a tier, whose digest is that of the +complete deny-all policy bound to its identity, in encoding version 3, or +version 4 with IPv6, is a NIC template, whose restore may install another +policy with `--restore-network-policy`. Earlier encodings do not record the +host-loopback action or the proxy endpoint, so they never identify a template, +and the format records nothing else for it. ## Device state (`state.bin`) diff --git a/Guide/src/reference/openvmm/management/cli.md b/Guide/src/reference/openvmm/management/cli.md index a711df360c1..ea9c0c92de9 100644 --- a/Guide/src/reference/openvmm/management/cli.md +++ b/Guide/src/reference/openvmm/management/cli.md @@ -246,8 +246,9 @@ describes the source definitions. endpoint-ready RX at the capture boundary, rewind unused guest RX descriptors, and recreate a fresh Consomme endpoint generation on restore. Restore of a networked snapshot requires `--network-profile portable` and, - unless `--restore-network-policy` replaces a platform snapshot's complete - deny-all, the same active egress policy rules. The policy digest binds the + unless `--restore-network-policy` replaces the complete deny-all of a + platform snapshot or of a snapshot without a tier, the same active egress + policy rules. The policy digest binds the saved static identity, including the IPv6 identity, and its derived ARP next hops, which are reconstructed before vCPUs start. A snapshot whose network predates IPv6 restores IPv4-only, with its original digest. Native @@ -464,7 +465,8 @@ describes the source definitions. When the snapshot contains virtio-net, restore also requires `--network-profile portable`; the snapshot's profile and canonical egress policy must match the supplied portable configuration, unless - `--restore-network-policy` installs a new policy on a platform snapshot. + `--restore-network-policy` installs a new policy on a platform snapshot or + a snapshot without a tier. For sandbox-block snapshots, restore repeats each read-only `--microvm-sandbox-block` argument. Its role, access, geometry, and SHA-256 @@ -536,26 +538,30 @@ describes the source definitions. for a cold boot, for a paired snapshot or one without sandbox blocks, and together with a scratch `--microvm-sandbox-block`. A scratch file, when given instead, keeps the exact captured length. -* `--restore-network-policy`: Restore a platform snapshot's NIC with the - network policy of this restore's options rather than the captured one. The - snapshot must be a NIC template: a platform snapshot whose NIC was captured - with a complete deny-all policy, `--network-egress deny` and - `--host-loopback deny` without egress rules, `--network-proxy`, or +* `--restore-network-policy`: Restore the NIC of a platform snapshot, or of a + snapshot without a tier, with the network policy of this restore's options + rather than the captured one. The snapshot must be a NIC template: one whose + NIC was captured with a complete deny-all policy, `--network-egress deny` + and `--host-loopback deny` without egress rules, `--network-proxy`, or ingress. Its guest learned no DNS server and announces no listeners, and its - endpoint held no flows, listeners, or forwards, so its clones may each + endpoint held no flows, listeners, or forwards, so its restores may each install their own policy, including `--network-ingress allow`, explicit - `--host-loopback allow`, and `--host-loopback-forward`. OpenVMM validates the - policy as at a cold boot, binds it to the captured identity, which the NIC - keeps, and installs it, with its forwards bound, before any vCPU runs; a - policy that fails validation, or a forward that cannot bind its port, fails - the restore. A restore that wants no connectivity passes the template's own - deny-all. The restore cannot readdress the NIC, and a policy may name IPv6 - destinations only when the captured identity has IPv6. OpenVMM rejects the - option before any vCPU starts for a cold boot, for a snapshot of another - tier, for a snapshot without a NIC, and for a NIC captured with any other - policy, including one whose digest encoding predates version 3. The guest - keeps its captured command line, so it learns its DNS server and whether - to announce its listeners from its host, after the restore. + `--host-loopback allow`, and `--host-loopback-forward`. A snapshot without a + tier may hold a workload that has already run, but under the complete + deny-all that workload had no connectivity, so nothing in the snapshot + depends on the policy, and such a restore grants the workload connectivity + that it never had. OpenVMM validates the policy as at a cold boot, binds it + to the captured identity, which the NIC keeps, and installs it, with its + forwards bound, before any vCPU runs; a policy that fails validation, or a + forward that cannot bind its port, fails the restore. A restore that wants + no connectivity passes the template's own deny-all. The restore cannot + readdress the NIC, and a policy may name IPv6 destinations only when the + captured identity has IPv6. OpenVMM rejects the option before any vCPU + starts for a cold boot, for a `workload-start` or `instance-checkpoint` + snapshot, for a snapshot without a NIC, and for a NIC captured with any + other policy, including one whose digest encoding predates version 3. The + guest keeps its captured command line, so it learns its DNS server and + whether to announce its listeners from its host, after the restore. * `--restore-gate-timeout-ms `: Bound microVM guest repair and gate acknowledgement after restore. The default is 60000 milliseconds. * `--cpu-profile `: Select the microVM's pinned CPU profile, `auto` diff --git a/Guide/src/user_guide/openvmm/snapshots.md b/Guide/src/user_guide/openvmm/snapshots.md index 2ba5ff07907..0bf1c66f6b2 100644 --- a/Guide/src/user_guide/openvmm/snapshots.md +++ b/Guide/src/user_guide/openvmm/snapshots.md @@ -199,14 +199,18 @@ block sizes. The device stays writable, as the guest negotiated it, but reads return zeros and writes fail with an I/O error: OpenVMM never drops a write or keeps one in host memory. Only a platform snapshot's fresh scratch may be absent, and a scratch file, when given, keeps the exact captured length. A -platform snapshot whose NIC was captured with a complete deny-all policy, -`--network-egress deny` and `--host-loopback deny` without egress rules, a -proxy, or ingress, is a NIC template: with `--restore-network-policy`, each -clone installs the policy of its own restore options, which may publish its -listeners and forward host ports, instead of the captured one. OpenVMM -validates that policy as at a cold boot and installs it before any vCPU runs, -and the NIC keeps its captured identity. Every other networked restore -requires the captured policy. An instance-checkpoint restore attempt atomically creates +platform snapshot, or a snapshot without a tier, whose NIC was captured with a +complete deny-all policy, `--network-egress deny` and `--host-loopback deny` +without egress rules, a proxy, or ingress, is a NIC template: with +`--restore-network-policy`, each restore installs the policy of its own +restore options, which may publish its listeners and forward host ports, +instead of the captured one. OpenVMM validates that policy as at a cold boot +and installs it before any vCPU runs, and the NIC keeps its captured identity. +A snapshot without a tier may hold a workload that has already run, but under +the complete deny-all that workload had no connectivity, so nothing in the +snapshot depends on the policy, and such a restore grants the workload +connectivity that it never had. Every other networked restore requires the +captured policy. An instance-checkpoint restore attempt atomically creates `resume.claim`; subsequent restores are rejected. The claim is committed after artifact and configuration validation but before worker construction, so the restore attempt remains consumed if later worker startup fails. diff --git a/openvmm/openvmm_entry/src/cli_args/microvm.rs b/openvmm/openvmm_entry/src/cli_args/microvm.rs index bad1f86deaa..0fa253ce9c5 100644 --- a/openvmm/openvmm_entry/src/cli_args/microvm.rs +++ b/openvmm/openvmm_entry/src/cli_args/microvm.rs @@ -297,7 +297,8 @@ pub struct MicrovmCli { pub restore_absent_scratch: bool, /// Install this restore's network policy in place of the complete - /// deny-all policy that a platform snapshot's NIC was captured with. + /// deny-all policy that the NIC of a platform snapshot, or of a snapshot + /// without a tier, was captured with. /// /// The NIC keeps its captured identity. The network policy options of /// this restore, which may include `--network-ingress allow`, diff --git a/openvmm/openvmm_entry/src/microvm/network.rs b/openvmm/openvmm_entry/src/microvm/network.rs index db84401bd24..264dc6999b9 100644 --- a/openvmm/openvmm_entry/src/microvm/network.rs +++ b/openvmm/openvmm_entry/src/microvm/network.rs @@ -238,8 +238,10 @@ pub(crate) mod tests { contract_with_policy(network, &policy) } - /// Returns the contract of a platform template whose NIC has the identity - /// `network` and the complete deny-all policy. + /// Returns the contract of a NIC template without sandbox blocks, as a + /// snapshot without a tier records one, whose NIC has the identity + /// `network` and the complete deny-all policy. A platform template's + /// contract adds sandbox blocks, which the network checks don't read. fn template_contract_for( network: &openvmm_defs::microvm::MicrovmNetworkConfig, ) -> openvmm_helpers::snapshot::microvm::SnapshotMachineContract { @@ -549,10 +551,17 @@ pub(crate) mod tests { #[cfg(guest_arch = "x86_64")] #[test] fn a_template_restore_installs_a_new_policy_and_keeps_the_identity() { + use openvmm_helpers::snapshot::format::SNAPSHOT_TIER_PLATFORM; + let network: openvmm_defs::microvm::MicrovmNetworkConfig = "10.0.0.2/24".parse().unwrap(); let contract = template_contract_for(&network); let template = openvmm_helpers::snapshot::microvm::microvm_network_template_policy(&network).unwrap(); + // The tier check is the only one that reads the tier, so one + // template, platform or without a tier, takes every policy. + for tier in [SNAPSHOT_TIER_PLATFORM, ""] { + super::super::restore::validate_network_policy_restore(tier, true).unwrap(); + } for arguments in [ // A clone without connectivity keeps the template's policy. &["--network-egress", "deny", "--host-loopback", "deny"][..], diff --git a/openvmm/openvmm_entry/src/microvm/restore.rs b/openvmm/openvmm_entry/src/microvm/restore.rs index e5ec2f573e2..6f245a447f1 100644 --- a/openvmm/openvmm_entry/src/microvm/restore.rs +++ b/openvmm/openvmm_entry/src/microvm/restore.rs @@ -201,8 +201,9 @@ pub(crate) fn restore_packet_base( /// platform snapshot's `distro` block keeps its captured capacity, whatever /// the length of the image that the restore attaches to it, and /// `--restore-absent-scratch` attaches its fresh scratch with no file. Only a -/// platform snapshot admits `--restore-network-policy`, and `--mount-aggregate` -/// on the filesystem slot that it left dormant. +/// platform snapshot, or one without a tier, admits +/// `--restore-network-policy`, and only a platform snapshot admits +/// `--mount-aggregate` on the filesystem slot that it left dormant. pub(crate) fn prepare_restore( opt: &mut Options, restore_snapshot: Option<&OpenedSnapshot>, @@ -405,18 +406,26 @@ pub(crate) fn prepare_restore( } /// Rejects `--restore-network-policy`, which `restore_network_policy` -/// reports, for a snapshot of a tier other than the platform tier. +/// reports, for a snapshot of the `workload-start` or `instance-checkpoint` +/// tier. /// -/// A platform snapshot is captured before any tenant input. Later tiers carry -/// a workload that ran under the captured policy, so they keep it. -fn validate_network_policy_restore( +/// A platform snapshot is captured before any tenant input. A snapshot +/// without a tier may hold a workload that has already run, but the option +/// requires a NIC captured with the complete deny-all, under which that +/// workload had no connectivity: its endpoint held no flows, listeners, or +/// forwards, and its guest learned no DNS server, so nothing in the snapshot +/// depends on the policy, and the restore grants the workload connectivity +/// that it never had. The later tiers continue a tenant's workload, which +/// keeps the policy that it was captured with. +pub(super) fn validate_network_policy_restore( snapshot_tier: &str, restore_network_policy: bool, ) -> anyhow::Result<()> { anyhow::ensure!( !restore_network_policy + || snapshot_tier.is_empty() || snapshot_tier == openvmm_helpers::snapshot::format::SNAPSHOT_TIER_PLATFORM, - "--restore-network-policy requires a platform snapshot" + "--restore-network-policy requires a platform snapshot or a snapshot without a tier" ); Ok(()) } @@ -910,25 +919,26 @@ pub(crate) mod tests { } #[test] - fn only_a_platform_restore_replaces_the_network_policy() { + fn only_a_platform_or_untiered_restore_replaces_the_network_policy() { use openvmm_helpers::snapshot::format::SNAPSHOT_TIER_INSTANCE_CHECKPOINT; use openvmm_helpers::snapshot::format::SNAPSHOT_TIER_PLATFORM; use openvmm_helpers::snapshot::format::SNAPSHOT_TIER_WORKLOAD_START; - for tier in [ - SNAPSHOT_TIER_PLATFORM, - SNAPSHOT_TIER_WORKLOAD_START, - SNAPSHOT_TIER_INSTANCE_CHECKPOINT, - "", + for (tier, replaces) in [ + (SNAPSHOT_TIER_PLATFORM, true), + ("", true), + (SNAPSHOT_TIER_WORKLOAD_START, false), + (SNAPSHOT_TIER_INSTANCE_CHECKPOINT, false), ] { validate_network_policy_restore(tier, false).unwrap(); let replaced = validate_network_policy_restore(tier, true); - if tier == SNAPSHOT_TIER_PLATFORM { + if replaces { replaced.unwrap(); } else { assert_eq!( replaced.unwrap_err().to_string(), - "--restore-network-policy requires a platform snapshot" + "--restore-network-policy requires a platform snapshot or a snapshot without a tier", + "{tier}" ); } } From 757a15fa36c973d2cdcd4da8d6e3c8d2e5036abb Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Fri, 30 Oct 2026 18:28:18 -0700 Subject: [PATCH 2/2] microVM: hand an untiered guest the network settings of its new policy A restore of a snapshot without a tier may now install a new network policy on a NIC template with --restore-network-policy, but its guest still configured its network from the captured command line, which a restore keeps: a cold boot names the guest's DNS server with virtnet_dns= and asks it to announce its listeners with virtnet_ingress=, and a template captured under the complete deny-all names neither. A platform clone learns both from its activation after the restore (microsoft/nvx#838, microsoft/nvx#850), but the non-sandbox mode has no activation, so its guest resolved no names, and ingress and explicit host loopback published none of its listeners (microsoft/nvx#943). Hand them to the guest's repair in the restore packet. At a restore that replaces the network policy of a snapshot without a tier, the packet becomes version 6: version 5's fields, with the network byte in place of the reserved byte, whose bits 0 and 1 give the family of the DNS server, none, IPv4, or IPv6, and whose bit 2 asks the guest to announce its listeners, and with a 16-byte DNS server after the entropy. OpenVMM derives both from the restore's policy as a cold boot does, with gateway_dns_server and microvm_ingress. Such a restore also gates host input, so that the guest acknowledges it once it has applied them, before any input reaches it. Every other restore, a platform clone's included, keeps version 5, and a guest that reads version 5 alone refuses version 6, so it fails such a restore rather than ignore its settings. The management RPC refuses snapshots that have a NIC, so its packets carry none. No status bit advertises the settings, since the guest reads every packet. The --restore-entropy help, which still named version 4, names both. RestorePacketV5 becomes RestorePacket, which encodes either version. Tests cover the golden encodings of version 6 with an IPv4 server and announcements, with an IPv6 server, and with neither; the refusal of malformed network bytes, padded or unspecified DNS servers, and lengths; the device serving a version 6 packet; the restore's decision for every tier; and the settings that the packet derives from each of ten policies that one template is restored with. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Guide/src/reference/openvmm/management/cli.md | 16 +- Guide/src/user_guide/openvmm/snapshots.md | 35 +- openvmm/openvmm_entry/src/cli_args/microvm.rs | 6 +- openvmm/openvmm_entry/src/microvm/config.rs | 12 +- openvmm/openvmm_entry/src/microvm/network.rs | 160 ++++++-- openvmm/openvmm_entry/src/microvm/restore.rs | 126 +++++- openvmm/openvmm_entry/src/ttrpc/microvm.rs | 3 + vm/devices/chipset/src/microvm.rs | 72 +++- vm/devices/chipset_resources/src/microvm.rs | 7 +- .../chipset_resources/src/microvm_time.rs | 373 ++++++++++++++++-- 10 files changed, 700 insertions(+), 110 deletions(-) diff --git a/Guide/src/reference/openvmm/management/cli.md b/Guide/src/reference/openvmm/management/cli.md index ea9c0c92de9..e51ad91700c 100644 --- a/Guide/src/reference/openvmm/management/cli.md +++ b/Guide/src/reference/openvmm/management/cli.md @@ -503,10 +503,12 @@ describes the source definitions. in progress; Windows flush completion waits for the named-pipe peer to consume the complete frame. * `--restore-entropy`: Accepted for compatibility; it has no effect. Every - microVM restore makes restore packet version 5 available on the private - portb restore channel. The packet carries 64 bytes of fresh entropy, the - online-VP target, the memory-expansion ranges, the downtime, and the TSC - rate deviation; the guest must consume it and explicitly reseed its RNG. + microVM restore makes restore packet version 5, or version 6 at a restore + that replaces the network policy of a snapshot without a tier, available on + the private portb restore channel. The packet carries 64 bytes of fresh + entropy, the online-VP target, the memory-expansion ranges, the downtime, + the TSC rate deviation, and, in version 6, the network settings of the + restore's policy; the guest must consume it and explicitly reseed its RNG. [Snapshots](../../../user_guide/openvmm/snapshots.md) documents its format, the portb status bits, and the time sample. Every microVM portb device also reports generation-ID support in status bit @@ -561,7 +563,11 @@ describes the source definitions. snapshot, for a snapshot without a NIC, and for a NIC captured with any other policy, including one whose digest encoding predates version 3. The guest keeps its captured command line, so it learns its DNS server and - whether to announce its listeners from its host, after the restore. + whether to announce its listeners after the restore: a platform clone from + its host, and the guest of a snapshot without a tier from restore packet + version 6, which carries them, and whose restore gates host input until the + guest acknowledges that it applied them. A guest that reads restore packet + version 5 alone fails such a restore rather than ignore them. * `--restore-gate-timeout-ms `: Bound microVM guest repair and gate acknowledgement after restore. The default is 60000 milliseconds. * `--cpu-profile `: Select the microVM's pinned CPU profile, `auto` diff --git a/Guide/src/user_guide/openvmm/snapshots.md b/Guide/src/user_guide/openvmm/snapshots.md index 0bf1c66f6b2..547e29f8b16 100644 --- a/Guide/src/user_guide/openvmm/snapshots.md +++ b/Guide/src/user_guide/openvmm/snapshots.md @@ -174,9 +174,11 @@ releasing gated input. The peer must accept and read while resume is in progress; on Windows, flush completion waits until the named-pipe peer consumes the complete frame. -For a tiered microVM restore, OpenVMM starts device workers with network, -control, and filesystem input gated. The guest performs post-restore repair -and writes the existing snapshot port (`0x605`) to acknowledge completion. +For a tiered microVM restore, and for a restore that replaces the network +policy of a snapshot without a tier, OpenVMM starts device workers with +network, control, and filesystem input gated. The guest performs post-restore +repair and writes the existing snapshot port (`0x605`) to acknowledge +completion. OpenVMM stops at that exact post-write boundary, completes the deferred write while the vCPU is stopped, and then releases device input before resuming the vCPU. Until then, the guest's virtio-fs requests wait on their virtqueues, so @@ -223,16 +225,18 @@ the ID from the first 16 bytes of the restore packet's fresh entropy, so the guest can reject an unchanged clone identity, reseed Linux, refresh runtime identifiers, and acknowledge the input gate without another PMIO transfer. -Every microVM restore exposes restore packet version 5. Status bit 1 -advertises it, and writing `0xa5` to the status port selects it for reading -from the data port. Its 24-byte little-endian header holds: +Every microVM restore exposes a restore packet: version 5, or version 6 at a +restore whose `--restore-network-policy` replaces the network policy of a +snapshot without a tier. Status bit 1 advertises it, and writing `0xa5` to the +status port selects it for reading from the data port. Its 24-byte +little-endian header holds: -- the magic `OVR` and version 5; +- the magic `OVR` and the version; - flags: bit 0, the downtime came from host UTC; bit 1, a memory target; bit 2, the guest must acknowledge the restore through port `0x605`; bit 3, a test hook is active; -- the online-VP target (0 for none), the memory-range count, and a reserved - zero byte; +- the online-VP target (0 for none), the memory-range count, and a byte that + is reserved and zero in version 5 and is the network byte in version 6; - the generation counter (`u32`), which is 0 at cold boot and one more than the snapshot's at each restore; - the TSC rate deviation (`i32`, ppm scaled by 2^16); and @@ -244,6 +248,19 @@ memory target, and one or more memory ranges, so a zero-range target can skip post-restore repair. The packet carries no host UTC: the guest sets its clock from a time sample. +Version 6 carries the network settings of the restore's policy, which a cold +boot names on the command line that the restore keeps, and which a NIC +template's guest never learned: its network byte's bits 0 and 1 give the +family of the guest's DNS server, 0 for none, 1 for IPv4, or 2 for IPv6, and +bit 2 asks the guest to announce its listeners, as `virtnet_dns=` and +`virtnet_ingress=` do at a cold boot; the other bits are zero. A 16-byte DNS +server follows the entropy: an IPv6 address, or an IPv4 address and 12 zero +bytes, or zeros without a server. Such a restore requires the guest's +acknowledgement, which the guest gives once it has applied the settings. A +guest that reads version 5 alone fails such a restore rather than ignore its +settings, and every other restore, a platform clone's included, keeps +version 5. + Status bit 6 advertises the time sample. Writing `0xa7` to the status port latches a 48-byte little-endian sample, version 2: diff --git a/openvmm/openvmm_entry/src/cli_args/microvm.rs b/openvmm/openvmm_entry/src/cli_args/microvm.rs index 0fa253ce9c5..4b5a059ca8b 100644 --- a/openvmm/openvmm_entry/src/cli_args/microvm.rs +++ b/openvmm/openvmm_entry/src/cli_args/microvm.rs @@ -274,9 +274,9 @@ impl From for MachineProfile { /// Options of the microVM machine profile. #[derive(clap::Args)] pub struct MicrovmCli { - /// Accepted for compatibility: every microVM restore exposes restore packet - /// version 4, which carries fresh entropy, through the private portb restore - /// channel. + /// Accepted for compatibility: every microVM restore exposes a restore + /// packet, version 5, or version 6 with network settings, which carries + /// fresh entropy, through the private portb restore channel. #[clap(long, requires = "restore_snapshot")] pub restore_entropy: bool, diff --git a/openvmm/openvmm_entry/src/microvm/config.rs b/openvmm/openvmm_entry/src/microvm/config.rs index e9fc9298d4c..cec64c846bb 100644 --- a/openvmm/openvmm_entry/src/microvm/config.rs +++ b/openvmm/openvmm_entry/src/microvm/config.rs @@ -20,6 +20,7 @@ use super::network::EffectiveMicrovmNetwork; use super::network::effective_microvm_network; use super::network::microvm_ingress; use super::network::microvm_network_endpoint; +use super::network::microvm_restore_network; use super::output::MicrovmOutputDrain; use super::output::OutputCompletion; use super::output::spawn_output; @@ -420,9 +421,18 @@ impl<'a> MicrovmConfigBuilder<'a> { let (drain, output_drain) = MicrovmOutputDrain::new(Some(output_completion)); self.resources.output_drain = Some(drain); let (generation_id, time_abi_restore) = if self.restore.machine_contract.is_some() { + let network = if self.restore.network_settings { + let network = self + .network + .as_ref() + .context("a restore that replaces the network policy requires a NIC")?; + Some(microvm_restore_network(&network.policy)) + } else { + None + }; let (generation_id, base) = self .restore - .time_abi_restore_packet(opt.microvm.restore_processors)?; + .time_abi_restore_packet(opt.microvm.restore_processors, network)?; ( generation_id, Some(self.resources.time_abi_restore_packet(base)), diff --git a/openvmm/openvmm_entry/src/microvm/network.rs b/openvmm/openvmm_entry/src/microvm/network.rs index 264dc6999b9..3106e9527a3 100644 --- a/openvmm/openvmm_entry/src/microvm/network.rs +++ b/openvmm/openvmm_entry/src/microvm/network.rs @@ -158,6 +158,19 @@ pub(super) fn microvm_ingress( } } +/// Returns the network settings that the restore packet hands the guest of a +/// restore that replaces its snapshot's policy with `policy`: the DNS server +/// and the listener announcement that a cold boot under `policy` names on the +/// command line, with `virtnet_dns=` and `virtnet_ingress=`. +pub(super) fn microvm_restore_network( + policy: &net_backend_resources::egress::EgressPolicy, +) -> chipset_resources::microvm_time::RestoreNetwork { + chipset_resources::microvm_time::RestoreNetwork { + dns_server: policy.gateway_dns_server().map(Into::into), + announce_listeners: microvm_ingress(policy).is_some(), + } +} + pub(super) fn microvm_network_endpoint( network: &openvmm_defs::microvm::MicrovmNetworkConfig, policy: &net_backend_resources::egress::EgressPolicy, @@ -562,46 +575,107 @@ pub(crate) mod tests { for tier in [SNAPSHOT_TIER_PLATFORM, ""] { super::super::restore::validate_network_policy_restore(tier, true).unwrap(); } - for arguments in [ + let ipv4_gateway: std::net::IpAddr = "10.0.0.1".parse().unwrap(); + let ipv6_gateway: std::net::IpAddr = "fd00::a00:1".parse().unwrap(); + // Each policy, with the DNS server and the listener announcement that + // a cold boot under it names, which the restore packet of a snapshot + // without a tier hands its guest. + for (arguments, dns_server, announce_listeners) in [ // A clone without connectivity keeps the template's policy. - &["--network-egress", "deny", "--host-loopback", "deny"][..], - &["--network-egress", "allow"][..], - &[ - "--network-egress", - "deny", - "--network-egress-allow", - "10.0.0.1/32:tcp:8080", - "--network-egress-allow", - "fd00::a00:1/128:tcp:8080", - "--host-loopback", - "deny", - ][..], - &[ - "--network-egress", - "deny", - "--allow-endpoint", - "192.0.2.7:443", - ][..], - &[ - "--network-egress", - "deny", - "--host-loopback", - "deny", - "--network-proxy", - "10.0.0.1:3128", - ][..], - &[ - "--network-egress", - "deny", - "--host-loopback", - "allow", - "--host-loopback-forward", - "tcp:3000:8080", - "--host-loopback-forward", - "udp:3000:8080", - ][..], - &["--network-ingress", "allow"][..], - &["--network-ingress", "allow", "--host-loopback", "allow"][..], + ( + &["--network-egress", "deny", "--host-loopback", "deny"][..], + None, + false, + ), + ( + &["--network-egress", "allow"][..], + Some(ipv4_gateway), + false, + ), + ( + &[ + "--network-egress", + "deny", + "--network-egress-allow", + "10.0.0.1/32:tcp:8080", + "--network-egress-allow", + "fd00::a00:1/128:tcp:8080", + "--host-loopback", + "deny", + ][..], + None, + false, + ), + ( + &[ + "--network-egress", + "deny", + "--network-egress-allow", + "10.0.0.1/32:udp:53", + "--host-loopback", + "deny", + ][..], + Some(ipv4_gateway), + false, + ), + ( + &[ + "--network-egress", + "deny", + "--network-egress-allow", + "fd00::a00:1/128:udp:53", + "--host-loopback", + "deny", + ][..], + Some(ipv6_gateway), + false, + ), + ( + &[ + "--network-egress", + "deny", + "--allow-endpoint", + "192.0.2.7:443", + ][..], + None, + false, + ), + ( + &[ + "--network-egress", + "deny", + "--host-loopback", + "deny", + "--network-proxy", + "10.0.0.1:3128", + ][..], + None, + false, + ), + ( + &[ + "--network-egress", + "deny", + "--host-loopback", + "allow", + "--host-loopback-forward", + "tcp:3000:8080", + "--host-loopback-forward", + "udp:3000:8080", + ][..], + None, + true, + ), + ( + &["--network-ingress", "allow"][..], + Some(ipv4_gateway), + true, + ), + ( + &["--network-ingress", "allow", "--host-loopback", "allow"][..], + Some(ipv4_gateway), + true, + ), ] { let options = restore_options(&[&["--restore-network-policy"][..], arguments].concat()); options.validate_microvm_options().unwrap(); @@ -623,6 +697,14 @@ pub(crate) mod tests { contract.microvm_network, "{arguments:?}" ); + assert_eq!( + microvm_restore_network(&restored.policy), + chipset_resources::microvm_time::RestoreNetwork { + dns_server: dns_server.map(Into::into), + announce_listeners, + }, + "{arguments:?}" + ); } let deny_all = restore_options(&[ "--restore-network-policy", diff --git a/openvmm/openvmm_entry/src/microvm/restore.rs b/openvmm/openvmm_entry/src/microvm/restore.rs index 6f245a447f1..afc1e479a97 100644 --- a/openvmm/openvmm_entry/src/microvm/restore.rs +++ b/openvmm/openvmm_entry/src/microvm/restore.rs @@ -15,6 +15,7 @@ use crate::storage_builder::microvm::MicrovmSandboxBlockGeometry; use anyhow::Context; use chipset_resources::microvm_time::RESTORE_ENTROPY_LEN; use chipset_resources::microvm_time::RestoreMemoryRange; +use chipset_resources::microvm_time::RestoreNetwork; use chipset_resources::microvm_time::RestorePacketBase; use chipset_resources::microvm_time::RestoreTimeRecord; use net_backend_resources::egress::EgressPolicy; @@ -102,6 +103,10 @@ pub(crate) struct MicrovmRestore { pub(crate) machine_contract: Option, /// Whether the guest must complete post-restore repair before readiness. pub(crate) gate_required: bool, + /// Whether the restore packet carries the network settings of the + /// restore's policy, which the guest applies before it acknowledges the + /// gated restore. + pub(crate) network_settings: bool, /// Whether `--restore-memory` selected an explicit RAM target. pub(crate) memory_target_requested: bool, /// Fresh private GPA ranges selected for this restore. @@ -124,12 +129,15 @@ impl MicrovmRestore { contract.time.as_ref().zip(contract.cpu_profile.as_ref()) } - /// Returns the controller's fields of restore packet version 5 for a time - /// ABI restore, and the generation ID they imply. The generation counter - /// is the snapshot's plus one. + /// Returns the controller's fields of the restore packet for a time ABI + /// restore, with `network`, the network settings of the restore's policy + /// when [`network_settings`](Self::network_settings) asks for them, which + /// make it version 6, and the generation ID they imply. The generation + /// counter is the snapshot's plus one. pub(crate) fn time_abi_restore_packet( &self, restore_online_vp_count: Option, + network: Option, ) -> anyhow::Result<([u8; 16], RestorePacketBase)> { let (time, _) = self .time_abi_records() @@ -140,21 +148,25 @@ impl MicrovmRestore { self.memory_target_requested, &self.memory_ranges, self.gate_required, + network, ) } } -/// Returns the controller's fields of restore packet version 5 for a time -/// ABI restore of a snapshot with the time contract `time`, and the -/// generation ID they imply. The generation counter is the snapshot's plus -/// one, and `ack_required` makes the guest acknowledge the restore through -/// port `0x605`. +/// Returns the controller's fields of the restore packet for a time ABI +/// restore of a snapshot with the time contract `time`, and the generation ID +/// they imply. The generation counter is the snapshot's plus one, +/// `ack_required` makes the guest acknowledge the restore through port +/// `0x605`, and `network`, the network settings of a restore that replaces +/// the network policy of a snapshot without a tier, makes the packet version +/// 6. pub(crate) fn restore_packet_base( time: &SnapshotTimeContract, restore_online_vp_count: Option, memory_target_requested: bool, memory_ranges: &[SnapshotMemoryExpansionRange], ack_required: bool, + network: Option, ) -> anyhow::Result<([u8; 16], RestorePacketBase)> { let generation = time.capture_generation.checked_add(1).ok_or_else(|| { virt::time_abi::TimeAbiError::new( @@ -186,9 +198,9 @@ pub(crate) fn restore_packet_base( }) .collect(), entropy, + network, }; - base.validate() - .context("restore packet version 5 is invalid")?; + base.validate().context("restore packet is invalid")?; generation_id_create.complete("restore", "generation_id_create", Default::default()); Ok((microvm_generation_id(&entropy), base)) } @@ -203,7 +215,10 @@ pub(crate) fn restore_packet_base( /// `--restore-absent-scratch` attaches its fresh scratch with no file. Only a /// platform snapshot, or one without a tier, admits /// `--restore-network-policy`, and only a platform snapshot admits -/// `--mount-aggregate` on the filesystem slot that it left dormant. +/// `--mount-aggregate` on the filesystem slot that it left dormant. A restore +/// that replaces the network policy of a snapshot without a tier hands the +/// guest the new policy's network settings in its restore packet, and gates +/// host input until the guest has applied them. pub(crate) fn prepare_restore( opt: &mut Options, restore_snapshot: Option<&OpenedSnapshot>, @@ -212,6 +227,7 @@ pub(crate) fn prepare_restore( let mut distro_capacity = None; let mut absent_scratch = None; let mut restore_gate_required = false; + let mut network_settings = false; let restore_memory_target_requested = opt.microvm.restore_memory.is_some(); let mut restore_memory_ranges = Vec::new(); if let Some(contract) = @@ -247,6 +263,13 @@ pub(crate) fn prepare_restore( &manifest.snapshot_tier, opt.microvm.restore_network_policy, )?; + if restore_packet_carries_network_settings( + &manifest.snapshot_tier, + opt.microvm.restore_network_policy, + ) { + network_settings = true; + restore_gate_required = true; + } validate_filesystem_aggregate_restore( &manifest.snapshot_tier, contract.microvm_filesystem.is_some(), @@ -397,6 +420,7 @@ pub(crate) fn prepare_restore( Ok(MicrovmRestore { machine_contract: restore_machine_contract, gate_required: restore_gate_required, + network_settings, memory_target_requested: restore_memory_target_requested, memory_ranges: restore_memory_ranges, private_scratch_dir, @@ -430,6 +454,26 @@ pub(super) fn validate_network_policy_restore( Ok(()) } +/// Returns whether the restore packet of a restore of a snapshot of tier +/// `snapshot_tier` carries the network settings of the restore's policy, +/// given whether `--restore-network-policy`, which `restore_network_policy` +/// reports, replaces the snapshot's policy. +/// +/// A cold boot names the guest's DNS server, and asks the guest to announce +/// its listeners, on its command line, which a restore keeps, and a NIC +/// template is captured under the complete deny-all, which asks neither. A +/// platform clone learns them from its activation, after the restore, but a +/// snapshot without a tier has no activation, so its restore packet carries +/// them, as version 6, and the restore gates host input until the guest has +/// applied them. A guest that reads version 5 alone fails such a restore +/// rather than ignore them. +fn restore_packet_carries_network_settings( + snapshot_tier: &str, + restore_network_policy: bool, +) -> bool { + restore_network_policy && snapshot_tier.is_empty() +} + /// Rejects `--mount-aggregate`, which `aggregate` reports, on the filesystem /// slot that a snapshot of a tier other than the platform tier left dormant, /// which `captured_filesystem` reports when false. @@ -944,6 +988,28 @@ pub(crate) mod tests { } } + #[test] + fn only_an_untiered_restore_that_replaces_the_policy_carries_network_settings() { + use openvmm_helpers::snapshot::format::SNAPSHOT_TIER_INSTANCE_CHECKPOINT; + use openvmm_helpers::snapshot::format::SNAPSHOT_TIER_PLATFORM; + use openvmm_helpers::snapshot::format::SNAPSHOT_TIER_WORKLOAD_START; + + for tier in [ + SNAPSHOT_TIER_PLATFORM, + "", + SNAPSHOT_TIER_WORKLOAD_START, + SNAPSHOT_TIER_INSTANCE_CHECKPOINT, + ] { + assert!(!restore_packet_carries_network_settings(tier, false)); + // A platform clone learns its settings from its activation. + assert_eq!( + restore_packet_carries_network_settings(tier, true), + tier.is_empty(), + "{tier}" + ); + } + } + #[test] fn only_a_platform_restore_attaches_an_aggregate_to_a_dormant_slot() { use openvmm_helpers::snapshot::format::SNAPSHOT_TIER_INSTANCE_CHECKPOINT; @@ -1001,6 +1067,7 @@ pub(crate) mod tests { MicrovmRestore { machine_contract: Some(contract), gate_required: true, + network_settings: false, memory_target_requested: true, memory_ranges: vec![SnapshotMemoryExpansionRange { gpa_start: 0x1_0000_0000, @@ -1015,7 +1082,7 @@ pub(crate) mod tests { #[test] fn time_abi_restore_packet_carries_the_controller_fields() { let restore = time_abi_restore(6); - let (generation_id, base) = restore.time_abi_restore_packet(Some(4)).unwrap(); + let (generation_id, base) = restore.time_abi_restore_packet(Some(4), None).unwrap(); assert_eq!(base.generation, 7); assert_eq!(base.online_vp_count, 4); assert!(base.memory_target && base.ack_required); @@ -1026,14 +1093,41 @@ pub(crate) mod tests { length: 0x4000_0000, }] ); + assert_eq!(base.network, None); + assert_eq!( + base.version(), + chipset_resources::microvm_time::RESTORE_PACKET_VERSION + ); assert_eq!(generation_id, microvm_generation_id(&base.entropy)); - let (_, other) = restore.time_abi_restore_packet(None).unwrap(); + let (_, other) = restore.time_abi_restore_packet(None, None).unwrap(); assert_eq!(other.online_vp_count, 0); assert_ne!(other.entropy, base.entropy); - assert!(restore.time_abi_restore_packet(Some(3)).is_err()); + // The network settings of an untiered restore that replaces its + // policy make the packet version 6. + let network = RestoreNetwork { + dns_server: Some(chipset_resources::microvm_time::RestoreDnsServer::Ipv4( + std::net::Ipv4Addr::new(10, 0, 0, 1), + )), + announce_listeners: true, + }; + let restore = MicrovmRestore { + network_settings: true, + ..time_abi_restore(6) + }; + let (_, base) = restore + .time_abi_restore_packet(None, Some(network)) + .unwrap(); + assert_eq!(base.network, Some(network)); + assert!(base.ack_required); + assert_eq!( + base.version(), + chipset_resources::microvm_time::RESTORE_PACKET_NETWORK_VERSION + ); + + assert!(restore.time_abi_restore_packet(Some(3), None).is_err()); let err = time_abi_restore(u32::MAX) - .time_abi_restore_packet(None) + .time_abi_restore_packet(None, None) .unwrap_err(); assert!( format!("{err:#}").contains("[E_GENERATION_EXHAUSTED]"), @@ -1041,7 +1135,7 @@ pub(crate) mod tests { ); assert!( MicrovmRestore::default() - .time_abi_restore_packet(None) + .time_abi_restore_packet(None, None) .is_err() ); } diff --git a/openvmm/openvmm_entry/src/ttrpc/microvm.rs b/openvmm/openvmm_entry/src/ttrpc/microvm.rs index e34a33fb6f5..42379ab8425 100644 --- a/openvmm/openvmm_entry/src/ttrpc/microvm.rs +++ b/openvmm/openvmm_entry/src/ttrpc/microvm.rs @@ -753,12 +753,15 @@ impl CreateVm { .authoritative_restore .as_ref() .context("time ABI restore is missing its snapshot")?; + // The management RPC refuses snapshots with a NIC, so its + // packets carry no network settings. let (generation_id, base) = crate::microvm::restore_packet_base( restore_time.contract(), self.restore_online_vp_count, restore.restore_memory_target_requested, &restore.restore_memory_ranges, gate_required, + None, )?; ( generation_id, diff --git a/vm/devices/chipset/src/microvm.rs b/vm/devices/chipset/src/microvm.rs index 25a938febcb..83c0b413980 100644 --- a/vm/devices/chipset/src/microvm.rs +++ b/vm/devices/chipset/src/microvm.rs @@ -14,8 +14,8 @@ use chipset_device::pio::PortIoIntercept; use chipset_device::poll_device::PollDevice; use chipset_resources::microvm::MicrovmPortbDrain; use chipset_resources::microvm::MicrovmPortbTimeAbi; +use chipset_resources::microvm_time::RestorePacket; use chipset_resources::microvm_time::RestorePacketBase; -use chipset_resources::microvm_time::RestorePacketV5; use chipset_resources::microvm_time::RestoreTimeRecord; use chipset_resources::microvm_time::TimeSample; use chipset_resources::microvm_time::TimeSampleSource; @@ -62,7 +62,7 @@ const STATUS_TIME_SAMPLE_AVAILABLE: u8 = chipset_resources::microvm_time::STATUS_TIME_SAMPLE_AVAILABLE; const BUFFER_MAX: usize = 1024 * 1024; -/// The restore packet version 5 of a restored VM process. +/// The restore packet of a restored VM process. enum RestorePacketState { /// No packet, or the packet was consumed. None, @@ -146,7 +146,7 @@ pub struct MicrovmPortb { impl MicrovmPortb { /// Creates a portb console using `io` as its host endpoint, with the NVX - /// time ABI 2 configuration `time_abi`: restore packet version 5 and the + /// time ABI 2 configuration `time_abi`: the restore packet and the /// time-sample window at port `0xeb`, which stays empty until /// [`Self::with_time_sample_source`] provides the host fields of its /// samples. @@ -224,7 +224,7 @@ impl MicrovmPortb { if let Some(selected) = time_abi.packet_selected.take() { selected.send(()); } - let packet = RestorePacketV5 { base, time }; + let packet = RestorePacket { base, time }; match packet.encode() { Ok(bytes) => self.restore_packet = bytes.into(), Err(error) => { @@ -1281,6 +1281,7 @@ mod tests { generation: 8, ranges: Vec::new(), entropy: [0x7e; 64], + network: None, }; let (send, recv) = mesh::oneshot(); let mut restored = MicrovmPortb::new( @@ -1325,7 +1326,7 @@ mod tests { IoResult::Ok )); let bytes = read_record(&mut restored, DATA_PORT, base.encoded_len()); - assert_eq!(RestorePacketV5::decode(&bytes).unwrap().base, base); + assert_eq!(RestorePacket::decode(&bytes).unwrap().base, base); } #[test] @@ -1389,6 +1390,7 @@ mod tests { generation: 7, ranges, entropy: [0x5a; 64], + network: None, }; let (send, recv) = mesh::oneshot(); let mut portb = time_abi_portb(Some( @@ -1416,7 +1418,7 @@ mod tests { IoResult::Ok )); let bytes = read_record(&mut portb, DATA_PORT, base.encoded_len()); - assert_eq!(RestorePacketV5::decode(&bytes).unwrap().base, base); + assert_eq!(RestorePacket::decode(&bytes).unwrap().base, base); // Reading the whole packet clears every target. assert_eq!( read_status(&mut portb), @@ -1425,6 +1427,60 @@ mod tests { } } + #[test] + fn a_restore_packet_with_network_settings_is_version_6() { + let base = RestorePacketBase { + online_vp_count: 0, + memory_target: false, + ack_required: true, + generation: 7, + ranges: Vec::new(), + entropy: [0x5a; 64], + network: Some(chipset_resources::microvm_time::RestoreNetwork { + dns_server: Some(chipset_resources::microvm_time::RestoreDnsServer::Ipv4( + std::net::Ipv4Addr::new(10, 0, 0, 1), + )), + announce_listeners: true, + }), + }; + let (send, recv) = mesh::oneshot(); + let mut portb = time_abi_portb(Some( + chipset_resources::microvm::MicrovmRestorePacketSource { + base: base.clone(), + time: recv, + selected: None, + }, + )); + send.send(RestoreTimeRecord { + downtime_ns: 0, + downtime_utc: false, + rate_deviation: 0, + test_hooks: false, + }); + // No status bit advertises the settings: the guest reads every packet. + assert_eq!( + read_status(&mut portb), + STATUS_RESTORE_PACKET_AVAILABLE + | STATUS_GENERATION_ID_AVAILABLE + | STATUS_TIME_SAMPLE_AVAILABLE + ); + assert!(matches!( + portb.io_write(STATUS_PORT, &[RESTORE_PACKET_SELECT]), + IoResult::Ok + )); + let bytes = read_record(&mut portb, DATA_PORT, base.encoded_len()); + assert_eq!(bytes.len(), 104); + assert_eq!( + bytes[3], + chipset_resources::microvm_time::RESTORE_PACKET_NETWORK_VERSION + ); + assert_eq!(RestorePacket::decode(&bytes).unwrap().base, base); + assert_eq!( + read_status(&mut portb), + STATUS_GENERATION_ID_AVAILABLE | STATUS_TIME_SAMPLE_AVAILABLE + ); + } + #[test] fn accepted_connection_is_polled_immediately() { let mut portb = MicrovmPortb::new( @@ -1583,6 +1639,7 @@ mod tests { generation: 7, ranges: Vec::new(), entropy: [0x3c; 64], + network: None, }; let time = RestoreTimeRecord { downtime_ns: 1_000_000, @@ -1625,7 +1682,7 @@ mod tests { )); assert!(matches!(selected.now_or_never(), Some(Ok(())))); let bytes = read_record(&mut portb, DATA_PORT, base.encoded_len()); - let packet = RestorePacketV5::decode(&bytes).unwrap(); + let packet = RestorePacket::decode(&bytes).unwrap(); assert_eq!((packet.base, packet.time), (base, time)); // The packet is consumed once read. @@ -1652,6 +1709,7 @@ mod tests { generation: 1, ranges: Vec::new(), entropy: [0; 64], + network: None, }, time: recv, selected: None, diff --git a/vm/devices/chipset_resources/src/microvm.rs b/vm/devices/chipset_resources/src/microvm.rs index 9276745a43f..aa4d8cd5e5d 100644 --- a/vm/devices/chipset_resources/src/microvm.rs +++ b/vm/devices/chipset_resources/src/microvm.rs @@ -55,8 +55,8 @@ pub struct MicrovmPortbHandle { /// Requests to deliver accepted output before a snapshot capture or before /// terminating the VM process. pub output_drain: Option>>, - /// NVX time ABI 2 configuration: restore packet version 5 and the time - /// samples at port `0xeb`, whose host fields the platform's + /// NVX time ABI 2 configuration: the restore packet and the time samples + /// at port `0xeb`, whose host fields the platform's /// [`TimeSampleSourceKind`](crate::microvm_time::TimeSampleSourceKind) /// provides. pub time_abi: MicrovmPortbTimeAbi, @@ -75,7 +75,8 @@ pub struct MicrovmPortbTimeAbi { pub restore: Option, } -/// The parts of a restore packet version 5. +/// The parts of a restore packet, version 5, or version 6 when its base +/// carries network settings. #[derive(MeshPayload)] pub struct MicrovmRestorePacketSource { /// The fields the controller knows before the worker starts. diff --git a/vm/devices/chipset_resources/src/microvm_time.rs b/vm/devices/chipset_resources/src/microvm_time.rs index bb31ae99705..d6423c9d737 100644 --- a/vm/devices/chipset_resources/src/microvm_time.rs +++ b/vm/devices/chipset_resources/src/microvm_time.rs @@ -1,14 +1,18 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -//! The microVM restore packet version 5 and the time sample of NVX time ABI -//! 2, as the guest reads them through the portb device, and the platform -//! resource that pairs each time sample's host UTC with the guest TSC. +//! The microVM restore packet, version 5 or, with network settings, version 6, +//! and the time sample of NVX time ABI 2, as the guest reads them through the +//! portb device, and the platform resource that pairs each time sample's host +//! UTC with the guest TSC. //! //! Every byte the guest reads is a port exit, so the records are compact and //! readable four bytes at a time. use mesh::MeshPayload; +use std::net::IpAddr; +use std::net::Ipv4Addr; +use std::net::Ipv6Addr; use std::sync::Arc; use thiserror::Error; use vm_resource::CanResolveTo; @@ -16,14 +20,24 @@ use vm_resource::ResourceKind; /// The restore packet magic. pub const RESTORE_PACKET_MAGIC: [u8; 3] = *b"OVR"; -/// The restore packet version. +/// The version of a restore packet without network settings. pub const RESTORE_PACKET_VERSION: u8 = 5; +/// The version of a restore packet that carries network settings: version +/// 5's fields, with the network byte in place of the reserved byte, and the +/// DNS server after the entropy. Only a restore that replaces the network +/// policy of a snapshot without a tier gives one, so that a guest that reads +/// version 5 alone fails such a restore rather than ignore its settings. +pub const RESTORE_PACKET_NETWORK_VERSION: u8 = 6; /// The length of the restore packet header. pub const RESTORE_PACKET_HEADER_LEN: usize = 24; /// The length of one memory range record. pub const RESTORE_RANGE_LEN: usize = 16; /// The length of the restore entropy. pub const RESTORE_ENTROPY_LEN: usize = 64; +/// The length of the DNS server record of a version 6 packet: an IPv6 +/// address, or an IPv4 address and twelve zero bytes, or zeros without a DNS +/// server. +pub const RESTORE_DNS_SERVER_LEN: usize = 16; /// The most memory ranges a packet can carry. pub const MAX_RESTORE_RANGES: usize = u8::MAX as usize; /// The online VP targets a packet can carry; 0 means no processor target. @@ -42,6 +56,17 @@ pub const FLAG_TEST_HOOKS: u8 = 1 << 3; const RESTORE_PACKET_FLAGS: u8 = FLAG_DOWNTIME_UTC | FLAG_MEMORY_TARGET | FLAG_ACK_REQUIRED | FLAG_TEST_HOOKS; +/// The bits of a version 6 packet's network byte that give the family of its +/// DNS server: 0 for none, [`NETWORK_DNS_IPV4`], or [`NETWORK_DNS_IPV6`]. +pub const NETWORK_DNS_MASK: u8 = 0x3; +/// The network byte's family of an IPv4 DNS server. +pub const NETWORK_DNS_IPV4: u8 = 1; +/// The network byte's family of an IPv6 DNS server. +pub const NETWORK_DNS_IPV6: u8 = 2; +/// The network byte's bit that asks the guest to announce its listeners. +pub const NETWORK_ANNOUNCE_LISTENERS: u8 = 1 << 2; +const NETWORK_BITS: u8 = NETWORK_DNS_MASK | NETWORK_ANNOUNCE_LISTENERS; + /// The portb selector of the restore packet. pub const RESTORE_PACKET_SELECT: u8 = 0xa5; /// The portb selector of the generation ID. @@ -89,6 +114,15 @@ pub enum RestorePacketError { /// The host clock status of a time sample is unknown to this version. #[error("unknown host clock status {0}")] Status(u8), + /// The network byte of a version 6 packet sets reserved bits, or names no + /// DNS server family. + #[error("invalid network byte {0:#04x}")] + Network(u8), + /// The DNS server of a version 6 packet is unspecified, pads an IPv4 + /// address with a byte that is not zero, or is not zero although the + /// network byte names no DNS server. + #[error("invalid DNS server")] + DnsServer, /// The record length does not match its counts. #[error("record is {actual} bytes, expected {expected}")] Length { @@ -108,6 +142,110 @@ pub struct RestoreMemoryRange { pub length: u64, } +/// The DNS server of a [`RestoreNetwork`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, MeshPayload)] +pub enum RestoreDnsServer { + /// An IPv4 server. + Ipv4(Ipv4Addr), + /// An IPv6 server. + Ipv6(Ipv6Addr), +} + +impl From for RestoreDnsServer { + fn from(address: IpAddr) -> Self { + match address { + IpAddr::V4(address) => Self::Ipv4(address), + IpAddr::V6(address) => Self::Ipv6(address), + } + } +} + +impl From for IpAddr { + fn from(server: RestoreDnsServer) -> Self { + match server { + RestoreDnsServer::Ipv4(address) => Self::V4(address), + RestoreDnsServer::Ipv6(address) => Self::V6(address), + } + } +} + +/// The network settings of a restore that replaces the network policy of a +/// snapshot without a tier, which a version 6 packet carries: what a cold +/// boot under the new policy names on the guest's command line, and the +/// guest learns from no other source after such a restore. +#[derive(Debug, Clone, Copy, PartialEq, Eq, MeshPayload)] +pub struct RestoreNetwork { + /// The guest's DNS server, one of its gateway's addresses, which a cold + /// boot names with `virtnet_dns=`. + pub dns_server: Option, + /// Whether the guest announces its listeners to its gateway, which a + /// cold boot asks with `virtnet_ingress=`. + pub announce_listeners: bool, +} + +impl RestoreNetwork { + /// Returns the network byte that encodes the settings. + fn byte(&self) -> u8 { + let dns = match self.dns_server { + None => 0, + Some(RestoreDnsServer::Ipv4(_)) => NETWORK_DNS_IPV4, + Some(RestoreDnsServer::Ipv6(_)) => NETWORK_DNS_IPV6, + }; + let announce = if self.announce_listeners { + NETWORK_ANNOUNCE_LISTENERS + } else { + 0 + }; + dns | announce + } + + /// Returns the DNS server record that encodes the settings. + fn dns_server_record(&self) -> [u8; RESTORE_DNS_SERVER_LEN] { + let mut record = [0; RESTORE_DNS_SERVER_LEN]; + match self.dns_server { + None => {} + Some(RestoreDnsServer::Ipv4(address)) => record[..4].copy_from_slice(&address.octets()), + Some(RestoreDnsServer::Ipv6(address)) => record = address.octets(), + } + record + } + + /// Decodes the settings from the network byte `byte` and the DNS server + /// record `record`. + fn decode(byte: u8, record: &[u8; RESTORE_DNS_SERVER_LEN]) -> Result { + if byte & !NETWORK_BITS != 0 { + return Err(RestorePacketError::Network(byte)); + } + let [a, b, c, d, padding @ ..] = *record; + let dns_server = match byte & NETWORK_DNS_MASK { + 0 if *record == [0; RESTORE_DNS_SERVER_LEN] => None, + NETWORK_DNS_IPV4 if padding == [0; RESTORE_DNS_SERVER_LEN - 4] => { + Some(RestoreDnsServer::Ipv4(Ipv4Addr::new(a, b, c, d))) + } + NETWORK_DNS_IPV6 => Some(RestoreDnsServer::Ipv6(Ipv6Addr::from(*record))), + NETWORK_DNS_MASK => return Err(RestorePacketError::Network(byte)), + _ => return Err(RestorePacketError::DnsServer), + }; + let network = Self { + dns_server, + announce_listeners: byte & NETWORK_ANNOUNCE_LISTENERS != 0, + }; + network.validate()?; + Ok(network) + } + + /// Checks that the DNS server, if any, is not unspecified. + fn validate(&self) -> Result<(), RestorePacketError> { + if self + .dns_server + .is_some_and(|server| IpAddr::from(server).is_unspecified()) + { + return Err(RestorePacketError::DnsServer); + } + Ok(()) + } +} + /// The restore packet fields the controller knows before the worker starts. #[derive(Debug, Clone, PartialEq, Eq, MeshPayload)] pub struct RestorePacketBase { @@ -123,6 +261,9 @@ pub struct RestorePacketBase { pub ranges: Vec, /// The restore entropy; its first 16 bytes are the generation ID. pub entropy: [u8; RESTORE_ENTROPY_LEN], + /// The network settings of a restore that replaces the network policy of + /// a snapshot without a tier, which make the packet version 6. + pub network: Option, } impl RestorePacketBase { @@ -139,12 +280,33 @@ impl RestorePacketBase { if self.generation == 0 { return Err(RestorePacketError::Generation); } + if let Some(network) = &self.network { + network.validate()?; + } Ok(()) } + /// Returns the version of the packet: [`RESTORE_PACKET_NETWORK_VERSION`] + /// with network settings, and [`RESTORE_PACKET_VERSION`] without. + pub fn version(&self) -> u8 { + if self.network.is_some() { + RESTORE_PACKET_NETWORK_VERSION + } else { + RESTORE_PACKET_VERSION + } + } + /// Returns the encoded length of the packet. pub fn encoded_len(&self) -> usize { - RESTORE_PACKET_HEADER_LEN + RESTORE_RANGE_LEN * self.ranges.len() + RESTORE_ENTROPY_LEN + let dns_server = if self.network.is_some() { + RESTORE_DNS_SERVER_LEN + } else { + 0 + }; + RESTORE_PACKET_HEADER_LEN + + RESTORE_RANGE_LEN * self.ranges.len() + + RESTORE_ENTROPY_LEN + + dns_server } } @@ -162,19 +324,20 @@ pub struct RestoreTimeRecord { pub test_hooks: bool, } -/// A complete restore packet version 5. +/// A complete restore packet: version 5, or version 6 when its base carries +/// network settings. /// /// It carries no host UTC: the guest takes a time sample, which pairs host /// UTC with its TSC, to set its clock. #[derive(Debug, Clone, PartialEq, Eq)] -pub struct RestorePacketV5 { +pub struct RestorePacket { /// The controller's fields. pub base: RestorePacketBase, /// The worker's sealed time fields. pub time: RestoreTimeRecord, } -impl RestorePacketV5 { +impl RestorePacket { /// Encodes the packet. pub fn encode(&self) -> Result, RestorePacketError> { self.base.validate()?; @@ -195,11 +358,11 @@ impl RestorePacketV5 { let mut bytes = Vec::with_capacity(self.base.encoded_len()); bytes.extend_from_slice(&RESTORE_PACKET_MAGIC); bytes.extend_from_slice(&[ - RESTORE_PACKET_VERSION, + self.base.version(), flags, self.base.online_vp_count, self.base.ranges.len() as u8, - 0, + self.base.network.map_or(0, |network| network.byte()), ]); bytes.extend_from_slice(&self.base.generation.to_le_bytes()); bytes.extend_from_slice(&self.time.rate_deviation.to_le_bytes()); @@ -209,6 +372,9 @@ impl RestorePacketV5 { bytes.extend_from_slice(&range.length.to_le_bytes()); } bytes.extend_from_slice(&self.base.entropy); + if let Some(network) = &self.base.network { + bytes.extend_from_slice(&network.dns_server_record()); + } debug_assert_eq!(bytes.len(), self.base.encoded_len()); Ok(bytes) } @@ -222,16 +388,24 @@ impl RestorePacketV5 { if header[..3] != RESTORE_PACKET_MAGIC { return Err(RestorePacketError::Magic); } - if header[3] != RESTORE_PACKET_VERSION { - return Err(RestorePacketError::Version(header[3])); - } + // Version 6 puts its network byte in version 5's reserved byte. + let network_byte = match header[3] { + RESTORE_PACKET_VERSION => None, + RESTORE_PACKET_NETWORK_VERSION => Some(header[7]), + version => return Err(RestorePacketError::Version(version)), + }; let flags = header[4]; - if flags & !RESTORE_PACKET_FLAGS != 0 || header[7] != 0 { + if flags & !RESTORE_PACKET_FLAGS != 0 || (network_byte.is_none() && header[7] != 0) { return Err(RestorePacketError::Reserved); } let range_count = usize::from(header[6]); - let expected = - RESTORE_PACKET_HEADER_LEN + RESTORE_RANGE_LEN * range_count + RESTORE_ENTROPY_LEN; + let entropy_offset = RESTORE_PACKET_HEADER_LEN + RESTORE_RANGE_LEN * range_count; + let dns_server_offset = entropy_offset + RESTORE_ENTROPY_LEN; + let expected = if network_byte.is_some() { + dns_server_offset + RESTORE_DNS_SERVER_LEN + } else { + dns_server_offset + }; if bytes.len() != expected { return Err(RestorePacketError::Length { expected, @@ -251,7 +425,11 @@ impl RestorePacketV5 { } }) .collect(); - let entropy_offset = expected - RESTORE_ENTROPY_LEN; + let network = network_byte + .map(|byte| { + RestoreNetwork::decode(byte, bytes[dns_server_offset..].try_into().unwrap()) + }) + .transpose()?; let packet = Self { base: RestorePacketBase { online_vp_count: header[5], @@ -259,7 +437,8 @@ impl RestorePacketV5 { ack_required: flags & FLAG_ACK_REQUIRED != 0, generation: u32_at(8), ranges, - entropy: bytes[entropy_offset..].try_into().unwrap(), + entropy: bytes[entropy_offset..dns_server_offset].try_into().unwrap(), + network, }, time: RestoreTimeRecord { downtime_ns: u64_at(16), @@ -455,8 +634,8 @@ mod tests { std::array::from_fn(|i| i as u8) } - fn packet_without_ranges() -> RestorePacketV5 { - RestorePacketV5 { + fn packet_without_ranges() -> RestorePacket { + RestorePacket { base: RestorePacketBase { online_vp_count: 2, memory_target: false, @@ -464,6 +643,7 @@ mod tests { generation: 7, ranges: Vec::new(), entropy: entropy(), + network: None, }, time: RestoreTimeRecord { downtime_ns: 0x0102_0304_0506_0708, @@ -474,8 +654,8 @@ mod tests { } } - fn packet_with_ranges() -> RestorePacketV5 { - RestorePacketV5 { + fn packet_with_ranges() -> RestorePacket { + RestorePacket { base: RestorePacketBase { online_vp_count: 0, memory_target: true, @@ -492,6 +672,7 @@ mod tests { }, ], entropy: entropy(), + network: None, }, time: RestoreTimeRecord { downtime_ns: 30_000_000_000, @@ -515,7 +696,7 @@ mod tests { let bytes = packet.encode().unwrap(); assert_eq!(bytes.len(), 88); assert_eq!(bytes, expected); - assert_eq!(RestorePacketV5::decode(&bytes).unwrap(), packet); + assert_eq!(RestorePacket::decode(&bytes).unwrap(), packet); } #[test] @@ -535,7 +716,7 @@ mod tests { let bytes = packet.encode().unwrap(); assert_eq!(bytes.len(), 120); assert_eq!(bytes, expected); - assert_eq!(RestorePacketV5::decode(&bytes).unwrap(), packet); + assert_eq!(RestorePacket::decode(&bytes).unwrap(), packet); } #[test] @@ -544,7 +725,7 @@ mod tests { let mutate = |offset: usize, value: u8| { let mut bytes = good.clone(); bytes[offset] = value; - RestorePacketV5::decode(&bytes).unwrap_err() + RestorePacket::decode(&bytes).unwrap_err() }; assert_eq!(mutate(0, b'X'), RestorePacketError::Magic); assert_eq!(mutate(3, 4), RestorePacketError::Version(4)); @@ -559,11 +740,11 @@ mod tests { RestorePacketError::RateDeviation(_) )); assert_eq!( - RestorePacketV5::decode(&good[..23]).unwrap_err(), + RestorePacket::decode(&good[..23]).unwrap_err(), RestorePacketError::Truncated(23) ); assert!(matches!( - RestorePacketV5::decode(&good[..good.len() - 1]).unwrap_err(), + RestorePacket::decode(&good[..good.len() - 1]).unwrap_err(), RestorePacketError::Length { .. } )); } @@ -594,6 +775,144 @@ mod tests { packet.encode().unwrap_err(), RestorePacketError::RateDeviation(_) )); + + for unspecified in [ + RestoreDnsServer::Ipv4(Ipv4Addr::UNSPECIFIED), + RestoreDnsServer::Ipv6(Ipv6Addr::UNSPECIFIED), + ] { + let packet = network_packet(RestoreNetwork { + dns_server: Some(unspecified), + announce_listeners: false, + }); + assert_eq!(packet.encode().unwrap_err(), RestorePacketError::DnsServer); + } + } + + fn network_packet(network: RestoreNetwork) -> RestorePacket { + let mut packet = packet_without_ranges(); + packet.base.network = Some(network); + packet + } + + #[test] + fn golden_packets_with_network_settings() { + let ipv6: Ipv6Addr = "fd00::a00:1".parse().unwrap(); + for (network, network_byte, dns_server) in [ + // An IPv4 DNS server, with the guest's listeners announced. + ( + RestoreNetwork { + dns_server: Some(RestoreDnsServer::Ipv4(Ipv4Addr::new(10, 0, 0, 1))), + announce_listeners: true, + }, + 0x05, + [10, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + ), + // An IPv6 DNS server, without announcements. + ( + RestoreNetwork { + dns_server: Some(RestoreDnsServer::Ipv6(ipv6)), + announce_listeners: false, + }, + 0x02, + ipv6.octets(), + ), + // A policy that names no DNS server and publishes no listener. + ( + RestoreNetwork { + dns_server: None, + announce_listeners: false, + }, + 0x00, + [0; RESTORE_DNS_SERVER_LEN], + ), + ] { + let mut expected = vec![ + b'O', + b'V', + b'R', + 6, + 0x05, + 2, + 0, + network_byte, // header + 7, + 0, + 0, + 0, // generation + 0xfe, + 0xff, + 0xff, + 0xff, // rate deviation + 0x08, + 0x07, + 0x06, + 0x05, + 0x04, + 0x03, + 0x02, + 0x01, // downtime + ]; + expected.extend(entropy()); + expected.extend(dns_server); + let packet = network_packet(network); + let bytes = packet.encode().unwrap(); + assert_eq!(bytes.len(), 104, "{network:?}"); + assert_eq!(bytes, expected, "{network:?}"); + assert_eq!( + RestorePacket::decode(&bytes).unwrap(), + packet, + "{network:?}" + ); + } + } + + #[test] + fn decode_rejects_malformed_network_settings() { + let good = network_packet(RestoreNetwork { + dns_server: Some(RestoreDnsServer::Ipv4(Ipv4Addr::new(10, 0, 0, 1))), + announce_listeners: true, + }) + .encode() + .unwrap(); + let dns_server = good.len() - RESTORE_DNS_SERVER_LEN; + let mutate = |offset: usize, value: u8| { + let mut bytes = good.clone(); + bytes[offset] = value; + RestorePacket::decode(&bytes).unwrap_err() + }; + // Reserved bits, and the DNS server family that no version defines. + assert_eq!(mutate(7, 0x0d), RestorePacketError::Network(0x0d)); + assert_eq!(mutate(7, 0x07), RestorePacketError::Network(0x07)); + // A nonzero byte after an IPv4 server, and a server where the network + // byte names none. + assert_eq!(mutate(dns_server + 4, 1), RestorePacketError::DnsServer); + assert_eq!(mutate(7, 0x04), RestorePacketError::DnsServer); + // Unspecified servers. + let mut unspecified = good.clone(); + unspecified[dns_server..dns_server + 4].fill(0); + assert_eq!( + RestorePacket::decode(&unspecified).unwrap_err(), + RestorePacketError::DnsServer + ); + unspecified[7] = 0x06; + assert_eq!( + RestorePacket::decode(&unspecified).unwrap_err(), + RestorePacketError::DnsServer + ); + // Version 5 keeps its reserved byte zero, and its packet ends with the + // entropy. + assert_eq!(mutate(3, 5), RestorePacketError::Reserved); + let mut version_5 = good.clone(); + version_5[3] = 5; + version_5[7] = 0; + assert!(matches!( + RestorePacket::decode(&version_5).unwrap_err(), + RestorePacketError::Length { .. } + )); + assert!(matches!( + RestorePacket::decode(&good[..dns_server]).unwrap_err(), + RestorePacketError::Length { .. } + )); } fn sample() -> TimeSample {