From 69ddc715b3247d5883eca44bf3ad13c83a6b9ff0 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Thu, 29 Oct 2026 10:02:06 -0700 Subject: [PATCH 1/4] openvmm_core: name the registers in which a recorded CPUID entry differs When the effective CPUID that a restore recomputes differs from the one that the snapshot recorded, the E_CPU_SURFACE error names each entry that differs. For an entry that both have, it printed all four results and masks of each side with {:#x?}, which pretty-prints each array over six lines, so that one entry took 24 lines of the error and the reader had to compare the arrays to find the register that differs. Name each register in which such an entry differs instead, on one line, with its value and mask on each side, as the check of VP 0's CPUID names the registers of the results that VP 0 does not observe: CPUID 0x0 EAX is 0x0000001b under mask 0xffffffff, and 0x0000001a under mask 0xffffffff in the snapshot An entry that only one side has is named as before. The error thus takes one line, whatever the number of entries that it names, which the next commit makes every differing entry rather than the first 16. The test of the recorded CPUID now checks that a flipped bit is named with its register, value, and mask, and that the entries of another topology are named on one line. Part of microsoft/nvx#568. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/worker/dispatch/time_abi.rs | 56 ++++++++++++++----- 1 file changed, 42 insertions(+), 14 deletions(-) diff --git a/openvmm/openvmm_core/src/worker/dispatch/time_abi.rs b/openvmm/openvmm_core/src/worker/dispatch/time_abi.rs index d6744c33788..a58844eb495 100644 --- a/openvmm/openvmm_core/src/worker/dispatch/time_abi.rs +++ b/openvmm/openvmm_core/src/worker/dispatch/time_abi.rs @@ -59,6 +59,9 @@ pub(super) const TIME_ABI_UNIT: &str = "time-abi"; /// The most differences an `E_CPU_SURFACE` read-back failure names. const MAX_REPORTED_CPUID_DIFFERENCES: usize = 16; +/// The names of the CPUID result registers, for messages. +const REGISTERS: [&str; 4] = ["EAX", "EBX", "ECX", "EDX"]; + /// The time ABI state of a loaded VM. pub(super) struct TimeAbiState { /// The `time-abi` state unit. @@ -661,7 +664,9 @@ fn record_entries(effective: &EffectiveCpuid) -> impl Iterator differences.push(format!("{} is not in the snapshot", name(entry_key))), - Some(theirs) if theirs != entry => differences.push(format!( - "{} is {:#x?} under mask {:#x?}, and {:#x?} under mask {:#x?} in the snapshot", - name(entry_key), - entry.result, - entry.mask, - theirs.result, - theirs.mask - )), + Some(theirs) if theirs != entry => { + let registers: Vec = (0..4) + .filter(|®ister| { + (entry.result[register], entry.mask[register]) + != (theirs.result[register], theirs.mask[register]) + }) + .map(|register| { + format!( + "{} is {:#010x} under mask {:#010x}, and {:#010x} under mask {:#010x} in the snapshot", + REGISTERS[register], + entry.result[register], + entry.mask[register], + theirs.result[register], + theirs.mask[register] + ) + }) + .collect(); + differences.push(format!("{} {}", name(entry_key), registers.join(", "))); + } Some(_) => {} } } @@ -756,7 +772,6 @@ fn check_presented( /// Lists every result of `effective` that `presented` does not match under /// the result's masks. fn cpuid_differences(presented: &CpuidLeafSet, effective: &EffectiveCpuid) -> Vec { - const REGISTERS: [&str; 4] = ["EAX", "EBX", "ECX", "EDX"]; effective .results() .filter_map(|expected| { @@ -1405,13 +1420,16 @@ mod tests { }; check_recorded_cpuid(&effective, &record).unwrap(); - // Another topology changes the topology leaves. + // Another topology changes the topology leaves, which are named on + // one line, with the registers in which they differ. let (_, _, other) = test_config(4); let message = format!("{:#}", check_recorded_cpuid(&other, &record).unwrap_err()); assert!(message.starts_with("[E_CPU_SURFACE]"), "{message}"); - assert!(message.contains("CPUID 0xb.1"), "{message}"); + assert!(message.contains("CPUID 0xb.1 E"), "{message}"); + assert!(!message.contains('\n'), "{message}"); - // One flipped bit is named. + // One flipped bit is named, with its register, value, and mask. + let leaf_0 = record_entries(&effective).next().unwrap(); let mut flipped = record.clone(); flipped.effective_cpuid[3 * 4] ^= 1; let message = format!( @@ -1419,7 +1437,17 @@ mod tests { check_recorded_cpuid(&effective, &flipped).unwrap_err() ); assert!(message.starts_with("[E_CPU_SURFACE]"), "{message}"); - assert!(message.contains("CPUID 0x0 is"), "{message}"); + assert!( + message.ends_with(&format!( + ": CPUID 0x0 EAX is {:#010x} under mask {:#010x}, and {:#010x} under mask \ + {:#010x} in the snapshot", + leaf_0.result[0], + leaf_0.mask[0], + leaf_0.result[0] ^ 1, + leaf_0.mask[0] + )), + "{message}" + ); // A missing entry, and the same entries in another order. let mut truncated = record.clone(); From c55db5d9e8e69a3488a7f688e5f48c9a9fbb0274 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Thu, 29 Oct 2026 10:08:09 -0700 Subject: [PATCH 2/4] openvmm_core: name every CPUID difference in E_CPU_SURFACE errors The CPU checks of a new partition are to report every violation together, naming the leaf, subleaf, register, and bit. Both E_CPU_SURFACE failures of the VM worker counted every difference but named at most 16 (MAX_REPORTED_CPUID_DIFFERENCES) and dropped the rest: the restore's comparison of the recomputed effective CPUID with the snapshot's record, and the comparison of VP 0's CPUID with the effective CPUID at every cold boot and restore. A host or a build that differed in more entries showed only the first 16, so a fix of those could fail again on entries that the error had not named. Name every difference. The lists stay bounded: the effective CPUID of each pinned profile has between 64 entries (Skylake-SP) and 94 (Turin), its table and the 22 topology and identity entries that OpenVMM adds, so VP 0's comparison names at most that many results. A snapshot's record holds at most MAX_EFFECTIVE_CPUID_ENTRIES (1024) entries, which the manifest decoder enforces, so the restore's comparison names at most that many more entries that only the snapshot has. The new tests make every result of a 2-VP Ice Lake-SP partition, 69 of them, differ from what VP 0 reads, and every entry from the snapshot's record, and check that the errors count and name each one, in order. Both fail with the limit restored. Part of microsoft/nvx#568. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/worker/dispatch/time_abi.rs | 125 ++++++++++++++++-- 1 file changed, 113 insertions(+), 12 deletions(-) diff --git a/openvmm/openvmm_core/src/worker/dispatch/time_abi.rs b/openvmm/openvmm_core/src/worker/dispatch/time_abi.rs index a58844eb495..545fa671b7b 100644 --- a/openvmm/openvmm_core/src/worker/dispatch/time_abi.rs +++ b/openvmm/openvmm_core/src/worker/dispatch/time_abi.rs @@ -56,9 +56,6 @@ use vmcore::save_restore::SavedStateBlob; /// The name of the time ABI state unit. pub(super) const TIME_ABI_UNIT: &str = "time-abi"; -/// The most differences an `E_CPU_SURFACE` read-back failure names. -const MAX_REPORTED_CPUID_DIFFERENCES: usize = 16; - /// The names of the CPUID result registers, for messages. const REGISTERS: [&str; 4] = ["EAX", "EBX", "ECX", "EDX"]; @@ -625,7 +622,7 @@ pub(super) fn declare_rates( /// Compares the effective CPUID recomputed for a restored partition with the /// one the snapshot recorded (`E_CPU_SURFACE`). It compares the binary /// records, about a microsecond of work, and decodes the snapshot's only to -/// name the differences. +/// name every difference, as [`recorded_cpuid_differences`] lists them. pub(super) fn check_recorded_cpuid( effective_cpuid: &EffectiveCpuid, record: &SnapshotCpuProfile, @@ -640,14 +637,13 @@ pub(super) fn check_recorded_cpuid( ) })?; let differences = recorded_cpuid_differences(effective_cpuid, &recorded); - let count = differences.len(); - let shown = &differences[..count.min(MAX_REPORTED_CPUID_DIFFERENCES)]; Err(TimeAbiError::new( TimeAbiCode::CpuSurface, format!( - "the effective CPUID of CPU profile {} differs from the snapshot's in {count} entries: {}", + "the effective CPUID of CPU profile {} differs from the snapshot's in {} entries: {}", record.id, - shown.join("; ") + differences.len(), + differences.join("; ") ), ) .into()) @@ -728,6 +724,10 @@ fn recorded_cpuid_differences( /// (`cpu_profile::unlisted_cpuid_candidates`); each must read zero, so that /// no reserved entry exposes a host feature (`E_CPU_UNLISTED`). A table /// backend reports none, and its guests read zero there by construction. +/// +/// An `E_CPU_SURFACE` failure names every result that VP 0 does not observe, +/// with the registers in which it differs, as [`cpuid_differences`] lists +/// them. pub(super) fn check_presented_cpuid( partition: &dyn HvlitePartition, hypervisor: &str, @@ -756,14 +756,13 @@ fn check_presented( cpu_profile::check_unlisted_cpuid(profile, &entries)?; return Ok(()); } - let count = differences.len(); - let shown = &differences[..count.min(MAX_REPORTED_CPUID_DIFFERENCES)]; Err(TimeAbiError::new( TimeAbiCode::CpuSurface, format!( - "VP 0 does not observe the effective CPUID of CPU profile {} in {count} results: {}", + "VP 0 does not observe the effective CPUID of CPU profile {} in {} results: {}", profile.id(), - shown.join("; ") + differences.len(), + differences.join("; ") ), ) .into()) @@ -1217,6 +1216,15 @@ mod tests { .map_or(message.clone(), |(code, _)| code.to_owned()) } + /// Returns the name of the CPUID entry of `function` and `index` in the + /// messages of the CPU checks. + fn leaf_name(function: u32, index: Option) -> String { + match index { + Some(index) => format!("CPUID {function:#x}.{index}"), + None => format!("CPUID {function:#x}"), + } + } + #[test] fn unit_saves_restores_and_resets_the_invariant_control() { futures::executor::block_on(async { @@ -1407,6 +1415,52 @@ mod tests { assert!(differences[0].starts_with("CPUID 0x80000002 ")); } + /// A failure names every result that VP 0 does not observe, however many + /// there are, and counts them. + #[test] + fn presented_cpuid_names_every_difference() { + let (_, _, effective) = test_config(2); + // VP 0 reads each result with the lowest bit of its first masked + // register flipped. + let mut expected = Vec::new(); + let presented = effective + .results() + .map(|result| { + let mut value = result.result; + if let Some(register) = (0..4).find(|®ister| result.mask[register] != 0) { + let mask = result.mask[register]; + value[register] ^= mask & mask.wrapping_neg(); + expected.push(format!( + "{} {} is {:#010x}, not {:#010x} under mask {mask:#010x}", + leaf_name(result.function, result.index), + REGISTERS[register], + value[register], + result.result[register] + )); + } + let leaf = CpuidLeaf::new(result.function, value); + match result.index { + Some(index) => leaf.indexed(index), + None => leaf, + } + }) + .collect(); + assert!(expected.len() > 16, "{expected:?}"); + let message = format!( + "{:#}", + check_presented(test_profile(), &effective, presented).unwrap_err() + ); + assert_eq!( + message, + format!( + "[E_CPU_SURFACE] VP 0 does not observe the effective CPUID of CPU profile \ + {TEST_PROFILE} in {} results: {}", + expected.len(), + expected.join("; ") + ) + ); + } + #[test] fn recorded_cpuid_must_match() { let (_, _, effective) = test_config(2); @@ -1476,6 +1530,53 @@ mod tests { ); } + /// A failure names every entry in which the snapshot's record differs, + /// however many there are, and counts them. + #[test] + fn recorded_cpuid_names_every_difference() { + let (_, _, effective) = test_config(2); + // The snapshot records each entry with the lowest bit of EAX flipped. + let mut expected = Vec::new(); + let recorded: Vec<_> = record_entries(&effective) + .map(|entry| { + let mut recorded = entry; + recorded.result[0] ^= 1; + expected.push(format!( + "{} EAX is {:#010x} under mask {:#010x}, and {:#010x} under mask {:#010x} \ + in the snapshot", + leaf_name(entry.function, entry.index), + entry.result[0], + entry.mask[0], + recorded.result[0], + recorded.mask[0] + )); + recorded + }) + .collect(); + assert!(expected.len() > 16, "{expected:?}"); + let pinned = cpu_profile::pinned_record(TEST_PROFILE).unwrap(); + let record = SnapshotCpuProfile { + id: TEST_PROFILE.to_owned(), + sha256: pinned.digest.to_vec(), + profile: pinned.encoding.to_vec(), + effective_cpuid: encode_effective_cpuid(recorded), + capture_cpu_signature: 0, + }; + let message = format!( + "{:#}", + check_recorded_cpuid(&effective, &record).unwrap_err() + ); + assert_eq!( + message, + format!( + "[E_CPU_SURFACE] the effective CPUID of CPU profile {TEST_PROFILE} differs from \ + the snapshot's in {} entries: {}", + expected.len(), + expected.join("; ") + ) + ); + } + /// Capture copies the pinned profile's precomputed record, so restore /// never encodes or hashes a profile. #[test] From e1fa0f02837747c984c4a884972554443d32fca8 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Thu, 29 Oct 2026 10:11:08 -0700 Subject: [PATCH 3/4] openvmm_core: report unlisted CPUID entries with VP 0's differences Before any VP runs, VP 0's CPUID, as the backend reports it, must equal the effective CPUID under its masks (E_CPU_SURFACE), and, on a pass-through backend, read zero at every host entry outside the profile's tables (E_CPU_UNLISTED). check_presented checked the second only once the first had passed, so a report that failed both named the results that VP 0 does not observe but none of the non-zero entries outside the profile, which surfaced only once the results were fixed. Name them together, as cpu_profile::verify_support names the non-zero entries outside the profile after the features, limits, and layouts that a backend lacks: an E_CPU_SURFACE failure names every result that VP 0 does not observe, then every non-zero entry outside the profile's tables, and a report whose only violations are such entries still fails with E_CPU_UNLISTED. No failure changes its code, so callers that match the code see the same one as before. The test of the entries outside the profile now drops two governed results from a report that also reads a non-zero entry past Ice Lake-SP's maximum basic leaf, and checks that the error names both results and then the entry. Part of microsoft/nvx#568. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/worker/dispatch/time_abi.rs | 59 ++++++++++++++----- 1 file changed, 43 insertions(+), 16 deletions(-) diff --git a/openvmm/openvmm_core/src/worker/dispatch/time_abi.rs b/openvmm/openvmm_core/src/worker/dispatch/time_abi.rs index 545fa671b7b..93ec0daa6c8 100644 --- a/openvmm/openvmm_core/src/worker/dispatch/time_abi.rs +++ b/openvmm/openvmm_core/src/worker/dispatch/time_abi.rs @@ -725,9 +725,11 @@ fn recorded_cpuid_differences( /// no reserved entry exposes a host feature (`E_CPU_UNLISTED`). A table /// backend reports none, and its guests read zero there by construction. /// -/// An `E_CPU_SURFACE` failure names every result that VP 0 does not observe, -/// with the registers in which it differs, as [`cpuid_differences`] lists -/// them. +/// A failure names every violation at once, as `cpu_profile::verify_support` +/// does: every result that VP 0 does not observe, with the registers in +/// which it differs, as [`cpuid_differences`] lists them, then every +/// non-zero entry outside the profile's tables. A report whose only +/// violations are such entries fails with `E_CPU_UNLISTED`. pub(super) fn check_presented_cpuid( partition: &dyn HvlitePartition, hypervisor: &str, @@ -747,22 +749,28 @@ fn check_presented( ) -> anyhow::Result<()> { let presented = CpuidLeafSet::new(leaves); let differences = cpuid_differences(&presented, effective_cpuid); + let entries: Vec<_> = presented + .leaves() + .iter() + .map(|leaf| cpu_profile::cpuid::CpuidEntry::new(leaf.function, leaf.index, leaf.result)) + .collect(); if differences.is_empty() { - let entries: Vec<_> = presented - .leaves() - .iter() - .map(|leaf| cpu_profile::cpuid::CpuidEntry::new(leaf.function, leaf.index, leaf.result)) - .collect(); cpu_profile::check_unlisted_cpuid(profile, &entries)?; return Ok(()); } + let unlisted = cpu_profile::unlisted_cpuid_violations(profile, &entries); Err(TimeAbiError::new( TimeAbiCode::CpuSurface, format!( "VP 0 does not observe the effective CPUID of CPU profile {} in {} results: {}", profile.id(), differences.len(), - differences.join("; ") + differences + .iter() + .chain(&unlisted) + .map(String::as_str) + .collect::>() + .join("; ") ), ) .into()) @@ -1304,8 +1312,9 @@ mod tests { /// A pass-through backend's report may hold VP 0's view at the host's /// entries outside the profile: zero there passes, and anything else is - /// `E_CPU_UNLISTED`. A report of the table alone passes, for every pinned - /// profile and topology. + /// `E_CPU_UNLISTED`, or is named after the governed differences of an + /// `E_CPU_SURFACE` failure. A report of the table alone passes, for every + /// pinned profile and topology. #[test] fn presented_cpuid_rejects_non_zero_unlisted_entries() { for profile in cpu_profile::pinned_profiles() { @@ -1337,11 +1346,29 @@ mod tests { "E_CPU_UNLISTED" ); - // A governed difference is reported first. - leaked.retain(|leaf| leaf.function != 0x8000_0002); - assert_eq!( - code(check_presented(profile, &effective, leaked)), - "E_CPU_SURFACE" + // Governed differences are reported first, then every non-zero entry + // outside the profile. + leaked.retain(|leaf| !matches!(leaf.function, 0x8000_0002 | 0x8000_0003)); + let message = format!( + "{:#}", + check_presented(profile, &effective, leaked).unwrap_err() + ); + assert!( + message.starts_with(&format!( + "[E_CPU_SURFACE] VP 0 does not observe the effective CPUID of CPU profile \ + {TEST_PROFILE} in 2 results: CPUID 0x80000002 EAX is 0x00000000, not " + )), + "{message}" + ); + assert!( + message.contains("; CPUID 0x80000003 EAX is 0x00000000, not "), + "{message}" + ); + assert!( + message.ends_with( + "; CPUID 0x1d is outside the profile and reads EAX 0x1, EBX 0x0, ECX 0x0, EDX 0x0" + ), + "{message}" ); } From e4c741e1860de1e2149b3dd5d41863468705b02c Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Thu, 29 Oct 2026 12:13:07 -0700 Subject: [PATCH 4/4] openvmm_core: report the failures of every CPU check of a partition together At every cold boot and restore, before any VP runs, the VM worker checks the partition's CPU against its CPU profile in three steps: on restore, that the effective CPUID it recomputes is the one that the snapshot recorded (E_CPU_SURFACE); that VP 0 observes the effective CPUID (E_CPU_SURFACE, or E_CPU_UNLISTED for a non-zero entry outside the profile's tables); and that the backend supports the profile (E_PROFILE_UNSUPPORTED). Each check names every violation that it finds, but the worker ran them in turn and stopped at the first that failed. A restore whose record differed therefore did not say whether VP 0 also read other values or the host also lacked a feature of the profile, which surfaced only once the first failure was fixed. Run every check and report their failures together, as CPU-profile verification is to report every violation together. check_cpu_profile runs the three checks in that order, even after one fails. report_together returns the failure of a check that fails alone unchanged, so that its type and code are what they were, and otherwise one error that gives each failure's chain in turn, joined with "; ". Such an error leads with the first failure, and so with the code that the worker reported before: the process's exit line, the hints of --cpu-profile auto, and petri's host-profile fallback all take the first bracketed code of the error chain. A partition without a time ABI backend fails both of the backend's checks with that failure, after any failure of the snapshot's record. The log line of checks that pass, with their timings, moves into check_cpu_profile. The error, which follows OpenVMM's exit line in its log, stays bounded: for the largest pinned profile, Turin, whose effective CPUID has 94 entries, an error in which every register of every entry differs in both comparisons and the backend supports nothing takes 61,717 bytes, on one line. The new test fails the snapshot's record, VP 0's view, and the backend's support of a 2-VP Ice Lake-SP partition, alone and in every combination, and runs a partition without a time ABI backend. It checks the combined errors and their leading code, and that a failure alone is the check's own error. It fails if the checks stop at the first failure. The Guide's snapshot page says that the checks report together. Part of microsoft/nvx#568. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Guide/src/user_guide/openvmm/snapshots.md | 6 + openvmm/openvmm_core/src/worker/dispatch.rs | 16 +- .../src/worker/dispatch/time_abi.rs | 318 ++++++++++++++++-- 3 files changed, 307 insertions(+), 33 deletions(-) diff --git a/Guide/src/user_guide/openvmm/snapshots.md b/Guide/src/user_guide/openvmm/snapshots.md index 01a023b8336..81cb5d0c490 100644 --- a/Guide/src/user_guide/openvmm/snapshots.md +++ b/Guide/src/user_guide/openvmm/snapshots.md @@ -362,6 +362,12 @@ a host of the profile's generation; and an identical effective CPUID. OpenVMM computes the effective CPUID, including its own leaves and the bits it owns, such as the cache topology, so an OpenVMM build that computes it differently rejects earlier snapshots with `E_CPU_SURFACE`. +These checks all run, even after one of them fails: on restore, the +comparison of the effective CPUID with the snapshot's, then, on every boot, +the check that VP 0 observes it and the check that the backend supports the +profile. Their error thus names every violation of each, leads with the code +of the first that fails, in that order, and goes on with the failures of the +others. Restore also requires: diff --git a/openvmm/openvmm_core/src/worker/dispatch.rs b/openvmm/openvmm_core/src/worker/dispatch.rs index fb3558cfb0f..797284ea5f7 100644 --- a/openvmm/openvmm_core/src/worker/dispatch.rs +++ b/openvmm/openvmm_core/src/worker/dispatch.rs @@ -2026,7 +2026,6 @@ impl InitializedVm { saved_state.is_some() == restore.is_some(), "a time ABI restore requires both the saved state and the time ABI restore inputs" ); - let checks_started = std::time::Instant::now(); if let Some(input) = restore { anyhow::ensure!( input.cpu_profile.id == profile.id(), @@ -2034,25 +2033,14 @@ impl InitializedVm { profile.id(), input.cpu_profile.id ); - time_abi::check_recorded_cpuid(&effective_cpuid, &input.cpu_profile)?; } - let recorded_done = std::time::Instant::now(); - time_abi::check_presented_cpuid( + time_abi::check_cpu_profile( partition.as_ref(), hypervisor, &profile, &effective_cpuid, + restore.map(|input| &input.cpu_profile), )?; - let presented_done = std::time::Instant::now(); - time_abi::check_profile_support(partition.as_ref(), hypervisor, &profile)?; - let support_done = std::time::Instant::now(); - tracing::info!( - cpu_profile = profile.id(), - recorded_cpuid_us = (recorded_done - checks_started).as_micros() as u64, - presented_cpuid_us = (presented_done - recorded_done).as_micros() as u64, - profile_support_us = (support_done - presented_done).as_micros() as u64, - "time ABI CPU checks passed" - ); let report = time_abi::declare_rates( partition.as_ref(), &msrs, diff --git a/openvmm/openvmm_core/src/worker/dispatch/time_abi.rs b/openvmm/openvmm_core/src/worker/dispatch/time_abi.rs index 93ec0daa6c8..26b54e11ada 100644 --- a/openvmm/openvmm_core/src/worker/dispatch/time_abi.rs +++ b/openvmm/openvmm_core/src/worker/dispatch/time_abi.rs @@ -619,11 +619,90 @@ pub(super) fn declare_rates( }) } +/// Checks the CPU of a new partition against its CPU profile, at every cold +/// boot and restore, before any VP runs: on restore, that the effective CPUID +/// recomputed for the partition is the one that the snapshot recorded in +/// `record` ([`check_recorded_cpuid`]), then that VP 0 observes the effective +/// CPUID ([`check_presented_cpuid`]), and then that the backend supports the +/// profile ([`check_profile_support`]). +/// +/// Every check runs, even after one fails, so that a failure names every +/// violation together ([`report_together`]): the error leads with the failure +/// of the first check that fails, in that order, and so with its code, as if +/// the checks had stopped there, and goes on with the failure of each later +/// check that fails, with its own code. +pub(super) fn check_cpu_profile( + partition: &dyn HvlitePartition, + hypervisor: &str, + profile: &CpuProfile, + effective_cpuid: &EffectiveCpuid, + record: Option<&SnapshotCpuProfile>, +) -> anyhow::Result<()> { + check_cpu_profile_with( + backend(partition, hypervisor), + hypervisor, + profile, + effective_cpuid, + record, + ) +} + +/// Runs the checks of [`check_cpu_profile`] with `backend`, the partition's +/// time ABI backend, or the failure of a partition without one, which fails +/// both of the backend's checks at once. +fn check_cpu_profile_with( + backend: Result<&dyn TimeAbiBackend, TimeAbiError>, + hypervisor: &str, + profile: &CpuProfile, + effective_cpuid: &EffectiveCpuid, + record: Option<&SnapshotCpuProfile>, +) -> anyhow::Result<()> { + let started = std::time::Instant::now(); + let recorded = record.map_or(Ok(()), |record| { + check_recorded_cpuid(effective_cpuid, record) + }); + let recorded_done = std::time::Instant::now(); + let backend = match backend { + Ok(backend) => backend, + Err(err) => return report_together([recorded, Err(err.into())]), + }; + let presented = check_presented_cpuid(backend, profile, effective_cpuid); + let presented_done = std::time::Instant::now(); + let support = check_profile_support(backend, hypervisor, profile); + let support_done = std::time::Instant::now(); + report_together([recorded, presented, support])?; + tracing::info!( + cpu_profile = profile.id(), + recorded_cpuid_us = (recorded_done - started).as_micros() as u64, + presented_cpuid_us = (presented_done - recorded_done).as_micros() as u64, + profile_support_us = (support_done - presented_done).as_micros() as u64, + "time ABI CPU checks passed" + ); + Ok(()) +} + +/// Returns the outcome of checks that ran together, in order: success if +/// every check passed, the failure of the only check that failed, unchanged, +/// and otherwise one error that gives the error chain of each failure in +/// turn, joined with `; `. Such an error leads with the first failure, whose +/// code is therefore the first one that the error names. +fn report_together(results: impl IntoIterator>) -> anyhow::Result<()> { + let mut failures: Vec<_> = results.into_iter().filter_map(Result::err).collect(); + if failures.len() <= 1 { + return failures.pop().map_or(Ok(()), Err); + } + let failures: Vec<_> = failures + .iter() + .map(|failure| format!("{failure:#}")) + .collect(); + Err(anyhow::Error::msg(failures.join("; "))) +} + /// Compares the effective CPUID recomputed for a restored partition with the /// one the snapshot recorded (`E_CPU_SURFACE`). It compares the binary /// records, about a microsecond of work, and decodes the snapshot's only to /// name every difference, as [`recorded_cpuid_differences`] lists them. -pub(super) fn check_recorded_cpuid( +fn check_recorded_cpuid( effective_cpuid: &EffectiveCpuid, record: &SnapshotCpuProfile, ) -> anyhow::Result<()> { @@ -730,14 +809,12 @@ fn recorded_cpuid_differences( /// which it differs, as [`cpuid_differences`] lists them, then every /// non-zero entry outside the profile's tables. A report whose only /// violations are such entries fails with `E_CPU_UNLISTED`. -pub(super) fn check_presented_cpuid( - partition: &dyn HvlitePartition, - hypervisor: &str, +fn check_presented_cpuid( + backend: &dyn TimeAbiBackend, profile: &CpuProfile, effective_cpuid: &EffectiveCpuid, ) -> anyhow::Result<()> { - let leaves = backend(partition, hypervisor)?.effective_cpuid()?; - check_presented(profile, effective_cpuid, leaves) + check_presented(profile, effective_cpuid, backend.effective_cpuid()?) } /// Checks a backend's report of VP 0's CPUID, as [`check_presented_cpuid`] @@ -812,22 +889,20 @@ fn cpuid_differences(presented: &CpuidLeafSet, effective: &EffectiveCpuid) -> Ve /// Checks that the backend supports the CPU profile, from the CPU surface it /// reports (`E_PROFILE_UNSUPPORTED`), at every cold boot and restore. A /// backend that reports no surface fails the check. -pub(super) fn check_profile_support( - partition: &dyn HvlitePartition, +fn check_profile_support( + backend: &dyn TimeAbiBackend, hypervisor: &str, profile: &CpuProfile, ) -> anyhow::Result<()> { - let surface = backend(partition, hypervisor)? - .supported_cpu_surface()? - .ok_or_else(|| { - TimeAbiError::new( - TimeAbiCode::ProfileUnsupported, - format!( - "the {hypervisor} backend reports no CPU surface to verify CPU profile {} against", - profile.id() - ), - ) - })?; + let surface = backend.supported_cpu_surface()?.ok_or_else(|| { + TimeAbiError::new( + TimeAbiCode::ProfileUnsupported, + format!( + "the {hypervisor} backend reports no CPU surface to verify CPU profile {} against", + profile.id() + ), + ) + })?; cpu_profile::verify_support(profile, &host_cpu_surface(&surface, hypervisor))?; Ok(()) } @@ -1124,8 +1199,13 @@ impl StateUnit for TimeAbiUnit { mod tests { use super::*; use openvmm_defs::time_abi::EFFECTIVE_CPUID_ENTRY_BYTES; + use virt::time_abi::BackendPreflight; + use virt::time_abi::HostTimeSample; + use virt::time_abi::TscAnchor; + use virt::time_abi::TscSetReport; use virt::time_abi::cpuid::effective_cpuid; use virt::time_abi::msr::MSR_TSC_INVARIANT_CONTROL; + use virt::time_abi::surface::SupportedMsrValue; use vm_topology::processor::ProcessorTopology; use vm_topology::processor::TopologyBuilder; use vm_topology::processor::VpIndex; @@ -1604,6 +1684,206 @@ mod tests { ); } + /// Returns the snapshot record of `effective`, an effective CPUID of the + /// test profile. + fn test_record(effective: &EffectiveCpuid) -> SnapshotCpuProfile { + let pinned = cpu_profile::pinned_record(TEST_PROFILE).unwrap(); + SnapshotCpuProfile { + id: TEST_PROFILE.to_owned(), + sha256: pinned.digest.to_vec(), + profile: pinned.encoding.to_vec(), + effective_cpuid: encode_effective_cpuid(record_entries(effective)), + capture_cpu_signature: 0, + } + } + + /// A time ABI backend for the CPU checks of a partition: VP 0 reads + /// `presented`, and the backend supports `surface`. + #[derive(Clone)] + struct CpuBackend { + presented: Vec, + surface: SupportedCpuSurface, + } + + impl CpuBackend { + /// Returns a backend whose VP 0 reads the effective CPUID that + /// `config` programs, as KVM reports it, and that supports exactly + /// the test profile. + fn new(config: &TimeAbiConfig) -> Self { + let profile = test_profile(); + let cpuid = profile + .cpuid() + .iter() + .map(|entry| { + let (function, index) = entry.key(); + let leaf = CpuidLeaf::new(function, entry.values()); + match index { + Some(index) => leaf.indexed(index), + None => leaf, + } + }) + .collect(); + let msrs = profile + .msrs() + .iter() + .map(|msr| SupportedMsrValue { + index: msr.index.0, + supported: msr.value.0, + controllable: !0, + }) + .collect(); + Self { + presented: config.cpuid.leaves().to_vec(), + surface: SupportedCpuSurface { + cpuid, + physical_address_width: profile.physical_address_width(), + msrs, + }, + } + } + } + + impl TimeAbiBackend for CpuBackend { + fn native_tsc_hz(&self) -> Result { + unreachable!() + } + + fn lapic_hz(&self) -> Result { + unreachable!() + } + + fn preflight(&self) -> Result { + unreachable!() + } + + fn effective_cpuid(&self) -> Result, TimeAbiError> { + Ok(self.presented.clone()) + } + + fn supported_cpu_surface(&self) -> Result, TimeAbiError> { + Ok(Some(self.surface.clone())) + } + + fn capture_anchor(&self) -> Result { + unreachable!() + } + + fn set_synchronized_tsc( + &self, + _target: &mut dyn FnMut(&HostTimeSample) -> Result, + ) -> Result { + unreachable!() + } + } + + /// Runs the CPU checks of a partition of the test profile whose effective + /// CPUID is `effective`, on `backend`, and on restore of `record`. + fn check_cpu( + backend: Result<&dyn TimeAbiBackend, TimeAbiError>, + effective: &EffectiveCpuid, + record: Option<&SnapshotCpuProfile>, + ) -> anyhow::Result<()> { + check_cpu_profile_with(backend, "kvm", test_profile(), effective, record) + } + + /// Every CPU check of a partition runs, so that a failure names the + /// violations of each check that fails, in order: here a snapshot record + /// that differs in one entry, a result that VP 0 does not observe, and a + /// feature that the backend lacks. The first failure leads, and so gives + /// the error its code, and a failure alone is the check's own error. + #[test] + fn cpu_checks_report_every_failure_together() { + let (_, config, effective) = test_config(2); + let record = test_record(&effective); + let backend = CpuBackend::new(&config); + check_cpu(Ok(&backend), &effective, Some(&record)).unwrap(); + check_cpu(Ok(&backend), &effective, None).unwrap(); + + // The snapshot records leaf 0 with EAX's lowest bit flipped, VP 0 + // reads zeros at the brand string's first leaf, and the backend lacks + // AVX2, which the profile sets. + let mut flipped = record.clone(); + flipped.effective_cpuid[3 * 4] ^= 1; + let mut unobserved = backend.clone(); + unobserved + .presented + .retain(|leaf| leaf.function != 0x8000_0002); + let mut lacking = backend.clone(); + for leaf in &mut lacking.surface.cpuid { + if (leaf.function, leaf.index) == (7, Some(0)) { + leaf.result[1] &= !(1 << 5); + } + } + let failing = CpuBackend { + presented: unobserved.presented.clone(), + surface: lacking.surface.clone(), + }; + + let failure = |backend: &CpuBackend, record: &SnapshotCpuProfile| { + check_cpu(Ok(backend), &effective, Some(record)).unwrap_err() + }; + let recorded = format!("{:#}", failure(&backend, &flipped)); + assert!( + recorded.starts_with(&format!( + "[E_CPU_SURFACE] the effective CPUID of CPU profile {TEST_PROFILE} differs \ + from the snapshot's in 1 entries: CPUID 0x0 EAX is " + )), + "{recorded}" + ); + let presented = format!("{:#}", failure(&unobserved, &record)); + assert!( + presented.starts_with(&format!( + "[E_CPU_SURFACE] VP 0 does not observe the effective CPUID of CPU profile \ + {TEST_PROFILE} in 1 results: CPUID 0x80000002 EAX is 0x00000000, not " + )), + "{presented}" + ); + let unsupported = failure(&lacking, &record); + let support = format!("{unsupported:#}"); + assert_eq!( + support, + format!( + "[E_PROFILE_UNSUPPORTED] the backend does not support CPU profile \ + {TEST_PROFILE}: CPUID 0x7.0 EBX bit 5 is not supported" + ) + ); + assert!(unsupported.downcast_ref::().is_some()); + assert_eq!( + leading_code(&unsupported), + Some(TimeAbiCode::ProfileUnsupported) + ); + + for (backend, snapshot, expected) in [ + ( + &failing, + &flipped, + format!("{recorded}; {presented}; {support}"), + ), + (&failing, &record, format!("{presented}; {support}")), + (&lacking, &flipped, format!("{recorded}; {support}")), + (&unobserved, &flipped, format!("{recorded}; {presented}")), + ] { + let err = failure(backend, snapshot); + assert_eq!(format!("{err:#}"), expected); + assert_eq!(leading_code(&err), Some(TimeAbiCode::CpuSurface)); + } + + // A partition whose backend does not implement the time ABI fails + // both of the backend's checks at once, after the snapshot's record. + let missing = TimeAbiError::new( + TimeAbiCode::TscSyncUnsupported, + "the kvm backend does not implement the time ABI", + ); + let without = |record| { + format!( + "{:#}", + check_cpu(Err(missing.clone()), &effective, Some(record)).unwrap_err() + ) + }; + assert_eq!(without(&record), missing.to_string()); + assert_eq!(without(&flipped), format!("{recorded}; {missing}")); + } + /// Capture copies the pinned profile's precomputed record, so restore /// never encodes or hashes a profile. #[test]