From 34c4b872d86794e2da84f2214a754c6eb739f409 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Thu, 29 Oct 2026 09:37:32 -0700 Subject: [PATCH 1/6] virt: share the host TSC bracket of the time ABI The KVM backend pairs a host time sample with the host TSC by reading the TSC on both sides of the sample, and pairs a vCPU's TSC with the host TSC the same way. Time ABI 2 pairs every time sample's host UTC with the guest TSC on every backend, and MSHV and WHP need the same bracket to measure the guest TSC's offset from the host TSC, so the bracket belongs with the time ABI rather than with one backend. Move HostTscBracket, which reads the host TSC around an operation with LFENCE on both sides, and the conversion of its half width to nanoseconds, cycles_to_ns_ceil and pair_host_sample, to virt::time_abi::tsc. Their arithmetic is available on every host, and only the TSC read itself on an x86-64 host. The KVM backend uses them unchanged. The cycle conversion test moves with them, and a new test checks the bracket's midpoint and its refusal of a TSC that went backwards. Part of microsoft/nvx#1009. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- vmm_core/virt/src/time_abi/mod.rs | 1 + vmm_core/virt/src/time_abi/tsc.rs | 105 ++++++++++++++++++ vmm_core/virt_kvm/src/arch/x86_64/time_abi.rs | 64 +---------- 3 files changed, 108 insertions(+), 62 deletions(-) create mode 100644 vmm_core/virt/src/time_abi/tsc.rs diff --git a/vmm_core/virt/src/time_abi/mod.rs b/vmm_core/virt/src/time_abi/mod.rs index 4aaee12b20c..cd336b1f687 100644 --- a/vmm_core/virt/src/time_abi/mod.rs +++ b/vmm_core/virt/src/time_abi/mod.rs @@ -21,6 +21,7 @@ pub mod identity; pub mod msr; pub mod rate; pub mod surface; +pub mod tsc; #[cfg(guest_arch = "x86_64")] pub use backend::BackendPreflight; diff --git a/vmm_core/virt/src/time_abi/tsc.rs b/vmm_core/virt/src/time_abi/tsc.rs new file mode 100644 index 00000000000..d625bc5dedc --- /dev/null +++ b/vmm_core/virt/src/time_abi/tsc.rs @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Host TSC reads, which pair the host's clocks and the guest TSC with an +//! instant on the host TSC. + +// UNSAFETY: Executing LFENCE, which orders the host TSC reads. +// xtask-fmt allow-target-arch cpu-intrinsic +#![cfg_attr(target_arch = "x86_64", expect(unsafe_code))] + +/// Two host TSC reads around another host operation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct HostTscBracket { + /// The host TSC before the operation. + pub start: u64, + /// The host TSC after the operation. + pub end: u64, +} + +/// Waits until every earlier instruction has completed locally, so that a +/// TSC read is not reordered with its neighbors. +// xtask-fmt allow-target-arch cpu-intrinsic +#[cfg(target_arch = "x86_64")] +fn lfence() { + // SAFETY: LFENCE only orders instruction execution. + unsafe { core::arch::asm!("lfence", options(nostack, preserves_flags)) }; +} + +impl HostTscBracket { + /// Runs `f` between two host TSC reads. + /// + /// Each read waits for every earlier instruction to complete, so that + /// neither is reordered with `f`. + // xtask-fmt allow-target-arch cpu-intrinsic + #[cfg(target_arch = "x86_64")] + pub fn around(f: impl FnOnce() -> T) -> (Self, T) { + lfence(); + let start = safe_intrinsics::rdtsc(); + lfence(); + let value = f(); + lfence(); + let end = safe_intrinsics::rdtsc(); + (Self { start, end }, value) + } + + /// The host TSC at the middle of the bracket, and the most the instant of + /// the bracketed operation can be from it, in cycles. `None` if the TSC + /// went backwards. + pub fn midpoint(&self) -> Option<(u64, u64)> { + let width = self.end.checked_sub(self.start)?; + Some((self.start + width / 2, width.div_ceil(2))) + } +} + +/// Converts `cycles` at `tsc_hz` to nanoseconds, rounding up. +pub fn cycles_to_ns_ceil(cycles: u64, tsc_hz: u64) -> u64 { + let ns = (u128::from(cycles) * 1_000_000_000).div_ceil(u128::from(tsc_hz.max(1))); + u64::try_from(ns).unwrap_or(u64::MAX) +} + +/// Pairs a host operation with the bracket of host TSC reads around it. +/// +/// Returns the host TSC at the middle of the bracket and the pairing +/// uncertainty, half the bracket, in nanoseconds at `tsc_hz`. `None` if the +/// TSC went backwards. +pub fn pair_host_sample(bracket: HostTscBracket, tsc_hz: u64) -> Option<(u64, u64)> { + let (host_tsc, half_width) = bracket.midpoint()?; + Some((host_tsc, cycles_to_ns_ceil(half_width, tsc_hz))) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cycle_conversion_rounds_up() { + assert_eq!(cycles_to_ns_ceil(0, 2_000_000_000), 0); + assert_eq!(cycles_to_ns_ceil(1, 2_000_000_000), 1); + // One millisecond at the bare-metal KVM host's rate, and a hair over a + // microsecond. + assert_eq!(cycles_to_ns_ceil(2_194_804, 2_194_804_000), 1_000_000); + assert_eq!(cycles_to_ns_ceil(2_195, 2_194_804_000), 1_001); + assert_eq!(cycles_to_ns_ceil(u64::MAX, 1), u64::MAX); + } + + #[test] + fn brackets_pair_at_their_midpoint() { + let bracket = HostTscBracket { + start: 1_000, + end: 1_005, + }; + assert_eq!(bracket.midpoint(), Some((1_002, 3))); + assert_eq!(pair_host_sample(bracket, 1_000_000_000), Some((1_002, 3))); + assert_eq!(HostTscBracket { start: 10, end: 9 }.midpoint(), None); + } + + // xtask-fmt allow-target-arch cpu-intrinsic + #[cfg(target_arch = "x86_64")] + #[test] + fn host_tsc_reads_advance() { + let (bracket, ()) = HostTscBracket::around(|| ()); + assert!(bracket.end >= bracket.start, "{bracket:?}"); + } + +} diff --git a/vmm_core/virt_kvm/src/arch/x86_64/time_abi.rs b/vmm_core/virt_kvm/src/arch/x86_64/time_abi.rs index 7cd752a07bc..30e2950b434 100644 --- a/vmm_core/virt_kvm/src/arch/x86_64/time_abi.rs +++ b/vmm_core/virt_kvm/src/arch/x86_64/time_abi.rs @@ -91,6 +91,8 @@ use virt::time_abi::msr::amd_config_msr_write; use virt::time_abi::rate::LAPIC_HZ_KVM; use virt::time_abi::surface::SupportedCpuSurface; use virt::time_abi::surface::SupportedMsrValue; +use virt::time_abi::tsc::HostTscBracket; +use virt::time_abi::tsc::pair_host_sample; use virt::x86::MsrError; use x86defs::cpuid::CpuidFunction; @@ -269,57 +271,6 @@ fn routing_error(message: impl Into) -> TimeAbiError { TimeAbiError::new(TimeAbiCode::IdentityRouting, message) } -/// Two host TSC reads around another host operation. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) struct HostTscBracket { - pub start: u64, - pub end: u64, -} - -/// Waits until every earlier instruction has completed locally, so that a -/// TSC read is not reordered with its neighbors. -fn lfence() { - // SAFETY: LFENCE only orders instruction execution. - unsafe { core::arch::asm!("lfence", options(nostack, preserves_flags)) }; -} - -impl HostTscBracket { - /// Runs `f` between two host TSC reads. - pub(crate) fn around(f: impl FnOnce() -> T) -> (Self, T) { - lfence(); - let start = safe_intrinsics::rdtsc(); - lfence(); - let value = f(); - lfence(); - let end = safe_intrinsics::rdtsc(); - (Self { start, end }, value) - } - - /// The host TSC at the middle of the bracket, and the most the instant of - /// the bracketed operation can be from it, in cycles. `None` if the TSC - /// went backwards. - pub(crate) fn midpoint(&self) -> Option<(u64, u64)> { - let width = self.end.checked_sub(self.start)?; - Some((self.start + width / 2, width.div_ceil(2))) - } -} - -/// Converts `cycles` at `tsc_hz` to nanoseconds, rounding up. -fn cycles_to_ns_ceil(cycles: u64, tsc_hz: u64) -> u64 { - let ns = (u128::from(cycles) * 1_000_000_000).div_ceil(u128::from(tsc_hz.max(1))); - u64::try_from(ns).unwrap_or(u64::MAX) -} - -/// Pairs a host time sample with the bracket of host TSC reads around it. -/// -/// Returns the host TSC at the middle of the bracket and the pairing -/// uncertainty, half the bracket, in nanoseconds at `tsc_hz`. `None` if the -/// TSC went backwards. -pub(crate) fn pair_host_sample(bracket: HostTscBracket, tsc_hz: u64) -> Option<(u64, u64)> { - let (host_tsc, half_width) = bracket.midpoint()?; - Some((host_tsc, cycles_to_ns_ceil(half_width, tsc_hz))) -} - /// A host time sample paired with the host TSC. #[derive(Debug, Clone, Copy)] struct PairedSample { @@ -1301,17 +1252,6 @@ mod tests { assert_eq!(read(MSR_IA32_TSC_ADJUST), Some(Err(()))); assert_eq!(read(0xc001_0010), None); } - - #[test] - fn cycle_conversion_rounds_up() { - assert_eq!(cycles_to_ns_ceil(0, 2_000_000_000), 0); - assert_eq!(cycles_to_ns_ceil(1, 2_000_000_000), 1); - // One millisecond at the bare-metal KVM host's rate, and a hair over a - // microsecond. - assert_eq!(cycles_to_ns_ceil(2_194_804, 2_194_804_000), 1_000_000); - assert_eq!(cycles_to_ns_ceil(2_195, 2_194_804_000), 1_001); - assert_eq!(cycles_to_ns_ceil(u64::MAX, 1), u64::MAX); - } } /// Tests against the host's KVM. From 7e59e9145e71d42b63d7dcf70e44fb0ddc7d13fa Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Thu, 29 Oct 2026 09:37:43 -0700 Subject: [PATCH 2/6] virt: read the discipline of host UTC Time ABI 2 publishes, with every time sample, how fast host UTC runs, so that the guest applies the host's clock correction instead of estimating it over minutes. The host already knows that correction: its time service steers host UTC by it. Add host_clock_discipline, which reads it from the host OS, whatever time service runs there, rather than from a hypervisor feature: - On Linux, adjtimex with no modes: the frequency correction plus the tick's deviation from its nominal 10 ms, the status (STA_CLOCKERR is unreliable, STA_UNSYNC free-running, anything else synchronized), and the maximum error that the time service sets. It also names the clock's basis from the current clocksource: the TSC at the kernel's kHz calibration, or the TSC at the hypervisor's rate through the Hyper-V TSC page. - On Windows, GetSystemTimeAdjustmentPrecise: its increment is the performance counter's frequency, and Windows counts a second of UTC for every adjustment of the counter's ticks, so UTC runs at the increment over the adjustment of the counter's rate, and a larger adjustment slows it. Over 20 s, UTC ran 0.45 ppm slow against the counter on a Windows host that reported an adjustment of 10,000,005 for an increment of 10,000,000, and 53 ppm slow on one whose time service moved its adjustment from 10,000,501 to 10,000,588. An adjustment that Windows does not apply is a free-running clock with no correction, and a zero one an unknown clock. Under Hyper-V the performance counter runs at the hypervisor's TSC rate. Add host_suspended_ns, how far the monotonic clock that counts suspended time has run ahead of one that does not (CLOCK_BOOTTIME minus CLOCK_MONOTONIC, and the interrupt time minus the unbiased interrupt time), from which a change reveals a suspension, and HostClockKind::follows_utc_rate, which says whether the anchor's monotonic clock runs at host UTC's corrected rate, as Linux's does and Windows' does not, so that a step of host UTC shows against it. The tests check the Windows adjustment's arithmetic, its direction with the readings of the host above, and its rounding, that each host reports a status and a plausible correction, and the clock kinds. Part of microsoft/nvx#1009. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- vmm_core/virt/src/time_abi/host.rs | 289 +++++++++++++++++++++++++++++ 1 file changed, 289 insertions(+) diff --git a/vmm_core/virt/src/time_abi/host.rs b/vmm_core/virt/src/time_abi/host.rs index 2a71e7e84f1..ecd8ca9f01c 100644 --- a/vmm_core/virt/src/time_abi/host.rs +++ b/vmm_core/virt/src/time_abi/host.rs @@ -39,6 +39,84 @@ impl HostClockKind { _ => None, } } + + /// Returns whether the clock runs at host UTC's corrected rate, so that + /// its difference from host UTC changes only when host UTC steps. + /// + /// Linux's corrections of `CLOCK_REALTIME`'s rate also apply to + /// `CLOCK_BOOTTIME`; Windows never corrects its interrupt time. + pub const fn follows_utc_rate(self) -> bool { + match self { + Self::LinuxBoottime => true, + Self::WindowsInterruptTime => false, + } + } +} + +/// The host clock kind of this host OS, as [`host_identity`] reports it. +pub const fn host_clock_kind() -> HostClockKind { + if cfg!(windows) { + HostClockKind::WindowsInterruptTime + } else { + HostClockKind::LinuxBoottime + } +} + +/// The status of host UTC, as the host's time service reports it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HostClockStatus { + /// The host does not report its clock's status. + Unknown, + /// A time service synchronizes host UTC with a reference. + Synchronized, + /// No time service synchronizes host UTC, which runs on its own. + FreeRunning, + /// The host reports a fault of its clock. + Unreliable, +} + +/// How the host converts its TSC to UTC. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HostClockBasis { + /// The TSC at the rate that the host kernel calibrated, to the kHz: the + /// `tsc` clocksource of Linux. + KernelTsc, + /// The TSC at the hypervisor's rate, to the Hz: the Hyper-V TSC page + /// clocksource of Linux, and the performance counter of Windows. + HypervisorTsc, + /// Another clock, which may not follow the TSC. + Other, +} + +/// What the host's time service makes host UTC do. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct HostClockDiscipline { + /// The status of host UTC. + pub status: HostClockStatus, + /// How much faster than the host's uncorrected clock host UTC runs, in + /// parts per million scaled by 2^16, the unit of Linux's frequency + /// correction. + pub correction: i64, + /// The time service's bound on the error of host UTC, in microseconds. + pub max_error_us: Option, + /// The clock from which the host derives UTC. + pub basis: HostClockBasis, +} + +/// Reads the discipline of host UTC: on Linux, the kernel's frequency +/// correction and status that `adjtimex` reports, which any time service +/// sets; on Windows, the time adjustment that `GetSystemTimeAdjustmentPrecise` +/// reports. A host that cannot report it reports +/// [`HostClockStatus::Unknown`] and no correction. +pub fn host_clock_discipline() -> HostClockDiscipline { + sys::clock_discipline() +} + +/// Returns how long the host has been suspended since it booted, in +/// nanoseconds: how far the host monotonic clock of [`host_clock_kind`], +/// which counts suspended time, has run ahead of one that does not. +pub fn host_suspended_ns() -> Result { + sys::suspended_ns() } /// The identity of a host and of its current boot. @@ -94,6 +172,28 @@ fn host_error(message: impl Into) -> TimeAbiError { TimeAbiError::new(TimeAbiCode::HostIdentity, message) } +/// One part per million in Linux's frequency correction, which scales it by +/// 2^16: the unit of [`HostClockDiscipline::correction`]. +pub const SCALED_PPM: i64 = 1 << 16; + +/// Returns the correction of a clock that counts a second for every +/// `adjustment` ticks of a counter that ticks `increment` times a second, as +/// Windows reports its precise time adjustment, in parts per million scaled +/// by 2^16, rounded to the nearest: the clock runs at `increment / +/// adjustment` of the counter's rate, so a larger adjustment slows it. +#[cfg(any(windows, test))] +fn adjustment_correction(adjustment: u64, increment: u64) -> i64 { + let scale = i128::from(SCALED_PPM) * 1_000_000; + let numerator = (i128::from(increment) - i128::from(adjustment)) * scale; + let adjustment = i128::from(adjustment.max(1)); + let rounded = if numerator >= 0 { + (numerator + adjustment / 2) / adjustment + } else { + (numerator - adjustment / 2) / adjustment + }; + rounded.clamp(i64::MIN.into(), i64::MAX.into()) as i64 +} + /// Parses 32 hexadecimal digits, ignoring dashes and a trailing newline, as /// in `/etc/machine-id`, the Linux boot ID, and the Windows machine GUID. #[cfg(any(target_os = "linux", windows, test))] @@ -126,7 +226,10 @@ fn zero_extended_id(value: u32) -> [u8; 16] { #[cfg(target_os = "linux")] mod sys { + use super::HostClockBasis; + use super::HostClockDiscipline; use super::HostClockKind; + use super::HostClockStatus; use super::HostIdentity; use super::host_error; use super::parse_hex_id; @@ -163,6 +266,58 @@ mod sys { clock_ns(libc::CLOCK_BOOTTIME, "CLOCK_BOOTTIME") } + /// The file that names the clocksource from which Linux derives UTC. + const CURRENT_CLOCKSOURCE: &str = + "/sys/devices/system/clocksource/clocksource0/current_clocksource"; + + /// Linux's `USER_HZ`, whose ticks `adjtimex` measures: 100 on x86-64. + const USER_HZ: i64 = 100; + + pub(super) fn clock_discipline() -> HostClockDiscipline { + let basis = match std::fs::read_to_string(CURRENT_CLOCKSOURCE) + .as_deref() + .map(str::trim) + { + Ok("tsc") => HostClockBasis::KernelTsc, + Ok("hyperv_clocksource_tsc_page") => HostClockBasis::HypervisorTsc, + _ => HostClockBasis::Other, + }; + // SAFETY: `timex` holds only integers, for which zero is valid. + let mut timex: libc::timex = unsafe { std::mem::zeroed() }; + // SAFETY: `timex` is a valid, writable timex for the duration of the + // call, and its zero `modes` only reads the kernel's state. + if unsafe { libc::adjtimex(&mut timex) } < 0 { + return HostClockDiscipline { + status: HostClockStatus::Unknown, + correction: 0, + max_error_us: None, + basis, + }; + } + let status = if timex.status & libc::STA_CLOCKERR != 0 { + HostClockStatus::Unreliable + } else if timex.status & libc::STA_UNSYNC != 0 { + HostClockStatus::FreeRunning + } else { + HostClockStatus::Synchronized + }; + // A second of host UTC lasts `tick` microseconds per `USER_HZ` tick, + // plus the frequency correction. + let tick_correction = (timex.tick * USER_HZ - 1_000_000) * super::SCALED_PPM; + HostClockDiscipline { + status, + correction: timex.freq.saturating_add(tick_correction), + max_error_us: u64::try_from(timex.maxerror).ok(), + basis, + } + } + + pub(super) fn suspended_ns() -> Result { + let monotonic = clock_ns(libc::CLOCK_MONOTONIC, "CLOCK_MONOTONIC")?; + let boottime = clock_ns(libc::CLOCK_BOOTTIME, "CLOCK_BOOTTIME")?; + Ok(boottime.saturating_sub(monotonic)) + } + fn read_id(path: &str) -> Result<[u8; 16], TimeAbiError> { use std::io::Read; @@ -196,7 +351,10 @@ mod sys { #[cfg(windows)] mod sys { + use super::HostClockBasis; + use super::HostClockDiscipline; use super::HostClockKind; + use super::HostClockStatus; use super::HostIdentity; use super::host_error; use super::parse_hex_id; @@ -208,8 +366,10 @@ mod sys { use windows_sys::Win32::System::Registry::RRF_RT_REG_DWORD; use windows_sys::Win32::System::Registry::RRF_RT_REG_SZ; use windows_sys::Win32::System::Registry::RegGetValueW; + use windows_sys::Win32::System::SystemInformation::GetSystemTimeAdjustmentPrecise; use windows_sys::Win32::System::SystemInformation::GetSystemTimePreciseAsFileTime; use windows_sys::Win32::System::WindowsProgramming::QueryInterruptTimePrecise; + use windows_sys::Win32::System::WindowsProgramming::QueryUnbiasedInterruptTimePrecise; /// 100 ns intervals from 1601-01-01 to 1970-01-01. const FILETIME_UNIX_EPOCH: u64 = 116_444_736_000_000_000; @@ -239,6 +399,56 @@ mod sys { .ok_or_else(|| host_error("host interrupt time overflows")) } + pub(super) fn clock_discipline() -> HostClockDiscipline { + let mut adjustment = 0; + let mut increment = 0; + let mut disabled = 0; + // SAFETY: the three outputs are valid, writable values for the + // duration of the call. + let read = unsafe { + GetSystemTimeAdjustmentPrecise(&mut adjustment, &mut increment, &mut disabled) + }; + let (status, correction) = if read == 0 || increment == 0 { + (HostClockStatus::Unknown, 0) + } else if disabled != 0 { + // No time service adjusts UTC, which follows the performance + // counter. + (HostClockStatus::FreeRunning, 0) + } else if adjustment == 0 { + (HostClockStatus::Unknown, 0) + } else { + // The increment is the performance counter's frequency, and UTC + // counts a second for every adjustment of its ticks. + ( + HostClockStatus::Synchronized, + super::adjustment_correction(adjustment, increment), + ) + }; + // Under Hyper-V, the performance counter runs at the TSC rate that + // the hypervisor reports. + HostClockDiscipline { + status, + correction, + max_error_us: None, + basis: HostClockBasis::HypervisorTsc, + } + } + + pub(super) fn suspended_ns() -> Result { + let mut unbiased = 0; + let mut interrupt = 0; + // SAFETY: `unbiased` and `interrupt` are valid, writable u64 values + // for the duration of the calls. + unsafe { + QueryUnbiasedInterruptTimePrecise(&mut unbiased); + QueryInterruptTimePrecise(&mut interrupt); + } + interrupt + .saturating_sub(unbiased) + .checked_mul(100) + .ok_or_else(|| host_error("host interrupt time overflows")) + } + fn wide(text: &str) -> Vec { text.encode_utf16().chain(Some(0)).collect() } @@ -308,6 +518,9 @@ mod sys { #[cfg(not(any(target_os = "linux", windows)))] mod sys { + use super::HostClockBasis; + use super::HostClockDiscipline; + use super::HostClockStatus; use super::HostIdentity; use super::host_error; use crate::time_abi::TimeAbiError; @@ -324,6 +537,21 @@ mod sys { )) } + pub(super) fn clock_discipline() -> HostClockDiscipline { + HostClockDiscipline { + status: HostClockStatus::Unknown, + correction: 0, + max_error_us: None, + basis: HostClockBasis::Other, + } + } + + pub(super) fn suspended_ns() -> Result { + Err(host_error( + "host time sampling is not supported on this host", + )) + } + pub(super) fn host_identity() -> Result { Err(host_error("host identity is not supported on this host")) } @@ -374,6 +602,67 @@ mod tests { assert_eq!(HostClockKind::from_manifest("utc"), None); } + #[test] + fn only_linux_boottime_follows_utc_rate() { + assert!(HostClockKind::LinuxBoottime.follows_utc_rate()); + assert!(!HostClockKind::WindowsInterruptTime.follows_utc_rate()); + } + + #[test] + fn windows_adjustments_are_scaled_ppm() { + // Windows's nominal rate: a second of UTC per second of the + // performance counter. + assert_eq!(adjustment_correction(10_000_000, 10_000_000), 0); + assert_eq!( + adjustment_correction(10_000_000, 10_000_100), + 10 * SCALED_PPM + ); + assert_eq!( + adjustment_correction(10_000_000, 9_999_900), + -10 * SCALED_PPM + ); + // A larger adjustment counts more of the counter's ticks as a second, + // so UTC runs slow: a Windows host that reported 10,000,501 for its + // 10 MHz counter ran UTC 53 ppm slow against the counter, between the + // -50.1 and -58.8 ppm of its adjustments over the 20 s measured. + assert_eq!(adjustment_correction(10_000_501, 10_000_000), -3_283_189); + assert_eq!( + adjustment_correction(10_000_005, 10_000_000), + -SCALED_PPM / 2 + ); + // The correction rounds to the nearest scaled unit, and half a unit + // away from zero. + assert_eq!(adjustment_correction(1_000_000, 1_000_001), SCALED_PPM); + assert_eq!(adjustment_correction(156_250, 156_251), 419_430); + assert_eq!(adjustment_correction(156_250, 156_249), -419_430); + assert_eq!(adjustment_correction(131_072_000_000, 131_072_000_001), 1); + assert_eq!(adjustment_correction(131_072_000_000, 131_071_999_999), -1); + // A zero adjustment, which the caller rejects, does not divide by + // zero. + assert_eq!(adjustment_correction(0, 1), SCALED_PPM * 1_000_000); + } + + #[cfg(any(target_os = "linux", windows))] + #[test] + fn host_clock_discipline_is_available() { + let discipline = host_clock_discipline(); + assert_ne!( + discipline.status, + HostClockStatus::Unknown, + "{discipline:?}" + ); + // Linux limits the correction to its tick range and ±500 ppm. + assert!( + discipline.correction.unsigned_abs() < 200_000 * SCALED_PPM.unsigned_abs(), + "{discipline:?}" + ); + if cfg!(windows) { + assert_eq!(discipline.basis, HostClockBasis::HypervisorTsc); + assert_eq!(discipline.max_error_us, None); + } + host_suspended_ns().unwrap(); + } + #[cfg(any(target_os = "linux", windows))] #[test] fn host_time_samples_advance() { From d4200ad70dde549aef42487417ec6035c7e5ae15 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Thu, 29 Oct 2026 09:38:03 -0700 Subject: [PATCH 3/6] virt: publish host UTC paired with the guest TSC, and its correction In time ABI 1 the guest pairs host UTC with its own clock: it reads its clock around the port write that latches a time sample, and accepts the pairing only within 1 ms at boot and restore, or 50 us for its discipline, so a port write that a loaded host serves late costs a sample, and three cost a boot or a restore. It also estimates over minutes how fast host UTC runs, although the host's time service knows. Add virt::time_abi::publish, which computes the host's fields of a time sample under time ABI 2: - Host UTC paired with the guest TSC on the host: the tightest of up to 16 pairings of a host time sample with the host TSC, plus the guest TSC's offset from the host TSC, and the pairing's bound, which adds the offset's uncertainty. GuestTscOffset holds that offset; it measures it from guest TSC reads between two host TSC reads and keeps the tightest, and says whether two measurements can be the same offset. - The correction: how fast host UTC runs against the guest TSC counted at the declared rate, minus one, in ppm scaled by 2^16, that is (1 + host correction) * F / F_d - 1. On a restore it includes the native rate's deviation from the declared one. - A bound on that correction's error: the rounding when the host's clock and the backend's native rate come from one calibration, the kernel's kHz rate for KVM and the hypervisor's rate for MSHV and WHP, one kHz of the native rate when they do not, and none when the host's clocksource may not follow the TSC. - The status and maximum error of host UTC, as its time service reports them. - A disruption counter, which changes when the host's suspended time changes, when host UTC moves against the host monotonic clock by 1 ms or more beyond what the correction explains (Linux corrects both alike; on Windows the correction may change between samples by up to 500 ppm), and when the guest TSC's offset changes. Add the host-rate-offset-ppm test hook, which publishes host UTC as running that many ppm faster than it does from the process's first sample on, with the correction to match, so that tests can check that a guest follows a host clock whose rate differs from the TSC's. The publisher applies utc-offset-ms too, which the portb device applied until now. The tests check the correction's arithmetic against exact values, the bounds, the pairing of a sample, the disruptions of Linux and Windows clocks, the hooks, and a real host sample's pairing. Part of microsoft/nvx#1009. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/worker/dispatch/time_abi.rs | 1 + vmm_core/virt/src/time_abi/backend.rs | 1 + vmm_core/virt/src/time_abi/mod.rs | 24 + vmm_core/virt/src/time_abi/publish.rs | 565 ++++++++++++++++++ vmm_core/virt/src/time_abi/tsc.rs | 127 ++++ 5 files changed, 718 insertions(+) create mode 100644 vmm_core/virt/src/time_abi/publish.rs diff --git a/openvmm/openvmm_core/src/worker/dispatch/time_abi.rs b/openvmm/openvmm_core/src/worker/dispatch/time_abi.rs index 74b5c6524d5..fae4bc6e638 100644 --- a/openvmm/openvmm_core/src/worker/dispatch/time_abi.rs +++ b/openvmm/openvmm_core/src/worker/dispatch/time_abi.rs @@ -1251,6 +1251,7 @@ mod tests { utc_offset_ms: 0, sample_delay_us: 0, host_cpu_unknown: false, + host_rate_offset_ppm: 0, }; /// The `host-cpu-unknown` test hook. diff --git a/vmm_core/virt/src/time_abi/backend.rs b/vmm_core/virt/src/time_abi/backend.rs index 69cad91b772..65e2e20ceee 100644 --- a/vmm_core/virt/src/time_abi/backend.rs +++ b/vmm_core/virt/src/time_abi/backend.rs @@ -270,6 +270,7 @@ mod tests { utc_offset_ms: 0, sample_delay_us: 0, host_cpu_unknown: false, + host_rate_offset_ppm: 0, }; #[test] diff --git a/vmm_core/virt/src/time_abi/mod.rs b/vmm_core/virt/src/time_abi/mod.rs index cd336b1f687..785b8f8e787 100644 --- a/vmm_core/virt/src/time_abi/mod.rs +++ b/vmm_core/virt/src/time_abi/mod.rs @@ -19,6 +19,8 @@ pub mod host; pub mod identity; #[cfg(guest_arch = "x86_64")] pub mod msr; +#[cfg(guest_arch = "x86_64")] +pub mod publish; pub mod rate; pub mod surface; pub mod tsc; @@ -262,10 +264,16 @@ pub struct TimeAbiTestHooks { /// profile on a host that a pinned profile serves. #[mesh(7)] pub host_cpu_unknown: bool, + /// Publish host UTC as running this many ppm faster than it does, from + /// the VM process's first time sample on, with the correction to match. + #[mesh(8)] + pub host_rate_offset_ppm: i32, } /// The largest accepted `dest-rate-offset-ppm` magnitude. pub const MAX_HOOK_RATE_OFFSET_PPM: i32 = 10_000; +/// The largest accepted `host-rate-offset-ppm` magnitude. +pub const MAX_HOOK_HOST_RATE_OFFSET_PPM: i32 = 10_000; /// The largest accepted `downtime-add-s` value: ten years. pub const MAX_HOOK_DOWNTIME_ADD_S: u64 = 10 * 365 * 24 * 60 * 60; /// The largest accepted `utc-offset-ms` magnitude: ten years. @@ -329,6 +337,14 @@ impl TimeAbiTestHooks { MAX_HOOK_RATE_OFFSET_PPM, )?; } + "host-rate-offset-ppm" => { + hooks.host_rate_offset_ppm = number( + value, + argument, + -MAX_HOOK_HOST_RATE_OFFSET_PPM, + MAX_HOOK_HOST_RATE_OFFSET_PPM, + )?; + } "downtime-add-s" => { hooks.downtime_add_s = number(value, argument, 0, MAX_HOOK_DOWNTIME_ADD_S)?; } @@ -572,6 +588,7 @@ mod tests { "utc-offset-ms=-5000", "sample-delay-us=3000", "host-cpu-unknown", + "host-rate-offset-ppm=-750", ]) .unwrap(); assert_eq!( @@ -584,6 +601,7 @@ mod tests { utc_offset_ms: -5000, sample_delay_us: 3000, host_cpu_unknown: true, + host_rate_offset_ppm: -750, } ); assert!(hooks.active()); @@ -592,6 +610,10 @@ mod tests { let host_cpu_unknown = TimeAbiTestHooks::parse(&["host-cpu-unknown"]).unwrap(); assert!(host_cpu_unknown.host_cpu_unknown); assert!(host_cpu_unknown.active()); + // So is the host rate hook. + let host_rate = TimeAbiTestHooks::parse(&["host-rate-offset-ppm=10000"]).unwrap(); + assert_eq!(host_rate.host_rate_offset_ppm, 10_000); + assert!(host_rate.active()); } #[test] @@ -605,6 +627,8 @@ mod tests { "dest-rate-offset-ppm=", "dest-rate-offset-ppm=x", "dest-rate-offset-ppm=10001", + "host-rate-offset-ppm", + "host-rate-offset-ppm=-10001", "downtime-add-s=-1", "downtime-add-s=315360001", "utc-offset-ms=315360000001", diff --git a/vmm_core/virt/src/time_abi/publish.rs b/vmm_core/virt/src/time_abi/publish.rs new file mode 100644 index 00000000000..9013d64a86a --- /dev/null +++ b/vmm_core/virt/src/time_abi/publish.rs @@ -0,0 +1,565 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! The host's side of a time sample under time ABI 2: host UTC paired with +//! the guest TSC, and the discipline of host UTC, which the VMM publishes so +//! that the guest applies the host's clock correction instead of estimating +//! it. + +use super::TimeAbiCode; +use super::TimeAbiError; +use super::TimeAbiTestHooks; +use super::host::HostClockBasis; +use super::host::HostClockDiscipline; +use super::host::HostClockKind; +use super::host::HostClockStatus; +use super::host::HostTimeSample; +use super::host::SCALED_PPM; +use super::tsc::GuestTscOffset; +use super::tsc::cycles_to_ns_ceil; + +/// One, in parts per million scaled by 2^16. +const ONE: i128 = SCALED_PPM as i128 * 1_000_000; + +/// A change of host UTC or of the host's suspended time at least this large, +/// in nanoseconds, beyond what the host's correction explains, is a +/// disruption. +pub const DISRUPTION_THRESHOLD_NS: u64 = 1_000_000; + +/// How far the correction of a host clock that does not follow host UTC's +/// rate may change between two samples, in parts per million: Linux's limit +/// of a frequency correction. +const CORRECTION_CHANGE_PPM: i128 = 500; + +/// The rounding of a published correction and of the host's conversion of +/// the TSC to time, in parts per million scaled by 2^16: one part per +/// billion. +pub const CORRECTION_ROUNDING: u64 = 66; + +/// What a backend reports as the native TSC rate `F_d`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NativeRateBasis { + /// The host kernel's calibration of the TSC, to the kHz, as KVM reports + /// it. + KernelKhz, + /// The hypervisor's TSC rate, to the Hz, as MSHV and WHP report it. + Hypervisor, +} + +/// The rates to which a time sample's correction refers. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SampleRates { + /// The declared TSC rate `F`, in Hz. + pub declared_hz: u64, + /// The native TSC rate `F_d`, in Hz. + pub native_hz: u64, + /// What `native_hz` measures. + pub native_basis: NativeRateBasis, +} + +/// The host's readings for one time sample. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct HostReadings { + /// The host TSC at the instant of `time`. + pub host_tsc: u64, + /// Host UTC and host monotonic time at that instant. + pub time: HostTimeSample, + /// The most the instants of `host_tsc` and `time` can differ, in + /// nanoseconds. + pub pairing_ns: u64, + /// The discipline of host UTC. + pub discipline: HostClockDiscipline, + /// The host's suspended time, as + /// [`host_suspended_ns`](super::host::host_suspended_ns) reports it. + pub suspended_ns: u64, +} + +/// The host's fields of a time sample. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PublishedSample { + /// Host UTC, in nanoseconds since the Unix epoch, after the test hooks. + pub utc_ns: u64, + /// The guest TSC at the instant of `utc_ns`. + pub tsc: u64, + /// The most the instants of `tsc` and `utc_ns` can differ, in + /// nanoseconds. + pub pairing_ns: u64, + /// The status of host UTC. + pub status: HostClockStatus, + /// A counter that changes when host UTC steps, when the host resumes + /// from a suspension, and when the guest TSC's offset from the host TSC + /// changes. + pub disruption: u32, + /// How fast host UTC runs against the guest TSC counted at the declared + /// rate `F`, minus one, in parts per million scaled by 2^16. + pub correction: i64, + /// A bound on the error of `correction`, in parts per million scaled by + /// 2^16, unless the host's clock may not follow the TSC. + pub correction_bound: Option, + /// The time service's bound on the error of host UTC, in microseconds. + pub max_error_us: Option, +} + +/// Returns how fast host UTC runs against the TSC counted at `declared_hz`, +/// minus one, in parts per million scaled by 2^16, rounded to the nearest, +/// for a host whose UTC runs `host_correction` faster than its uncorrected +/// clock, which counts the TSC at `native_hz`, and whose published UTC the +/// rate hook makes run `rate_offset_ppm` faster still. +/// +/// That is `(1 + host_correction) * F / F_d * (1 + rate_offset) - 1`: on a +/// restore, it includes the deviation of the native rate from the declared +/// one, which the restore packet reports. +pub fn sample_correction( + host_correction: i64, + declared_hz: u64, + native_hz: u64, + rate_offset_ppm: i32, +) -> i64 { + let million = 1_000_000; + let native_hz = i128::from(native_hz.max(1)); + let numerator = (ONE + i128::from(host_correction)) + * i128::from(declared_hz) + * (million + i128::from(rate_offset_ppm)) + - ONE * native_hz * million; + let denominator = native_hz * million; + let rounded = if numerator >= 0 { + (numerator + denominator / 2) / denominator + } else { + (numerator - denominator / 2) / denominator + }; + rounded.clamp(i64::MIN.into(), i64::MAX.into()) as i64 +} + +/// Returns a bound on the error of a published correction, in parts per +/// million scaled by 2^16, for a host that derives UTC from `host` and a +/// backend whose native rate `native_hz` measures `native`, or `None` if the +/// host's clock may not follow the TSC at all. +/// +/// When the two rates come from the same calibration, only the rounding +/// remains; otherwise one is the other to the kHz. +pub fn correction_bound( + host: HostClockBasis, + native: NativeRateBasis, + native_hz: u64, +) -> Option { + match (host, native) { + (HostClockBasis::KernelTsc, NativeRateBasis::KernelKhz) + | (HostClockBasis::HypervisorTsc, NativeRateBasis::Hypervisor) => Some(CORRECTION_ROUNDING), + (HostClockBasis::KernelTsc | HostClockBasis::HypervisorTsc, _) => { + let khz = (1_000 * ONE as u128).div_ceil(u128::from(native_hz.max(1))); + Some(u64::try_from(khz).unwrap_or(u64::MAX) + CORRECTION_ROUNDING) + } + (HostClockBasis::Other, _) => None, + } +} + +/// The host clock readings that the disruption check compares. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct ClockMarks { + utc_ns: u64, + monotonic_ns: u64, + suspended_ns: u64, + correction: i64, +} + +/// Returns whether host UTC stepped, or the host was suspended, between the +/// readings `last` and `now` of a host whose monotonic clock is `clock`. +fn disrupted(last: &ClockMarks, now: &ClockMarks, clock: HostClockKind) -> bool { + let threshold = i128::from(DISRUPTION_THRESHOLD_NS); + if now.suspended_ns.abs_diff(last.suspended_ns) >= DISRUPTION_THRESHOLD_NS { + return true; + } + let Some(elapsed) = now.monotonic_ns.checked_sub(last.monotonic_ns) else { + return true; + }; + let elapsed = i128::from(elapsed); + let utc_elapsed = i128::from(now.utc_ns) - i128::from(last.utc_ns); + let (expected, tolerance) = if clock.follows_utc_rate() { + (elapsed, threshold) + } else { + // The host corrects UTC but not the monotonic clock, and the + // correction may have changed since the last sample. + ( + elapsed + elapsed * i128::from(now.correction) / ONE, + threshold + elapsed * CORRECTION_CHANGE_PPM / 1_000_000, + ) + }; + (utc_elapsed - expected).abs() >= tolerance +} + +/// Publishes the host's side of the time samples of one VM process. +#[derive(Debug)] +pub struct HostClockPublisher { + /// The host monotonic clock that the readings carry. + clock: HostClockKind, + /// The `utc-offset-ms` hook, in nanoseconds. + utc_offset_ns: i128, + /// The `host-rate-offset-ppm` hook. + rate_offset_ppm: i32, + /// The host UTC of the first sample, from which the rate hook skews + /// published UTC. + skew_origin_ns: Option, + /// The readings of the last sample. + last: Option, + /// The disruption counter. + disruption: u32, +} + +impl HostClockPublisher { + /// Returns a publisher for a host whose monotonic clock is `clock`, with + /// the test hooks `hooks`. + pub fn new(clock: HostClockKind, hooks: &TimeAbiTestHooks) -> Self { + Self { + clock, + utc_offset_ns: i128::from(hooks.utc_offset_ms) * 1_000_000, + rate_offset_ppm: hooks.host_rate_offset_ppm, + skew_origin_ns: None, + last: None, + disruption: 0, + } + } + + /// Records a disruption that the host clock readings cannot show, such + /// as a change of the guest TSC's offset from the host TSC. + pub fn disrupt(&mut self) { + self.disruption = self.disruption.wrapping_add(1); + } + + /// Returns the host's fields of a time sample from `readings`, with the + /// guest TSC's offset `offset` and the rates `rates`. + pub fn publish( + &mut self, + readings: &HostReadings, + offset: GuestTscOffset, + rates: SampleRates, + ) -> PublishedSample { + let marks = ClockMarks { + utc_ns: readings.time.utc_ns, + monotonic_ns: readings.time.monotonic_ns, + suspended_ns: readings.suspended_ns, + correction: readings.discipline.correction, + }; + if let Some(last) = self.last.replace(marks) + && disrupted(&last, &marks, self.clock) + { + self.disrupt(); + } + let utc_ns = readings.time.utc_ns; + let origin = *self.skew_origin_ns.get_or_insert(utc_ns); + let skew = (i128::from(utc_ns) - i128::from(origin)) * i128::from(self.rate_offset_ppm) + / 1_000_000; + let utc_ns = (i128::from(utc_ns) + skew + self.utc_offset_ns).clamp(1, u64::MAX.into()); + PublishedSample { + utc_ns: utc_ns as u64, + tsc: offset.guest_tsc(readings.host_tsc), + pairing_ns: readings + .pairing_ns + .saturating_add(cycles_to_ns_ceil(offset.uncertainty, rates.native_hz)), + status: readings.discipline.status, + disruption: self.disruption, + correction: sample_correction( + readings.discipline.correction, + rates.declared_hz, + rates.native_hz, + self.rate_offset_ppm, + ), + correction_bound: correction_bound( + readings.discipline.basis, + rates.native_basis, + rates.native_hz, + ), + max_error_us: readings.discipline.max_error_us, + } + } + + /// Takes the host's fields of a time sample now: pairs host UTC with the + /// host TSC and reads the discipline of host UTC. + // xtask-fmt allow-target-arch cpu-intrinsic + #[cfg(target_arch = "x86_64")] + pub fn sample( + &mut self, + offset: GuestTscOffset, + rates: SampleRates, + ) -> Result { + let (host_tsc, time, pairing_ns) = pair_host_time(rates.native_hz)?; + let readings = HostReadings { + host_tsc, + time, + pairing_ns, + discipline: super::host::host_clock_discipline(), + suspended_ns: super::host::host_suspended_ns()?, + }; + Ok(self.publish(&readings, offset, rates)) + } +} + +/// Host time pairings that a sample tries. +const PAIRING_ATTEMPTS: usize = 16; + +/// A pairing this tight, in nanoseconds, ends the attempts early. +const PAIRING_TARGET_NS: u64 = 1_000; + +/// Pairs a host time sample with the host TSC, keeping the tightest of +/// [`PAIRING_ATTEMPTS`] pairings, and returns the host TSC, the sample, and +/// their pairing uncertainty in nanoseconds at `tsc_hz`. +// xtask-fmt allow-target-arch cpu-intrinsic +#[cfg(target_arch = "x86_64")] +fn pair_host_time(tsc_hz: u64) -> Result<(u64, HostTimeSample, u64), TimeAbiError> { + use super::tsc::HostTscBracket; + use super::tsc::pair_host_sample; + + let mut tightest: Option<(u64, HostTimeSample, u64)> = None; + for _ in 0..PAIRING_ATTEMPTS { + let (bracket, time) = HostTscBracket::around(super::host::sample_host_time); + let time = time?; + let Some((host_tsc, pairing_ns)) = pair_host_sample(bracket, tsc_hz) else { + continue; + }; + if tightest.is_none_or(|(_, _, tightest)| pairing_ns < tightest) { + tightest = Some((host_tsc, time, pairing_ns)); + } + if pairing_ns <= PAIRING_TARGET_NS { + break; + } + } + tightest.ok_or_else(|| { + TimeAbiError::new( + TimeAbiCode::TscAnchor, + "the host TSC went backwards around every host time sample", + ) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + const PPM: i64 = SCALED_PPM; + + fn discipline(correction: i64) -> HostClockDiscipline { + HostClockDiscipline { + status: HostClockStatus::Synchronized, + correction, + max_error_us: Some(1_500), + basis: HostClockBasis::KernelTsc, + } + } + + fn readings(utc_ns: u64, monotonic_ns: u64, correction: i64) -> HostReadings { + HostReadings { + host_tsc: 1_000_000, + time: HostTimeSample { + utc_ns, + monotonic_ns, + }, + pairing_ns: 40, + discipline: discipline(correction), + suspended_ns: 0, + } + } + + const RATES: SampleRates = SampleRates { + declared_hz: 2_000_000_000, + native_hz: 2_000_000_000, + native_basis: NativeRateBasis::KernelKhz, + }; + + const OFFSET: GuestTscOffset = GuestTscOffset { + offset: 500, + uncertainty: 2_000, + }; + + #[test] + fn corrections_fold_the_declared_rate_and_the_hook() { + let f = 2_000_000_000; + assert_eq!(sample_correction(0, f, f, 0), 0); + assert_eq!(sample_correction(-12 * PPM, f, f, 0), -12 * PPM); + // A native rate 100 ppm above the declared one: the guest's clock + // runs 99.99 ppm fast unless it corrects. + assert_eq!(sample_correction(0, f, f + 200_000, 0), -6_552_945); + // The hook multiplies: (1 + 100 ppm)(1 + 200 ppm) - 1. + assert_eq!(sample_correction(100 * PPM, f, f, 200), 300 * PPM + 1_311); + assert_eq!(sample_correction(0, f, f, -50), -50 * PPM); + // Linux's widest tick and Windows' adjustments stay in range. + assert_eq!(sample_correction(100_000 * PPM, f, f, 0), 100_000 * PPM); + } + + #[test] + fn bounds_depend_on_the_rates_calibration() { + let hz = 2_621_440_000; + assert_eq!( + correction_bound(HostClockBasis::KernelTsc, NativeRateBasis::KernelKhz, hz), + Some(CORRECTION_ROUNDING) + ); + assert_eq!( + correction_bound( + HostClockBasis::HypervisorTsc, + NativeRateBasis::Hypervisor, + hz + ), + Some(CORRECTION_ROUNDING) + ); + // One kHz of 2.62144 GHz is 0.381... ppm, or 25,000 scaled units. + for (host, native) in [ + (HostClockBasis::HypervisorTsc, NativeRateBasis::KernelKhz), + (HostClockBasis::KernelTsc, NativeRateBasis::Hypervisor), + ] { + assert_eq!( + correction_bound(host, native, hz), + Some(25_000 + CORRECTION_ROUNDING) + ); + } + assert_eq!( + correction_bound(HostClockBasis::Other, NativeRateBasis::KernelKhz, hz), + None + ); + } + + #[test] + fn samples_pair_utc_with_the_guest_tsc() { + let mut publisher = + HostClockPublisher::new(HostClockKind::LinuxBoottime, &TimeAbiTestHooks::default()); + let sample = publisher.publish( + &readings(1_700_000_000_000_000_000, 5_000, 0), + OFFSET, + RATES, + ); + assert_eq!( + sample, + PublishedSample { + utc_ns: 1_700_000_000_000_000_000, + tsc: 1_000_500, + // 40 ns of pairing and 2,000 cycles, 1 µs, of offset. + pairing_ns: 1_040, + status: HostClockStatus::Synchronized, + disruption: 0, + correction: 0, + correction_bound: Some(CORRECTION_ROUNDING), + max_error_us: Some(1_500), + } + ); + } + + #[test] + fn linux_steps_and_suspensions_are_disruptions() { + fn publish( + publisher: &mut HostClockPublisher, + utc_ns: u64, + monotonic_ns: u64, + suspended_ns: u64, + ) -> u32 { + let mut readings = readings(utc_ns, monotonic_ns, 25 * PPM); + readings.suspended_ns = suspended_ns; + publisher.publish(&readings, OFFSET, RATES).disruption + } + + let mut publisher = + HostClockPublisher::new(HostClockKind::LinuxBoottime, &TimeAbiTestHooks::default()); + let p = &mut publisher; + let start = 1_700_000_000_000_000_000; + assert_eq!(publish(p, start, 1_000_000_000, 0), 0); + // Both clocks advance together, whatever the correction. + assert_eq!(publish(p, start + 64_000_000_000, 65_000_000_000, 0), 0); + // A step just under the threshold, then one at it. + assert_eq!(publish(p, start + 128_000_999_999, 129_000_000_000, 0), 0); + assert_eq!(publish(p, start + 192_001_999_999, 193_000_000_000, 0), 1); + // A suspension of 30 s advances both clocks and the suspended time. + assert_eq!( + publish(p, start + 286_001_999_999, 287_000_000_000, 30_000_000_000), + 2 + ); + // A monotonic clock that goes backwards is a disruption too. + assert_eq!( + publish(p, start + 286_001_999_999, 286_000_000_000, 30_000_000_000), + 3 + ); + p.disrupt(); + assert_eq!( + publish(p, start + 287_001_999_999, 287_000_000_000, 30_000_000_000), + 4 + ); + } + + #[test] + fn windows_utc_may_run_at_its_correction() { + let mut publisher = HostClockPublisher::new( + HostClockKind::WindowsInterruptTime, + &TimeAbiTestHooks::default(), + ); + let start = 1_700_000_000_000_000_000; + let mut publish = |utc_ns, monotonic_ns, correction| { + publisher + .publish(&readings(utc_ns, monotonic_ns, correction), OFFSET, RATES) + .disruption + }; + assert_eq!(publish(start, 1_000_000_000, 100 * PPM), 0); + // 64 s of interrupt time at a correction of 100 ppm: 6.4 ms more UTC. + assert_eq!( + publish(start + 64_006_400_000, 65_000_000_000, 100 * PPM), + 0 + ); + // Within the 32 ms that a correction change could explain. + assert_eq!(publish(start + 128_030_000_000, 129_000_000_000, 0), 0); + // A 40 ms step is not. + assert_eq!(publish(start + 192_070_000_000, 193_000_000_000, 0), 1); + } + + #[test] + fn hooks_offset_and_skew_published_utc() { + let hooks = TimeAbiTestHooks { + utc_offset_ms: -5, + host_rate_offset_ppm: 250, + ..Default::default() + }; + let mut publisher = HostClockPublisher::new(HostClockKind::LinuxBoottime, &hooks); + let start = 1_700_000_000_000_000_000; + let first = publisher.publish(&readings(start, 1_000_000_000, 0), OFFSET, RATES); + assert_eq!(first.utc_ns, start - 5_000_000); + assert_eq!(first.correction, 250 * PPM); + // 100 s later, published UTC is 25 ms further ahead, and the skew is + // no disruption. + let later = publisher.publish( + &readings(start + 100_000_000_000, 101_000_000_000, 0), + OFFSET, + RATES, + ); + assert_eq!( + later.utc_ns, + start + 100_000_000_000 + 25_000_000 - 5_000_000 + ); + assert_eq!(later.disruption, 0); + } + + #[test] + fn published_utc_is_never_zero() { + let hooks = TimeAbiTestHooks { + utc_offset_ms: -10, + ..Default::default() + }; + let mut publisher = HostClockPublisher::new(HostClockKind::LinuxBoottime, &hooks); + let sample = publisher.publish(&readings(1_000, 0, 0), OFFSET, RATES); + assert_eq!(sample.utc_ns, 1); + } + + // xtask-fmt allow-target-arch cpu-intrinsic + #[cfg(all(target_arch = "x86_64", any(target_os = "linux", windows)))] + #[test] + fn host_samples_pair_tightly() { + let mut publisher = HostClockPublisher::new( + super::super::host::host_clock_kind(), + &TimeAbiTestHooks::default(), + ); + let sample = publisher + .sample( + GuestTscOffset { + offset: 0, + uncertainty: 0, + }, + RATES, + ) + .unwrap(); + // After 2020-01-01. + assert!(sample.utc_ns > 1_577_836_800_000_000_000); + assert!(sample.pairing_ns < 100_000, "{sample:?}"); + } +} diff --git a/vmm_core/virt/src/time_abi/tsc.rs b/vmm_core/virt/src/time_abi/tsc.rs index d625bc5dedc..2a6f40a2ee1 100644 --- a/vmm_core/virt/src/time_abi/tsc.rs +++ b/vmm_core/virt/src/time_abi/tsc.rs @@ -68,6 +68,76 @@ pub fn pair_host_sample(bracket: HostTscBracket, tsc_hz: u64) -> Option<(u64, u6 Some((host_tsc, cycles_to_ns_ceil(half_width, tsc_hz))) } +/// Guest TSC reads that a backend's offset measurement tries. +pub const OFFSET_ATTEMPTS: usize = 8; + +/// An offset measurement this tight, in cycles, ends the reads early. +pub const OFFSET_TARGET_CYCLES: u64 = 2_000; + +/// The offset of a guest's TSC from the host TSC that this process reads. +/// +/// The guest TSC is never scaled, so the offset holds until the VMM writes a +/// guest TSC again, or stops and resumes partition time. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct GuestTscOffset { + /// The guest TSC is the host TSC plus this offset, wrapping. + pub offset: u64, + /// The most the true offset can differ from `offset`, in cycles. + pub uncertainty: u64, +} + +impl GuestTscOffset { + /// Returns the guest TSC at the host TSC `host_tsc`. + pub fn guest_tsc(&self, host_tsc: u64) -> u64 { + host_tsc.wrapping_add(self.offset) + } + + /// Returns the offset of a guest TSC that was `guest_tsc` within + /// `bracket`, or `None` if the host TSC went backwards. + pub fn from_bracket(guest_tsc: u64, bracket: HostTscBracket) -> Option { + let (host_tsc, uncertainty) = bracket.midpoint()?; + Some(Self { + offset: guest_tsc.wrapping_sub(host_tsc), + uncertainty, + }) + } + + /// Returns whether `other` can be the same offset, within both + /// uncertainties. + pub fn agrees_with(&self, other: &Self) -> bool { + // The offsets wrap, so their difference is the shorter way round. + let difference = self.offset.wrapping_sub(other.offset) as i64; + difference.unsigned_abs() <= self.uncertainty.saturating_add(other.uncertainty) + } + + /// Measures the offset of a guest TSC that `read` returns, reading it + /// between two host TSC reads up to `attempts` times, and keeps the + /// tightest reading. It stops early at an uncertainty of `target` cycles. + /// Returns `None` if the host TSC went backwards around every read. + // xtask-fmt allow-target-arch cpu-intrinsic + #[cfg(target_arch = "x86_64")] + pub fn measure( + attempts: usize, + target: u64, + mut read: impl FnMut() -> Result, + ) -> Result, E> { + let mut tightest: Option = None; + for _ in 0..attempts { + let (bracket, guest_tsc) = HostTscBracket::around(&mut read); + let Some(candidate) = Self::from_bracket(guest_tsc?, bracket) else { + continue; + }; + if tightest.is_none_or(|tightest| candidate.uncertainty < tightest.uncertainty) { + tightest = Some(candidate); + } + if candidate.uncertainty <= target { + break; + } + } + Ok(tightest) + } +} + #[cfg(test)] mod tests { use super::*; @@ -102,4 +172,61 @@ mod tests { assert!(bracket.end >= bracket.start, "{bracket:?}"); } + #[test] + fn offsets_pair_the_guest_tsc_with_the_bracket_midpoint() { + let bracket = HostTscBracket { + start: 1_000, + end: 1_010, + }; + let offset = GuestTscOffset::from_bracket(5_000, bracket).unwrap(); + assert_eq!( + offset, + GuestTscOffset { + offset: 3_995, + uncertainty: 5 + } + ); + assert_eq!(offset.guest_tsc(2_005), 6_000); + // A guest TSC below the host TSC wraps. + let behind = GuestTscOffset::from_bracket(5, bracket).unwrap(); + assert_eq!(behind.guest_tsc(1_005), 5); + assert_eq!( + GuestTscOffset::from_bracket(0, HostTscBracket { start: 10, end: 9 }), + None + ); + } + + #[test] + fn offsets_agree_within_their_uncertainties() { + let offset = |offset: u64, uncertainty| GuestTscOffset { + offset, + uncertainty, + }; + assert!(offset(1_000, 5).agrees_with(&offset(1_010, 5))); + assert!(!offset(1_000, 5).agrees_with(&offset(1_011, 5))); + assert!(offset(0, 0).agrees_with(&offset(0, 0))); + // Across the wrap. + assert!(offset(u64::MAX, 1).agrees_with(&offset(0, 0))); + assert!(!offset(u64::MAX - 1, 0).agrees_with(&offset(1, 1))); + } + + // xtask-fmt allow-target-arch cpu-intrinsic + #[cfg(target_arch = "x86_64")] + #[test] + fn measurement_keeps_the_tightest_reading() { + // A guest TSC that runs one million cycles ahead of the host TSC. + let read = || Ok::<_, ()>(safe_intrinsics::rdtsc().wrapping_add(1_000_000)); + let offset = GuestTscOffset::measure(8, 0, read).unwrap().unwrap(); + // The read lies between the two host reads, so the offset is within + // the uncertainty of one million cycles. + assert!( + offset.agrees_with(&GuestTscOffset { + offset: 1_000_000, + uncertainty: 0 + }), + "{offset:?}" + ); + assert_eq!(GuestTscOffset::measure(4, 0, || Err::(7)), Err(7)); + assert_eq!(GuestTscOffset::measure(0, 0, read), Ok(None)); + } } From d2905b582606deb6f7d15e06b52f85bd1785ec26 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Thu, 29 Oct 2026 09:38:18 -0700 Subject: [PATCH 4/6] virt: measure the guest TSC's offset from the host TSC on every backend Pairing a time sample's host UTC with the guest TSC on the host needs the guest TSC at an instant of the host TSC while the VPs run. The guest TSC is never scaled, so it is the host TSC plus an offset that holds until the VMM writes a guest TSC or stops partition time; what differs between backends is how to learn it. Add TimeAbiBackend::guest_tsc_offset, which core calls at every start of the VM, with every VP bound and stopped, after the TSC writes of a cold boot, a restore, or a reset, and before any VP runs: - KVM returns VP 0's KVM_VCPU_TSC_OFFSET, which preflight verifies is the unscaled offset, so it is exact. - MSHV and WHP read VP 0's TSC between two host TSC reads, up to eight times, and keep the tightest reading; it ends early within 2,000 cycles. Both freeze partition time until a partition starts (MSHV after preflight and a reset, WHP from creation and after a reset, and both when the partition unit stops), so the measurement resumes it first, as the partition's start would right after, and the offset holds once the VPs run. Resuming time that runs is harmless on both. Preflight also reports what each backend's native rate measures, which bounds the error of the published correction: KVM_GET_TSC_KHZ is the kernel's calibration of the TSC to the kHz, and MSHV's and WHP's processor clock frequency is the hypervisor's rate to the Hz. The MSHV hardware test checks that the measurement resumes partition time and agrees with a direct read of VP 0's TSC, and a new WHP hardware test checks that the offset holds while time runs: on a Windows 11 host, two measurements 500 ms apart differed by 87 cycles, within the 3,700 cycles of their uncertainty. Part of microsoft/nvx#1009. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/worker/dispatch/time_abi.rs | 4 ++ vmm_core/virt/src/time_abi/backend.rs | 19 ++++++ vmm_core/virt_kvm/src/arch/x86_64/time_abi.rs | 21 +++++- vmm_core/virt_mshv/src/x86_64/time_abi.rs | 53 +++++++++++++++ vmm_core/virt_whp/src/time_abi.rs | 64 +++++++++++++++++++ 5 files changed, 160 insertions(+), 1 deletion(-) diff --git a/openvmm/openvmm_core/src/worker/dispatch/time_abi.rs b/openvmm/openvmm_core/src/worker/dispatch/time_abi.rs index fae4bc6e638..98111f24406 100644 --- a/openvmm/openvmm_core/src/worker/dispatch/time_abi.rs +++ b/openvmm/openvmm_core/src/worker/dispatch/time_abi.rs @@ -1991,6 +1991,10 @@ mod tests { ) -> Result { unreachable!() } + + fn guest_tsc_offset(&self) -> Result { + unreachable!() + } } /// Runs the CPU checks of a partition of the test profile whose effective diff --git a/vmm_core/virt/src/time_abi/backend.rs b/vmm_core/virt/src/time_abi/backend.rs index 65e2e20ceee..545d69ac5f0 100644 --- a/vmm_core/virt/src/time_abi/backend.rs +++ b/vmm_core/virt/src/time_abi/backend.rs @@ -8,7 +8,9 @@ use super::HostTimeSample; use super::RateCheck; use super::TimeAbiError; use super::TimeAbiTestHooks; +use super::publish::NativeRateBasis; use super::rate; +use super::tsc::GuestTscOffset; use crate::CpuidLeaf; use vm_topology::processor::VpIndex; @@ -95,6 +97,17 @@ pub trait TimeAbiBackend: Send + Sync { &self, target: &mut dyn FnMut(&HostTimeSample) -> Result, ) -> Result; + + /// Returns the offset of VP 0's TSC from the host TSC that this process + /// reads, which pairs the guest TSC with host UTC in every time sample. + /// + /// Core calls it at every start of the VM, with all VPs stopped: after + /// every instantiated VP is bound, and after the TSC writes of a cold + /// boot, a restore's synchronized set, or a reset, and before any VP runs. + /// A backend that measures the offset reads VP 0's TSC between two host + /// TSC reads and keeps the tightest of a bounded number of reads. Fails + /// with `E_TSC_ANCHOR`. + fn guest_tsc_offset(&self) -> Result; } /// How a backend delivers the identity MSR range to [`super::TimeAbiMsrs`]. @@ -125,6 +138,8 @@ pub struct BackendPreflight { pub msr_route: IdentityMsrRoute, /// The synchronized TSC set method. pub sync: TscSyncMethod, + /// What the backend reports as the native TSC rate. + pub native_rate: NativeRateBasis, } /// The largest accepted anchor pairing, in nanoseconds. A backend samples the @@ -253,6 +268,10 @@ mod tests { ) -> Result { unreachable!() } + + fn guest_tsc_offset(&self) -> Result { + unreachable!() + } } fn backend(tsc_hz: u64, lapic_hz: u64) -> FakeBackend { diff --git a/vmm_core/virt_kvm/src/arch/x86_64/time_abi.rs b/vmm_core/virt_kvm/src/arch/x86_64/time_abi.rs index 30e2950b434..0387e4e3644 100644 --- a/vmm_core/virt_kvm/src/arch/x86_64/time_abi.rs +++ b/vmm_core/virt_kvm/src/arch/x86_64/time_abi.rs @@ -44,7 +44,8 @@ //! never scaled: the backend never calls `KVM_SET_TSC_KHZ`, and `F_d` is //! VP 0's `KVM_GET_TSC_KHZ`. The synchronized set writes one //! `KVM_VCPU_TSC_OFFSET` (Linux 5.16 and later) to every vCPU, computed from -//! one host TSC read, and reads every offset back. +//! one host TSC read, and reads every offset back. VP 0's offset pairs the +//! guest TSC with host UTC in every time sample. //! - **LAPIC.** The in-kernel LAPIC timer counts KVM's fixed 1 GHz bus clock; //! `KVM_CAP_X86_APIC_BUS_CYCLES_NS` is never set. //! @@ -88,9 +89,11 @@ use virt::time_abi::msr::IDENTITY_MSR_RANGE; use virt::time_abi::msr::MSR_TSC_INVARIANT_CONTROL; use virt::time_abi::msr::amd_config_msr_read; use virt::time_abi::msr::amd_config_msr_write; +use virt::time_abi::publish::NativeRateBasis; use virt::time_abi::rate::LAPIC_HZ_KVM; use virt::time_abi::surface::SupportedCpuSurface; use virt::time_abi::surface::SupportedMsrValue; +use virt::time_abi::tsc::GuestTscOffset; use virt::time_abi::tsc::HostTscBracket; use virt::time_abi::tsc::pair_host_sample; use virt::x86::MsrError; @@ -613,6 +616,7 @@ impl TimeAbiBackend for KvmPartitionInner { Ok(BackendPreflight { msr_route: IdentityMsrRoute::ExitToVmm, sync: TscSyncMethod::CommonOffset, + native_rate: NativeRateBasis::KernelKhz, }) } @@ -722,6 +726,21 @@ impl TimeAbiBackend for KvmPartitionInner { method: TscSyncMethod::CommonOffset, }) } + + fn guest_tsc_offset(&self) -> Result { + // Preflight verified that VP 0's TSC is the host TSC plus this + // offset, unscaled, so the offset is exact. + let offset = self.vp_kvm(VpIndex::BSP).tsc_offset().map_err(|err| { + TimeAbiError::new( + TimeAbiCode::TscAnchor, + format!("cannot read VP 0's TSC offset: {err:#}"), + ) + })?; + Ok(GuestTscOffset { + offset, + uncertainty: 0, + }) + } } /// Rejects partition capabilities that model Hyper-V (`hv1`), which the diff --git a/vmm_core/virt_mshv/src/x86_64/time_abi.rs b/vmm_core/virt_mshv/src/x86_64/time_abi.rs index 9fdb828b3ac..9b3dbbe16e6 100644 --- a/vmm_core/virt_mshv/src/x86_64/time_abi.rs +++ b/vmm_core/virt_mshv/src/x86_64/time_abi.rs @@ -36,6 +36,11 @@ //! every created VP, reads it back, and resumes partition time right after a //! good read-back. Guest time therefore runs from the restore anchor, as on //! the other backends, instead of waiting for the first VP run. +//! - With partition time running, VP 0's TSC is the host TSC plus a constant +//! offset. The backend measures it at every start by reading VP 0's TSC +//! between two host TSC reads, after resuming partition time, which +//! preflight, a reset, and a stop freeze; the offset pairs the guest TSC +//! with host UTC in every time sample. //! //! These semantics were verified on hypervisor builds 10.0.26100.30000 (bare //! metal) and 10.0.26100.9444 (Azure nested), where they are identical. @@ -86,7 +91,11 @@ use virt::time_abi::msr::MSR_TSC_INVARIANT_CONTROL; use virt::time_abi::msr::MSR_VP_INDEX; use virt::time_abi::msr::amd_config_msr_read; use virt::time_abi::msr::amd_config_msr_write; +use virt::time_abi::publish::NativeRateBasis; use virt::time_abi::surface::SupportedCpuSurface; +use virt::time_abi::tsc::GuestTscOffset; +use virt::time_abi::tsc::OFFSET_ATTEMPTS; +use virt::time_abi::tsc::OFFSET_TARGET_CYCLES; /// The identity MSRs that OpenVMM serves. The hypervisor raises #GP for /// every other MSR of the identity range. @@ -1153,6 +1162,7 @@ impl TimeAbiBackend for MshvPartition { Ok(BackendPreflight { msr_route: IdentityMsrRoute::ExitToVmm, sync: TscSyncMethod::FrozenWrite, + native_rate: NativeRateBasis::Hypervisor, }) } @@ -1306,6 +1316,37 @@ impl TimeAbiBackend for MshvPartition { method: TscSyncMethod::FrozenWrite, }) } + + fn guest_tsc_offset(&self) -> Result { + let anchor_error = |message: String| TimeAbiError::new(TimeAbiCode::TscAnchor, message); + // Partition time stays frozen after preflight, a reset, or a stop + // until the partition starts, which resumes it before any VP runs. + // Resume it now, so that the guest TSC keeps the measured offset. + self.inner.thaw_time().map_err(|error| { + anchor_error(format!( + "cannot resume partition time: {}", + error_chain(&error) + )) + })?; + let offset = GuestTscOffset::measure(OFFSET_ATTEMPTS, OFFSET_TARGET_CYCLES, || { + get_vp_tsc(&self.inner.vmfd, VpIndex::BSP) + }) + .map_err(|error| { + anchor_error(format!( + "cannot read the TSC of VP 0: {}", + error_chain(&error) + )) + })? + .ok_or_else(|| { + anchor_error("the host TSC went backwards around every read of VP 0's TSC".into()) + })?; + tracing::info!( + offset = format_args!("{:#x}", offset.offset), + uncertainty_cycles = offset.uncertainty, + "time ABI: guest TSC offset" + ); + Ok(offset) + } } #[cfg(test)] @@ -1845,6 +1886,7 @@ mod hw { use virt::time_abi::identity::check_time_bits; use virt::time_abi::rate::LAPIC_HZ_HYPERV; use virt::time_abi::rate::check_plausible_tsc_hz; + use virt::time_abi::tsc::HostTscBracket; use vm_topology::memory::MemoryLayout; use vm_topology::processor::TopologyBuilder; use vm_topology::processor::x86::X2ApicState; @@ -2022,11 +2064,22 @@ mod hw { BackendPreflight { msr_route: IdentityMsrRoute::ExitToVmm, sync: TscSyncMethod::FrozenWrite, + native_rate: NativeRateBasis::Hypervisor, } ); // Preflight leaves partition time frozen, as a reset does, until // the synchronized TSC set or the first VP run resumes it. assert!(*partition.inner.time_frozen.lock()); + // The offset measurement resumes it, so that the offset holds. + let offset = backend.guest_tsc_offset().unwrap(); + assert!(!*partition.inner.time_frozen.lock()); + let (bracket, tsc) = + HostTscBracket::around(|| get_vp_tsc(&partition.inner.vmfd, VpIndex::BSP).unwrap()); + let read = GuestTscOffset::from_bracket(tsc, bracket).unwrap(); + assert!( + offset.agrees_with(&read), + "{offset:x?} {read:x?} with {created} VPs" + ); partition.inner.thaw_time().unwrap(); let tsc_hz = backend.native_tsc_hz().unwrap(); check_plausible_tsc_hz(tsc_hz).unwrap(); diff --git a/vmm_core/virt_whp/src/time_abi.rs b/vmm_core/virt_whp/src/time_abi.rs index b9d9e6e4fef..78f114eaa97 100644 --- a/vmm_core/virt_whp/src/time_abi.rs +++ b/vmm_core/virt_whp/src/time_abi.rs @@ -48,6 +48,11 @@ //! resumes explicitly before any VP runs, so the guest TSC runs from the //! restore anchor. Every VP exists from partition creation, so no VP can be //! created after the set. +//! - **Guest TSC offset.** With partition time running, VP 0's TSC is the +//! host TSC plus a constant offset. The backend measures it at every start +//! by reading VP 0's TSC between two host TSC reads, after resuming +//! partition time, which a stop, a reset, and a new partition suspend; the +//! offset pairs the guest TSC with host UTC in every time sample. //! //! These semantics were measured on Windows Server 2025 bare metal (Xeon Gold //! 6138, Skylake-SP) and on Azure runners (Xeon Platinum 8370C and 8573C). @@ -88,7 +93,11 @@ use virt::time_abi::identity::IDENTITY_CPUID_RANGE; use virt::time_abi::identity::IDENTITY_MAX_LEAF; use virt::time_abi::msr::amd_config_msr_read; use virt::time_abi::msr::amd_config_msr_write; +use virt::time_abi::publish::NativeRateBasis; use virt::time_abi::surface::SupportedCpuSurface; +use virt::time_abi::tsc::GuestTscOffset; +use virt::time_abi::tsc::OFFSET_ATTEMPTS; +use virt::time_abi::tsc::OFFSET_TARGET_CYCLES; use virt::x86::topology::per_vp_cpuid_bits; use whp::abi::WHV_CPUID_OUTPUT; use whp::abi::WHV_EXTENDED_VM_EXITS; @@ -1048,6 +1057,7 @@ impl TimeAbiBackend for WhpPartitionInner { Ok(BackendPreflight { msr_route: IdentityMsrRoute::ExitToVmm, sync: TscSyncMethod::FrozenWrite, + native_rate: NativeRateBasis::Hypervisor, }) } @@ -1196,6 +1206,38 @@ impl TimeAbiBackend for WhpPartitionInner { method: TscSyncMethod::FrozenWrite, }) } + + fn guest_tsc_offset(&self) -> Result { + self.time_abi_state()?; + // Partition time stays suspended after creation, a reset, or a stop + // until the partition starts, which resumes it before any VP runs. + // Resume it now, so that the guest TSC keeps the measured offset. + for (_, vtlp) in self.vtlps() { + vtlp.set_time_frozen(false); + } + let bsp = self.vtl0.whp.vp(VpIndex::BSP.index()); + let offset = GuestTscOffset::measure(OFFSET_ATTEMPTS, OFFSET_TARGET_CYCLES, || { + bsp.get_register(whp::Register64::Tsc) + }) + .map_err(|err| { + TimeAbiError::new( + TimeAbiCode::TscAnchor, + format!("cannot read the TSC of VP 0: {err}"), + ) + })? + .ok_or_else(|| { + TimeAbiError::new( + TimeAbiCode::TscAnchor, + "the host TSC went backwards around every read of VP 0's TSC", + ) + })?; + tracing::info!( + offset = format_args!("{:#x}", offset.offset), + uncertainty_cycles = offset.uncertainty, + "time ABI: guest TSC offset" + ); + Ok(offset) + } } impl WhpProcessor<'_> { @@ -2992,4 +3034,26 @@ mod whp_tests { let anchor = select_anchor(&candidates, MAX_ANCHOR_PAIRING_NS).unwrap(); println!("pairing {} ns", anchor.pairing_ns); } + + #[test] + #[ignore = "requires WHP"] + fn guest_tsc_keeps_its_offset_from_the_host_tsc() { + let Some(Partition { partition, .. }) = partition(1) else { + return; + }; + partition.resume_time().unwrap(); + let bsp = partition.vp(0); + let measure = || { + GuestTscOffset::measure(OFFSET_ATTEMPTS, OFFSET_TARGET_CYCLES, || { + bsp.get_register(whp::Register64::Tsc) + }) + .unwrap() + .unwrap() + }; + let first = measure(); + std::thread::sleep(std::time::Duration::from_millis(500)); + let second = measure(); + println!("offsets: {first:x?}, {second:x?}"); + assert!(first.agrees_with(&second), "{first:x?} {second:x?}"); + } } From 69f7ab266f443b7493e39f1134a5263202f6591b Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Thu, 29 Oct 2026 09:38:44 -0700 Subject: [PATCH 5/6] microVM: publish the host's clock correction with each time sample Move every microVM to NVX time ABI 2 (microsoft/nvx#1009). In time ABI 1, the guest estimates how fast its TSC runs against host UTC: it pairs host UTC samples with its own clock, bracketing each port write with reads of that clock, and steers the kernel's phase-locked loop over minutes. The host already knows that rate, because its time service steers host UTC by it, and the guest's brackets depend on how fast the host serves the port write. Time ABI 2 has the VMM pair host UTC with the guest TSC and publish the host's correction, which the guest applies instead of estimating it. The time sample, which writing 0xa7 to the portb status port latches, becomes version 2, 48 bytes: the generation counter, host UTC and the guest TSC at the same instant, the bound of their pairing, the status of host UTC, the disruption counter, the correction against the TSC at the declared rate, and the bounds of the correction's error and of host UTC's error. The portb device takes these host fields from a platform resource, TimeSampleSourceKind, which the VM worker resolves to the time ABI's clock; the device keeps the generation, the test-hook flag, and the sample-delay-us hook, which delays the latch. The worker's clock pairs host UTC with the guest TSC through the offset that the time-abi state unit measures at every start, before any VP runs: the partition unit now depends on the time-abi unit, so the measurement follows the TSC writes of a cold boot, a restore's synchronized set, or a reset, and precedes the VPs' release on every start, including a resume after a failed capture. An offset that changes across a start is a disruption. Until the first measurement, or when the host cannot pair a sample, the window stays empty, and the guest rejects the sample. The restore packet becomes version 5, which drops host UTC: the guest takes a time sample for its repair, which pairs host UTC with its TSC on the host, so the packet no longer latches UTC at its first selection, and its header shrinks to 24 bytes. CPUID leaf 0x40000002 reports time ABI 2.0 in EBX (0x00020000), and snapshots record time ABI version 2. A snapshot of time ABI 1 fails restore with E_MANIFEST_TIME and must be recaptured: its guest decodes only version-1 samples, and a version-4 packet with UTC. The CPU time bits do not change, so every pinned CPU profile serves time ABI 2 as it is. The portb device no longer takes utc-offset-ms, which the publisher applies, so petri's microVM configuration drops it too, and the guest of the ttrpc microVM test reads restore packet 5: version 5, and 17 header bytes after the range count instead of 25. The tests encode and decode the version-2 sample and the version-5 packet byte for byte, check that the portb device latches the source's fields into its window and leaves it empty without a sample, check that the unit's start measures the offset that samples use and that a changed offset is a disruption, and check that restore rejects a version-1 time contract. Part of microsoft/nvx#1009. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- openvmm/openvmm_core/src/worker/dispatch.rs | 42 ++- .../src/worker/dispatch/time_abi.rs | 234 ++++++++++++++- openvmm/openvmm_defs/src/microvm.rs | 4 +- openvmm/openvmm_defs/src/time_abi.rs | 2 +- openvmm/openvmm_entry/src/microvm/mod.rs | 4 +- openvmm/openvmm_entry/src/microvm/restore.rs | 10 +- openvmm/openvmm_helpers/src/snapshot/time.rs | 8 +- petri/src/vm/openvmm/microvm.rs | 1 - vm/devices/chipset/src/microvm.rs | 181 ++++++++---- vm/devices/chipset/src/microvm/resolver.rs | 10 +- vm/devices/chipset_resources/src/microvm.rs | 14 +- .../chipset_resources/src/microvm_time.rs | 273 ++++++++++++++---- vmm_core/virt/src/time_abi/identity.rs | 6 +- vmm_core/virt/src/time_abi/mod.rs | 16 +- vmm_core/virt_kvm/src/arch/x86_64/time_abi.rs | 2 +- vmm_core/virt_mshv/src/x86_64/time_abi.rs | 2 +- vmm_core/virt_whp/src/time_abi.rs | 2 +- .../vmm_tests/tests/tests/ttrpc/microvm.rs | 4 +- 18 files changed, 653 insertions(+), 162 deletions(-) diff --git a/openvmm/openvmm_core/src/worker/dispatch.rs b/openvmm/openvmm_core/src/worker/dispatch.rs index 797284ea5f7..6b6f9f54549 100644 --- a/openvmm/openvmm_core/src/worker/dispatch.rs +++ b/openvmm/openvmm_core/src/worker/dispatch.rs @@ -2052,18 +2052,40 @@ impl InitializedVm { }), ¶meters.hooks, )?; + // The portb device takes the host fields of its time samples + // from the clock, which pairs host UTC with the guest TSC + // through the offset that the unit measures. + let clock = Arc::new(time_abi::TimeAbiClock::new( + msrs.clone(), + &report, + ¶meters.hooks, + )); + resolver.add_resolver(time_abi::TimeSampleSourceResolver(clock.clone())); + let measure_offset: time_abi::MeasureOffset = { + let partition = partition.clone(); + let hypervisor = hypervisor.clone(); + Box::new(move || time_abi::guest_tsc_offset(partition.as_ref(), &hypervisor)) + }; let unit = state_units .add(time_abi::TIME_ABI_UNIT) .spawn(driver_source.simple(), { let msrs = msrs.clone(); let report = report.clone(); move |recv| { - state_unit::run_unit(time_abi::TimeAbiUnit { msrs, report }, recv) + state_unit::run_unit( + time_abi::TimeAbiUnit { + msrs, + report, + clock, + measure_offset, + }, + recv, + ) } }) .unwrap(); Some(time_abi::TimeAbiState { - _unit: unit, + unit, msrs, profile, effective_cpuid, @@ -3376,12 +3398,20 @@ impl InitializedVm { // create a new channel to intercept guest resets let (halt_send, halt_recv) = mesh::channel(); + let partition_unit_builder = state_units + .add("partition") + .depends_on(devices.chipset_unit()) + .depends_on(vmtime.handle()); + // The time ABI unit measures the guest TSC's offset for the time + // samples at every start, before any VP runs. + #[cfg(guest_arch = "x86_64")] + let partition_unit_builder = match &time_abi_state { + Some(time_abi) => partition_unit_builder.depends_on(time_abi.unit.handle()), + None => partition_unit_builder, + }; let (partition_unit, vp_runners) = PartitionUnit::new( driver_source.simple(), - state_units - .add("partition") - .depends_on(devices.chipset_unit()) - .depends_on(vmtime.handle()), + partition_unit_builder, partition.clone().into_vm_partition(), PartitionUnitParams { processor_topology: &processor_topology, diff --git a/openvmm/openvmm_core/src/worker/dispatch/time_abi.rs b/openvmm/openvmm_core/src/worker/dispatch/time_abi.rs index 98111f24406..019e4a3b850 100644 --- a/openvmm/openvmm_core/src/worker/dispatch/time_abi.rs +++ b/openvmm/openvmm_core/src/worker/dispatch/time_abi.rs @@ -1,16 +1,22 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -//! NVX time ABI v1 in the VM worker, which every microVM uses: the partition +//! NVX time ABI 2 in the VM worker, which every microVM uses: the partition //! configuration, the backend preflight and rate policy, the `time-abi` state -//! unit, the capture records, and the restore clock. +//! unit, which measures the guest TSC's offset from the host TSC at every +//! start, the source of the host fields of time samples, the capture records, +//! and the restore clock. #![cfg(guest_arch = "x86_64")] use super::LoadedVm; use crate::partition::HvlitePartition; use anyhow::Context as _; +use chipset_resources::microvm_time::HostTimeFields; +use chipset_resources::microvm_time::ResolvedTimeSampleSource; use chipset_resources::microvm_time::RestoreTimeRecord; +use chipset_resources::microvm_time::TimeSampleSource; +use chipset_resources::microvm_time::TimeSampleSourceKind; use cpu_profile::CpuProfile; use cpu_profile::EffectiveCpuid; use cpu_profile::HostCpuSignature; @@ -29,6 +35,7 @@ use openvmm_defs::time_abi::TimeCapture; use openvmm_defs::time_abi::decode_effective_cpuid; use openvmm_defs::time_abi::encode_effective_cpuid; use openvmm_defs::worker::SavedState; +use parking_lot::Mutex; use state_unit::SpawnedUnit; use state_unit::StateUnit; use std::collections::BTreeMap; @@ -48,7 +55,15 @@ use virt::time_abi::TimeAbiError; use virt::time_abi::TimeAbiMsrs; use virt::time_abi::TimeAbiTestHooks; use virt::time_abi::TscSyncMethod; +use virt::time_abi::host::HostClockStatus; +use virt::time_abi::publish::HostClockPublisher; +use virt::time_abi::publish::NativeRateBasis; +use virt::time_abi::publish::PublishedSample; +use virt::time_abi::publish::SampleRates; use virt::time_abi::surface::SupportedCpuSurface; +use virt::time_abi::tsc::GuestTscOffset; +use vm_resource::PlatformResource; +use vm_resource::ResolveResource; use vmcore::save_restore::RestoreError; use vmcore::save_restore::SaveError; use vmcore::save_restore::SavedStateBlob; @@ -62,7 +77,7 @@ const REGISTERS: [&str; 4] = ["EAX", "EBX", "ECX", "EDX"]; /// The time ABI state of a loaded VM. pub(super) struct TimeAbiState { /// The `time-abi` state unit. - pub _unit: SpawnedUnit, + pub unit: SpawnedUnit, /// The identity MSR handler, holding the declared rates. pub msrs: Arc, /// The CPU profile. @@ -562,6 +577,15 @@ fn backend<'a>( }) } +/// Measures the guest TSC's offset from the host TSC, with every VP +/// stopped. +pub(super) fn guest_tsc_offset( + partition: &dyn HvlitePartition, + hypervisor: &str, +) -> Result { + backend(partition, hypervisor)?.guest_tsc_offset() +} + /// What the time ABI preflight established for a VM process, exposed through /// inspect at `time-abi` together with the declared rates. #[derive(Debug, Clone, Inspect)] @@ -576,6 +600,9 @@ pub(super) struct TimeAbiReport { /// The synchronized TSC set method. #[inspect(debug)] pub sync: TscSyncMethod, + /// What the backend reports as the native TSC rate. + #[inspect(debug)] + pub native_rate: NativeRateBasis, /// The backend's native TSC rate `F_d`, after the rate test hook. pub native_tsc_hz: u64, /// On restore, the accepted rate check against the snapshot's rate. @@ -614,6 +641,7 @@ pub(super) fn declare_rates( cpu_profile, msr_route: preflight.msr_route, sync: preflight.sync, + native_rate: preflight.native_rate, native_tsc_hz: rates.native_tsc_hz, rate_check: rates.check, }) @@ -1179,12 +1207,137 @@ impl LoadedVm { } } +/// The time ABI's source of the host fields of time samples. +/// +/// It pairs host UTC with the guest TSC through the guest TSC's offset from +/// the host TSC, which the `time-abi` unit measures at every start, before +/// any VP runs, and publishes the discipline of host UTC. +pub(super) struct TimeAbiClock { + /// The identity MSR handler, which holds the declared rates. + msrs: Arc, + /// The backend's native TSC rate `F_d`, after the rate test hook. + native_hz: u64, + /// What the backend reports as the native rate. + native_rate: NativeRateBasis, + /// The measured offset and the publisher. + state: Mutex, +} + +/// The mutable state of a [`TimeAbiClock`]. +struct ClockState { + /// The guest TSC's offset from the host TSC, once measured. + offset: Option, + /// The publisher of the host fields. + publisher: HostClockPublisher, +} + +impl TimeAbiClock { + /// Returns the clock of a VM whose identity MSR handler is `msrs`, whose + /// preflight reported `report`, with the test hooks `hooks`. + pub(super) fn new( + msrs: Arc, + report: &TimeAbiReport, + hooks: &TimeAbiTestHooks, + ) -> Self { + Self { + msrs, + native_hz: report.native_tsc_hz, + native_rate: report.native_rate, + state: Mutex::new(ClockState { + offset: None, + publisher: HostClockPublisher::new(virt::time_abi::host::host_clock_kind(), hooks), + }), + } + } + + /// Records the guest TSC's offset that the `time-abi` unit measured. An + /// offset that cannot be the last one is a disruption. + fn set_offset(&self, offset: GuestTscOffset) { + let mut state = self.state.lock(); + if state.offset.is_some_and(|last| !last.agrees_with(&offset)) { + state.publisher.disrupt(); + } + state.offset = Some(offset); + } +} + +impl TimeSampleSource for TimeAbiClock { + fn sample(&self) -> Option { + let declared = self.msrs.declared()?; + let rates = SampleRates { + declared_hz: declared.tsc_hz, + native_hz: self.native_hz, + native_basis: self.native_rate, + }; + let mut state = self.state.lock(); + let offset = state.offset?; + match state.publisher.sample(offset, rates) { + Ok(sample) => Some(host_fields(&sample)), + Err(error) => { + tracelimit::error_ratelimited!( + error = &error as &dyn std::error::Error, + "time ABI time sample failed" + ); + None + } + } + } +} + +/// Returns the time sample fields of a published sample, saturating the +/// values that the sample carries in 32 bits. +fn host_fields(sample: &PublishedSample) -> HostTimeFields { + use chipset_resources::microvm_time::HostClockStatus as SampleStatus; + + let saturate = |value: u64| u32::try_from(value).unwrap_or(u32::MAX); + HostTimeFields { + utc_ns: sample.utc_ns, + tsc: sample.tsc, + pairing_ns: saturate(sample.pairing_ns), + status: match sample.status { + HostClockStatus::Unknown => SampleStatus::Unknown, + HostClockStatus::Synchronized => SampleStatus::Synchronized, + HostClockStatus::FreeRunning => SampleStatus::FreeRunning, + HostClockStatus::Unreliable => SampleStatus::Unreliable, + }, + disruption: sample.disruption, + correction: sample.correction, + correction_bound: sample.correction_bound.map(saturate), + max_error_us: sample.max_error_us.map(saturate), + } +} + +/// Resolves the platform's time sample source, which the microVM portb device +/// uses, to the time ABI's clock. +pub(super) struct TimeSampleSourceResolver(pub Arc); + +impl ResolveResource for TimeSampleSourceResolver { + type Output = ResolvedTimeSampleSource; + type Error = std::convert::Infallible; + + fn resolve( + &self, + _resource: PlatformResource, + _input: (), + ) -> Result { + Ok(ResolvedTimeSampleSource(self.0.clone())) + } +} + +/// Measures the guest TSC's offset from the host TSC. +pub(super) type MeasureOffset = Box Result + Send>; + /// The `time-abi` state unit. It saves and restores /// `HV_X64_MSR_TSC_INVARIANT_CONTROL`, which is partition-wide guest state, -/// and inspects the declared rates and the preflight report. +/// inspects the declared rates and the preflight report, and measures the +/// guest TSC's offset from the host TSC for the time samples at every start. pub(super) struct TimeAbiUnit { pub msrs: Arc, pub report: TimeAbiReport, + /// The time sample source, which uses the measured offset. + pub clock: Arc, + /// Measures the offset. + pub measure_offset: MeasureOffset, } impl InspectMut for TimeAbiUnit { @@ -1194,7 +1347,12 @@ impl InspectMut for TimeAbiUnit { } impl StateUnit for TimeAbiUnit { + /// Measures the guest TSC's offset. The partition unit depends on this + /// one, so every start follows the TSC writes of a cold boot, a restore, + /// or a reset, and precedes any VP run. async fn start(&mut self) -> anyhow::Result<()> { + let offset = (self.measure_offset)()?; + self.clock.set_offset(offset); Ok(()) } @@ -1218,6 +1376,8 @@ impl StateUnit for TimeAbiUnit { mod tests { use super::*; use openvmm_defs::time_abi::EFFECTIVE_CPUID_ENTRY_BYTES; + use std::sync::atomic::AtomicU64; + use std::sync::atomic::Ordering; use virt::time_abi::BackendPreflight; use virt::time_abi::HostTimeSample; use virt::time_abi::TscAnchor; @@ -1302,20 +1462,69 @@ mod tests { (msrs, config, effective) } + fn test_report() -> TimeAbiReport { + TimeAbiReport { + hypervisor: "kvm".to_owned(), + cpu_profile: TEST_PROFILE.to_owned(), + msr_route: IdentityMsrRoute::ExitToVmm, + sync: TscSyncMethod::CommonOffset, + native_rate: NativeRateBasis::KernelKhz, + native_tsc_hz: 2_100_000_000, + rate_check: None, + } + } + fn test_unit(msrs: Arc) -> TimeAbiUnit { + test_unit_measuring(msrs, Arc::new(AtomicU64::new(0))) + } + + /// Returns a unit whose offset measurement reads `offset`. + fn test_unit_measuring(msrs: Arc, offset: Arc) -> TimeAbiUnit { + let report = test_report(); TimeAbiUnit { + clock: Arc::new(TimeAbiClock::new(msrs.clone(), &report, &NO_HOOKS)), msrs, - report: TimeAbiReport { - hypervisor: "kvm".to_owned(), - cpu_profile: TEST_PROFILE.to_owned(), - msr_route: IdentityMsrRoute::ExitToVmm, - sync: TscSyncMethod::CommonOffset, - native_tsc_hz: 2_100_000_000, - rate_check: None, - }, + report, + measure_offset: Box::new(move || { + Ok(GuestTscOffset { + offset: offset.load(Ordering::Relaxed), + uncertainty: 1_000, + }) + }), } } + #[test] + fn unit_start_measures_the_offset_that_samples_use() { + futures::executor::block_on(async { + let (msrs, _, _) = test_config(1); + msrs.declare(DeclaredRates::new(2_100_000_000, 1_000_000_000).unwrap()) + .unwrap(); + let offset = Arc::new(AtomicU64::new(1 << 40)); + let mut unit = test_unit_measuring(msrs, offset.clone()); + // No sample before the first start measures the offset. + assert_eq!(unit.clock.sample(), None); + unit.start().await.unwrap(); + let first = unit.clock.sample().unwrap(); + // The guest TSC is ahead of the host TSC by the offset, and the + // offset's uncertainty of 1,000 cycles widens the pairing. + assert!(first.tsc > 1 << 40, "{first:?}"); + assert!(first.pairing_ns >= 477, "{first:?}"); + assert_eq!(first.disruption, 0); + + // A start that measures the same offset is no disruption; one that + // measures another is. + unit.stop().await; + offset.fetch_add(1_000, Ordering::Relaxed); + unit.start().await.unwrap(); + assert_eq!(unit.clock.sample().unwrap().disruption, 0); + unit.stop().await; + offset.fetch_add(2_001, Ordering::Relaxed); + unit.start().await.unwrap(); + assert_eq!(unit.clock.sample().unwrap().disruption, 1); + }); + } + fn code(result: anyhow::Result) -> String { let message = format!("{:#}", result.unwrap_err()); message @@ -1370,6 +1579,7 @@ mod tests { "cpu_profile", "msr_route", "sync", + "native_rate", "native_tsc_hz", ] { assert!(names.contains(&name), "{names:?}"); diff --git a/openvmm/openvmm_defs/src/microvm.rs b/openvmm/openvmm_defs/src/microvm.rs index 40721550ace..114455bd981 100644 --- a/openvmm/openvmm_defs/src/microvm.rs +++ b/openvmm/openvmm_defs/src/microvm.rs @@ -1816,8 +1816,8 @@ pub struct MicrovmConfig { pub snapshot_memory_ranges: Vec, /// Fresh private GPA ranges selected for this restore launch. pub restore_memory_ranges: Vec, - /// The NVX time ABI v1 parameters. Every microVM uses the time ABI, so - /// the microVM profile requires them. + /// The NVX time ABI parameters. Every microVM uses the time ABI, so the + /// microVM profile requires them. pub time_abi: Option, } diff --git a/openvmm/openvmm_defs/src/time_abi.rs b/openvmm/openvmm_defs/src/time_abi.rs index c0a2c0723bc..344a1c46bb6 100644 --- a/openvmm/openvmm_defs/src/time_abi.rs +++ b/openvmm/openvmm_defs/src/time_abi.rs @@ -20,7 +20,7 @@ use virt::time_abi::downtime::MAX_DOWNTIME_NS; #[derive(Clone, Debug, PartialEq, Eq, Protobuf)] #[mesh(package = "openvmm.snapshot")] pub struct SnapshotTimeContract { - /// The time ABI version: 1. + /// The time ABI version: 2. #[mesh(1)] pub time_abi_version: u32, /// `F_s`, the declared TSC rate of the snapshot lineage, in Hz. diff --git a/openvmm/openvmm_entry/src/microvm/mod.rs b/openvmm/openvmm_entry/src/microvm/mod.rs index 2d86ac08aa1..d1eae534816 100644 --- a/openvmm/openvmm_entry/src/microvm/mod.rs +++ b/openvmm/openvmm_entry/src/microvm/mod.rs @@ -147,7 +147,8 @@ impl MicrovmResources { /// Configures the chipset devices of a microVM for the time ABI: puts the PIT /// in strict mode, which forbids a periodic channel 0 at capture and restore /// (`E_PIT_ACTIVE`), and returns the portb device's time ABI configuration, -/// with the restore packet of a restore. +/// with the restore packet of a restore. The VM worker provides the host +/// fields of the time samples, and applies the hooks that alter them. pub(crate) fn time_abi_chipset( chipset_devices: &mut [vmotherboard::ChipsetDeviceHandle], hooks: &virt::time_abi::TimeAbiTestHooks, @@ -166,7 +167,6 @@ pub(crate) fn time_abi_chipset( generation: restore .as_ref() .map_or(0, |restore| restore.base.generation), - utc_offset_ms: hooks.utc_offset_ms, sample_delay_us: hooks.sample_delay_us, test_hooks: hooks.active(), restore, diff --git a/openvmm/openvmm_entry/src/microvm/restore.rs b/openvmm/openvmm_entry/src/microvm/restore.rs index 7388ab8b0c2..e5ec2f573e2 100644 --- a/openvmm/openvmm_entry/src/microvm/restore.rs +++ b/openvmm/openvmm_entry/src/microvm/restore.rs @@ -124,7 +124,7 @@ impl MicrovmRestore { contract.time.as_ref().zip(contract.cpu_profile.as_ref()) } - /// Returns the controller's fields of restore packet version 4 for a time + /// Returns the controller's fields of restore packet version 5 for a time /// ABI restore, and the generation ID they imply. The generation counter /// is the snapshot's plus one. pub(crate) fn time_abi_restore_packet( @@ -144,7 +144,7 @@ impl MicrovmRestore { } } -/// Returns the controller's fields of restore packet version 4 for a time +/// Returns the controller's fields of restore packet version 5 for a time /// ABI restore of a snapshot with the time contract `time`, and the /// generation ID they imply. The generation counter is the snapshot's plus /// one, and `ack_required` makes the guest acknowledge the restore through @@ -188,7 +188,7 @@ pub(crate) fn restore_packet_base( entropy, }; base.validate() - .context("restore packet version 4 is invalid")?; + .context("restore packet version 5 is invalid")?; generation_id_create.complete("restore", "generation_id_create", Default::default()); Ok((microvm_generation_id(&entropy), base)) } @@ -700,7 +700,7 @@ pub(crate) mod tests { /// Returns a valid time contract for test machine contracts. pub(crate) fn test_time_contract() -> SnapshotTimeContract { SnapshotTimeContract { - time_abi_version: 1, + time_abi_version: virt::time_abi::TIME_ABI_VERSION, tsc_frequency_hz: 2_100_000_000, tsc_tolerance_ppm: 250, apic_frequency_hz: virt::time_abi::rate::LAPIC_HZ_HYPERV, @@ -968,7 +968,7 @@ pub(crate) mod tests { fn time_abi_restore(capture_generation: u32) -> MicrovmRestore { let mut contract: SnapshotMachineContract = mesh::payload::decode(&[]).unwrap(); contract.time = Some(SnapshotTimeContract { - time_abi_version: 1, + time_abi_version: virt::time_abi::TIME_ABI_VERSION, tsc_frequency_hz: 2_100_000_000, tsc_tolerance_ppm: 250, apic_frequency_hz: 200_000_000, diff --git a/openvmm/openvmm_helpers/src/snapshot/time.rs b/openvmm/openvmm_helpers/src/snapshot/time.rs index 54e955691fe..130a54be973 100644 --- a/openvmm/openvmm_helpers/src/snapshot/time.rs +++ b/openvmm/openvmm_helpers/src/snapshot/time.rs @@ -282,7 +282,7 @@ pub(super) mod tests { pub(in crate::snapshot) fn test_time_contract() -> SnapshotTimeContract { SnapshotTimeContract { - time_abi_version: 1, + time_abi_version: TIME_ABI_VERSION, tsc_frequency_hz: 2_100_000_000, tsc_tolerance_ppm: 250, apic_frequency_hz: rate::LAPIC_HZ_HYPERV, @@ -383,8 +383,10 @@ pub(super) mod tests { #[test] fn time_contract_rules() { validate_time_contract(&test_time_contract()).unwrap(); - let mutations: [(fn(&mut SnapshotTimeContract), TimeAbiCode); 8] = [ - (|c| c.time_abi_version = 2, TimeAbiCode::ManifestTime), + let mutations: [(fn(&mut SnapshotTimeContract), TimeAbiCode); 9] = [ + // A snapshot of time ABI 1 must be recaptured. + (|c| c.time_abi_version = 1, TimeAbiCode::ManifestTime), + (|c| c.time_abi_version = 3, TimeAbiCode::ManifestTime), (|c| c.tsc_tolerance_ppm = 251, TimeAbiCode::ManifestTime), (|c| c.tsc_frequency_hz = 100, TimeAbiCode::ManifestTime), ( diff --git a/petri/src/vm/openvmm/microvm.rs b/petri/src/vm/openvmm/microvm.rs index 066852ff618..f64adc5d3e2 100644 --- a/petri/src/vm/openvmm/microvm.rs +++ b/petri/src/vm/openvmm/microvm.rs @@ -399,7 +399,6 @@ pub(super) fn attach_chipset_devices( output_drain: None, time_abi: MicrovmPortbTimeAbi { generation: 0, - utc_offset_ms: 0, sample_delay_us: 0, test_hooks: false, restore: None, diff --git a/vm/devices/chipset/src/microvm.rs b/vm/devices/chipset/src/microvm.rs index c9edcaaf135..25a938febcb 100644 --- a/vm/devices/chipset/src/microvm.rs +++ b/vm/devices/chipset/src/microvm.rs @@ -15,9 +15,10 @@ use chipset_device::poll_device::PollDevice; use chipset_resources::microvm::MicrovmPortbDrain; use chipset_resources::microvm::MicrovmPortbTimeAbi; use chipset_resources::microvm_time::RestorePacketBase; -use chipset_resources::microvm_time::RestorePacketV4; +use chipset_resources::microvm_time::RestorePacketV5; use chipset_resources::microvm_time::RestoreTimeRecord; use chipset_resources::microvm_time::TimeSample; +use chipset_resources::microvm_time::TimeSampleSource; use futures::AsyncRead; use futures::AsyncWrite; use inspect::InspectMut; @@ -31,10 +32,10 @@ use std::io; use std::io::ErrorKind; use std::ops::RangeInclusive; use std::pin::Pin; +use std::sync::Arc; use std::task::Context; use std::task::Poll; use std::task::Waker; -use std::time::SystemTime; use vmcore::device_state::ChangeDeviceState; use vmcore::save_restore::NoSavedState; use vmcore::save_restore::RestoreError; @@ -61,43 +62,31 @@ const STATUS_TIME_SAMPLE_AVAILABLE: u8 = chipset_resources::microvm_time::STATUS_TIME_SAMPLE_AVAILABLE; const BUFFER_MAX: usize = 1024 * 1024; -/// The restore packet version 4 of a restored VM process. +/// The restore packet version 5 of a restored VM process. enum RestorePacketState { /// No packet, or the packet was consumed. None, /// Waiting for the worker to seal the time fields. Pending(RestorePacketBase, mesh::OneshotReceiver), - /// Sealed and not yet selected; host UTC is latched at the first - /// selection. + /// Sealed and not yet selected. Sealed(RestorePacketBase, RestoreTimeRecord), /// Encoded into the restore-packet queue. Encoded, } -/// NVX time ABI v1 state of the portb device. +/// NVX time ABI 2 state of the portb device. struct PortbTimeAbi { generation: u32, - utc_offset_ns: i128, sample_delay: std::time::Duration, test_hooks: bool, + /// The platform's source of the host fields of time samples. + source: Option>, restore: RestorePacketState, /// Notified at the guest's first selection of the restore packet. packet_selected: Option>, } impl PortbTimeAbi { - /// Returns host UTC in nanoseconds, after the test hooks' delay and - /// offset. - fn latch_utc_ns(&self) -> u64 { - if !self.sample_delay.is_zero() { - std::thread::sleep(self.sample_delay); - } - let now = SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .map_or(0, |since_epoch| since_epoch.as_nanos() as i128); - (now + self.utc_offset_ns).clamp(0, u64::MAX.into()) as u64 - } - /// Returns whether a sealed packet waits for its first selection, /// receiving the sealed time fields if they arrived. fn poll_sealed(&mut self) -> bool { @@ -157,8 +146,10 @@ pub struct MicrovmPortb { impl MicrovmPortb { /// Creates a portb console using `io` as its host endpoint, with the NVX - /// time ABI v1 configuration `time_abi`: restore packet version 4 and the - /// time-sample window at port `0xeb`. + /// time ABI 2 configuration `time_abi`: restore packet version 5 and the + /// time-sample window at port `0xeb`, which stays empty until + /// [`Self::with_time_sample_source`] provides the host fields of its + /// samples. pub fn new( io: Box, generation_id: [u8; GENERATION_ID_SIZE], @@ -166,7 +157,6 @@ impl MicrovmPortb { ) -> Self { let MicrovmPortbTimeAbi { generation, - utc_offset_ms, sample_delay_us, test_hooks, restore, @@ -201,9 +191,9 @@ impl MicrovmPortb { input_gated: false, time_abi: PortbTimeAbi { generation, - utc_offset_ns: i128::from(utc_offset_ms) * 1_000_000, sample_delay: std::time::Duration::from_micros(sample_delay_us.into()), test_hooks, + source: None, restore, packet_selected, }, @@ -215,8 +205,14 @@ impl MicrovmPortb { } } - /// Handles selector `0xa5`: encodes the sealed packet with host UTC at - /// its first selection, then selects it. + /// Installs the platform's source of the host fields of time samples. + pub fn with_time_sample_source(mut self, source: Arc) -> Self { + self.time_abi.source = Some(source); + self + } + + /// Handles selector `0xa5`: encodes the sealed packet at its first + /// selection, then selects it. fn select_restore_packet(&mut self) { let time_abi = &mut self.time_abi; if time_abi.poll_sealed() { @@ -228,11 +224,7 @@ impl MicrovmPortb { if let Some(selected) = time_abi.packet_selected.take() { selected.send(()); } - let packet = RestorePacketV4 { - base, - time, - utc_ns: time_abi.latch_utc_ns(), - }; + let packet = RestorePacketV5 { base, time }; match packet.encode() { Ok(bytes) => self.restore_packet = bytes.into(), Err(error) => { @@ -251,14 +243,27 @@ impl MicrovmPortb { } } - /// Handles selector `0xa7`: latches a fresh time sample into the window. + /// Handles selector `0xa7`: latches a fresh time sample into the window, + /// or empties the window if the platform cannot take one. fn latch_time_sample(&mut self) { - let sample = TimeSample { - test_hooks: self.time_abi.test_hooks, - generation: self.time_abi.generation, - utc_ns: self.time_abi.latch_utc_ns(), + let time_abi = &self.time_abi; + if !time_abi.sample_delay.is_zero() { + std::thread::sleep(time_abi.sample_delay); + } + let host = time_abi.source.as_ref().and_then(|source| source.sample()); + self.time_window = match host { + Some(host) => TimeSample { + test_hooks: time_abi.test_hooks, + generation: time_abi.generation, + host, + } + .encode() + .into(), + None => { + tracelimit::error_ratelimited!("microVM time sample is unavailable"); + VecDeque::new() + } }; - self.time_window = sample.encode().into(); } fn clear_restore_packet(&mut self) { @@ -884,6 +889,9 @@ impl SaveRestore for MicrovmSnapshotRequest { #[cfg(test)] mod tests { use super::*; + use chipset_resources::microvm_time::HostClockStatus; + use chipset_resources::microvm_time::HostTimeFields; + use chipset_resources::microvm_time::TIME_SAMPLE_LEN; use futures::AsyncRead; use futures::AsyncWrite; use futures::FutureExt; @@ -893,7 +901,6 @@ mod tests { use serial_core::serial_io::Connected; use std::io; use std::pin::Pin; - use std::sync::Arc; use std::task::Context; use std::task::Poll; use test_with_tracing::test; @@ -905,7 +912,6 @@ mod tests { ) -> MicrovmPortbTimeAbi { MicrovmPortbTimeAbi { generation: 7, - utc_offset_ms: 0, sample_delay_us: 0, test_hooks: true, restore, @@ -1319,7 +1325,7 @@ mod tests { IoResult::Ok )); let bytes = read_record(&mut restored, DATA_PORT, base.encoded_len()); - assert_eq!(RestorePacketV4::decode(&bytes).unwrap().base, base); + assert_eq!(RestorePacketV5::decode(&bytes).unwrap().base, base); } #[test] @@ -1348,7 +1354,7 @@ mod tests { } #[test] - fn restore_packet_v4_advertises_its_targets() { + fn restore_packet_v5_advertises_its_targets() { let range = chipset_resources::microvm_time::RestoreMemoryRange { gpa_start: 0x2000_0000, length: 0x2000_0000, @@ -1410,7 +1416,7 @@ mod tests { IoResult::Ok )); let bytes = read_record(&mut portb, DATA_PORT, base.encoded_len()); - assert_eq!(RestorePacketV4::decode(&bytes).unwrap().base, base); + assert_eq!(RestorePacketV5::decode(&bytes).unwrap().base, base); // Reading the whole packet clears every target. assert_eq!( read_status(&mut portb), @@ -1463,16 +1469,42 @@ mod tests { bytes } - fn now_ns() -> u64 { - SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .unwrap() - .as_nanos() as u64 + /// A time sample source that returns its host fields with a count of the + /// samples taken in the disruption counter. + struct FakeSource { + host: Option, + samples: Mutex, } + impl TimeSampleSource for FakeSource { + fn sample(&self) -> Option { + let mut samples = self.samples.lock(); + *samples += 1; + self.host.map(|host| HostTimeFields { + disruption: *samples, + ..host + }) + } + } + + const HOST_FIELDS: HostTimeFields = HostTimeFields { + utc_ns: 1_700_000_000_123_456_789, + tsc: 0x1234_5678_9abc, + pairing_ns: 640, + status: HostClockStatus::Synchronized, + disruption: 0, + correction: -12 * 65_536, + correction_bound: Some(66), + max_error_us: Some(2_000), + }; + #[test] fn time_abi_portb_cold_boot_offers_time_samples() { - let mut portb = time_abi_portb(None); + let source = Arc::new(FakeSource { + host: Some(HOST_FIELDS), + samples: Mutex::new(0), + }); + let mut portb = time_abi_portb(None).with_time_sample_source(source.clone()); assert_eq!(portb.get_static_regions()[0].1, 0xe9..=0xeb); assert_eq!( read_status(&mut portb), @@ -1487,15 +1519,23 @@ mod tests { // The window is empty until a sample is latched. assert_eq!(read_record(&mut portb, TIME_WINDOW_PORT, 4), [0; 4]); - let before = now_ns(); + assert_eq!(*source.samples.lock(), 0); assert!(matches!( portb.io_write(STATUS_PORT, &[TIME_SAMPLE_SELECT]), IoResult::Ok )); - let after = now_ns(); - let sample = TimeSample::decode(&read_record(&mut portb, TIME_WINDOW_PORT, 16)).unwrap(); - assert_eq!((sample.generation, sample.test_hooks), (7, true)); - assert!((before..=after).contains(&sample.utc_ns)); + let bytes = read_record(&mut portb, TIME_WINDOW_PORT, TIME_SAMPLE_LEN); + assert_eq!( + TimeSample::decode(&bytes).unwrap(), + TimeSample { + test_hooks: true, + generation: 7, + host: HostTimeFields { + disruption: 1, + ..HOST_FIELDS + }, + } + ); // Reads past the end are zero, and selecting a sample keeps the // generation-ID selection. assert_eq!(read_record(&mut portb, TIME_WINDOW_PORT, 4), [0; 4]); @@ -1507,11 +1547,35 @@ mod tests { portb.io_write(STATUS_PORT, &[TIME_SAMPLE_SELECT]), IoResult::Ok )); + assert_eq!(*source.samples.lock(), 2); assert_eq!(read_record(&mut portb, DATA_PORT, 16), TEST_GENERATION_ID); } #[test] - fn time_abi_portb_serves_restore_packet_v4() { + fn time_abi_portb_window_stays_empty_without_a_sample() { + let mut portb = time_abi_portb(None); + assert!(matches!( + portb.io_write(STATUS_PORT, &[TIME_SAMPLE_SELECT]), + IoResult::Ok + )); + assert_eq!(read_record(&mut portb, TIME_WINDOW_PORT, 4), [0; 4]); + + // A sample the platform cannot take empties a latched window. + let source = Arc::new(FakeSource { + host: None, + samples: Mutex::new(0), + }); + let mut portb = time_abi_portb(None).with_time_sample_source(source.clone()); + assert!(matches!( + portb.io_write(STATUS_PORT, &[TIME_SAMPLE_SELECT]), + IoResult::Ok + )); + assert_eq!(*source.samples.lock(), 1); + assert_eq!(read_record(&mut portb, TIME_WINDOW_PORT, 4), [0; 4]); + } + + #[test] + fn time_abi_portb_serves_restore_packet_v5() { let base = RestorePacketBase { online_vp_count: 2, memory_target: false, @@ -1555,17 +1619,14 @@ mod tests { "selection is notified only at the first selection" ); - let before = now_ns(); assert!(matches!( portb.io_write(STATUS_PORT, &[RESTORE_PACKET_SELECT]), IoResult::Ok )); - let after = now_ns(); assert!(matches!(selected.now_or_never(), Some(Ok(())))); let bytes = read_record(&mut portb, DATA_PORT, base.encoded_len()); - let packet = RestorePacketV4::decode(&bytes).unwrap(); + let packet = RestorePacketV5::decode(&bytes).unwrap(); assert_eq!((packet.base, packet.time), (base, time)); - assert!((before..=after).contains(&packet.utc_ns)); // The packet is consumed once read. assert_eq!( @@ -1605,7 +1666,11 @@ mod tests { #[test] fn time_abi_portb_window_is_not_saved() { - let mut portb = time_abi_portb(None); + let source = Arc::new(FakeSource { + host: Some(HOST_FIELDS), + samples: Mutex::new(0), + }); + let mut portb = time_abi_portb(None).with_time_sample_source(source); assert!(matches!( portb.io_write(STATUS_PORT, &[TIME_SAMPLE_SELECT]), IoResult::Ok diff --git a/vm/devices/chipset/src/microvm/resolver.rs b/vm/devices/chipset/src/microvm/resolver.rs index 8384912f93d..1c7404e624e 100644 --- a/vm/devices/chipset/src/microvm/resolver.rs +++ b/vm/devices/chipset/src/microvm/resolver.rs @@ -12,6 +12,7 @@ use chipset_device_resources::ResolvedChipsetDevice; use chipset_resources::microvm::MicrovmPortbHandle; use chipset_resources::microvm::MicrovmShutdownHandle; use chipset_resources::microvm::MicrovmSnapshotRequestHandle; +use chipset_resources::microvm_time::TimeSampleSourceKind; use power_resources::PowerRequestHandleKind; use serial_core::resources::ResolveSerialBackendParams; use thiserror::Error; @@ -37,6 +38,8 @@ declare_static_async_resolver! { pub enum ResolveMicrovmPortbError { #[error("failed to resolve microVM portb backend")] ResolveBackend(#[source] ResolveError), + #[error("failed to resolve the platform's time sample source")] + ResolveTimeSampleSource(#[source] ResolveError), } #[async_trait] @@ -60,8 +63,13 @@ impl AsyncResolveResource for Micro ) .await .map_err(ResolveMicrovmPortbError::ResolveBackend)?; + let source = resolver + .resolve::(PlatformResource.into_resource(), ()) + .await + .map_err(ResolveMicrovmPortbError::ResolveTimeSampleSource)?; let portb = MicrovmPortb::new(io.0.into_io(), resource.generation_id, resource.time_abi) - .with_output_drain(resource.output_drain); + .with_output_drain(resource.output_drain) + .with_time_sample_source(source.0); Ok(portb.into()) } } diff --git a/vm/devices/chipset_resources/src/microvm.rs b/vm/devices/chipset_resources/src/microvm.rs index cc913c41015..9276745a43f 100644 --- a/vm/devices/chipset_resources/src/microvm.rs +++ b/vm/devices/chipset_resources/src/microvm.rs @@ -55,19 +55,19 @@ pub struct MicrovmPortbHandle { /// Requests to deliver accepted output before a snapshot capture or before /// terminating the VM process. pub output_drain: Option>>, - /// NVX time ABI v1 configuration: restore packet version 4 and the time - /// samples at port `0xeb`. + /// NVX time ABI 2 configuration: restore packet version 5 and the time + /// samples at port `0xeb`, whose host fields the platform's + /// [`TimeSampleSourceKind`](crate::microvm_time::TimeSampleSourceKind) + /// provides. pub time_abi: MicrovmPortbTimeAbi, } -/// NVX time ABI v1 configuration of the portb device. +/// NVX time ABI 2 configuration of the portb device. #[derive(MeshPayload)] pub struct MicrovmPortbTimeAbi { /// The generation counter of this VM process. pub generation: u32, - /// Test hook: milliseconds added to every host UTC reading. - pub utc_offset_ms: i64, - /// Test hook: microseconds of delay before latching a UTC reading. + /// Test hook: microseconds of delay before latching a time sample. pub sample_delay_us: u32, /// Whether a test hook is active. pub test_hooks: bool, @@ -75,7 +75,7 @@ pub struct MicrovmPortbTimeAbi { pub restore: Option, } -/// The parts of a restore packet version 4. +/// The parts of a restore packet version 5. #[derive(MeshPayload)] pub struct MicrovmRestorePacketSource { /// The fields the controller knows before the worker starts. diff --git a/vm/devices/chipset_resources/src/microvm_time.rs b/vm/devices/chipset_resources/src/microvm_time.rs index ae195e2ecdb..bb31ae99705 100644 --- a/vm/devices/chipset_resources/src/microvm_time.rs +++ b/vm/devices/chipset_resources/src/microvm_time.rs @@ -1,21 +1,25 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -//! The microVM restore packet version 4 and the time sample of the NVX time -//! ABI v1, as the guest reads them through the portb device. +//! The microVM restore packet version 5 and the time sample of NVX time ABI +//! 2, as the guest reads them through the portb device, and the platform +//! resource that pairs each time sample's host UTC with the guest TSC. //! //! Every byte the guest reads is a port exit, so the records are compact and //! readable four bytes at a time. use mesh::MeshPayload; +use std::sync::Arc; use thiserror::Error; +use vm_resource::CanResolveTo; +use vm_resource::ResourceKind; /// The restore packet magic. pub const RESTORE_PACKET_MAGIC: [u8; 3] = *b"OVR"; /// The restore packet version. -pub const RESTORE_PACKET_VERSION: u8 = 4; +pub const RESTORE_PACKET_VERSION: u8 = 5; /// The length of the restore packet header. -pub const RESTORE_PACKET_HEADER_LEN: usize = 32; +pub const RESTORE_PACKET_HEADER_LEN: usize = 24; /// The length of one memory range record. pub const RESTORE_RANGE_LEN: usize = 16; /// The length of the restore entropy. @@ -49,9 +53,11 @@ pub const TIME_WINDOW_PORT: u16 = 0xeb; /// The portb status bit advertising the time-sample window. pub const STATUS_TIME_SAMPLE_AVAILABLE: u8 = 1 << 6; /// The time sample version. -pub const TIME_SAMPLE_VERSION: u8 = 1; +pub const TIME_SAMPLE_VERSION: u8 = 2; /// The length of a time sample. -pub const TIME_SAMPLE_LEN: usize = 16; +pub const TIME_SAMPLE_LEN: usize = 48; +/// The encoding of an unknown bound in a time sample. +pub const TIME_SAMPLE_UNKNOWN: u32 = u32::MAX; /// A restore packet or time sample is malformed. #[derive(Debug, Error, Clone, PartialEq, Eq)] @@ -80,6 +86,9 @@ pub enum RestorePacketError { /// The rate deviation is beyond the tolerance. #[error("rate deviation {0} is beyond the tolerance")] RateDeviation(i32), + /// The host clock status of a time sample is unknown to this version. + #[error("unknown host clock status {0}")] + Status(u8), /// The record length does not match its counts. #[error("record is {actual} bytes, expected {expected}")] Length { @@ -153,19 +162,19 @@ pub struct RestoreTimeRecord { pub test_hooks: bool, } -/// A complete restore packet version 4. +/// A complete restore packet version 5. +/// +/// It carries no host UTC: the guest takes a time sample, which pairs host +/// UTC with its TSC, to set its clock. #[derive(Debug, Clone, PartialEq, Eq)] -pub struct RestorePacketV4 { +pub struct RestorePacketV5 { /// The controller's fields. pub base: RestorePacketBase, /// The worker's sealed time fields. pub time: RestoreTimeRecord, - /// Host UTC when the guest first selected the packet, in nanoseconds - /// since the Unix epoch. - pub utc_ns: u64, } -impl RestorePacketV4 { +impl RestorePacketV5 { /// Encodes the packet. pub fn encode(&self) -> Result, RestorePacketError> { self.base.validate()?; @@ -195,7 +204,6 @@ impl RestorePacketV4 { bytes.extend_from_slice(&self.base.generation.to_le_bytes()); bytes.extend_from_slice(&self.time.rate_deviation.to_le_bytes()); bytes.extend_from_slice(&self.time.downtime_ns.to_le_bytes()); - bytes.extend_from_slice(&self.utc_ns.to_le_bytes()); for range in &self.base.ranges { bytes.extend_from_slice(&range.gpa_start.to_le_bytes()); bytes.extend_from_slice(&range.length.to_le_bytes()); @@ -259,7 +267,6 @@ impl RestorePacketV4 { rate_deviation: u32_at(12) as i32, test_hooks: flags & FLAG_TEST_HOOKS != 0, }, - utc_ns: u64_at(24), }; packet.base.validate()?; if packet.time.rate_deviation.unsigned_abs() > MAX_RATE_DEVIATION.unsigned_abs() { @@ -271,26 +278,111 @@ impl RestorePacketV4 { } } -/// A time sample, latched by selector [`TIME_SAMPLE_SELECT`] and read from -/// port [`TIME_WINDOW_PORT`]. +/// The status of host UTC in a time sample, as the host's time service +/// reports it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HostClockStatus { + /// The host does not report its clock's status. + Unknown, + /// A time service synchronizes host UTC with a reference. + Synchronized, + /// No time service synchronizes host UTC, which runs on its own. + FreeRunning, + /// The host reports a fault of its clock. + Unreliable, +} + +impl HostClockStatus { + /// Returns the encoding of the status. + pub const fn to_byte(self) -> u8 { + match self { + Self::Unknown => 0, + Self::Synchronized => 1, + Self::FreeRunning => 2, + Self::Unreliable => 3, + } + } + + /// Returns the status that `byte` encodes, if any. + pub const fn from_byte(byte: u8) -> Option { + match byte { + 0 => Some(Self::Unknown), + 1 => Some(Self::Synchronized), + 2 => Some(Self::FreeRunning), + 3 => Some(Self::Unreliable), + _ => None, + } + } +} + +/// The host's fields of a time sample: host UTC paired with the guest TSC, +/// and the discipline of host UTC. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct HostTimeFields { + /// Host UTC at the latch, in nanoseconds since the Unix epoch. + pub utc_ns: u64, + /// The guest TSC at the instant of `utc_ns`. + pub tsc: u64, + /// The most the instants of `tsc` and `utc_ns` can differ, in + /// nanoseconds. + pub pairing_ns: u32, + /// The status of host UTC. + pub status: HostClockStatus, + /// A counter that changes when host UTC steps, the host resumes from a + /// suspension, or the guest TSC's offset from the host TSC changes. + pub disruption: u32, + /// How fast host UTC runs against the guest TSC counted at the declared + /// rate `F`, minus one, in ppm scaled by 2^16. + pub correction: i64, + /// A bound on the error of `correction`, in ppm scaled by 2^16, if + /// known. + pub correction_bound: Option, + /// The time service's bound on the error of host UTC, in microseconds, + /// if it reports one. + pub max_error_us: Option, +} + +/// A time sample version 2, latched by selector [`TIME_SAMPLE_SELECT`] and +/// read from port [`TIME_WINDOW_PORT`]. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct TimeSample { /// Whether a test hook is active. pub test_hooks: bool, /// The generation counter `g` of this VM process. pub generation: u32, - /// Host UTC at the latch, in nanoseconds since the Unix epoch. - pub utc_ns: u64, + /// The host's fields. + pub host: HostTimeFields, +} + +/// Encodes an optional bound, saturating a known one below +/// [`TIME_SAMPLE_UNKNOWN`]. +fn encode_bound(bound: Option) -> u32 { + bound.map_or(TIME_SAMPLE_UNKNOWN, |bound| { + bound.min(TIME_SAMPLE_UNKNOWN - 1) + }) +} + +/// Decodes an optional bound. +fn decode_bound(value: u32) -> Option { + (value != TIME_SAMPLE_UNKNOWN).then_some(value) } impl TimeSample { /// Encodes the sample. pub fn encode(&self) -> [u8; TIME_SAMPLE_LEN] { + let host = &self.host; let mut bytes = [0; TIME_SAMPLE_LEN]; bytes[0] = TIME_SAMPLE_VERSION; bytes[1] = if self.test_hooks { FLAG_TEST_HOOKS } else { 0 }; + bytes[2] = host.status.to_byte(); bytes[4..8].copy_from_slice(&self.generation.to_le_bytes()); - bytes[8..16].copy_from_slice(&self.utc_ns.to_le_bytes()); + bytes[8..16].copy_from_slice(&host.utc_ns.to_le_bytes()); + bytes[16..24].copy_from_slice(&host.tsc.to_le_bytes()); + bytes[24..28].copy_from_slice(&host.pairing_ns.to_le_bytes()); + bytes[28..32].copy_from_slice(&host.disruption.to_le_bytes()); + bytes[32..40].copy_from_slice(&host.correction.to_le_bytes()); + bytes[40..44].copy_from_slice(&encode_bound(host.correction_bound).to_le_bytes()); + bytes[44..48].copy_from_slice(&encode_bound(host.max_error_us).to_le_bytes()); bytes } @@ -304,17 +396,57 @@ impl TimeSample { if bytes[0] != TIME_SAMPLE_VERSION { return Err(RestorePacketError::Version(bytes[0])); } - if bytes[1] & !FLAG_TEST_HOOKS != 0 || bytes[2..4] != [0, 0] { + if bytes[1] & !FLAG_TEST_HOOKS != 0 || bytes[3] != 0 { return Err(RestorePacketError::Reserved); } + let status = + HostClockStatus::from_byte(bytes[2]).ok_or(RestorePacketError::Status(bytes[2]))?; + let u32_at = + |offset: usize| u32::from_le_bytes(bytes[offset..offset + 4].try_into().unwrap()); + let u64_at = + |offset: usize| u64::from_le_bytes(bytes[offset..offset + 8].try_into().unwrap()); Ok(Self { test_hooks: bytes[1] & FLAG_TEST_HOOKS != 0, - generation: u32::from_le_bytes(bytes[4..8].try_into().unwrap()), - utc_ns: u64::from_le_bytes(bytes[8..16].try_into().unwrap()), + generation: u32_at(4), + host: HostTimeFields { + utc_ns: u64_at(8), + tsc: u64_at(16), + pairing_ns: u32_at(24), + status, + disruption: u32_at(28), + correction: u64_at(32) as i64, + correction_bound: decode_bound(u32_at(40)), + max_error_us: decode_bound(u32_at(44)), + }, }) } } +/// The platform's source of the host's fields of time samples, which pairs +/// host UTC with the guest TSC. +pub trait TimeSampleSource: Send + Sync { + /// Pairs host UTC with the guest TSC now and reads the discipline of + /// host UTC, or returns `None` if the host cannot, which leaves the time + /// window empty. + fn sample(&self) -> Option; +} + +/// The resource kind of the platform's [`TimeSampleSource`], which the +/// microVM portb device resolves from +/// [`PlatformResource`](vm_resource::PlatformResource). +pub enum TimeSampleSourceKind {} + +impl ResourceKind for TimeSampleSourceKind { + const NAME: &'static str = "microvm_time_sample_source"; +} + +impl CanResolveTo for TimeSampleSourceKind { + type Input<'a> = (); +} + +/// A resolved [`TimeSampleSourceKind`] resource. +pub struct ResolvedTimeSampleSource(pub Arc); + #[cfg(test)] mod tests { use super::*; @@ -323,8 +455,8 @@ mod tests { std::array::from_fn(|i| i as u8) } - fn packet_without_ranges() -> RestorePacketV4 { - RestorePacketV4 { + fn packet_without_ranges() -> RestorePacketV5 { + RestorePacketV5 { base: RestorePacketBase { online_vp_count: 2, memory_target: false, @@ -339,12 +471,11 @@ mod tests { rate_deviation: -2, test_hooks: false, }, - utc_ns: 0x1122_3344_5566_7788, } } - fn packet_with_ranges() -> RestorePacketV4 { - RestorePacketV4 { + fn packet_with_ranges() -> RestorePacketV5 { + RestorePacketV5 { base: RestorePacketBase { online_vp_count: 0, memory_target: true, @@ -368,35 +499,32 @@ mod tests { rate_deviation: MAX_RATE_DEVIATION, test_hooks: true, }, - utc_ns: 1_700_000_000_000_000_000, } } #[test] fn golden_packet_without_ranges() { let mut expected = vec![ - b'O', b'V', b'R', 4, 0x05, 2, 0, 0, // header + b'O', b'V', b'R', 5, 0x05, 2, 0, 0, // header 7, 0, 0, 0, // generation 0xfe, 0xff, 0xff, 0xff, // rate deviation 0x08, 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, // downtime - 0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11, // UTC ]; expected.extend(entropy()); let packet = packet_without_ranges(); let bytes = packet.encode().unwrap(); - assert_eq!(bytes.len(), 96); + assert_eq!(bytes.len(), 88); assert_eq!(bytes, expected); - assert_eq!(RestorePacketV4::decode(&bytes).unwrap(), packet); + assert_eq!(RestorePacketV5::decode(&bytes).unwrap(), packet); } #[test] fn golden_packet_with_ranges() { let mut expected = vec![ - b'O', b'V', b'R', 4, 0x0a, 0, 2, 0, // header + b'O', b'V', b'R', 5, 0x0a, 0, 2, 0, // header 1, 0, 0, 0, // generation 0x00, 0x00, 0xfa, 0x00, // rate deviation 250 * 2^16 0x00, 0xac, 0x23, 0xfc, 0x06, 0x00, 0x00, 0x00, // downtime 30 s - 0x00, 0x00, 0x2a, 0x36, 0xfe, 0x9c, 0x97, 0x17, // UTC 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, // range 0 start 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, // range 0 length 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, // range 1 start @@ -405,9 +533,9 @@ mod tests { expected.extend(entropy()); let packet = packet_with_ranges(); let bytes = packet.encode().unwrap(); - assert_eq!(bytes.len(), 128); + assert_eq!(bytes.len(), 120); assert_eq!(bytes, expected); - assert_eq!(RestorePacketV4::decode(&bytes).unwrap(), packet); + assert_eq!(RestorePacketV5::decode(&bytes).unwrap(), packet); } #[test] @@ -416,10 +544,10 @@ mod tests { let mutate = |offset: usize, value: u8| { let mut bytes = good.clone(); bytes[offset] = value; - RestorePacketV4::decode(&bytes).unwrap_err() + RestorePacketV5::decode(&bytes).unwrap_err() }; assert_eq!(mutate(0, b'X'), RestorePacketError::Magic); - assert_eq!(mutate(3, 3), RestorePacketError::Version(3)); + assert_eq!(mutate(3, 4), RestorePacketError::Version(4)); assert_eq!(mutate(4, 0x1a), RestorePacketError::Reserved); assert_eq!(mutate(7, 1), RestorePacketError::Reserved); assert_eq!(mutate(5, 3), RestorePacketError::OnlineVpCount(3)); @@ -431,11 +559,11 @@ mod tests { RestorePacketError::RateDeviation(_) )); assert_eq!( - RestorePacketV4::decode(&good[..31]).unwrap_err(), - RestorePacketError::Truncated(31) + RestorePacketV5::decode(&good[..23]).unwrap_err(), + RestorePacketError::Truncated(23) ); assert!(matches!( - RestorePacketV4::decode(&good[..good.len() - 1]).unwrap_err(), + RestorePacketV5::decode(&good[..good.len() - 1]).unwrap_err(), RestorePacketError::Length { .. } )); } @@ -468,27 +596,48 @@ mod tests { )); } - #[test] - fn golden_time_sample() { - let sample = TimeSample { + fn sample() -> TimeSample { + TimeSample { test_hooks: true, generation: 3, - utc_ns: 0x0102_0304_0506_0708, - }; + host: HostTimeFields { + utc_ns: 0x0102_0304_0506_0708, + tsc: 0x1112_1314_1516_1718, + pairing_ns: 0x2122_2324, + status: HostClockStatus::Synchronized, + disruption: 0x3132_3334, + correction: -2, + correction_bound: Some(66), + max_error_us: None, + }, + } + } + + #[test] + fn golden_time_sample() { + let sample = sample(); let bytes = sample.encode(); assert_eq!( bytes, [ - 1, 0x08, 0, 0, 3, 0, 0, 0, 0x08, 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01 + 2, 0x08, 1, 0, // version, flags, status, reserved + 3, 0, 0, 0, // generation + 0x08, 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, // UTC + 0x18, 0x17, 0x16, 0x15, 0x14, 0x13, 0x12, 0x11, // TSC + 0x24, 0x23, 0x22, 0x21, // pairing + 0x34, 0x33, 0x32, 0x31, // disruption + 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, // correction + 66, 0, 0, 0, // correction bound + 0xff, 0xff, 0xff, 0xff, // maximum error, unknown ] ); assert_eq!(TimeSample::decode(&bytes).unwrap(), sample); let mut bad = bytes; - bad[0] = 2; + bad[0] = 1; assert_eq!( TimeSample::decode(&bad).unwrap_err(), - RestorePacketError::Version(2) + RestorePacketError::Version(1) ); let mut bad = bytes; bad[1] = 0x01; @@ -502,9 +651,33 @@ mod tests { TimeSample::decode(&bad).unwrap_err(), RestorePacketError::Reserved ); + let mut bad = bytes; + bad[2] = 4; + assert_eq!( + TimeSample::decode(&bad).unwrap_err(), + RestorePacketError::Status(4) + ); assert!(matches!( - TimeSample::decode(&bytes[..15]).unwrap_err(), + TimeSample::decode(&bytes[..47]).unwrap_err(), RestorePacketError::Length { .. } )); } + + #[test] + fn known_bounds_never_encode_as_unknown() { + let mut sample = sample(); + sample.host.correction_bound = Some(u32::MAX); + sample.host.max_error_us = Some(0); + let decoded = TimeSample::decode(&sample.encode()).unwrap(); + assert_eq!(decoded.host.correction_bound, Some(u32::MAX - 1)); + assert_eq!(decoded.host.max_error_us, Some(0)); + for status in [ + HostClockStatus::Unknown, + HostClockStatus::Synchronized, + HostClockStatus::FreeRunning, + HostClockStatus::Unreliable, + ] { + assert_eq!(HostClockStatus::from_byte(status.to_byte()), Some(status)); + } + } } diff --git a/vmm_core/virt/src/time_abi/identity.rs b/vmm_core/virt/src/time_abi/identity.rs index 82e8dcda73d..4ba6f7cf424 100644 --- a/vmm_core/virt/src/time_abi/identity.rs +++ b/vmm_core/virt/src/time_abi/identity.rs @@ -28,8 +28,8 @@ pub const VENDOR_SIGNATURE: [u32; 3] = [0x7263_694d, 0x666f_736f, 0x7648_2074]; pub const INTERFACE_SIGNATURE: u32 = 0x3123_7648; /// `"NVX"` in EAX of leaf `0x40000002`. pub const NVX_BUILD_SIGNATURE: u32 = 0x0058_564e; -/// Time ABI 1.0 in EBX of leaf `0x40000002`. -pub const TIME_ABI_VERSION_EBX: u32 = 0x0001_0000; +/// Time ABI 2.0 in EBX of leaf `0x40000002`. +pub const TIME_ABI_VERSION_EBX: u32 = 0x0002_0000; /// `AccessHypercallMsrs`, `AccessVpIndex`, `AccessFrequencyRegs`, and /// `AccessTscInvariantControls` in EAX of leaf `0x40000003`. pub const PRIVILEGES_EAX: u32 = 0x0000_8860; @@ -298,7 +298,7 @@ mod tests { [0x4000_0005, 0x7263_694d, 0x666f_736f, 0x7648_2074] ), (0x4000_0001, [0x3123_7648, 0, 0, 0]), - (0x4000_0002, [0x0058_564e, 0x0001_0000, 0, 0]), + (0x4000_0002, [0x0058_564e, 0x0002_0000, 0, 0]), (0x4000_0003, [0x0000_8860, 0, 0, 0x0000_0100]), (0x4000_0004, [0, 0xffff_ffff, 0, 0]), (0x4000_0005, [4, 4, 0, 0]), diff --git a/vmm_core/virt/src/time_abi/mod.rs b/vmm_core/virt/src/time_abi/mod.rs index 785b8f8e787..16dca712bbe 100644 --- a/vmm_core/virt/src/time_abi/mod.rs +++ b/vmm_core/virt/src/time_abi/mod.rs @@ -1,13 +1,15 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -//! NVX time ABI v1. +//! NVX time ABI 2. //! //! The guest-visible time contract of the microVM: a minimal Hyper-V //! frequency identity, the identity MSRs, the TSC rate policy, downtime -//! selection, and the backend primitives that snapshot restore needs. A host, -//! backend, or snapshot that cannot meet the contract fails with a stable -//! [`TimeAbiCode`], and nothing falls back to another behavior. +//! selection, the time samples that pair host UTC with the guest TSC and +//! publish the host's clock correction, and the backend primitives that +//! snapshot restore needs. A host, backend, or snapshot that cannot meet the +//! contract fails with a stable [`TimeAbiCode`], and nothing falls back to +//! another behavior. #[cfg(guest_arch = "x86_64")] pub mod backend; @@ -56,8 +58,10 @@ pub use rate::RateCheck; use mesh_protobuf::Protobuf; use std::fmt; -/// The time ABI version this OpenVMM implements. -pub const TIME_ABI_VERSION: u32 = 1; +/// The time ABI version this OpenVMM implements: 2 publishes the host's clock +/// correction with every time sample, and pairs its host UTC with the guest +/// TSC on the host. +pub const TIME_ABI_VERSION: u32 = 2; macro_rules! time_abi_codes { ($($(#[doc = $doc:literal])* $variant:ident => $name:literal,)*) => { diff --git a/vmm_core/virt_kvm/src/arch/x86_64/time_abi.rs b/vmm_core/virt_kvm/src/arch/x86_64/time_abi.rs index 0387e4e3644..1de4599c109 100644 --- a/vmm_core/virt_kvm/src/arch/x86_64/time_abi.rs +++ b/vmm_core/virt_kvm/src/arch/x86_64/time_abi.rs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -//! The NVX time ABI v1 on KVM. +//! The NVX time ABI on KVM. //! //! Active only for partitions built with //! [`ProtoPartitionConfig::time_abi`](virt::ProtoPartitionConfig::time_abi). diff --git a/vmm_core/virt_mshv/src/x86_64/time_abi.rs b/vmm_core/virt_mshv/src/x86_64/time_abi.rs index 9b3dbbe16e6..05623b2d610 100644 --- a/vmm_core/virt_mshv/src/x86_64/time_abi.rs +++ b/vmm_core/virt_mshv/src/x86_64/time_abi.rs @@ -1556,7 +1556,7 @@ mod tests { [0x4000_0005, 0x7263_694d, 0x666f_736f, 0x7648_2074] ); assert_eq!(result(0x4000_0001), [0x3123_7648, 0, 0, 0]); - assert_eq!(result(0x4000_0002), [0x0058_564e, 0x0001_0000, 0, 0]); + assert_eq!(result(0x4000_0002), [0x0058_564e, 0x0002_0000, 0, 0]); assert_eq!(result(0x4000_0003), [0x8860, 0, 0, 0x100]); assert_eq!(result(0x4000_0004), [0, 0xffff_ffff, 0, 0]); assert_eq!(result(0x4000_0005), [VP_COUNT, VP_COUNT, 0, 0]); diff --git a/vmm_core/virt_whp/src/time_abi.rs b/vmm_core/virt_whp/src/time_abi.rs index 78f114eaa97..43238a6f0ec 100644 --- a/vmm_core/virt_whp/src/time_abi.rs +++ b/vmm_core/virt_whp/src/time_abi.rs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -//! The NVX time ABI v1 on WHP. +//! The NVX time ABI on WHP. //! //! Active only for partitions built with //! [`ProtoPartitionConfig::time_abi`](virt::ProtoPartitionConfig::time_abi). diff --git a/vmm_tests/vmm_tests/tests/tests/ttrpc/microvm.rs b/vmm_tests/vmm_tests/tests/tests/ttrpc/microvm.rs index a06bb41e33e..bcff5798b80 100644 --- a/vmm_tests/vmm_tests/tests/tests/ttrpc/microvm.rs +++ b/vmm_tests/vmm_tests/tests/tests/ttrpc/microvm.rs @@ -397,12 +397,12 @@ magic="$(read_port 233) $(read_port 233) $(read_port 233) $(read_port 233)" flags=$(read_port 233) target=$(read_port 233) range_count=$(read_port 233) -if [ "$magic" != "79 86 82 4" ] || [ $((flags & 4)) -eq 0 ]; then +if [ "$magic" != "79 86 82 5" ] || [ $((flags & 4)) -eq 0 ]; then {} write_port 1540 255 while :; do sleep 1; done fi -remaining=$((25 + range_count * 16 + 64)) +remaining=$((17 + range_count * 16 + 64)) index=0 while [ "$index" -lt "$remaining" ]; do read_port 233 >/dev/null From 37ff2f9a5fb2c44963745965247e1398d3333076 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Thu, 29 Oct 2026 09:38:53 -0700 Subject: [PATCH 6/6] Guide: describe time ABI 2's time samples and restore packet 5 The snapshots page described restore packet version 4, with host UTC latched at its first selection, and the 16-byte time sample of time ABI 1. Describe restore packet version 5, which carries no UTC, and the 48-byte sample of time ABI 2, field by field: host UTC paired with the guest TSC on the host, the pairing's bound, the status of host UTC, the disruption counter, the host's clock correction against the TSC at the declared rate, and the bounds of the correction's and of host UTC's errors. Say how OpenVMM pairs the guest TSC (KVM's offset, and MSHV's and WHP's offset measured from VP 0's TSC at every start) and where it reads the correction (adjtimex on Linux, GetSystemTimeAdjustmentPrecise on Windows), that the correction includes a restore's rate deviation, that the samples discipline only the guest's wall clock and are no paravirtual clock, and that a snapshot of time ABI 1 must be recaptured. The CLI and gRPC references name restore packet version 5. Part of microsoft/nvx#1009. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Guide/src/reference/openvmm/management/cli.md | 2 +- .../src/reference/openvmm/management/grpc.md | 4 +- Guide/src/user_guide/openvmm/snapshots.md | 63 ++++++++++++++----- 3 files changed, 51 insertions(+), 18 deletions(-) diff --git a/Guide/src/reference/openvmm/management/cli.md b/Guide/src/reference/openvmm/management/cli.md index 11bda46d1e3..6c55ed6c1ba 100644 --- a/Guide/src/reference/openvmm/management/cli.md +++ b/Guide/src/reference/openvmm/management/cli.md @@ -496,7 +496,7 @@ describes the source definitions. in progress; Windows flush completion waits for the named-pipe peer to consume the complete frame. * `--restore-entropy`: Accepted for compatibility; it has no effect. Every - microVM restore makes restore packet version 4 available on the private + microVM restore makes restore packet version 5 available on the private portb restore channel. The packet carries 64 bytes of fresh entropy, the online-VP target, the memory-expansion ranges, the downtime, and the TSC rate deviation; the guest must consume it and explicitly reseed its RNG. diff --git a/Guide/src/reference/openvmm/management/grpc.md b/Guide/src/reference/openvmm/management/grpc.md index d046e9c54d5..a66cccefcaa 100644 --- a/Guide/src/reference/openvmm/management/grpc.md +++ b/Guide/src/reference/openvmm/management/grpc.md @@ -183,7 +183,7 @@ restore flow over both transports: client requires the matching path configuration. * `restore_entropy` is accepted for compatibility and is valid only with `restore_path`; it has no effect, because every restore delivers restore - packet version 4 with fresh entropy. Every microVM process also receives a + packet version 5 with fresh entropy. Every microVM process also receives a fresh, non-serialized 16-byte generation ID through the fixed portb selector. On restore, the ID is the first 16 bytes of the restore packet's entropy. * `restore_processor_count` requests restore-time activation of the contiguous @@ -198,7 +198,7 @@ restore flow over both transports: snapshot base and immutable capacity. Zero selects the base. Expansion uses fresh private zeroed backing, implies the post-restore repair gate, and is rejected for snapshots without a RAM capacity contract. A nonzero value sets - the memory-target flag of restore packet version 4, with the expansion + the memory-target flag of restore packet version 5, with the expansion ranges (none at the base size); zero leaves the flag clear. * `restore_gate_timeout_ms` bounds gated guest repair. Zero selects the 60-second default; a nonzero value is valid only with `restore_path`. diff --git a/Guide/src/user_guide/openvmm/snapshots.md b/Guide/src/user_guide/openvmm/snapshots.md index e701a1bc238..2ba5ff07907 100644 --- a/Guide/src/user_guide/openvmm/snapshots.md +++ b/Guide/src/user_guide/openvmm/snapshots.md @@ -219,11 +219,11 @@ the ID from the first 16 bytes of the restore packet's fresh entropy, so the guest can reject an unchanged clone identity, reseed Linux, refresh runtime identifiers, and acknowledge the input gate without another PMIO transfer. -Every microVM restore exposes restore packet version 4. Status bit 1 +Every microVM restore exposes restore packet version 5. Status bit 1 advertises it, and writing `0xa5` to the status port selects it for reading -from the data port. Its 32-byte little-endian header holds: +from the data port. Its 24-byte little-endian header holds: -- the magic `OVR` and version 4; +- the magic `OVR` and version 5; - flags: bit 0, the downtime came from host UTC; bit 1, a memory target; bit 2, the guest must acknowledge the restore through port `0x605`; bit 3, a test hook is active; @@ -231,22 +231,38 @@ from the data port. Its 32-byte little-endian header holds: zero byte; - the generation counter (`u32`), which is 0 at cold boot and one more than the snapshot's at each restore; -- the TSC rate deviation (`i32`, ppm scaled by 2^16); -- the downtime in nanoseconds (`u64`); and -- host UTC in nanoseconds since the Unix epoch (`u64`), latched when the guest - first selects the packet. +- the TSC rate deviation (`i32`, ppm scaled by 2^16); and +- the downtime in nanoseconds (`u64`). The `(u64 GPA start, u64 length)` memory ranges and 64 bytes of fresh entropy follow the header. Status bits 2, 3, and 4 report an online-VP target, a memory target, and one or more memory ranges, so a zero-range target can skip -post-restore repair. +post-restore repair. The packet carries no host UTC: the guest sets its clock +from a time sample. Status bit 6 advertises the time sample. Writing `0xa7` to the status port -latches a 16-byte sample: version 1, flags (bit 3: a test hook is active), two -reserved zero bytes, the generation counter (`u32`), and host UTC in -nanoseconds (`u64`). The guest reads it from window port `0xeb`, so samples -never interleave with console input on the data port. The selected packet and -the window can be read one, two, or four bytes at a time. +latches a 48-byte little-endian sample, version 2: + +- the version, flags (bit 3: a test hook is active), the status of host UTC + (0 unknown, 1 synchronized, 2 free-running, 3 unreliable), and a reserved + zero byte; +- the generation counter (`u32`); +- host UTC in nanoseconds since the Unix epoch (`u64`), and the guest TSC at + the same instant (`u64`), which OpenVMM pairs on the host; +- the most those two instants can differ, in nanoseconds (`u32`); +- a disruption counter (`u32`), which changes when host UTC steps, the host + resumes from a suspension, or the guest TSC's offset from the host TSC + changes; +- the host's clock correction (`i64`): how fast host UTC runs against the + guest TSC counted at the declared rate, minus one, in ppm scaled by 2^16; + and +- a bound on the error of that correction (`u32`, ppm scaled by 2^16) and + the time service's bound on the error of host UTC (`u32`, microseconds), + each `0xffffffff` when unknown. + +The guest reads it from window port `0xeb`, so samples never interleave with +console input on the data port. The selected packet and the window can be +read one, two, or four bytes at a time. A microVM template may opt into restore-time vCPU activation by booting with an explicit canonical `maxcpus=1`, `2`, `4`, or `8` value below or equal to its @@ -300,7 +316,7 @@ validation error and refuse to start. ## Time and CPU compatibility -Every microVM uses the NVX time ABI v1. The guest sees a minimal Hyper-V +Every microVM uses the NVX time ABI 2. The guest sees a minimal Hyper-V identity: CPUID leaves `0x40000000` through `0x40000005`, explicit zero leaves `0x40000006` through `0x4000000f` and `0x40000080` through `0x40000082`, and identity MSRs that return the declared TSC rate (`0x40000022`) and LAPIC rate @@ -314,6 +330,20 @@ HWCR (`0xc0010015`) only `TscFreqSel`, so Linux takes the TSC to count at the P0 frequency, and DE_CFG (`0xc0011029`) only `LFENCE` serialization. Writing the value read back changes nothing; any other write raises #GP. +Each time sample pairs host UTC with the guest TSC on the host, so the guest +needs no tight bracket of its own around the port write: the guest TSC is the +host TSC plus an offset, which KVM reports, and which MSHV and WHP measure +from VP 0's TSC at every start, before any VP runs. Each sample also +publishes the host's clock correction, which the guest applies instead of +estimating it. OpenVMM reads it from the host's time service: on Linux, the +frequency correction, tick, status, and maximum error that `adjtimex` +reports, and on Windows, the time adjustment that +`GetSystemTimeAdjustmentPrecise` reports, with an adjustment that Windows +does not apply reported as a free-running clock. The correction refers to +the declared rate, so on a restore it includes the deviation of the native +rate. A paravirtual clock is still not exposed: the samples only discipline +the guest's wall clock, and the TSC remains its clocksource. + The guest CPU is a pinned CPU profile: one per CPU generation, the same on every backend. OpenVMM pins `intel.skylake-sp.v1` (Xeon Scalable, first generation: family 6, model 85, steppings 0 to 4), `intel.icelake-sp.v1` @@ -375,6 +405,8 @@ others. Restore also requires: +- a snapshot of the same time ABI version, so a snapshot of time ABI 1 must + be recaptured (`E_MANIFEST_TIME`); - the backend that captured the snapshot; - a native TSC rate within 250 ppm of the snapshot's (`E_TSC_RATE_TOLERANCE`), and the identical LAPIC rate @@ -383,7 +415,8 @@ Restore also requires: `E_DOWNTIME_EXCESSIVE`). The declared rates never change across a restore; the restore packet reports -the rate deviation so the guest can compensate. The downtime is host monotonic +the rate deviation, and every time sample's correction includes it, so the +guest can compensate. The downtime is host monotonic time when capture and restore run on the same host boot, and host UTC otherwise. Capture and restore reject an armed periodic or TSC-deadline LAPIC timer and a periodically counting PIT channel 0 (`E_LAPIC_PERIODIC`,