From c0279c42ce433a7ec344f62547a4ba5c64d2edd9 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Fri, 13 Nov 2026 20:42:33 -0800 Subject: [PATCH] guest: hold the declared TSC rate through a capture request A restore advanced the guest's monotonic and boot-time clocks by the downtime D times one plus the frequency correction that the guest held at its capture, not by D. The kernel counts a TSC advance whole, at the rate of its clocks' multiplier when it first reads the TSC afterwards, which is their rate at the capture, and nvx-time holds the host's clock correction, within 500 ppm, as that rate. Over the year that the time ABI now admits, the clocks stood up to 4.4 hours from the downtime. The restore-downtime scenario's simulated year ended short of it by 461 s on KVM, 1,390 s on MSHV, and 13 to 24 s on WHP, which hold -14.7, -44.3, and -0.6 ppm (microsoft/nvx#516). Make the capture request hold the declared rate. Just before it writes the snapshot port, under the port-B lock that keeps the discipline's polls out until the request returns, it queries the discipline and pins it with one adjtimex call: a step by zero, which clears the kernel's NTP state and the phase slew that the tick length caches until its next second, with the frequency correction F_k / F - 1 that counts the TSC at exactly the declared rate F, for the kHz F_k to which the kernel rounds it, and the status and error estimates that the query found. The monotonic and boot-time clocks then advance by D, within the 1 ppm of the kernel's clocksource multiplier. A restore's repair holds the host's correction again, as it did, and a request that returns in a VM that continues sets back the frequency that it found. The next poll slews the phase that the step cleared. A repair that fails before its step leaves the declared rate in force, with the clock unstepped, and enforcing mode stops the VM. A hold that fails fails the request before it writes the port, as a violation of the existing check C10, so that the request adds no API, event, or error code. The request now needs CAP_SYS_TIME in a VM that continues too, which the repair of a restore already needed to set the clock, and the rustdoc of TimeAbi lists it. The step by zero notifies a workload that watches the clock for steps with TFD_TIMER_CANCEL_ON_SET, as restore repair's step does; 3.2.10 says so. Clearing the phase alone, with ADJ_OFFSET, is not enough: the kernel keeps the cached slew, up to 500 ppm of a year, in force until its next second. No contract version changes. The hold belongs to the guest's capture and repair steps, on which no VMM or snapshot format depends, and a restore still advances the TSC by D at the declared rate. It holds only for a snapshot whose guest has it: a snapshot of an earlier guest keeps counting the downtime at its correction until it is captured again. 3.2.8 documents this, 5.3 and 5.5 the capture and repair steps. The restore-downtime scenario now checks the fix. Each capture's OpenVMM boots with the host-rate-offset-ppm hook at 100 ppm, so that the guest holds a large correction, and the guest leaves an adjtime slew of 500 ppm in flight at its request. The restored guest's monotonic and boot-time clocks must advance by the simulated downtime, within 1 s plus 1 ppm, whatever it held before; its raw clock keeps the declared-rate check. The wait for the daemon's next poll, which made the recorded correction the held one, goes, which makes the scenario about a minute shorter. With this harness, the unfixed guest ends 17,490 s beyond the downtime on MSHV, and a variant that holds the rate but leaves the slew in flight 15,770 s. With the fix, all four restores end 1.7 s short of it on MSHV, 2.5 s short on WHP, and 1.1 s beyond on KVM. Tests: the capture's adjustments in each mode; that a declared rate of 2,300,000,123 Hz holds -3,505 scaled ppm, a whole-kHz rate none, and that a hold that fails reports its cause, and touches nothing without a rate; that a kernel that refuses the hold fails the request before the snapshot port, with its event on the debug console; the discipline's modes and status; the harness's bounds on both clocks, and its pre-capture script's slew. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- doc/ci.md | 25 +- doc/design/3.2-clock-and-time.md | 48 +++- doc/design/5.3-snapshot-protocol.md | 14 +- doc/design/5.5-restore-protocol.md | 3 +- nvx-rs/README.md | 8 + nvx-rs/src/guest/sys/discipline.rs | 49 +++- nvx-rs/src/guest/sys/discipline/tests.rs | 28 +++ nvx-rs/src/guest/time/api.rs | 69 +++++- nvx-rs/src/guest/time/api/tests.rs | 93 +++++++- nvx-rs/src/guest/time/system.rs | 57 +++++ nvx-rs/src/guest/time/system/tests.rs | 96 +++++++- nvx-rs/src/guest/time/test_mode.rs | 6 +- .../restore-year-capture.sh | 30 +-- scripts/nvx_tools/microvm_tests.py | 62 +++-- scripts/test_microvm_tests.py | 222 +++++++++++------- 15 files changed, 636 insertions(+), 174 deletions(-) diff --git a/doc/ci.md b/doc/ci.md index abb5d9bb7..0f328421a 100644 --- a/doc/ci.md +++ b/doc/ci.md @@ -152,14 +152,15 @@ monotonic time advanced by the downtime. The captures share one downtime window, so the scenario adds about a minute per backend. Before each capture, the guest starts a CPU-bound workload in a cgroup of its own and freezes it, as guest preparation freezes the workload for paired -scratch, arms a one-hour timer, and, once its time daemon's next discipline -poll has passed, records its clocks, its frequency correction, the workload's -CPU time and cgroup usage, and the length of its kernel log. The daemon changes -the correction only at its polls, which take the port lock that the capture -takes too, 16 s apart at first and 64 s apart once it has settled, so the -correction recorded is the one that the capture holds. Each capture comes -within seconds of its guest's boot, so the wait adds up to about 16 s to it, -and the capture allows this preparation 90 s beyond the scenario's timeout. The +scratch, arms a one-hour timer, records its clocks, its frequency correction, +the workload's CPU time and cgroup usage, and the length of its kernel log, and +leaves a slew of its clock in flight: an adjtime of 5 ms, which the kernel +slews at 500 ppm for 10 s, and whose rate it caches in its tick length from the +next second on, which a second's wait lets pass. Each capture's OpenVMM boots +with the `host-rate-offset-ppm` test hook at 100 ppm, so that the guest holds a +large frequency correction, which the host's own, at most 250 ppm on a +qualified host, composes with under Linux's 500 ppm limit, and the capture +allows this preparation 15 s beyond the scenario's timeout. The scenario then restores each snapshot again after a simulated year: OpenVMM's `downtime-add-s` test hook adds what brings the downtime to 60 s below the time ABI's @@ -172,8 +173,10 @@ downtime from the expected source. Against its realtime clock, which must stand within 10 s of host UTC, its raw monotonic clock must have advanced by the simulated downtime at the declared TSC rate, to the kHz, within 1 s plus 0.5 ppm for the kernel's rounded clocksource multiplier, and its monotonic and -boot-time clocks by that times one plus the frequency correction that it held -before the capture, within 5 ppm more for a phase slew then. The timer must +boot-time clocks by the simulated downtime itself, within 1 s plus 1 ppm, +whatever frequency correction and slew it held before the capture, since the +capture holds the declared rate: clocks that kept the 100 ppm and the slew, +about 555 ppm, would stand some 17,000 s beyond it. The timer must fire within 10 s of the restore's clock step, before or after it, the frozen workload must gain no CPU time or cgroup usage across the restore and about a second's in the second that it then runs, and the restore must print its @@ -191,7 +194,7 @@ arithmetic: about one restore in twenty then dereferenced NULL in `pick_task_fair()` and left its processor spinning on the runqueue lock. Last, OpenVMM must refuse a restore of the first snapshot with the bound plus one second, with `E_DOWNTIME_EXCESSIVE`, before the guest runs. The simulated -restores and the captures' waits add about two minutes per backend. +restores add about a minute per backend. The `time-abi-conformance` scenario boots the largest requested vCPU count and runs the guest's exhaustive CI check, `/sbin/nvx-time exhaustive`, which the boot check leaves to CI. On every online CPU it checks every leaf diff --git a/doc/design/3.2-clock-and-time.md b/doc/design/3.2-clock-and-time.md index 9c39121d8..371208462 100644 --- a/doc/design/3.2-clock-and-time.md +++ b/doc/design/3.2-clock-and-time.md @@ -320,14 +320,26 @@ the TSC has advanced by `D` plus at most the restore latency `R`. The guest observes no other discontinuity between the snapshot request and the first restored instruction. -The guest kernel counts that advance as it counts any run of the TSC: its raw -monotonic clock at the declared rate, to the kHz, and its monotonic and -boot-time clocks at the frequency correction that the guest held at the -capture, which follows the host's -[clock correction](#329-time-samples) and stays within the kernel's ±500 ppm -limit. Those two clocks therefore advance by `D` times one plus that -correction, which over a year differs from `D` by up to 4.4 hours. Restore -steps the realtime clock to host UTC, so it keeps no such difference. +The guest kernel counts that advance as it counts any run of the TSC, whole, +at the rate of its clocks when it first reads the TSC after the restore, which +is their rate at the capture. Its raw monotonic clock counts it at the +declared rate, to the kHz. Its monotonic and boot-time clocks count it at the +rate that the kernel's frequency correction gives them, which the guest holds +at the host's [clock correction](#329-time-samples), within the kernel's +±500 ppm limit, and which would make them advance by `D` times one plus that +correction, up to 4.4 hours more or less than `D` over a year. The guest +therefore holds the correction that counts the TSC at exactly the declared +rate, `F_k / F - 1` for the kHz `F_k` to which the kernel rounds `F`, from its +capture request to the capture anchor, in place of the host's, and clears the +phase slew that its discipline may have under way, because the kernel keeps +adding that slew's rate to its clocks' for up to a second after the phase +offset is cleared. Its monotonic and boot-time clocks then advance by `D`, to +the resolution of the kernel's clocksource multiplier: within 1 ppm of `D`, 32 s +over a year. The guest holds the host's correction again when restore repair +steps the clock, or when a capture that continues without a restore returns, +as [guest preparation](5.3-snapshot-protocol.md#532-guest-preparation) and +[guest repair](5.5-restore-protocol.md#557-guest-repair) say. Restore steps +the realtime clock to host UTC, so it keeps no such difference. The guest's scheduler clock reads the raw TSC, which advances by the downtime, so the downtime would be runtime of the task that each processor ran at the @@ -394,6 +406,15 @@ can oops before its guest rejects the packet, which hung about one such restore in twenty in qualification. Snapshots captured before the change therefore keep their 30-day window; only a recapture gets the longer one. +Holding the declared rate through the capture request changes no contract +version either: it belongs to the guest's own capture and repair steps, on +which no VMM or snapshot format depends, and a restore still advances the TSC +by `D` at the declared rate. It too holds only for a snapshot whose guest has +it: a snapshot of an earlier guest, which kept the host's correction through +its capture, counts the downtime in its monotonic and boot-time clocks as that +guest's kernel did, `D` times one plus that correction, until it is captured +again. + The restore anchor is the instant of the synchronized TSC set. Guest time runs from that instant, or, on a backend that writes the TSCs with partition time frozen, from the instant at which partition time resumes after the write, and @@ -598,10 +619,13 @@ the kernel's limit is held at the limit and reported in the state file, so an offset can grow until the discipline steps it. A sample whose generation differs from the recorded one is discarded, because a restore happened and restore repair resets the discipline. The discipline -follows host wall-clock steps and never powers off the guest. Restore repair -and the discipline step the clock through the kernel, so a workload can -observe every step through the kernel's clock-set notification, such as a -`timerfd` armed with `TFD_TIMER_CANCEL_ON_SET`. +follows host wall-clock steps and never powers off the guest. Restore repair, +the discipline, and a capture request step the clock through the kernel, so a +workload can observe every step through the kernel's clock-set notification, +such as a `timerfd` armed with `TFD_TIMER_CANCEL_ON_SET`; the capture +request's step is by zero, and keeps the clock as it is. The capture request +also holds the clock at the declared rate until the capture ends, as +[Downtime and Continuity](#328-downtime-and-continuity) describes. The [snapshot protocol](5.3-snapshot-protocol.md) and [guest repair](5.5-restore-protocol.md#557-guest-repair) define the snapshot diff --git a/doc/design/5.3-snapshot-protocol.md b/doc/design/5.3-snapshot-protocol.md index f31ac2009..f87de6025 100644 --- a/doc/design/5.3-snapshot-protocol.md +++ b/doc/design/5.3-snapshot-protocol.md @@ -114,6 +114,12 @@ Before they write the snapshot port, the guest agents: 4. Write back the pages that shared mappings dirtied on the mounted [host shares](3.6-host-filesystem.md), and wait for the writes. 5. Write the capture request, with its scratch policy, to the snapshot port. + Just before the write, the time tools hold the wall clock at the declared + TSC rate, with no phase slew in flight, under the byte channel's lock, which + keeps the discipline's polls from changing the rate before the capture + anchor, so that the kernel counts the restore's advance of the TSC by the + downtime in its monotonic and boot-time clocks as the downtime itself; see + [Downtime and Continuity](3.2-clock-and-time.md#328-downtime-and-continuity). Step 4 keeps captured memory from holding file data that the host files lack, which every restore would otherwise write back to the live host files. The @@ -143,9 +149,11 @@ A committed capture terminates the source, so the write returns in the source only when the capture had no destination or failed before its commit point. The guest agents tell the cases apart by the restore-packet status bit, which is clear in a resumed source and set in a restored process. In a resumed -source, they remove their barriers and restore the detector settings, and a -failure to do so powers off the guest. A rejected capture likewise thaws every -guest-owned barrier and then releases the stall-detector suppression. +source, the time tools hold the host's clock correction again before the +request returns, and the guest agents remove their barriers and restore the +detector settings, and a failure to do so, or to hold the correction, powers +off the guest. A rejected capture likewise thaws every guest-owned barrier and +then releases the stall-detector suppression. ## 5.3.3 Capture Transaction diff --git a/doc/design/5.5-restore-protocol.md b/doc/design/5.5-restore-protocol.md index 7db5452a0..956a36e38 100644 --- a/doc/design/5.5-restore-protocol.md +++ b/doc/design/5.5-restore-protocol.md @@ -326,7 +326,8 @@ runs before the acknowledgement: packet's generation, until one pairs host UTC with the guest TSC within 100 µs, step the clock by its offset, and hold the frequency correction that its host clock correction gives, which carries the rate deviation, - within ±500 ppm. + within ±500 ppm, in place of the declared-rate correction that the capture + request held. 4. Activate processors and memory as requested, then apply the network settings of a packet that carries them as the guest's init applies a cold boot's: name the DNS server in `/etc/resolv.conf`, and, when the packet asks for the diff --git a/nvx-rs/README.md b/nvx-rs/README.md index e7cb2e8b7..ba4ca809f 100644 --- a/nvx-rs/README.md +++ b/nvx-rs/README.md @@ -1384,6 +1384,14 @@ The request's mode selects how a restore hands over its fresh entropy: the file mode, which `nvx-time` keeps, writes it to a file, and the in-memory mode returns it as the `Restore`'s owned `Seed`, and writes no file. +Just before it writes the snapshot port, a `CaptureRequest` holds the wall +clock at the declared TSC rate, without the host's clock correction and without +a phase slew in flight, so that a restore's advance of the TSC by the downtime +counts in the monotonic and boot-time clocks as the downtime itself; the repair +of a restore holds the host's correction again, and so does a request that +returns in a VM that continues. A hold that fails is a violation of +conformance check C10, which fails the request before it writes the port. + A violation of the ABI stops the VM in enforcing mode, with the time ABI's status, as `nvx-time` does; in strict mode, `TimeAbi::strict`, which a caller that owns the VM's exit status and shutdown selects, the call fails with the diff --git a/nvx-rs/src/guest/sys/discipline.rs b/nvx-rs/src/guest/sys/discipline.rs index b743cdf8d..32e7fd3f9 100644 --- a/nvx-rs/src/guest/sys/discipline.rs +++ b/nvx-rs/src/guest/sys/discipline.rs @@ -42,6 +42,21 @@ pub(crate) enum ClockAdjustment { /// The frequency correction and the error estimates. hold: FrequencyHold, }, + /// Sets the frequency correction alone, and keeps the status, the error estimates, and the + /// phase offset. + SetFrequency(i64), + /// Makes the wall clock run at the frequency correction of `hold` alone, from the moment of + /// the call: it steps the wall clock by zero, which clears the kernel's NTP state, the phase + /// offset and the tick length's cached correction of it included, which a clearing of the + /// phase offset alone leaves in force until the kernel's next second, and then sets the + /// frequency correction and the error estimates of `hold`, and `status`, which the step reset + /// to unsynchronized. + Pin { + /// The frequency correction and the error estimates. + hold: FrequencyHold, + /// The discipline's status, as the kernel reports it. + status: i32, + }, } /// The frequency correction that the discipline holds, and the error estimates that it sets with @@ -68,14 +83,17 @@ impl ClockAdjustment { Self::Frequency(_) => HOLD, Self::ClearPhase => libc::ADJ_OFFSET | libc::ADJ_NANO, Self::Slew { .. } => HOLD | libc::ADJ_OFFSET | libc::ADJ_TIMECONST | libc::ADJ_NANO, + Self::SetFrequency(_) => libc::ADJ_FREQUENCY, + Self::Pin { .. } => HOLD | libc::ADJ_SETOFFSET | libc::ADJ_NANO, } } /// Returns the discipline status that the adjustment sets, or zero for one that sets none. pub(crate) fn status(self) -> i32 { match self { - Self::Query | Self::Step { .. } | Self::ClearPhase => 0, + Self::Query | Self::Step { .. } | Self::ClearPhase | Self::SetFrequency(_) => 0, Self::Frequency(_) | Self::Slew { .. } => PLL_NANOSECONDS, + Self::Pin { status, .. } => status, } } } @@ -85,6 +103,24 @@ impl ClockAdjustment { pub(crate) struct ClockDiscipline { /// Frequency correction in parts per million, scaled by 2^16. pub(crate) scaled_ppm: i64, + /// Maximum error in microseconds. + pub(crate) maximum_error_us: i64, + /// Estimated error in microseconds. + pub(crate) estimated_error_us: i64, + /// The discipline's status bits, `STA_*`. + pub(crate) status: i32, +} + +impl ClockDiscipline { + /// Returns the hold of the frequency correction `scaled_ppm` with this discipline's error + /// estimates. + pub(crate) fn hold(self, scaled_ppm: i64) -> FrequencyHold { + FrequencyHold { + scaled_ppm, + maximum_error_us: self.maximum_error_us, + estimated_error_us: self.estimated_error_us, + } + } } impl FrequencyHold { @@ -104,7 +140,7 @@ pub(crate) fn adjust_clock(adjustment: ClockAdjustment) -> Result {} + ClockAdjustment::Query | ClockAdjustment::ClearPhase => {} ClockAdjustment::Step { seconds, nanoseconds, @@ -115,8 +151,10 @@ pub(crate) fn adjust_clock(adjustment: ClockAdjustment) -> Result hold.apply(&mut value), - ClockAdjustment::ClearPhase => {} + ClockAdjustment::Frequency(hold) | ClockAdjustment::Pin { hold, .. } => { + hold.apply(&mut value); + } + ClockAdjustment::SetFrequency(scaled_ppm) => value.freq = scaled_ppm, ClockAdjustment::Slew { offset_ns, time_constant, @@ -131,6 +169,9 @@ pub(crate) fn adjust_clock(adjustment: ClockAdjustment) -> Result = Token { /// restore's finish, which run in the calling process. /// /// A call needs the privileges of `nvx-time`: `CAP_SYS_RAWIO` for OpenVMM's I/O ports, and -/// write access to `/run/nvx`. It takes the permission for the ports only for each of its port -/// transactions, on the calling thread, and returns only once the thread holds none of it, or -/// fails with [`TimeError::PortRelease`], after which the thread must start neither a thread -/// nor a process, which would inherit the permission; [`Command`](crate::guest::spawn::Command) -/// refuses to start one from it. No call waits for a child that it did not start. +/// write access to `/run/nvx`; the cold boot and the capture request, which set the wall clock or +/// its rate, need `CAP_SYS_TIME` too, the capture request whether or not the VM then restores, +/// since it holds the clock's rate before it writes the snapshot port. A call takes the +/// permission for the ports only for each of its port transactions, on the calling thread, and +/// returns only once the thread holds none of it, or fails with [`TimeError::PortRelease`], +/// after which the thread must start neither a thread nor a process, which would inherit the +/// permission; [`Command`](crate::guest::spawn::Command) refuses to start one from it. No call +/// waits for a child that it did not start. /// /// A violation of the time ABI is reported by a violation event on the console, in the kernel /// log, and on OpenVMM's debug console. In enforcing mode, the event stops the VM, so the call @@ -182,6 +185,19 @@ impl TimeAbi { /// slews the wall clock, hands over the packet's fresh entropy, and records the restore, /// repair steps 1 to 3. /// + /// Just before it writes the port, the call holds the wall clock at the declared TSC rate, + /// without the host's clock correction and without a phase slew in flight, so that the + /// kernel counts a restore's advance of the TSC by the downtime in its monotonic and + /// boot-time clocks as the downtime itself. The hold sets the clock's discipline, which + /// needs `CAP_SYS_TIME`, in a VM that continues without a restore as in one that a snapshot + /// restored. The repair holds the host's correction again in a restored VM, and the call + /// does in a VM that continues without a restore, before it returns + /// [`CaptureOutcome::Continued`]. A hold that fails, because the kernel refuses it or the + /// declared rate is unknown, is a violation of check C10 that fails the call before it + /// writes the port, and so is a refusal to hold the host's correction again afterwards. A + /// repair that fails before it steps the clock leaves the declared rate in force; the failure + /// stops the VM, unless the call is report-only or strict. + /// /// The request's mode selects how the entropy is handed over. In the file mode of /// [`CaptureRequest::new`], the call creates, or truncates, the entropy file before it /// requests the capture, writes the entropy to it, and removes it again when the VM @@ -781,6 +797,23 @@ fn capture(runtime: &Runtime<'_>, request: CaptureRequest) -> Result hold, + Err(detail) => { + entropy.discard(); + // The failure's event writes the debug console in a transaction of its own. + drop(ports); + return Err(check_failed( + runtime, + Phase::Capture, + CheckId::C10, + prefixed(b"cannot hold the clock at the declared rate: ", &detail), + )); + } + }; ports.write_u8(Port::Snapshot, scratch.request()); let start = clock_ns(sys::Clock::Monotonic); let read = (ports.read_u8(Port::PortbStatus) & PORTB_PACKET_AVAILABLE != 0) @@ -788,7 +821,7 @@ fn capture(runtime: &Runtime<'_>, request: CaptureRequest) -> Result, request: CaptureRequest) -> Result resume_rate(runtime, hold), + restored @ CaptureOutcome::Restored(_) => Ok(restored), + } +} + +/// Ends a capture that continued without a restore, once the clock holds the host's correction +/// again, which the capture's [hold](system::hold_declared_rate) interrupted. +fn resume_rate(runtime: &Runtime<'_>, hold: system::RateHold) -> Result { + match system::resume_rate(runtime, hold) { + Ok(()) => Ok(CaptureOutcome::Continued), + Err(detail) => Err(check_failed( + runtime, + Phase::Capture, + CheckId::C10, + prefixed(b"cannot resume the host's clock correction: ", &detail), + )), + } +} + +/// Returns `detail` after `prefix`, as a violation's diagnostic. +fn prefixed(prefix: &[u8], detail: &[u8]) -> Vec { + [prefix, detail].concat() } /// Loads the persisted generation, and the declared TSC rate that the boot checks recorded, with diff --git a/nvx-rs/src/guest/time/api/tests.rs b/nvx-rs/src/guest/time/api/tests.rs index 772f2981f..7b3a53f9e 100644 --- a/nvx-rs/src/guest/time/api/tests.rs +++ b/nvx-rs/src/guest/time/api/tests.rs @@ -5,7 +5,7 @@ use std::path::Path; use std::sync::atomic::Ordering; use super::*; -use crate::guest::sys::{ClockAdjustment, FrequencyHold}; +use crate::guest::sys::{ClockAdjustment, ClockDiscipline, FrequencyHold}; use crate::guest::time::decoders::{HostClock, PACKET_DNS_SIZE}; use crate::guest::time::state::HostClockStatus; use crate::guest::time::system::PORTB_TIME_WINDOW; @@ -613,7 +613,8 @@ fn holds(directory: &Path, secret: &[u8]) -> bool { } /// Captures through the fakes, in the in-memory mode and then in the file mode, a restore, -/// then a VM that continues, on a tmpfs at `/run` in a mount namespace of the process's own. +/// then a VM that continues, and then a hold that the kernel refuses, on a tmpfs at `/run` in a +/// mount namespace of the process's own. fn capture_in_each_mode() { sys::enter_private_mount_namespace().unwrap(); let _run = sys::TmpfsMount::mount(c"nvx-test", c"/run", c"mode=0755").unwrap(); @@ -623,35 +624,81 @@ fn capture_in_each_mode() { let entropy: [u8; Seed::LENGTH] = std::array::from_fn(|index| index as u8 + 2); let secret = &entropy[GenerationId::LENGTH..]; let path = Path::new("/run/nvx/restore-entropy"); - // Returns the capture's outcome, and what it wrote to the kernel log and the console. + // The discipline that the capture finds: the frequency correction that the host's asks for, + // and the status and error estimates of a synchronized clock. + let found = ClockDiscipline { + scaled_ppm: -983_040, + maximum_error_us: 40, + estimated_error_us: 2, + status: 0x2081, + }; + // Returns the capture's outcome, what it wrote to the kernel log and the console, and its + // adjustments of the clock. let capture_with = |input: Vec, request| { let ops = FakeOps::new(input); + *ops.discipline.lock().unwrap() = found; let outcome = capture(&Runtime::new(&ops, true), request).unwrap(); let mut logged = ops.kernel_log.lock().unwrap().clone(); logged.extend_from_slice(&ops.console.lock().unwrap()); - (outcome, logged) + let adjustments = ops.adjustments.lock().unwrap().clone(); + (outcome, logged, adjustments) + }; + // Every request first holds the clock at the declared rate, which the fake's 2 GHz is the + // kernel's own, without the correction that it found, and keeps the status and the error + // estimates that it found. + let held = FrequencyHold { + scaled_ppm: 0, + maximum_error_us: 40, + estimated_error_us: 2, + }; + let pin = ClockAdjustment::Pin { + hold: held, + status: 0x2081, }; // The in-memory mode returns the entropy as the restore's seed, and writes no file. let request = CaptureRequest::in_memory(ScratchPolicy::Fresh, previous); - let (CaptureOutcome::Restored(mut restore), logged) = + let (CaptureOutcome::Restored(mut restore), logged, adjustments) = capture_with(restore_input(1, &entropy), request) else { panic!("the in-memory capture restored nothing"); }; assert_eq!(logged, b""); + // The repair of the restore reads the clock's frequency, which it pairs its time sample at, + // and holds the host's correction after its step, as it did before. + assert_eq!(adjustments[..2], [ClockAdjustment::Query, pin]); + assert!( + matches!( + adjustments[2..], + [ + ClockAdjustment::Query, + ClockAdjustment::Step { .. }, + ClockAdjustment::Frequency(_) + ] + ), + "{adjustments:?}" + ); let id = restore.generation_id().to_bytes(); assert_eq!(id[..], entropy[..GenerationId::LENGTH]); assert_eq!(restore.take_seed().map(|seed| *seed.bytes()), Some(entropy)); assert!(!path.exists() && !holds(Path::new("/run"), secret)); let request = CaptureRequest::in_memory(ScratchPolicy::Fresh, previous); - let (outcome, _) = capture_with(vec![0], request); + let (outcome, _, adjustments) = capture_with(vec![0], request); assert!(matches!(outcome, CaptureOutcome::Continued), "{outcome:?}"); + // A VM that continues resumes the frequency correction that it found. + assert_eq!( + adjustments, + [ + ClockAdjustment::Query, + pin, + ClockAdjustment::SetFrequency(found.scaled_ppm), + ] + ); // The file mode, which the tools keep, writes the entropy to a file that only its owner // may read and write, and the restore holds no seed. let request = CaptureRequest::new(ScratchPolicy::Paired, previous, path.to_owned()); - let (CaptureOutcome::Restored(mut restore), logged) = + let (CaptureOutcome::Restored(mut restore), logged, _) = capture_with(restore_input(2, &entropy), request) else { panic!("the file mode's capture restored nothing"); @@ -663,9 +710,39 @@ fn capture_in_each_mode() { assert_eq!(mode & 0o7777, 0o600); // A VM that continues has the file removed. let request = CaptureRequest::new(ScratchPolicy::Fresh, previous, path.to_owned()); - let (outcome, _) = capture_with(vec![0], request); + let (outcome, _, _) = capture_with(vec![0], request); assert!(matches!(outcome, CaptureOutcome::Continued), "{outcome:?}"); assert!(!path.exists()); + + // A kernel that refuses the hold fails the request before it writes the snapshot port, and + // the failure's event reaches the debug console in a port transaction of its own. + let mut ops = FakeOps::new([]); + ops.adjust_error = Some(Errno::EPERM); + let runtime = Runtime::new(&ops, true); + let request = CaptureRequest::new(ScratchPolicy::Fresh, previous, path.to_owned()); + let error = capture(&runtime, request).unwrap_err(); + let TimeError::Violation(violation) = &error else { + panic!("{error:?}"); + }; + assert_eq!( + (violation.status(), violation.code()), + (status::TIME_CONFORMANCE, "G_CONFORMANCE_C10") + ); + assert_eq!( + violation.detail(), + format!( + "cannot hold the clock at the declared rate: adjtimex: {}", + Errno::EPERM.desc() + ) + ); + let output = ops.output.lock().unwrap(); + assert!( + output.iter().all(|&(port, _)| port == Port::PortbData), + "{output:?}" + ); + let event = String::from_utf8(output.iter().map(|&(_, byte)| byte).collect()).unwrap(); + assert!(event.contains("G_CONFORMANCE_C10"), "{event}"); + assert!(!path.exists()); } #[test] diff --git a/nvx-rs/src/guest/time/system.rs b/nvx-rs/src/guest/time/system.rs index b2e109462..68f121288 100644 --- a/nvx-rs/src/guest/time/system.rs +++ b/nvx-rs/src/guest/time/system.rs @@ -959,6 +959,63 @@ pub(crate) fn synchronize(runtime: &Runtime<'_>, sample: &BestSample) -> Result< ) } +/// What a capture's [hold on the clock's rate](hold_declared_rate) interrupted: the frequency +/// correction that the discipline held before it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct RateHold { + /// The held frequency correction, in scaled ppm, that a capture that continues resumes. + frequency: i64, +} + +/// Holds the wall clock at the declared TSC rate for a capture request, without the host's +/// clock correction, and without a phase offset in flight, until [`resume_rate`] or the repair +/// of a restore holds the host's correction again. +/// +/// The kernel counts a TSC advance at the rate of its clocks' multiplier at that moment, whole, +/// when it first reads the TSC afterwards. A restore advances the TSC by the downtime at the +/// declared rate, so a clock whose frequency correction is the host's, as the discipline sets it, +/// counts the downtime times one plus that correction, which differs from the downtime by up to +/// the correction's 500 ppm of it. Holding the correction that counts the TSC at exactly the +/// declared rate, `F_k / F - 1` for the kernel's rounding `F_k` of the rate `F` to the kHz, makes +/// the monotonic and boot-time clocks advance by the downtime. +/// +/// The discipline's phase-locked loop adds its slew of a phase offset to the clock's rate, and +/// the kernel caches that slew in its tick length until its next second, so the hold steps the +/// clock by zero, which clears the cache, as [`ClockAdjustment::Pin`] describes. It keeps the +/// discipline's status and error estimates, and the caller holds the port-B lock, so that no +/// poll of the time daemon changes the rate before the capture's anchor. +pub(crate) fn hold_declared_rate(runtime: &Runtime<'_>) -> Result> { + let tsc_hz = runtime + .tsc_hz() + .map_err(|error| error.to_string().into_bytes())?; + let kept = runtime + .ops + .adjust_clock(ClockAdjustment::Query) + .map_err(|error| describe(b"adjtimex: ", error))?; + let (frequency, _) = decoders::guest_frequency(0, tsc_hz); + runtime + .ops + .adjust_clock(ClockAdjustment::Pin { + hold: kept.hold(frequency), + status: kept.status, + }) + .map_err(|error| describe(b"adjtimex: ", error))?; + Ok(RateHold { + frequency: kept.scaled_ppm, + }) +} + +/// Resumes the frequency correction that a [hold on the clock's rate](hold_declared_rate) +/// interrupted, for a capture that continues without a restore. The discipline's next poll slews +/// the phase that the hold cleared. +pub(crate) fn resume_rate(runtime: &Runtime<'_>, hold: RateHold) -> Result<(), Vec> { + runtime + .ops + .adjust_clock(ClockAdjustment::SetFrequency(hold.frequency)) + .map(drop) + .map_err(|error| describe(b"adjtimex: ", error)) +} + /// Takes the initial sample within the bound, steps realtime to it, and holds the frequency /// correction that the host's clock correction asks for. pub(crate) fn initial_sample(runtime: &Runtime<'_>) -> Result> { diff --git a/nvx-rs/src/guest/time/system/tests.rs b/nvx-rs/src/guest/time/system/tests.rs index 7aca1d5c7..3538d3a6d 100644 --- a/nvx-rs/src/guest/time/system/tests.rs +++ b/nvx-rs/src/guest/time/system/tests.rs @@ -29,6 +29,10 @@ pub(crate) struct FakeOps { pub(crate) output: Mutex>, /// Clock adjustments, in order. pub(crate) adjustments: Mutex>, + /// The discipline that every adjustment reports, as the kernel reports it after one. + pub(crate) discipline: Mutex, + /// The error with which the kernel refuses every adjustment but a query, if any. + pub(crate) adjust_error: Option, /// The last power-off status, or zero. pub(crate) power: AtomicUsize, /// Kernel log records, in order. @@ -62,6 +66,8 @@ impl FakeOps { input: Mutex::new(input.into_iter().collect()), output: Mutex::new(Vec::new()), adjustments: Mutex::new(Vec::new()), + discipline: Mutex::new(ClockDiscipline::default()), + adjust_error: None, power: AtomicUsize::new(0), kernel_log: Mutex::new(Vec::new()), console: Mutex::new(Vec::new()), @@ -180,7 +186,10 @@ impl PrivilegedOps for FakeOps { fn adjust_clock(&self, adjustment: ClockAdjustment) -> Result { self.adjustments.lock().unwrap().push(adjustment); - Ok(ClockDiscipline::default()) + match self.adjust_error { + Some(error) if adjustment != ClockAdjustment::Query => Err(error), + _ => Ok(*self.discipline.lock().unwrap()), + } } fn power_off(&self, status: PowerOffStatus) { @@ -575,6 +584,91 @@ fn a_synchronization_steps_then_holds_the_host_frequency() { ); } +#[test] +fn a_capture_holds_the_declared_rate_and_keeps_the_discipline_that_it_found() { + // The kernel counts a declared rate of 2,300,000,123 Hz as 2.3 GHz, 123 Hz short, so the + // correction that counts the TSC at the declared rate is 0.0535 ppm negative: -3,505 in the + // kernel's scaled ppm, which count 65,536 to the ppm. + let mut ops = FakeOps::new([]); + ops.tsc_hz = Ok(2_300_000_123); + let found = ClockDiscipline { + scaled_ppm: -2_000_000, + maximum_error_us: 70, + estimated_error_us: 3, + status: 0x2081, + }; + *ops.discipline.lock().unwrap() = found; + let runtime = Runtime::new(&ops, false); + let hold = hold_declared_rate(&runtime).unwrap(); + assert_eq!( + ops.adjustments.lock().unwrap().as_slice(), + [ + ClockAdjustment::Query, + ClockAdjustment::Pin { + hold: FrequencyHold { + scaled_ppm: -3_505, + maximum_error_us: 70, + estimated_error_us: 3, + }, + status: 0x2081, + }, + ] + ); + // A capture that continues resumes the frequency correction that it found, and no more. + resume_rate(&runtime, hold).unwrap(); + assert_eq!( + ops.adjustments.lock().unwrap().last(), + Some(&ClockAdjustment::SetFrequency(-2_000_000)) + ); + assert_eq!(ops.adjustments.lock().unwrap().len(), 3); +} + +#[test] +fn a_capture_holds_no_correction_at_a_rate_that_the_kernel_counts_exactly() { + // A rate of a whole number of kHz is the kernel's own. + let ops = FakeOps::new([]); + let runtime = Runtime::new(&ops, false); + hold_declared_rate(&runtime).unwrap(); + let adjustments = ops.adjustments.lock().unwrap(); + assert!( + matches!( + adjustments.as_slice(), + [ + ClockAdjustment::Query, + ClockAdjustment::Pin { + hold: FrequencyHold { scaled_ppm: 0, .. }, + .. + }, + ] + ), + "{adjustments:?}" + ); +} + +#[test] +fn a_capture_that_cannot_hold_the_rate_reports_why() { + // Without the declared rate, which neither the boot checks nor the MSR gave, the capture + // touches nothing. + let mut ops = FakeOps::new([]); + ops.tsc_hz = Err(Errno::EACCES); + let runtime = Runtime::new(&ops, false); + let detail = hold_declared_rate(&runtime).unwrap_err(); + assert_eq!( + String::from_utf8(detail).unwrap(), + format!("MSR 0x40000022 read: {}", Errno::EACCES.desc()) + ); + assert!(ops.adjustments.lock().unwrap().is_empty()); + // A kernel that refuses the pin, or the resumption, is the diagnostic of its error. + let mut ops = FakeOps::new([]); + ops.adjust_error = Some(Errno::EPERM); + let runtime = Runtime::new(&ops, false); + let refused = format!("adjtimex: {}", Errno::EPERM.desc()); + let detail = hold_declared_rate(&runtime).unwrap_err(); + assert_eq!(String::from_utf8(detail).unwrap(), refused); + let detail = resume_rate(&runtime, RateHold { frequency: 7 }).unwrap_err(); + assert_eq!(String::from_utf8(detail).unwrap(), refused); +} + #[test] fn repair_failures_report_restore_events_and_power_off_only_when_enforcing() { for (report_only, strict, prefix, power) in [ diff --git a/nvx-rs/src/guest/time/test_mode.rs b/nvx-rs/src/guest/time/test_mode.rs index bfc324a52..5717776f6 100644 --- a/nvx-rs/src/guest/time/test_mode.rs +++ b/nvx-rs/src/guest/time/test_mode.rs @@ -480,7 +480,11 @@ fn test_correction(arguments: &[&[u8]], out: &mut dyn Write) -> Result<(), Comma hold.estimated_error_us ); } - ClockAdjustment::Query | ClockAdjustment::Frequency(_) | ClockAdjustment::ClearPhase => {} + ClockAdjustment::Query + | ClockAdjustment::Frequency(_) + | ClockAdjustment::ClearPhase + | ClockAdjustment::SetFrequency(_) + | ClockAdjustment::Pin { .. } => {} } Ok(()) } diff --git a/scripts/nvx_tools/microvm_test_scripts/restore-year-capture.sh b/scripts/nvx_tools/microvm_test_scripts/restore-year-capture.sh index 1a59c5300..e895d0480 100644 --- a/scripts/nvx_tools/microvm_test_scripts/restore-year-capture.sh +++ b/scripts/nvx_tools/microvm_test_scripts/restore-year-capture.sh @@ -1,9 +1,9 @@ # The simulated one-year restore's preparation before the capture # (doc/ci.md): a CPU-bound workload that the capture freezes, as the guest # freezes its workload cgroup for a capture with paired scratch, a timer that -# the simulated downtime expires, and, once the time daemon's next poll has -# passed, the clocks, the frequency correction, the workload's CPU time, and -# the kernel log's length before the capture. +# the simulated downtime expires, the clocks, the frequency correction, the +# workload's CPU time, and the kernel log's length before the capture, and a +# slew of the clock that is still in flight when the capture starts. set -eu year=/run/nvx-year workload=/sys/fs/cgroup/nvx-year-workload @@ -23,25 +23,15 @@ until grep -qx 'frozen 1' "$workload/cgroup.events"; do [ "$attempts" -le 500 ] sleep 0.01 done -# The time daemon changes the frequency correction only at its discipline -# polls, which hold the port lock that the capture takes too. It polls 16 s -# after its last poll at first, and 64 s after it once it has settled. Right -# after its next poll, the correction that this records is the one that the -# capture holds, since no other poll can come between them. A poll counts one -# accepted or one rejected time sample. The wait gives up after 70 s, within -# what the capture allows this preparation. -polls() { - grep -E '^(samples|rejected_samples)=' /run/nvx/time/state -} -polled=$(polls) -attempts=0 -until [ "$(polls)" != "$polled" ]; do - attempts=$((attempts + 1)) - [ "$attempts" -le 700 ] - sleep 0.1 -done cut -d' ' -f14,15 "/proc/$(cat "$year/busy.pid")/stat" >"$year/ticks" sed -n 's/^usage_usec //p' "$workload/cpu.stat" >"$year/usage" dmesg | wc -l >"$year/dmesg" sed -n 's/^frequency_ppb=//p' /run/nvx/time/state >"$year/frequency" /sbin/nvx-time-probe clock --measure-ms 0 >"$year/clock" +# A slew in flight at the capture, as the discipline's polls leave one: an +# adjtime of 5 ms, which the kernel slews at 500 ppm for 10 s, and whose rate +# it caches in its tick length from the next second on. The wait lets that +# second pass, and the capture, a fraction of a second later, must keep the slew +# out of the monotonic clocks' advance across the downtime. +adjtimex -o 5000 >/dev/null +sleep 1.2 diff --git a/scripts/nvx_tools/microvm_tests.py b/scripts/nvx_tools/microvm_tests.py index 034d58941..8bc324226 100644 --- a/scripts/nvx_tools/microvm_tests.py +++ b/scripts/nvx_tools/microvm_tests.py @@ -680,19 +680,29 @@ class MicrovmVirtioDevice(NamedTuple): RESTORE_YEAR_PATH = "/tmp/nvx-restore-year" RESTORE_YEAR_COMPLETION_MARKER = b"NVX-RESTORE-YEAR-OK" RESTORE_YEAR_REPORT_PREFIX = "NVX-RESTORE-YEAR " +# The host clock correction, in ppm, that the simulated year's captures boot +# with, through OpenVMM's host-rate-offset-ppm hook, so that the guest holds a +# large frequency correction when it requests its capture, as a host whose +# clock runs far from its TSC's declared rate would give it: a restore would +# count the downtime in the monotonic clocks at it, as a share of a year, if +# the capture did not hold the declared rate. The host's own correction, at +# most 250 ppm on a qualified host, composes with it within Linux's 500 ppm +# limit. +RESTORE_YEAR_HOST_PPM = 100 # How far, in seconds, the guest's raw monotonic clock may advance from the # real downtime plus the simulated one, against its realtime clock, at the # declared TSC rate as the kernel counts it, to the kHz, beyond the ppb by # which the kernel's clocksource multiplier, rounded to the nearest for its # 600 s range, counts cycles off that rate at any plausible rate up to 10 GHz; -# how far, in ppm of the simulated downtime, its frequency-corrected -# monotonic and boot-time clocks may advance beyond that rate times the -# frequency correction that it held before the capture, by the phase slew -# that its discipline ran then; and how far its realtime clock may stand -# from host UTC once the report arrives. +# how far, in ppb of the simulated downtime, its monotonic and boot-time +# clocks may advance from the downtime itself, since the guest holds its +# frequency correction at that rate through the capture, but the kernel's tick +# length rounds the multiplier of that rate down, up to one more unit than the +# raw clock's rounding to the nearest; and how far its realtime clock may +# stand from host UTC once the report arrives. RESTORE_YEAR_CLOCK_TOLERANCE_SECONDS = 1 RESTORE_YEAR_MULTIPLIER_PPB = 500 -RESTORE_YEAR_SLEW_PPM = 5 +RESTORE_YEAR_MONOTONIC_PPB = 1_000 RESTORE_YEAR_REALTIME_TOLERANCE_SECONDS = 10 # The timer that the simulated downtime expires: one hour, armed before the # capture, which must fire within this many seconds of the restore's clock @@ -702,11 +712,9 @@ class MicrovmVirtioDevice(NamedTuple): RESTORE_YEAR_TIMER_DELAY_SECONDS = 10 # How long, in seconds, the capture of each restore-downtime snapshot gives the # year's pre-capture preparation beyond the scenario's timeout: a second of its -# workload, up to 5 s for the freeze, and the wait for the time daemon's next -# discipline poll, which comes 16 s after the last one at first and 64 s after -# it once the daemon has settled, and which the preparation gives up on after -# 700 sleeps of 0.1 s. -RESTORE_YEAR_PREPARATION_SECONDS = 90 +# workload, up to 5 s for the freeze, and the second and a fifth that it lets +# pass for the slew that it leaves in flight. +RESTORE_YEAR_PREPARATION_SECONDS = 15 # How much longer, in seconds, a simulated one-year restore may take to print # the restore marker than the restore of the same snapshot after the real # downtime: Linux accumulates the year's TSC advance at its first tick, 128 @@ -8156,7 +8164,12 @@ def run_restore_downtime( # harness can stage its downtime check; the post-restore status # query waits for the restore to finish first. capture_snapshot( - [*boot_command, "--snapshot-destination", str(snapshot_path)], + [ + *boot_command, + *time_host_correction_hook(RESTORE_YEAR_HOST_PPM), + "--snapshot-destination", + str(snapshot_path), + ], snapshot_path, timeout=timeout, processors=processors, @@ -8352,8 +8365,9 @@ def check_simulated_year( advanced by the simulated downtime more than its realtime clock, which stands on host UTC, ``host_utc_ns`` once the report arrived: its raw monotonic clock by the TSC's advance at the declared rate, as the kernel - counts it to the kHz, and its monotonic and boot-time clocks by that - advance at the frequency correction that it held before the capture; the + counts it to the kHz, and its monotonic and boot-time clocks by the + downtime itself, whatever frequency correction the guest held before the + capture, since it held the declared rate through the capture; the one-hour timer that it armed before the capture fired within RESTORE_YEAR_TIMER_DELAY_SECONDS of the restore's clock step, before or after it; a frozen workload that gained no CPU time or cgroup usage while @@ -8388,26 +8402,26 @@ def check_simulated_year( after = {clock: _clock_ns(report[f"{clock}_after"]) for clock in clocks} real = after["realtime"] - before["realtime"] # The restore advanced the TSC by the downtime at the declared rate, - # which the kernel counts to the kHz, raw and at its frequency - # correction. + # which the kernel counts to the kHz in its raw clock. The guest held + # its frequency correction at that rate across the capture, so the + # kernel counts the advance in its monotonic and boot-time clocks at + # it too: they advance by the downtime. tsc_hz = int(report["tsc_hz"]) kernel_hz = tsc_hz // 1000 * 1000 frequency_ppb = int(report["frequency_ppb"]) raw = simulated * tsc_hz // kernel_hz - corrected = raw + raw * frequency_ppb // second - multiplier = simulated * RESTORE_YEAR_MULTIPLIER_PPB // second allowances = { - "monotonic_raw": RESTORE_YEAR_CLOCK_TOLERANCE_SECONDS * second + multiplier, + "monotonic_raw": RESTORE_YEAR_CLOCK_TOLERANCE_SECONDS * second + + simulated * RESTORE_YEAR_MULTIPLIER_PPB // second, "monotonic": RESTORE_YEAR_CLOCK_TOLERANCE_SECONDS * second - + multiplier - + simulated * RESTORE_YEAR_SLEW_PPM // 1_000_000, + + simulated * RESTORE_YEAR_MONOTONIC_PPB // second, } allowances["boottime"] = allowances["monotonic"] gains: list[str] = [] for clock, expected in ( ("monotonic_raw", raw), - ("monotonic", corrected), - ("boottime", corrected), + ("monotonic", simulated), + ("boottime", simulated), ): gained = after[clock] - before[clock] - real gains.append(f"{clock} {gained - simulated:+d} ns") @@ -8419,7 +8433,7 @@ def check_simulated_year( summary.append( "beyond the simulated downtime, " + ", ".join(gains) - + f" at {tsc_hz} Hz and {frequency_ppb} ppb" + + f" at {tsc_hz} Hz, after {frequency_ppb} ppb before the capture" ) lag = host_utc_ns - after["realtime"] if abs(lag) > RESTORE_YEAR_REALTIME_TOLERANCE_SECONDS * second: diff --git a/scripts/test_microvm_tests.py b/scripts/test_microvm_tests.py index 4a121f76a..abcc6b5c3 100644 --- a/scripts/test_microvm_tests.py +++ b/scripts/test_microvm_tests.py @@ -15019,6 +15019,18 @@ def test_restore_downtime_shares_one_window_and_waits_for_the_restore_marker( ], ) for call in capture.call_args_list: + # Every capture boots with a large host clock correction, so that + # the guest holds a large frequency correction at its request. + self.assertEqual( + call.args[0][:5], + [ + "openvmm", + "boot", + "--x-time-abi-test-hook", + f"host-rate-offset-ppm={microvm_tests.RESTORE_YEAR_HOST_PPM}", + "--snapshot-destination", + ], + ) self.assertEqual(call.kwargs["teardown_mode"], "host-terminate") self.assertEqual( call.kwargs["post_restore_script"], @@ -15292,10 +15304,14 @@ def test_restore_downtime_script_rejects_stalls_and_a_frozen_clock(self): ) # A restore after a simulated year: the hook's seconds, a TSC rate whose kHz - # truncate 476 Hz, and the frequency correction that the guest held. + # truncate 476 Hz, the frequency correction that the guest held before the + # capture, which it held at the declared rate through it, and how far, in + # ns, the monotonic and boot-time clocks stand short of the downtime, as the + # kernel's multiplier truncates the declared rate. _YEAR_SIMULATED = 31_535_910 _YEAR_TSC_HZ = 2_299_999_476 - _YEAR_FREQUENCY_PPB = -44_500 + _YEAR_FREQUENCY_PPB = 55_300 + _YEAR_SHORT_NS = 1_735_000_000 @staticmethod def _seconds(nanoseconds: int) -> str: @@ -15307,14 +15323,14 @@ def _year_report(self) -> tuple[dict[str, str], int]: second = 1_000_000_000 simulated = self._YEAR_SIMULATED * second raw = simulated * self._YEAR_TSC_HZ // 2_299_999_000 - corrected = raw + raw * self._YEAR_FREQUENCY_PPB // second + held = simulated - self._YEAR_SHORT_NS real = 73 * second realtime = 1_794_375_529_848_200_279 monotonic = 1_683_905_063 boottime = 1_683_910_779 raw_before = 1_500_000_000 step_realtime = realtime + real - 2_280_000_000 - boottime_after = boottime + real + corrected + boottime_after = boottime + real + held stepped = boottime_after - (realtime + real - step_realtime) report = { "stalls": "0", @@ -15325,7 +15341,7 @@ def _year_report(self) -> tuple[dict[str, str], int]: "monotonic_raw_before": self._seconds(raw_before), "boottime_before": self._seconds(boottime), "realtime_after": self._seconds(realtime + real), - "monotonic_after": self._seconds(monotonic + real + corrected), + "monotonic_after": self._seconds(monotonic + real + held), "monotonic_raw_after": self._seconds(raw_before + real + raw), "boottime_after": self._seconds(boottime_after), "ticks_before": "100", @@ -15356,13 +15372,14 @@ def test_simulated_year_check_accepts_clocks_at_their_rates(self): report, host_utc_ns = self._year_report() summary = self._check_year(report, host_utc_ns) # Raw time counts the TSC at the declared rate to the kHz, 476 Hz slow, - # and corrected time at the guest's -44.5 ppm correction besides. + # and the monotonic and boot-time clocks the downtime itself, whatever + # frequency correction the guest held before the capture. self.assertTrue( summary.startswith( "downtime 31535980000000000 ns; beyond the simulated downtime, " - "monotonic_raw +6526565081 ns, monotonic -1396821720352 ns, " - "boottime -1396821720352 ns at 2299999476 Hz and -44500 ppb; " - "timer " + "monotonic_raw +6526565081 ns, monotonic -1735000000 ns, " + "boottime -1735000000 ns at 2299999476 Hz, after 55300 ppb " + "before the capture; timer " ), summary, ) @@ -15375,6 +15392,47 @@ def test_simulated_year_check_accepts_clocks_at_their_rates(self): summary, ) + def test_simulated_year_check_bounds_the_monotonic_clocks_by_the_multiplier(self): + # The monotonic and boot-time clocks may stand a second and 1 ppm of the + # simulated downtime from it, as the kernel's multiplier truncates the + # declared rate, but no farther. + second = 1_000_000_000 + report, host_utc_ns = self._year_report() + simulated = self._YEAR_SIMULATED * second + real = microvm_tests._clock_ns( + report["realtime_after"] + ) - microvm_tests._clock_ns(report["realtime_before"]) + limit = second + simulated // 1_000_000 + for clock in ("monotonic", "boottime"): + before = microvm_tests._clock_ns(report[f"{clock}_before"]) + for error, accepted in ( + (limit, True), + (-limit, True), + (limit + 1, False), + (-limit - 1, False), + ): + with self.subTest(clock=clock, error=error): + shifted = { + **report, + f"{clock}_after": self._seconds( + before + real + simulated + error + ), + } + if clock == "boottime": + # The timer's delay counts from the boot-time clock. + shifted["timer_s"] = self._seconds( + microvm_tests._clock_ns(report["timer_s"]) + + error + + self._YEAR_SHORT_NS + ) + if accepted: + self._check_year(shifted, host_utc_ns) + else: + with self.assertRaisesRegex( + RuntimeError, f"{clock} time advanced " + ): + self._check_year(shifted, host_utc_ns) + def test_simulated_year_check_names_every_failure(self): second = 1_000_000_000 report, host_utc_ns = self._year_report() @@ -15400,10 +15458,18 @@ def test_simulated_year_check_names_every_failure(self): host_utc_ns, "the downtime is ", ), - # Monotonic time that advanced by the downtime exactly, not at the - # guest's frequency correction. + # Monotonic time that advanced at the frequency correction that the + # guest held before the capture, as when its kernel counts the + # downtime at it, not at the declared rate: 55 ppm of a year. ( - {"monotonic_after": self._seconds(before + 73 * second + simulated)}, + { + "monotonic_after": self._seconds( + before + + 73 * second + + simulated + + simulated * self._YEAR_FREQUENCY_PPB // second + ) + }, host_utc_ns, "monotonic time advanced ", ), @@ -15613,80 +15679,72 @@ def clock_report(suffix: str) -> str: self.assertEqual(result.returncode, 0, result.stdout + result.stderr) return result.stdout - def test_simulated_year_capture_records_the_correction_after_the_next_poll(self): - # The time daemon changes the frequency correction only at its polls, - # so the capture's preparation waits for the next one before it records - # the correction that the capture then holds: here at its third pause, - # as at the daemon's first cadence, or at its 640th, 64 s on, as at the - # settled daemon's. + def test_simulated_year_capture_leaves_a_slew_in_flight_after_it_records(self): + # The preparation records the clocks and the frequency correction at + # once, without waiting for the time daemon's next poll, and then + # starts an adjtime slew, which it lets a second pass over before the + # capture. shell = _posix_shell() if shell is None: self.skipTest("POSIX shell is unavailable") - # A poll counts an accepted sample, or a rejected one. - for poll_at, polled in ( - (3, "samples=2\\nrejected_samples=0"), - (640, "samples=1\\nrejected_samples=1"), - ): - with ( - self.subTest(poll_at=poll_at), - tempfile.TemporaryDirectory() as temporary, + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + year = root / "year" + year.mkdir() + (year / "stat").write_bytes(f"42 (sh) R {'0 ' * 10}97 3 0\n".encode()) + (root / "state").write_bytes( + b"generation=0\nsamples=1\nrejected_samples=0\nfrequency_ppb=-13553\n" + ) + posix = root.as_posix() + script = microvm_tests._read_script("restore-year-capture.sh") + for guest, host in ( + ("/run/nvx-year", f"{posix}/year"), + ("/sys/fs/cgroup/nvx-year-workload", f"{posix}/workload"), + ( + 'mkdir "$workload"', + "mkdir \"$workload\" && printf 'frozen 1\\n' " + '>"$workload/cgroup.events" && ' + "printf 'usage_usec 1008498\\n' >\"$workload/cpu.stat\"", + ), + ("sh -c 'while :; do :; done' &", ": &"), + ('"/proc/$(cat "$year/busy.pid")/stat"', '"$year/stat"'), + ("/run/nvx/time/state", f"{posix}/state"), + ("/sbin/nvx-time-probe clock --measure-ms 0", "nvx_time_probe"), + ("nvx-exit", "nvx_exit"), ): - root = Path(temporary) - year = root / "year" - year.mkdir() - (year / "stat").write_bytes(f"42 (sh) R {'0 ' * 10}97 3 0\n".encode()) - state = root / "state" - state.write_bytes( - b"generation=0\nsamples=1\nrejected_samples=0\n" - b"frequency_ppb=-13553\n" - ) - posix = root.as_posix() - script = microvm_tests._read_script("restore-year-capture.sh") - for guest, host in ( - ("/run/nvx-year", f"{posix}/year"), - ("/sys/fs/cgroup/nvx-year-workload", f"{posix}/workload"), - ( - 'mkdir "$workload"', - "mkdir \"$workload\" && printf 'frozen 1\\n' " - '>"$workload/cgroup.events" && ' - "printf 'usage_usec 1008498\\n' >\"$workload/cpu.stat\"", - ), - ("sh -c 'while :; do :; done' &", ": &"), - ('"/proc/$(cat "$year/busy.pid")/stat"', '"$year/stat"'), - ("/run/nvx/time/state", f"{posix}/state"), - ("/sbin/nvx-time-probe clock --measure-ms 0", "nvx_time_probe"), - ("nvx-exit", "nvx_exit"), - ): - self.assertIn(guest, script) - script = script.replace(guest, host) - prelude = ( - "waited=0\n" - "mountpoint() { :; }\n" - 'sleep() { case "$1" in\n' - " 3600) return 1 ;;\n" - " 0.1) waited=$((waited + 1))\n" - f' [ "$waited" -ne {poll_at} ] || printf ' - f"'generation=0\\n{polled}\\n" - f"frequency_ppb=-21000\\n' >'{posix}/state' ;;\n" - "esac; }\n" - "dmesg() { printf '[ 1.0] boot\\n[ 2.0] clocksource: tsc\\n'; }\n" - "nvx_time_probe() { printf 'nvx-time-probe 1.1 clock\\n'; }\n" - 'nvx_exit() { printf "NVX-EXIT %s\\n" "$1"; }\n' - ) - result = subprocess.run( - [shell], - input=prelude + script, - text=True, - capture_output=True, - timeout=300, - check=False, - ) - self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - self.assertNotIn("NVX-RESTORE-YEAR-FAIL", result.stdout) - self.assertEqual((year / "frequency").read_bytes(), b"-21000\n") - self.assertEqual((year / "ticks").read_bytes(), b"97 3\n") - self.assertEqual((year / "usage").read_bytes(), b"1008498\n") - self.assertEqual((year / "dmesg").read_bytes().strip(), b"2") + self.assertIn(guest, script) + script = script.replace(guest, host) + calls = root / "calls" + prelude = ( + "mountpoint() { :; }\n" + 'sleep() { [ "$1" != 3600 ] || return 1; ' + f"echo \"sleep $1\" >>'{posix}/calls'; }}\n" + f"adjtimex() {{ echo \"adjtimex $*\" >>'{posix}/calls'; }}\n" + "dmesg() { printf '[ 1.0] boot\\n[ 2.0] clocksource: tsc\\n'; }\n" + f"nvx_time_probe() {{ echo probe >>'{posix}/calls'; " + "printf 'nvx-time-probe 1.1 clock\\n'; }\n" + 'nvx_exit() { printf "NVX-EXIT %s\\n" "$1"; }\n' + ) + result = subprocess.run( + [shell], + input=prelude + script, + text=True, + capture_output=True, + timeout=300, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertNotIn("NVX-RESTORE-YEAR-FAIL", result.stdout) + self.assertEqual((year / "frequency").read_bytes(), b"-13553\n") + self.assertEqual((year / "ticks").read_bytes(), b"97 3\n") + self.assertEqual((year / "usage").read_bytes(), b"1008498\n") + self.assertEqual((year / "dmesg").read_bytes().strip(), b"2") + # The workload's second, the clocks, then the slew and the second + # that lets it take hold: no wait for a poll. + self.assertEqual( + calls.read_text().split("\n")[:-1], + ["sleep 1", "probe", "adjtimex -o 5000", "sleep 1.2"], + ) @staticmethod def _exhaustive_report(