diff --git a/CMakeLists.txt b/CMakeLists.txt index da5cecc42..4a40aeafd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -460,6 +460,12 @@ add_python_test(python_test_order_check ${CMAKE_SOURCE_DIR}/tests/python_test_or add_python_test(python_test_order_check_self ${CMAKE_SOURCE_DIR}/tests/python_test_order_check.py --self-test) add_python_test(ci_targets ${CMAKE_SOURCE_DIR}/tools/ci/ci_targets_check.py --check --root ${CMAKE_SOURCE_DIR}) add_python_test(ci_targets_self ${CMAKE_SOURCE_DIR}/tools/ci/ci_targets_check.py --self-test) +# engine#2641: #2625 merged with its own conflict markers still in docs/nvme-iommu-fault-triage.md, inside a ```bash +# block, so the page did not render and nothing noticed -- the merge kept them because the file was never read back. +# The check is stateful on purpose: a bare `=======` is a Markdown underline or a shell rule (the first version +# flagged 10 of those on a clean tree), so a separator counts only inside an unterminated conflict block. +add_python_test(conflict_marker ${CMAKE_SOURCE_DIR}/tools/ci/conflict_marker_check.py --check --root ${CMAKE_SOURCE_DIR}) +add_python_test(conflict_marker_self ${CMAKE_SOURCE_DIR}/tools/ci/conflict_marker_check.py --self-test) # engine#803 gate, second half: the comparator that turns two --dump-logits directories into the KLD verdict the # P4 gate asks for. No device and no model, so the gate is one command when a ticket is free; the self-test pins # the four cases a KLD implementation gets wrong (identical -> exactly 0, a shift is positive and directional, a diff --git a/docs/nvme-iommu-fault-triage.md b/docs/nvme-iommu-fault-triage.md index 95ada4752..cc96d97d8 100644 --- a/docs/nvme-iommu-fault-triage.md +++ b/docs/nvme-iommu-fault-triage.md @@ -134,8 +134,10 @@ only if you want to confirm that, not as a discriminator. ```bash # 1. pick the top faulting IOVA from the census -<<<<<<< HEAD -======= +# 2. kprobe iommu_unmap_fast on that iova alone (filter), stacktrace on, capture for minutes in the background +# 3. in the same capture window, note the fault timestamps for that iova +``` + **The interval is seconds, not milliseconds (measured 2026-11-16).** A filtered probe on one faulting IOVA (`0xe1e18000`), captured over a boot's first 28 minutes, found **60 unmaps of that page, every one a single 4 KiB page, 59 of them from `` (interrupt context)**: the NVMe completion path repeatedly maps and tears down one @@ -144,20 +146,83 @@ pooled page. **Three of the four faults on it** followed an unmap of that same p page within 60 s** -- the nearest is **+74 s**, after the fault -- so this capture correlates three faults and shows the ceiling on the fourth; a wider window does not help it, and the earlier "19.60 s" I quoted for it came from a different capture and does not reproduce on this one. So `tools/diag/unmap_fault_order.py` defaults to **30 s** and -the page's own numbers are **5-25 s**: a millisecond window reports "nothing correlated" on a capture that contains -the correlation. Because the same sentence is printed whether the window is too narrow or the fault genuinely has no -nearby unmap, **the negative now names the window it used and the measured interval** -- a default alone does not fix -the failure, only a negative that says *at what scale* it found nothing does. +the page's own numbers are **5-25 s on that sparse stream**: a millisecond window reports "nothing correlated" on a +capture that contains the correlation. Because the same sentence is printed whether the window is too narrow or the +fault genuinely has no nearby unmap, **the negative names the window it used and the measured interval** -- a default +alone does not fix the failure, only a negative that says *at what scale* it found nothing does. + +**But "seconds" is not the rate of this stream -- it is the rate of that capture.** The same page measured again on a +busier boot is mapped and unmapped **about 470 times a second**: `nvme_prep_rq` maps it through `dma_map_phys` -> +`iommu_map`, and `nvme_pci_complete_batch` unmaps it from the completion IRQ through both `dma_unmap_phys` and +`dma_iova_unlink`/`dma_iova_destroy` (150 s, one IOVA, filtered, counts 27644 / 35907 / 33141). **So a wide window is +the wrong instrument in the other direction**: at that rate a +/-30 s window holds ~14,000 unmaps of the page and "the +nearest one was 31 us before the fault" is not evidence of anything. The tool therefore **carries the count** and flags +a pairing whose window held more than three unmaps of the same IOVA: +`[1 of N unmaps of this iova in the window -- NOT discriminating, narrow --window]`. On the burst capture above it +fires on the -5.65 s pairing immediately (`1 of 7`), which is the honest reading: **the interval is real, its precision +is not.** + +**Three ways a filtered kprobe silently measures nothing**, all hit while taking the counts above: + +1. **`tracing_on=0` with `enable=1`** records nothing, and the empty buffer reads exactly like "0 events". Check + `tracing_on` and `entries-written` on every capture, and run a **positive control** first (30 s of events, abort if + zero). +2. **A `stacktrace` trigger suppresses the event's own line.** The dump is `` blocks with no `iova=` text + at all, so a filter that is working perfectly looks like it never applied. Count events *without* the trigger first. + And `stacktrace:unlimited` must be cleared (`echo '!stacktrace:0' > .../trigger`) before `-:kprobes/x` will delete + the event -- otherwise the delete fails with `echo: I/O error` and the probe stays armed. +3. **tracefs rejects range filters, and the failed write leaves the probe UNFILTERED.** + `(iova + size > 0xX && iova <= 0xX)` gives `parse_error: Invalid operator`; the event then records **every** unmap + (1,947 in 20 s, against 18 in 30 s for the working exact filter). **Read the filter file back after writing it**, and + keep in mind that an exact-iova filter cannot see a *larger* mapping that covers the page -- for this driver that is + tolerable because its unmaps are all 4 KiB (60 of 60 measured), but it is a measured caveat, not a proof. + +**Arm the map side too, unfiltered, and the argument for one address stops being needed (measured 11-16 06:42Z).** +A filtered capture on five addresses chosen from the fault census recorded **zero events in 30 s** and aborted on its +own positive control -- because **the fault target moves**: the last three faults were all on an address the list did +not contain. The capture that worked armed `iommu_map` and `iommu_unmap_fast` with **no filter** for 13 minutes +(44 unmaps/s is what a quiet minute looks like; a busy one is ~1,000/s, and the per-CPU ring still covered all 795 s). +Of the 8 faults in that window: + +``` +0xebf50000 x4 faults NO map or unmap of this iova anywhere in the 795 s +0x6925f000 x1 nearest map +230 ms AFTER the fault +0xe407f000 x2 nearest map/unmap -58 s +0xe1d00000 x1 nearest -347 s +``` + +**Four of eight faults hit an IOVA that was never mapped and never unmapped at any point in 795 seconds of a capture +holding 252,704 map/unmap events** -- so the translation is not stale, it is **absent, and nothing re-creates it**. That +moves the question to the descriptor: the device writes to an address the driver has already finished with, which is a +DMA-mapping lifetime bug, not a TLB or lazy-invalidation one. `0x6925f000` is the exception and worth separating: it is +**re-mapped 230 ms after** the fault, the profile of a live pool page faulting before its re-map. + +The kprobe recipes, corrected against the kernel's own argument order -- the first one was wrong in this page's tool: + +``` +iommu_map(domain, iova, paddr, size, prot) -> iova=%si size=%cx # %dx is PADDR, and a probe + # using it printed size=0x138ec90000 +iommu_unmap_fast(domain, iova, size, ...) -> iova=%si size=%dx +``` + +**A task name can contain a space** (`next-server (v1-8386`), so a regex anchored with `^\s+\S+\s+\[` silently dropped +**2,176 of 115,941 map lines** -- 1.9 %, which is exactly the fraction that can contain the event being looked for. +Anchor on the timestamp instead, and reconcile the parse count against `grep -c` before believing a negative. + +**And the reverse is just as wrong: a positive must say whether it discriminates.** The same page's cadence is +workload-dependent -- a later capture on this box (05:40Z, 150 s, one IOVA filtered) found the NVMe paths mapping it +from `nvme_prep_rq` through `dma_map_phys` and unmapping it from `nvme_pci_complete_batch` through +`iommu_unmap_fast`, **about 470 times a second**. At that rate a ±30 s window holds ~14,000 unmaps of the page, so +"the nearest one was 31 us before the fault" is not evidence of anything. **The tool now carries the count**: a +pairing whose window held more than three unmaps of the same IOVA is printed with +`[1 of N unmaps of this iova in the window -- NOT discriminating, narrow --window]`. On the burst capture above that +fires immediately -- the -5.65 s pairing is `1 of 7` -- which is the honest reading of it. **One probe owns the shared buffer.** A kprobe on `iommu_unmap_fast` with **no filter** wrote 579,000 events and left a filtered probe on the same function with **zero** -- the trace ring buffer is shared, so the *narrow* probe (one IOVA, one call site) is the one that should be armed, and checking `grep -c ':' trace` before trusting a capture is the same "prove the plumbing" rule two paragraphs up. ->>>>>>> 2b1faf1b2 (diag: a negative names the window it used, and the window reads in seconds) -# 2. kprobe iommu_unmap_fast on that iova alone (filter), stacktrace on, capture for minutes in the background -# 3. in the same capture window, note the fault timestamps for that iova -``` The question the ordering answers: does a fault follow an unmap of the *same* address, or does it arrive while the address is still mapped? The first is a write-after-unmap — the device dirtying a buffer the CPU released, or the driver diff --git a/docs/pre-pr-checks.md b/docs/pre-pr-checks.md index f03fda321..0182d6702 100644 --- a/docs/pre-pr-checks.md +++ b/docs/pre-pr-checks.md @@ -32,6 +32,7 @@ below says why. The table's order is the runner's order: cheapest and most commo | `python3 tools/ci/tests_referenced_check.py` | a test, tool self-test or python test exists that nothing runs | | `python3 tools/ci/ci_targets_check.py --check` | a CI row names a target or path that does not exist | | `python3 tools/ci/tmp_path_check.py --check` | a new `/tmp` or `/dev/shm` path, where builds and dumps belong under `~/wt` or `~/.cache/scratch-*` | +| `python3 tools/ci/conflict_marker_check.py --check` | a merge-conflict marker committed at the start of a line -- #2625 shipped `<<<<<<< HEAD` inside a ```bash block and the page stopped rendering, and nothing read it back | | `python3 tools/ci/submodule_pin_check.py --check` | a submodule's checkout is not the commit the tree pins | | `python3 tools/ci/cmake_configure_check.py --check` | `CMakeLists.txt` no longer configures -- CI's Configure step would fail, and no text check can see a line that starts correctly and ends wrong | | `python3 tools/ci/registration_loss_check.py --check --base origin/main` | your branch would drop rows `main` has | diff --git a/scripts/pre-pr-checks.sh b/scripts/pre-pr-checks.sh index 23b107889..27ab399c9 100755 --- a/scripts/pre-pr-checks.sh +++ b/scripts/pre-pr-checks.sh @@ -63,6 +63,7 @@ checks=( "tools/ci/tests_referenced_check.py" "tools/ci/ci_targets_check.py --check" "tools/ci/tmp_path_check.py --check" + "tools/ci/conflict_marker_check.py --check" "tools/ci/submodule_pin_check.py --check" # engine#2445: the tree must CONFIGURE, not just parse as text. A rebase resolution left one line # ending `…_test.py)ms)` and every check above stayed green -- they read files, run Python and diff diff --git a/tools/ci/conflict_marker_check.py b/tools/ci/conflict_marker_check.py new file mode 100644 index 000000000..5b085d599 --- /dev/null +++ b/tools/ci/conflict_marker_check.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Merge-conflict markers committed to the tree. + +Engine#2625 merged with its own conflict markers still in `docs/nvme-iommu-fault-triage.md` -- `<<<<<<< HEAD` at +line 137 and `>>>>>>> 2b1faf1b2 …` at line 157, both **inside a ```bash block**, so the numbered recipe was split by +paragraphs of prose, the fence closed in the wrong place, and the page other lanes reason from did not render. The +merge kept the markers because the conflict was resolved by hand in the wrong place and the file was never read back: +the same shape as engine#2622 (a parser returning `None` for every real line), #2630 (`0 unmaps` indistinguishable +from a dead probe) and llama.cpp#406 (counters that do not sum) -- an output nobody checked. + +The check is deliberately narrow. It looks for a marker at the **start of a line** (`<<<<<<<`, `=======`, `>>>>>>>`, +`|||||||`), which is what git writes; prose is free to quote markers mid-line, and this file is such prose. + +Usage: + tools/ci/conflict_marker_check.py --check [--root .] # rc 1 on any marker, 0 when clean + tools/ci/conflict_marker_check.py --self-test # rc 1 unless the arms below behave +""" +from __future__ import annotations + +import argparse +import os +import pathlib +import re +import sys + +# `=======` is only a conflict separator INSIDE a conflict block. On its own it is a Markdown setext underline or a +# shell rule, and the first version of this check flagged 10 of those across 4 files on a tree with no conflicts at +# all -- a check that fails on clean input is one nobody keeps. So the scan is stateful: `open` counts the markers +# git writes, and `=======` is reported only while a `<<<<<<<` is still open. +MARKERS = { + "<<<<<<<": "conflict start", + "|||||||": "conflict base", + ">>>>>>>": "conflict end", +} +EQUALS = "=======" + +SKIP_DIRS = {".git", "build", "build-host", "build-shp", "__pycache__", "third_party", "vendor"} +SKIP_SUFFIX = {".onnx", ".gguf", ".q4nx", ".fconst", ".bin", ".so", ".a", ".o", ".hsaco", ".png", ".jpg", ".pdf"} + + +def scan(root: pathlib.Path) -> list[tuple[str, int, str]]: + """[(relpath, line number, marker kind)] for every marker at the start of a line in a text file.""" + hits: list[tuple[str, int, str]] = [] + for dirpath, dirnames, filenames in os.walk(root): + dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS] + for fn in filenames: + p = pathlib.Path(dirpath) / fn + if p.suffix.lower() in SKIP_SUFFIX: + continue + try: + text = p.read_text(errors="replace") + except OSError: + continue + if "\0" in text[:4096]: # binary that slipped past the suffix list + continue + open_block = False + for i, line in enumerate(text.splitlines(), 1): + head = line[:8] + if head.startswith("<<<<<<<"): + open_block = True + hits.append((str(p.relative_to(root)), i, "conflict start")) + elif head.startswith("|||||||"): + hits.append((str(p.relative_to(root)), i, "conflict base")) + elif head.startswith(">>>>>>>"): + open_block = False + hits.append((str(p.relative_to(root)), i, "conflict end")) + elif open_block and head.startswith(EQUALS): + hits.append((str(p.relative_to(root)), i, "conflict separator")) + return hits + + +def main(argv: list[str]) -> int: + g = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + g.add_argument("--check", action="store_true") + g.add_argument("--self-test", action="store_true") + g.add_argument("--root", default=".") + a = g.parse_args(argv) + if a.self_test: + return self_test() + if not a.check: + g.print_help() + return 2 + root = pathlib.Path(a.root) + if not root.is_dir(): + print(f"conflict_marker: no such root {root}") + return 2 + hits = scan(root) + if hits: + for rel, n, what in hits[:40]: + print(f" {rel}:{n}: {what}") + if len(hits) > 40: + print(f" ... and {len(hits) - 40} more") + print(f"conflict_marker: {len(hits)} marker(s) committed in {len({h[0] for h in hits})} file(s)") + return 1 + print("conflict_marker: 0 marker(s) at the start of any line") + return 0 + + +def self_test() -> int: + import tempfile + + ok = True + + def chk(cond: bool, what: str) -> None: + nonlocal ok + print(("ok " if cond else "FAIL ") + what) + ok = ok and cond + + with tempfile.TemporaryDirectory() as td: + root = pathlib.Path(td) + (root / "docs").mkdir() + (root / "build").mkdir() + + clean = ( + "# Title\n\n" + "A paragraph that QUOTES the markers mid-line, as this tool's own docstring does: `<<<<<<< HEAD` is what\n" + "git writes, and a `=======` rule inside a sentence is not a conflict.\n\n" + "```\n# not a conflict either\n=======\n```\n" + ) + # 1. a document that only mentions the markers mid-line is clean -- otherwise this check would fail on its + # own docstring, and a check that cannot pass on a clean tree is a check nobody keeps. + (root / "docs" / "clean.md").write_text(clean) + chk(scan(root) == [], "markers quoted mid-line, and a bare ======= line with no open block, are not conflicts") + (root / "docs" / "rule.md").write_text("Title\n=====\n\nsome prose\n\n=======\n\nmore prose\n") + chk(scan(root) == [], "a setext underline and a rule that look like markers are not conflicts either") + (root / "docs" / "rule.md").unlink() + + # 2. the shape engine#2625 shipped: markers at the start of a line, inside a fence + (root / "docs" / "broken.md").write_text("```bash\n# 1. pick the IOVA\n<<<<<<< HEAD\n=======\nprose\n>>>>>>> 2b1faf1b2 (msg)\n# 2. arm it\n```\n") + hits = scan(root) + kinds = {h[2] for h in hits} + chk(len(hits) == 3 and kinds == {"conflict start", "conflict separator", "conflict end"}, + "the #2625 shape yields start, separator and end (3 markers)") + chk(any(h[0].endswith("broken.md") and h[1] == 3 for h in hits), "... and the line number is the marker's own") + + # 3. build dirs and binaries are not scanned: a build tree holds no committed text, and a binary can carry + # bytes that look like a marker. + (root / "build" / "x.md").write_text("<<<<<<< HEAD\n") + (root / "blob.bin").write_bytes(b"<<<<<<< HEAD\n") + hits = scan(root) + chk(len(hits) == 3, "build/ and .bin are skipped") + + # 4. rc follows: 0 clean, 1 dirty, 2 no root + chk(main(["--check", "--root", str(root)]) == 1, "--check returns 1 when a marker is committed") + (root / "docs" / "broken.md").unlink() + chk(main(["--check", "--root", str(root)]) == 0, "--check returns 0 on the clean tree") + chk(main(["--check", "--root", str(root / "nope")]) == 2, "--check returns 2 for a root that is not a directory") + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/tools/diag/unmap_fault_order.py b/tools/diag/unmap_fault_order.py index d51d660c7..f1acf072b 100755 --- a/tools/diag/unmap_fault_order.py +++ b/tools/diag/unmap_fault_order.py @@ -105,19 +105,24 @@ def parse_faults(lines, journal_epoch): def correlate(unmaps, faults, window_us): - """[(fault, delta_us, unmap)] -- the nearest unmap of the same iova within the window, or None.""" + """[(fault, delta_us, unmap, n)] -- the nearest unmap of the same iova within the window, or None, and how many + unmaps of that iova the window held. `n` is the discriminator: a page recycled ~470 times a second puts thousands + of unmaps inside a 30 s window, and then "the nearest one" is not evidence of anything.""" by_iova = {} for t, iova, size in unmaps: by_iova.setdefault(iova, []).append((t, size)) rows = [] for t, iova, text in faults: best = None + n = 0 if t is not None: for ut, size in by_iova.get(iova, ()): d = (ut - t) * 1e6 # negative = the unmap came first - if abs(d) <= window_us and (best is None or abs(d) < abs(best[1])): - best = ((ut, iova, size), d) - rows.append((t, iova, text, best)) + if abs(d) <= window_us: + n += 1 + if best is None or abs(d) < abs(best[1]): + best = ((ut, iova, size), d) + rows.append((t, iova, text, best, n)) return rows @@ -146,15 +151,19 @@ def render(rows, unmaps, window_us): lines.append(f" sizes: {sorted({u[2] for u in unmaps})[0]:#x} .. {sorted({u[2] for u in unmaps})[-1]:#x}") hit = [r for r in rows if r[3] is not None] lines.append(f"faults parsed: {len(rows)}; with an unmap of the same iova within +-{_span(window_us)}: {len(hit)}") - for t, iova, _text, best in hit[:20]: + for t, iova, _text, best, n in hit[:20]: (ut, _u, size), d = best verdict = "unmap BEFORE fault (translation gone)" if d < 0 else "unmap AFTER fault (stale mapping or other cause)" - lines.append(f" iova {iova:#x} size {size:#x}: unmap {d:+.0f} us, {verdict}") + # A positive must say whether it discriminates. Measured 2026-11-16: one NVMe pool page is mapped and unmapped + # ~470 times a second, so a 30 s window holds thousands of unmaps of it and "the nearest" means nothing. + disc = "" if n <= 3 else f" [1 of {n} unmaps of this iova in the window -- NOT discriminating, narrow --window]" + lines.append(f" iova {iova:#x} size {size:#x}: unmap {d:+.0f} us, {verdict}{disc}") if rows and not hit: # A negative must name its own condition. The same sentence at a 5 ms window and at 30 s read alike, and the # 5 ms one was simply wrong, so the scale is in the line and the remedy is named with it. lines.append(f" no fault shares an iova with an unmap inside +-{_span(window_us)}: ordering not shown by " - f"this capture AT THIS WINDOW -- the measured unmap-to-fault interval on this box is 5-25 s, " + f"this capture AT THIS WINDOW -- the interval is seconds on a sparse stream and sub-millisecond " + f"where the page is recycled (measured 470/s on one NVMe pool page), " f"so widen --window before reading this as 'no unmap preceded the fault'") return lines @@ -203,7 +212,7 @@ def chk(cond, msg): # The fault is 100 s from any unmap: at 30 s that is still a miss, which is the shape a too-narrow window gives. out = " ".join(render(correlate(unm, [(300.000, 0x4128000, "f5")], 30_000_000), unm, 30_000_000)) chk("+-30 s" in out and "30000000 us" not in out, "a 30 s window reads as '+-30 s', not '+-30000000 us'") - chk("AT THIS WINDOW" in out and "5-25 s" in out, + chk("AT THIS WINDOW" in out and "sparse stream" in out, "a negative names the window it used and the measured interval, not 'nothing correlated' bare") chk("+-5000 us" in " ".join(render(correlate(unm, [(100.500, 0x4128000, "f3")], 5000), unm, 5000)), "a sub-second window still reads in microseconds") @@ -221,6 +230,15 @@ def chk(cond, msg): hit = correlate(near, pf[:1], 30_000_000) chk(hit and hit[0][3] is not None and hit[0][3][1] < 0, "a padded fault correlates with an unmap 31 us earlier (BEFORE), not an uncorrelated negative") + # 9. a POSITIVE must say whether it discriminates: a page recycled hundreds of times a second puts thousands of + # unmaps in a wide window, and then "the nearest one" is not evidence. Measured 2026-11-16: one NVMe pool page + # is mapped and unmapped ~470 times a second (nvme_prep_rq maps it, nvme_pci_complete_batch unmaps it). + hot = [(100.0 + i * 0.001, 0x4128000, 0x1000) for i in range(500)] # 500 unmaps over 0.5 s + hr = correlate(hot, [(100.4, 0x4128000, "f6")], 30_000_000) + chk(hr[0][4] == 500 and "NOT discriminating" in " ".join(render(hr, hot, 30_000_000)), + "a page recycled hundreds of times a second is flagged as not discriminating") + chk("NOT discriminating" not in " ".join(render(correlate(unm, [(100.0005, 0x4128000, "f1")], 5000), unm, 5000)), + "a lone unmap in the window is not flagged") return 0 if ok else 1 @@ -232,7 +250,9 @@ def main(): # 5.6 s / 24.7 s / 24.7 s earlier, and a 5 ms default answered "no fault shares an iova with an unmap" on the # very capture that contains them. The interval is seconds here; a small default hides the finding. ap.add_argument("--window", type=float, default=30_000_000.0, - help="window in microseconds (default 30,000,000 = 30 s; measured interval on this box is 5-25 s)") + help="window in microseconds (default 30,000,000 = 30 s). The interval is seconds on a sparse stream " + "and sub-millisecond on a recycled page (470/s measured), so the count of unmaps in the " + "window is reported: a pairing with many candidates is not evidence") ap.add_argument("--epoch-offset", type=float, default=0.0, help="seconds to add to journal stamps to reach the trace's boot clock") ap.add_argument("--self-test", action="store_true")