From ec7bfd7c85755ab1957846ac09e85fd2dc3bda75 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 19:02:58 -0400 Subject: [PATCH 1/3] diag: correlate an unmap with the fault it might have caused (engine#2478/#2568) The fault stream on this box is not driven by volume, and that is now measured four ways, each with its own window: 20 min of O_DIRECT reads (3 TiB), 10 min of mmap/touch/munmap churn (157 TiB), a 213 s build issuing 60,459 dma_iova_destroy calls, and three boots of uptime. Every one produced zero faults, counted with printk's suppressed counter as well as the printed lines. What is left is ordering: a fault needs the unmap of a page to land while a command that references it is still outstanding. No load can produce that; it needs both events in one capture. This tool reads a tracefs capture with a kprobe on the unmap path plus the kernel log, and for every fault reports the nearest unmap of the SAME iova with a signed delta in microseconds -- negative means the unmap came first (the translation was gone when the device wrote), positive means it followed. Three refusal paths, each of them a lesson this week already paid for: * an empty capture exits 3 with "an empty capture is not a negative, was the probe armed before the load" rather than printing zero correlated faults; * a fault with no timestamp is counted but not correlated, and the note says so; * the wall-vs-boot clock mismatch is named when the journal is on ISO stamps, because deltas across two clocks are meaningless numbers that look like data. --self-test pins the four shapes (before / after / outside the window / no timestamp), plus the tracefs line shape; my first version of the parser anchored the timestamp at the start of the line and parsed nothing, because tracefs writes it after the comm/pid/cpu/flags fields. --- tools/diag/unmap_fault_order.py | 190 ++++++++++++++++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100755 tools/diag/unmap_fault_order.py diff --git a/tools/diag/unmap_fault_order.py b/tools/diag/unmap_fault_order.py new file mode 100755 index 000000000..2d5ffcc4c --- /dev/null +++ b/tools/diag/unmap_fault_order.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-bong +# SPDX-License-Identifier: Apache-2.0 +"""Engine#2478/#2568: does an unmap of a page land while a command still references it? + +The fault stream on this box is NOT driven by volume -- measured, each with its own window: 20 min of `O_DIRECT` +reads (3 TiB), 10 min of mmap/touch/munmap churn (157 TiB), a 213 s build issuing **60,459** `dma_iova_destroy` +calls, and three boots' worth of uptime. Every one of them produced zero faults (printed *and* suppressed), so what +is left is **ordering**: a fault needs the unmap of a page to land while a command that references it is still +outstanding. That is not measurable by a load; it needs the two events in one capture. + +This reads two inputs and correlates them: + + * `--trace FILE` a `tracefs` capture with a kprobe on the unmap path, e.g. + echo 'p:unm iommu_unmap_fast iova=%di size=%si' >> kprobe_events + ... (run the workload) ... cat trace > cap.txt + * `--journal FILE` (or stdin) kernel-log lines carrying `IO_PAGE_FAULT ... address=0x...` + +For every fault it reports the nearest unmap of the SAME iova, with the signed delta in microseconds: +negative = the unmap preceded the fault (the translation was gone when the device wrote), positive = the unmap +followed it (a stale mapping or a different cause). `--window US` bounds what counts as "the same event". + +Exit status: 0 when both inputs parsed (the tool reports, it does not judge -- a zero-correlation capture and a +correlated one are both valid observations), 2 on an unreadable input, 3 when the trace has no unmap events at all +(an empty capture is not a negative -- engine#2531's first trap). +""" +import argparse +import re +import sys + +FAULT_RE = re.compile(r"IO_PAGE_FAULT domain=\S+ address=(0x[0-9a-fA-F]+)") +UNMAP_RE = re.compile(r"unm:.*iova=(0x[0-9a-fA-F]+)(?:.*size=(0x[0-9a-fA-F]+))?") +# tracefs puts the timestamp AFTER the comm/pid/cpu/flags fields (` bash-1 [001] .... 100.000000: ev: ...`), +# so it is searched mid-line, not anchored -- my first version anchored it and parsed nothing. +EPOCH_RE = re.compile(r"(\d+\.\d+):") +# Two stamp shapes, because the two sources that carry these lines disagree: `journalctl -k` writes an ISO stamp +# and a `dmesg`/trace-style capture writes `[1234.567890]` seconds-since-boot. A fault line is only correlatable +# when its clock is the trace's, so the seconds form is preferred when both appear. +ISO_RE = re.compile(r"(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d+)?)") +BOOT_RE = re.compile(r"^\s*\[(\d+\.\d+)\]") + + +def read_lines(path): + if path in (None, "-"): + return sys.stdin.read().splitlines() + with open(path, "r", errors="replace") as f: + return f.read().splitlines() + + +def parse_trace(lines): + """[(t, iova, size)] from a tracefs capture. Lines without a leading timestamp are skipped.""" + out = [] + for line in lines: + m = EPOCH_RE.search(line) + if not m: + continue + mu = UNMAP_RE.search(line) + if not mu: + continue + size = int(mu.group(2), 16) if mu.group(2) else 0x1000 + out.append((float(m.group(1)), int(mu.group(1), 16), size)) + return out + + +def parse_faults(lines, journal_epoch): + """[(t, iova, text)] from kernel-log lines; `journal_epoch` maps an ISO stamp to seconds.""" + out = [] + for line in lines: + mf = FAULT_RE.search(line) + if not mf: + continue + mb = BOOT_RE.search(line) + if mb: # already on the trace's boot clock + t = float(mb.group(1)) + else: + mi = ISO_RE.search(line) + t = journal_epoch(mi.group(1)) if mi else None + out.append((t, int(mf.group(1), 16), line.strip())) + return out + + +def correlate(unmaps, faults, window_us): + """[(fault, delta_us, unmap)] -- the nearest unmap of the same iova within the window, or None.""" + by_iova = {} + for t, iova, size in unmaps: + by_iova.setdefault(iova, []).append((t, size)) + rows = [] + for t, iova, text in faults: + best = None + if t is not None: + for ut, size in by_iova.get(iova, ()): + d = (ut - t) * 1e6 # negative = the unmap came first + if abs(d) <= window_us and (best is None or abs(d) < abs(best[1])): + best = ((ut, iova, size), d) + rows.append((t, iova, text, best)) + return rows + + +def render(rows, unmaps, window_us): + lines = [f"unmap events parsed: {len(unmaps)} over {len({u[1] for u in unmaps})} distinct iova(s)"] + if unmaps: + lines.append(f" sizes: {sorted({u[2] for u in unmaps})[0]:#x} .. {sorted({u[2] for u in unmaps})[-1]:#x}") + hit = [r for r in rows if r[3] is not None] + lines.append(f"faults parsed: {len(rows)}; with an unmap of the same iova within +-{window_us:.0f} us: {len(hit)}") + for t, iova, _text, best in hit[:20]: + (ut, _u, size), d = best + verdict = "unmap BEFORE fault (translation gone)" if d < 0 else "unmap AFTER fault (stale mapping or other cause)" + lines.append(f" iova {iova:#x} size {size:#x}: unmap {d:+.0f} us, {verdict}") + if rows and not hit: + lines.append(" no fault shares an iova with an unmap inside the window: ordering not shown by this capture") + return lines + + +def self_test(): + """Four shapes, each asserted: before, after, out-of-window, and an empty capture.""" + ok = True + + def chk(cond, msg): + nonlocal ok + print(("ok " if cond else "FAIL ") + msg) + ok = ok and cond + + unm = [(100.000000, 0x4128000, 0x1000), (200.000000, 0x5128000, 0x1000)] + # 1. unmap precedes the fault by 500 us -> reported as BEFORE + rows = correlate(unm, [(100.0005, 0x4128000, "f1")], 5000) + chk(rows[0][3] is not None and rows[0][3][1] < 0, "an unmap 500 us before the fault is reported as BEFORE") + # 2. unmap follows it + rows = correlate(unm, [(199.9995, 0x5128000, "f2")], 5000) + chk(rows[0][3] is not None and rows[0][3][1] > 0, "an unmap 500 us after the fault is reported as AFTER") + # 3. same iova but far outside the window -> not correlated + rows = correlate(unm, [(100.500, 0x4128000, "f3")], 5000) + chk(rows[0][3] is None, "an unmap 500 ms away is not counted in a 5 ms window") + # 4. an unread fault time cannot be correlated (and is kept in the count) + rows = correlate(unm, [(None, 0x4128000, "f4")], 5000) + chk(rows[0][3] is None and len(rows) == 1, "a fault with no timestamp is counted but not correlated") + # 5. the parser reads a real tracefs line shape + got = parse_trace([" bash-1234 [001] .... 100.000000: unm: (iommu_unmap_fast+0x0/0x20) iova=0x4128000 size=0x1000"]) + chk(len(got) == 1 and got[0][1] == 0x4128000 and got[0][2] == 0x1000, "a tracefs unmap line parses (iova, size)") + return 0 if ok else 1 + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--trace", help="a tracefs capture with a kprobe on the unmap path") + ap.add_argument("--journal", action="append", help="kernel-log lines (repeatable; default stdin)") + ap.add_argument("--window", type=float, default=5000.0, help="window in microseconds (default 5000)") + ap.add_argument("--epoch-offset", type=float, default=0.0, + help="seconds to add to journal stamps to reach the trace's boot clock") + ap.add_argument("--self-test", action="store_true") + args = ap.parse_args() + if args.self_test: + return self_test() + if not args.trace: + print("unmap_fault_order: --trace is required (or --self-test)", file=sys.stderr) + return 2 + try: + unmaps = parse_trace(read_lines(args.trace)) + except OSError as e: + print(f"unmap_fault_order: {e}", file=sys.stderr) + return 2 + if not unmaps: + print("unmap_fault_order: no unmap events in the trace -- an empty capture is not a negative; was the " + "probe armed before the load, and did it fire? (see --self-test for the four shapes)", file=sys.stderr) + return 3 + lines = [] + for j in (args.journal or [None]): + try: + lines += read_lines(j) + except OSError as e: + print(f"unmap_fault_order: {e}", file=sys.stderr) + return 2 + + def journal_epoch(iso): + import datetime + dt = datetime.datetime.fromisoformat(iso) + return dt.timestamp() + args.epoch_offset + + # journal stamps are wall-clock, the trace is boot-relative: align on the FIRST fault and its capture, which the + # caller supplies with --epoch-offset. Without it the deltas are meaningless, so say so rather than print zeros. + faults = parse_faults(lines, journal_epoch) + if args.epoch_offset == 0.0 and any(ISO_RE.search(l) and not BOOT_RE.search(l) for l in lines): + print("unmap_fault_order: note: the journal is on the wall clock and the trace on the boot clock; pass " + "--epoch-offset (or feed a `[seconds]` capture) or the deltas are not meaningful", file=sys.stderr) + for l in render(correlate(unmaps, faults, args.window), unmaps, args.window): + print(l) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 037c190ba255e690fdbae5c645f8c9217b4a1fd9 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 19:12:22 -0400 Subject: [PATCH 2/3] diag: unmap_fault_order carries the canonical copyright and Apollo notice (CI caught it) --- tools/diag/unmap_fault_order.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tools/diag/unmap_fault_order.py b/tools/diag/unmap_fault_order.py index 2d5ffcc4c..5709ec967 100755 --- a/tools/diag/unmap_fault_order.py +++ b/tools/diag/unmap_fault_order.py @@ -1,6 +1,18 @@ #!/usr/bin/env python3 -# Copyright 2026 bong-water-bong +# Copyright 2026 bong-water-water-bong # SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """Engine#2478/#2568: does an unmap of a page land while a command still references it? The fault stream on this box is NOT driven by volume -- measured, each with its own window: 20 min of `O_DIRECT` From ae8659ca56b2ca0f20cfc0d7a9700da42baed4f5 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 19:32:24 -0400 Subject: [PATCH 3/3] diag: register unmap_fault_order's self-test as a ctest row (tests_referenced caught it) CI's tests_referenced row failed this PR with exactly the rule this lane has been arguing all day: 'declares a self-test and no runner names it -- add a ctest row, or the arm is dead code'. The self-test existed and nothing ran it. Registered next to the other tools/diag self-tests; tests_referenced now reports all 67 tool self-tests registered. --- CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index f8b06e833..b122a0440 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -340,6 +340,12 @@ add_test(NAME driver_refusal_census_selftest # live logs (a CI runner has no /dev/accel), and it pins the window arithmetic with hand-checked cases. add_test(NAME accel_attribution_selftest COMMAND python3 ${CMAKE_SOURCE_DIR}/tools/diag/accel_attribution.py --self-test) +# engine#2478/#2568: the unmap/fault ordering correlator. Volume is ruled out four ways (3 TiB O_DIRECT, 157 TiB of +# mmap churn, three boots of uptime, 60,459 dma_iova_destroy calls in one build), so what is left is ordering -- and +# this is the tool for it. Self-test only: it pins the four shapes (before / after / out-of-window / no timestamp) and +# the tracefs line shape, which is where my own first parser failed (timestamps are not at the start of the line). +add_test(NAME unmap_fault_order_selftest + COMMAND python3 ${CMAKE_SOURCE_DIR}/tools/diag/unmap_fault_order.py --self-test) add_test(NAME server_launch_check_selftest COMMAND python3 ${CMAKE_SOURCE_DIR}/tools/ci/server_launch_check.py --self-test) # scripts/lemonade-hrx: the PPA lemond's HRX recipe runs our llama-server (docs/lemonade.md): no GPU. add_test(NAME lemonade_hrx COMMAND ${CMAKE_SOURCE_DIR}/tests/lemonade_hrx.sh $)