From a7a1e3d83040dae0f3b89db5178426954e30a00f Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Mon, 16 Nov 2026 05:01:29 -0400 Subject: [PATCH 1/7] diag: iommu_map_side asks whether the faulting IOVA was ever mapped engine#2478 spent a day on "a stale translation: find the buffer". The map side settles it in one capture: for each fault in the window, the nearest iommu_map and iommu_unmap_fast of the SAME IOVA, with the count of each, and whether the ring covers that fault at all. Measured 2026-11-16, unfiltered 795 s, 252,704 map/unmap events, 8 faults: 0xebf50000 x4 faults NEVER MAPPED OR UNMAPPED anywhere in the window 0x6925f000 x1 nearest map +230 ms AFTER the fault 0xe407f000 x2 nearest -58 s 0xe1d00000 x1 nearest -347 s so four of eight faults are on a translation that is absent and that nothing re-creates -- a descriptor carrying an address the driver has finished with, not a TLB or lazy-invalidation question. The docstring carries the capture recipe because it is where the traps are: iommu_map's size is %cx (%dx is paddr, and a probe using it printed size=0x138ec90000 for an 84 GB 'size'), a positive control must fire before the real window, and an address list chosen minutes earlier goes stale. Self-test: 8 arms, no device and no git -- including a task name containing a space, which is the shape that silently dropped 2,176 of 115,941 lines on a real capture, an empty capture refused as evidence rather than read as a negative, and a fault outside the ring reported as uncovered. --- CMakeLists.txt | 4 + tools/diag/iommu_map_side.py | 228 +++++++++++++++++++++++++++++++++++ 2 files changed, 232 insertions(+) create mode 100644 tools/diag/iommu_map_side.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 92890a859..8f6ecea67 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -466,6 +466,10 @@ add_python_test(ci_targets_self ${CMAKE_SOURCE_DIR}/tools/ci/ci_targets_check.py # flagged 10 of those on a clean tree), so a separator counts only inside an unterminated conflict block. add_python_test(conflict_marker ${CMAKE_SOURCE_DIR}/tools/ci/conflict_marker_check.py --check --root ${CMAKE_SOURCE_DIR}) add_python_test(conflict_marker_self ${CMAKE_SOURCE_DIR}/tools/ci/conflict_marker_check.py --self-test) +# engine#2478: the map side, so the "stale translation" question can be asked of a capture instead of argued about. +# An unfiltered 795 s capture with 252,704 map/unmap events showed four of eight faults on an IOVA that was never +# mapped and never unmapped anywhere in the window -- absent and never re-created, not stale. +add_python_test(iommu_map_side_self ${CMAKE_SOURCE_DIR}/tools/diag/iommu_map_side.py --self-test) # engine#803 gate, second half: the comparator that turns two --dump-logits directories into the KLD verdict the # P4 gate asks for. No device and no model, so the gate is one command when a ticket is free; the self-test pins # the four cases a KLD implementation gets wrong (identical -> exactly 0, a shift is positive and directional, a diff --git a/tools/diag/iommu_map_side.py b/tools/diag/iommu_map_side.py new file mode 100644 index 000000000..d83e2f0e8 --- /dev/null +++ b/tools/diag/iommu_map_side.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Ask the map side about each IO_PAGE_FAULT: was that IOVA ever mapped, and when. + +`tools/diag/unmap_fault_order.py` asks when the page was unmapped. This asks the other half: for every fault in +the capture window, the nearest `iommu_map` and `iommu_unmap_fast` of the **same IOVA**, with how many there were +and whether the window even covers the fault. That is what separated "a stale translation" (which engine#2478 +assumed for a day) from the answer: on 2026-11-16 an unfiltered 795 s capture held 252,704 map/unmap events, and +**four of eight faults were on an IOVA that was never mapped and never unmapped anywhere in the window** -- an +absent translation that nothing re-creates, not a stale entry. + +The capture this reads is three files, and the recipe matters more than the parsing: + + TR=/sys/kernel/tracing + echo 0 > $TR/tracing_on; echo > $TR/trace + echo 'p:kprobes/capmap iommu_map iova=%si size=%cx' >> $TR/kprobe_events # %cx is SIZE; %dx is paddr + echo 'p:kprobes/capunm iommu_unmap_fast iova=%si size=%dx' >> $TR/kprobe_events + echo 1 > $TR/events/kprobes/capmap/enable; echo 1 > $TR/events/kprobes/capunm/enable + journalctl -k -b 0 -o short-monotonic > before.txt + echo 1 > $TR/tracing_on; sleep 300; echo 0 > $TR/tracing_on + cat $TR/trace > capture.txt; journalctl -k -b 0 -o short-monotonic > after.txt + +Two rules, both learned by losing runs: **check a positive control first** (30 s of events; if the probe records +nothing, the probe is wrong, not the device), and **never filter on an address list chosen minutes earlier** -- +the faulting IOVAs move faster than that (measured: a five-address filter saw 8 events, then 0 five minutes later). + +Usage: + tools/diag/iommu_map_side.py --trace capture.txt --before before.txt --after after.txt + tools/diag/iommu_map_side.py --self-test +""" +from __future__ import annotations + +import argparse +import bisect +import collections +import re +import sys + +# Anchored on the timestamp, NOT on the task name: a task name can contain a space ("next-server (v1-8386"), and +# anchoring on `^\s+\S+\s+\[` silently dropped 2,176 of 115,941 map lines on a real capture -- 1.9 %, which is +# exactly the fraction that can contain the event being looked for. +EVENT = re.compile(r"(\d+\.\d+):\s+\S*cap(map|unm):\s+\([^)]*\)\s+iova=(0x[0-9a-f]+)\s+size=(0x[0-9a-f]+)") +FAULT = re.compile(r"^\[\s*(\d+\.\d+)\].*IO_PAGE_FAULT.*address=(0x[0-9a-f]+)") + + +def parse_events(path): + """([(t, iova, size)] maps, [(t, iova, size)] unmaps) -- insertion order, which is log order.""" + maps, unmaps = [], [] + try: + fh = open(path, errors="replace") + except OSError as e: + raise SystemExit(f"iommu_map_side: cannot read {path}: {e}") + with fh: + for line in fh: + m = EVENT.search(line) + if not m: + continue + t, kind, iova, size = float(m.group(1)), m.group(2), int(m.group(3), 16), int(m.group(4), 16) + (maps if kind == "map" else unmaps).append((t, iova, size)) + return maps, unmaps + + +def parse_faults(path): + """[(boot_seconds, iova)] for every IO_PAGE_FAULT line in a `journalctl -o short-monotonic` capture.""" + out = [] + try: + fh = open(path, errors="replace") + except OSError as e: + raise SystemExit(f"iommu_map_side: cannot read {path}: {e}") + with fh: + for line in fh: + m = FAULT.match(line) + if m: + out.append((float(m.group(1)), int(m.group(2), 16))) + return out + + +def _nearest(times, t): + """(delta_seconds or None, count of events for this iova)""" + if not times: + return None, 0 + i = bisect.bisect_left(times, t) + best = None + for j in (i - 1, i): + if 0 <= j < len(times): + d = times[j] - t + if best is None or abs(d) < abs(best): + best = d + return best, len(times) + + +def analyse(maps, unmaps, faults, before, after): + """[(t, iova, text)] -- one line per fault inside the capture window, in the words the census reads.""" + if not maps and not unmaps: + return [], "no capture events: the probe recorded nothing, so this file says nothing about the device" + times = [e[0] for e in maps + unmaps] + lo, hi = min(times), max(times) + win = {f for f in after if f not in set(before)} + by_map = collections.defaultdict(list) + for t, iova, _s in sorted(maps): + by_map[iova].append(t) + by_unm = collections.defaultdict(list) + for t, iova, _s in sorted(unmaps): + by_unm[iova].append(t) + rows = [] + for t, iova in win: + if not (lo <= t <= hi): + rows.append((t, iova, "OUTSIDE the buffered range: the ring held %.0f s and this fault is not in it" + % (hi - lo))) + continue + dm, nm = _nearest(by_map.get(iova), t) + du, nu = _nearest(by_unm.get(iova), t) + if dm is None and du is None: + rows.append((t, iova, "NEVER MAPPED OR UNMAPPED in %.0f s of capture: the translation is absent and " + "nothing re-creates it" % (hi - lo))) + continue + parts = [] + for label, d, n in (("map", dm, nm), ("unmap", du, nu)): + if d is None: + parts.append(f"no {label} of this iova") + else: + parts.append(f"nearest {label} {d * 1e6:+.0f} us (of {n})") + rows.append((t, iova, " | ".join(parts))) + return rows, (f"buffered {lo:.1f}..{hi:.1f} s ({hi - lo:.0f} s); {len(maps)} map, {len(unmaps)} unmap; " + f"{len(rows)} fault(s) in the window") + + +def render(maps, unmaps, before, after, faults_rows, header): + lines = [header] + for t, iova, text in faults_rows: + lines.append(f" t={t:.3f} iova={iova:#x}: {text}") + if not faults_rows: + lines.append(" no fault inside the capture window: this capture cannot answer, whatever it contains") + return lines + + +def self_test() -> int: + import tempfile + import pathlib + + ok = True + + def chk(cond, what): + nonlocal ok + print(("ok " if cond else "FAIL ") + what) + ok = ok and cond + + with tempfile.TemporaryDirectory() as td: + d = pathlib.Path(td) + # A task name WITH A SPACE, which is the shape that silently dropped 1.9 % of lines on a real capture. + cap = ( + " next-server (v1-8386 [000] ..... 1000.000000: capmap: (iommu_map+0x4/0x80) iova=0x4128000 size=0x1000\n" + " -0 [001] ..s1. 1000.500000: capunm: (iommu_unmap_fast+0x4/0x10) iova=0x4128000 size=0x1000\n" + " -0 [001] ..s1. 1005.000000: capmap: (iommu_map+0x4/0x80) iova=0x5128000 size=0x1000\n" + ) + (d / "cap.txt").write_text(cap) + before = "[1000.000000] kernel: x\n" + after = ( + before + + "[1001.000000] strixhalo kernel: nvme 0000:c4:00.0: AMD-Vi: Event logged [IO_PAGE_FAULT domain=0x0015 address=0x4128000 flags=0x0000]\n" + + "[1002.000000] strixhalo kernel: nvme 0000:c4:00.0: AMD-Vi: Event logged [IO_PAGE_FAULT domain=0x0015 address=0x9999000 flags=0x0000]\n" + ) + (d / "before.txt").write_text(before) + (d / "after.txt").write_text(after) + + maps, unmaps = parse_events(str(d / "cap.txt")) + chk(len(maps) == 2 and len(unmaps) == 1, "a task name containing a space still parses (2 map, 1 unmap)") + chk(all(e[2] == 0x1000 for e in maps + unmaps), "size parses as 4 KiB") + + faults = parse_faults(str(d / "after.txt")) + chk(len(faults) == 2, "two faults parse with their boot stamps") + + # the fault on the mapped-and-unmapped page: both neighbours reported, in microseconds + dm, nm = _nearest([1000.0], 1001.0) + chk(dm == -1.0 and nm == 1, "nearest map is 1 s before the fault, and it is counted") + rows, _hdr = analyse(maps, unmaps, faults, parse_faults(str(d / "before.txt")), faults) + by_iova = {r[1]: r[2] for r in rows} + chk("nearest map -1000000 us" in by_iova[0x4128000], "a mapped page reports its nearest map and unmap") + chk("NEVER MAPPED OR UNMAPPED" in by_iova[0x9999000], + "a page with no map or unmap anywhere in the capture is named as absent, not as uncorrelated") + + # an empty capture must not read as "the device is fine" + empty, hdr2 = analyse([], [], faults, [], faults) + chk(empty == [] and "says nothing about the device" in hdr2, + "an empty capture is refused as evidence rather than reported as a negative") + + # a fault outside the buffered range is named as such, not silently correlated + rows3, _ = analyse(maps, unmaps, [(9000.0, 0x4128000)], [], [(9000.0, 0x4128000)]) + chk("OUTSIDE the buffered range" in rows3[0][2], "a fault the ring does not cover says so") + return 0 if ok else 1 + + +def main(argv): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--trace") + ap.add_argument("--before") + ap.add_argument("--after") + ap.add_argument("--self-test", action="store_true") + a = ap.parse_args(argv) + if a.self_test: + return self_test() + if not (a.trace and a.before and a.after): + ap.print_help() + return 2 + maps, unmaps = parse_events(a.trace) + rows, header = analyse(maps, unmaps, parse_faults(a.after), parse_faults(a.before), parse_faults(a.after)) + for line in render(maps, unmaps, a.before, a.after, rows, header): + print(line) + if not maps and not unmaps: + return 3 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) From 8b9fff631608dc58b134a7a2f2f27d99a2224909 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Mon, 16 Nov 2026 05:32:46 -0400 Subject: [PATCH 2/7] diag: the kprobe delete can lie, so count the probes back Measured 2026-11-16 while running this tool's own recipe verbatim (45 s, the two probes from the docstring, 40,745 map and 41,174 unmap events): fresh event, never enabled -> deletes recorded 810 events -> deletes recorded 1,148 events, printed EIO x2 -> WAS GONE ANYWAY recorded ~40,000 events -> stayed armed through 3 deletes, "echo > trace" and a free_buffer kick so "echo: I/O error" from the deletion line is unreliable in both directions, and the only trustworthy check is a count of the event lines in /sys/kernel/tracing/kprobe_events before and after. A stuck event is enable=0 and records nothing, but it holds a kprobe slot on iommu_map/iommu_unmap_fast until reboot. Recipe guidance updated: keep the capture as short as the question allows, and arm the one probe the question needs (the unmap side answers most of them) rather than two. The mechanism is not isolated: filter damage (#2649), fetcharg shape (%cx vs %dx) and two-vs-one fetchargs were each tested and each refuted as the sole cause; volume is the only correlate, and that is a correlation. --- tools/diag/iommu_map_side.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tools/diag/iommu_map_side.py b/tools/diag/iommu_map_side.py index d83e2f0e8..cece378ae 100644 --- a/tools/diag/iommu_map_side.py +++ b/tools/diag/iommu_map_side.py @@ -33,9 +33,16 @@ echo 1 > $TR/tracing_on; sleep 300; echo 0 > $TR/tracing_on cat $TR/trace > capture.txt; journalctl -k -b 0 -o short-monotonic > after.txt -Two rules, both learned by losing runs: **check a positive control first** (30 s of events; if the probe records -nothing, the probe is wrong, not the device), and **never filter on an address list chosen minutes earlier** -- -the faulting IOVAs move faster than that (measured: a five-address filter saw 8 events, then 0 five minutes later). +Three rules, all learned by losing runs: **check a positive control first** (30 s of events; if the probe records +nothing, the probe is wrong, not the device); **never filter on an address list chosen minutes earlier** -- the +faulting IOVAs move faster than that (measured: a five-address filter saw 8 events, then 0 five minutes later); and +**count the probes back after cleanup, because the delete can lie**. Measured 2026-11-16 on one box, five tries: +a fresh event deletes; an event that recorded 810 or 1,148 events deletes; one that reported `echo: I/O error` was +**gone anyway**; and an event that had recorded ~40,000 events stayed armed through three deletes, a buffer clear and +a `free_buffer` kick -- `enable=0`, so it records nothing, but it holds a kprobe slot on `iommu_map` until reboot. +So: the error message is wrong in both directions, and **only `grep -c . kprobe_events` before and after tells you +whether you cleaned up**. Keep the capture as short as the question allows, and prefer the one probe the question +needs (the unmap side answers most of them) over two. Usage: tools/diag/iommu_map_side.py --trace capture.txt --before before.txt --after after.txt From fcc7b77848aa86508bcbaf5532273abcfc0308e3 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Mon, 16 Nov 2026 06:54:16 -0400 Subject: [PATCH 3/7] diag: read the probed function, not the event's name The map/unmap class now comes from the function in the event text (iommu_map / iommu_map_sg vs iommu_unmap / iommu_unmap_fast), not from the name the lane gave its probe. Any lane names its probe differently -- capmap, unm, b2map, prs2mp -- and a tool that only reads its own convention cannot read the captures that already exist on the box. Measured on the 65 MB capture from another lane's unmap-only probe, which the first version of this file refused outright: ~/.cache/scratch-i679/burst.trace -> 0 map, 579219 unmaps parsed The function is the fact; the name is a convention. A self-test arm asserts the same events under invented names parse identically. --- tools/diag/iommu_map_side.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/tools/diag/iommu_map_side.py b/tools/diag/iommu_map_side.py index cece378ae..c3a278d05 100644 --- a/tools/diag/iommu_map_side.py +++ b/tools/diag/iommu_map_side.py @@ -59,7 +59,13 @@ # Anchored on the timestamp, NOT on the task name: a task name can contain a space ("next-server (v1-8386"), and # anchoring on `^\s+\S+\s+\[` silently dropped 2,176 of 115,941 map lines on a real capture -- 1.9 %, which is # exactly the fraction that can contain the event being looked for. -EVENT = re.compile(r"(\d+\.\d+):\s+\S*cap(map|unm):\s+\([^)]*\)\s+iova=(0x[0-9a-f]+)\s+size=(0x[0-9a-f]+)") +# +# The map/unmap distinction comes from the PROBED FUNCTION in the event text, not from the event's name: any lane +# names its probe differently (`capmap`, `unm`, `b2map`, `prs2mp`), and a tool that only reads its own naming +# cannot read the captures that already exist on the box. The function is the fact; the name is a convention. +EVENT = re.compile(r"(\d+\.\d+):\s+\S*:\s+\((\w+)\+[^)]*\)\s+iova=(0x[0-9a-f]+)\s+size=(0x[0-9a-f]+)") +MAP_FUNCS = ("iommu_map", "iommu_map_sg") +UNMAP_FUNCS = ("iommu_unmap", "iommu_unmap_fast") FAULT = re.compile(r"^\[\s*(\d+\.\d+)\].*IO_PAGE_FAULT.*address=(0x[0-9a-f]+)") @@ -75,8 +81,11 @@ def parse_events(path): m = EVENT.search(line) if not m: continue - t, kind, iova, size = float(m.group(1)), m.group(2), int(m.group(3), 16), int(m.group(4), 16) - (maps if kind == "map" else unmaps).append((t, iova, size)) + t, fn, iova, size = float(m.group(1)), m.group(2), int(m.group(3), 16), int(m.group(4), 16) + if fn in MAP_FUNCS: + maps.append((t, iova, size)) + elif fn in UNMAP_FUNCS: + unmaps.append((t, iova, size)) return maps, unmaps @@ -185,6 +194,14 @@ def chk(cond, what): maps, unmaps = parse_events(str(d / "cap.txt")) chk(len(maps) == 2 and len(unmaps) == 1, "a task name containing a space still parses (2 map, 1 unmap)") + # 2b. the same events under ANOTHER lane's naming (`unm:`/`map:`/`prs2mp:`) parse identically, because the + # class comes from the probed function. A tool that only reads its own event names cannot read the + # captures that already exist on the box -- measured: 579,219 unmaps in ~/.cache/scratch-*/burst.trace + # are named `unm:`, and the first version of this file refused them. + other = pathlib.Path(td) / "other.txt" + other.write_text(cap.replace("capmap:", "whatever:").replace("capunm:", "unm:")) + m2, u2 = parse_events(str(other)) + chk((m2, u2) == (maps, unmaps), "event names invented by another lane parse by the probed function, not the name") chk(all(e[2] == 0x1000 for e in maps + unmaps), "size parses as 4 KiB") faults = parse_faults(str(d / "after.txt")) From 76816d7b452d9712b72f46e338e90ec73e3981bd Mon Sep 17 00:00:00 2001 From: pi-os3 Date: Mon, 16 Nov 2026 08:07:56 -0400 Subject: [PATCH 4/7] diag: iommu_map_side: a one-sided capture cannot claim absence, and an unknown map function is a map Two answers from the review at fcc7b77, both host-only: 1. An unmap-only capture read as "NEVER MAPPED OR UNMAPPED ... nothing re-creates it" at rc 0. Measured on ~/.cache/scratch-i679/burst.trace (579,219 iommu_unmap_fast, 0 maps): the map side was never probed, so absence is not knowable. The row now says which half is missing and that the question is not answerable here, the header repeats it, and the exit code is 4 (0 both sides, 3 nothing captured, 4 one side absent). 2. The class came from two exact-name lists, so __iommu_map, iommu_map_pages and iommu_map.part.0 were dropped without a count -- a lane that probed any of them read "0 map" and then hit (1). Classification is now by substring (unmap before map, since every unmap name contains map) with the lists kept as documentation, the probed-function regex allows dots, and every iova line whose function is still unrecognised is counted and named in the header. Self-test: 9 arms -> 17, rc 0. --- tools/diag/iommu_map_side.py | 108 +++++++++++++++++++++++++++++------ 1 file changed, 90 insertions(+), 18 deletions(-) diff --git a/tools/diag/iommu_map_side.py b/tools/diag/iommu_map_side.py index c3a278d05..7a405229d 100644 --- a/tools/diag/iommu_map_side.py +++ b/tools/diag/iommu_map_side.py @@ -63,15 +63,29 @@ # The map/unmap distinction comes from the PROBED FUNCTION in the event text, not from the event's name: any lane # names its probe differently (`capmap`, `unm`, `b2map`, `prs2mp`), and a tool that only reads its own naming # cannot read the captures that already exist on the box. The function is the fact; the name is a convention. -EVENT = re.compile(r"(\d+\.\d+):\s+\S*:\s+\((\w+)\+[^)]*\)\s+iova=(0x[0-9a-f]+)\s+size=(0x[0-9a-f]+)") +EVENT = re.compile(r"(\d+\.\d+):\s+\S*:\s+\(([\w.]+)\+[^)]*\)\s+iova=(0x[0-9a-f]+)\s+size=(0x[0-9a-f]+)") +# Any line whose event text names a function and an iova. A function this file cannot classify is COUNTED, not +# dropped: `__iommu_map`, `iommu_map_pages` and `iommu_map.part.0` are maps the exact-name lists below never held, +# and a silent drop made a lane's map probe read as "0 map" -- which then produced the unanswerable claim in (1). +IOVA_LINE = re.compile(r"\(\s*([\w.]+)\+[^)]*\)\s+iova=") +# Documentation of what is known to exist. NOT the classifier: see classify(). MAP_FUNCS = ("iommu_map", "iommu_map_sg") UNMAP_FUNCS = ("iommu_unmap", "iommu_unmap_fast") + + +def classify(fn): + """`unmap` is tested first because every unmap name contains `map`.""" + if "unmap" in fn: + return "unmap" + if "map" in fn: + return "map" + return "other" FAULT = re.compile(r"^\[\s*(\d+\.\d+)\].*IO_PAGE_FAULT.*address=(0x[0-9a-f]+)") def parse_events(path): - """([(t, iova, size)] maps, [(t, iova, size)] unmaps) -- insertion order, which is log order.""" - maps, unmaps = [], [] + """([(t, iova, size)] maps, [...] unmaps, Counter of unrecognised functions) -- in log order.""" + maps, unmaps, other = [], [], collections.Counter() try: fh = open(path, errors="replace") except OSError as e: @@ -80,13 +94,19 @@ def parse_events(path): for line in fh: m = EVENT.search(line) if not m: + loose = IOVA_LINE.search(line) + if loose: + other[loose.group(1)] += 1 continue t, fn, iova, size = float(m.group(1)), m.group(2), int(m.group(3), 16), int(m.group(4), 16) - if fn in MAP_FUNCS: + kind = classify(fn) + if kind == "map": maps.append((t, iova, size)) - elif fn in UNMAP_FUNCS: + elif kind == "unmap": unmaps.append((t, iova, size)) - return maps, unmaps + else: + other[fn] += 1 + return maps, unmaps, other def parse_faults(path): @@ -118,7 +138,7 @@ def _nearest(times, t): return best, len(times) -def analyse(maps, unmaps, faults, before, after): +def analyse(maps, unmaps, faults, before, after, other=()): """[(t, iova, text)] -- one line per fault inside the capture window, in the words the census reads.""" if not maps and not unmaps: return [], "no capture events: the probe recorded nothing, so this file says nothing about the device" @@ -140,8 +160,19 @@ def analyse(maps, unmaps, faults, before, after): dm, nm = _nearest(by_map.get(iova), t) du, nu = _nearest(by_unm.get(iova), t) if dm is None and du is None: - rows.append((t, iova, "NEVER MAPPED OR UNMAPPED in %.0f s of capture: the translation is absent and " - "nothing re-creates it" % (hi - lo))) + # Only a capture that probed BOTH sides may claim absence. A one-sided capture knows nothing about the + # other half, and saying "never mapped" from an unmap-only trace is the instrument answering a question + # it was never asked -- measured: 579,219 unmaps and 0 maps in scratch-i679/burst.trace read as + # "NEVER MAPPED OR UNMAPPED" at rc 0. + if not maps: + rows.append((t, iova, "not in this capture's map side: no map events in %.0f s of capture, so whether " + "this iova was ever mapped is not answerable here" % (hi - lo))) + elif not unmaps: + rows.append((t, iova, "not in this capture's unmap side: no unmap events in %.0f s of capture, so " + "whether this iova was released is not answerable here" % (hi - lo))) + else: + rows.append((t, iova, "NEVER MAPPED OR UNMAPPED in %.0f s of capture: the translation is absent and " + "nothing re-creates it" % (hi - lo))) continue parts = [] for label, d, n in (("map", dm, nm), ("unmap", du, nu)): @@ -150,8 +181,15 @@ def analyse(maps, unmaps, faults, before, after): else: parts.append(f"nearest {label} {d * 1e6:+.0f} us (of {n})") rows.append((t, iova, " | ".join(parts))) - return rows, (f"buffered {lo:.1f}..{hi:.1f} s ({hi - lo:.0f} s); {len(maps)} map, {len(unmaps)} unmap; " - f"{len(rows)} fault(s) in the window") + hdr = (f"buffered {lo:.1f}..{hi:.1f} s ({hi - lo:.0f} s); {len(maps)} map, {len(unmaps)} unmap; " + f"{len(rows)} fault(s) in the window") + if other: + hdr += "; %d events with an unrecognised function (e.g. %s)" % (sum(other.values()), next(iter(other))) + if not maps: + hdr += "; no map events in this capture (the map side is unanswerable)" + elif not unmaps: + hdr += "; no unmap events in this capture (the unmap side is unanswerable)" + return rows, hdr def render(maps, unmaps, before, after, faults_rows, header): @@ -192,7 +230,7 @@ def chk(cond, what): (d / "before.txt").write_text(before) (d / "after.txt").write_text(after) - maps, unmaps = parse_events(str(d / "cap.txt")) + maps, unmaps, _other = parse_events(str(d / "cap.txt")) chk(len(maps) == 2 and len(unmaps) == 1, "a task name containing a space still parses (2 map, 1 unmap)") # 2b. the same events under ANOTHER lane's naming (`unm:`/`map:`/`prs2mp:`) parse identically, because the # class comes from the probed function. A tool that only reads its own event names cannot read the @@ -200,10 +238,36 @@ def chk(cond, what): # are named `unm:`, and the first version of this file refused them. other = pathlib.Path(td) / "other.txt" other.write_text(cap.replace("capmap:", "whatever:").replace("capunm:", "unm:")) - m2, u2 = parse_events(str(other)) + m2, u2, _o2 = parse_events(str(other)) chk((m2, u2) == (maps, unmaps), "event names invented by another lane parse by the probed function, not the name") chk(all(e[2] == 0x1000 for e in maps + unmaps), "size parses as 4 KiB") + # 2c. functions the exact-name lists never held are MAPS, and one this file cannot classify is COUNTED. + # A silent drop made a lane's probe read as "0 map", which is what produced the unanswerable claim below. + odd = pathlib.Path(td) / "odd.txt" + odd.write_text( + " -0 [001] ..s1. 1000.000000: p: (__iommu_map+0x4/0x80) iova=0x1000 size=0x1000\n" + " -0 [001] ..s1. 1001.000000: p: (iommu_map_pages+0x4/0x80) iova=0x2000 size=0x1000\n" + " -0 [001] ..s1. 1002.000000: p: (iommu_map.part.0+0x4/0x80) iova=0x3000 size=0x1000\n" + " -0 [001] ..s1. 1003.000000: p: (iommu_tlb_sync+0x4/0x80) iova=0x4000 size=0x1000\n") + m4, u4, o4 = parse_events(str(odd)) + chk(len(m4) == 3, "__iommu_map, iommu_map_pages and iommu_map.part.0 are all counted as maps (3)") + chk(u4 == [] and o4 == {"iommu_tlb_sync": 1}, + "a function with neither word is counted, not dropped: %s" % dict(o4)) + _r4, hdr4 = analyse(m4, u4, [(1004.0, 0x9999000)], [], [(1004.0, 0x9999000)], o4) + chk("1 events with an unrecognised function (e.g. iommu_tlb_sync)" in hdr4, + "the unrecognised event is named in the header: %s" % hdr4) + + # 2d. ONE-SIDED captures: an unmap-only trace cannot say a page was never mapped, and says so. + rows5, hdr5 = analyse([], [(1000.0, 0x4128000, 0x1000), (1002.0, 0x5128000, 0x1000)], + [(1001.0, 0xdeadb000)], [], [(1001.0, 0xdeadb000)]) + chk("map side" in rows5[0][2] and "not answerable" in rows5[0][2], + "an unmap-only capture refuses the map-side claim: %s" % rows5[0][2][:60]) + chk("NEVER MAPPED OR UNMAPPED" not in rows5[0][2], "...and does not say NEVER MAPPED") + chk("no map events in this capture" in hdr5, "...and says it in the header too") + chk(exit_code([], [(1, 2, 3)]) == 4, "a one-sided capture exits 4, not 0") + chk(exit_code(maps, unmaps) == 0 and exit_code([], []) == 3, "both sides 0, nothing captured 3") + faults = parse_faults(str(d / "after.txt")) chk(len(faults) == 2, "two faults parse with their boot stamps") @@ -227,6 +291,15 @@ def chk(cond, what): return 0 if ok else 1 +def exit_code(maps, unmaps): + """0 both sides probed; 3 nothing captured; 4 ONE side absent, so the missing half cannot be claimed.""" + if not maps and not unmaps: + return 3 + if not maps or not unmaps: + return 4 + return 0 + + def main(argv): ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument("--trace") @@ -239,13 +312,12 @@ def main(argv): if not (a.trace and a.before and a.after): ap.print_help() return 2 - maps, unmaps = parse_events(a.trace) - rows, header = analyse(maps, unmaps, parse_faults(a.after), parse_faults(a.before), parse_faults(a.after)) + maps, unmaps, other = parse_events(a.trace) + rows, header = analyse(maps, unmaps, parse_faults(a.after), parse_faults(a.before), parse_faults(a.after), + other) for line in render(maps, unmaps, a.before, a.after, rows, header): print(line) - if not maps and not unmaps: - return 3 - return 0 + return exit_code(maps, unmaps) if __name__ == "__main__": From 440e4b7fde0352e232fbfbde63dd198a784ba5d1 Mon Sep 17 00:00:00 2001 From: pi-os3 Date: Mon, 16 Nov 2026 08:16:21 -0400 Subject: [PATCH 5/7] diag: iommu_map_side: an arm for main(), and render() stops taking what it ignores The previous commit changed main() (rc 3/4) and the header shape, and self_test() exercises only the pieces -- parse_events, analyse, exit_code -- so nothing asserted the thing a lane actually runs. The new arm calls main() on a one-sided fixture and checks both the printed header and rc 4, plus that the absence claim is never printed for it. 17 arms -> 19. render() was passed (maps, unmaps, before, after) and used none of them; it now takes the rows and the header it prints. The signature read as though the renderer considered the capture. --- tools/diag/iommu_map_side.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/tools/diag/iommu_map_side.py b/tools/diag/iommu_map_side.py index 7a405229d..7ae97ecb7 100644 --- a/tools/diag/iommu_map_side.py +++ b/tools/diag/iommu_map_side.py @@ -192,7 +192,7 @@ def analyse(maps, unmaps, faults, before, after, other=()): return rows, hdr -def render(maps, unmaps, before, after, faults_rows, header): +def render(faults_rows, header): lines = [header] for t, iova, text in faults_rows: lines.append(f" t={t:.3f} iova={iova:#x}: {text}") @@ -288,6 +288,25 @@ def chk(cond, what): # a fault outside the buffered range is named as such, not silently correlated rows3, _ = analyse(maps, unmaps, [(9000.0, 0x4128000)], [], [(9000.0, 0x4128000)]) chk("OUTSIDE the buffered range" in rows3[0][2], "a fault the ring does not cover says so") + # 3. the ASSEMBLY: main() is what a lane runs, and until this arm nothing exercised it end to end -- + # the exit code and the header shape of a one-sided capture were asserted only piecewise. + import contextlib + import io + one_sided = d / "one_sided.txt" + one_sided.write_text( + " -0 [001] ..s1. 1000.000000: capunm: (iommu_unmap_fast+0x4/0x10) iova=0x4128000 size=0x1000\n" + " -0 [001] ..s1. 1002.000000: capunm: (iommu_unmap_fast+0x4/0x10) iova=0x5128000 size=0x1000\n") + (d / "f.txt").write_text( + "[1001.000000] strixhalo kernel: nvme 0000:c4:00.0: AMD-Vi: Event logged [IO_PAGE_FAULT domain=0x0015 address=0xdeadb000 flags=0x0000]\n") + (d / "none.txt").write_text("") + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + rc_main = main(["--trace", str(one_sided), "--before", str(d / "none.txt"), "--after", str(d / "f.txt")]) + printed = buf.getvalue() + chk(rc_main == 4 and "no map events in this capture" in printed and "not answerable here" in printed, + "main() on an unmap-only capture prints the map-side refusal and exits 4 (rc=%d)" % rc_main) + chk("NEVER MAPPED OR UNMAPPED" not in printed, "...and main() never prints the absence claim for it") + return 0 if ok else 1 @@ -315,7 +334,7 @@ def main(argv): maps, unmaps, other = parse_events(a.trace) rows, header = analyse(maps, unmaps, parse_faults(a.after), parse_faults(a.before), parse_faults(a.after), other) - for line in render(maps, unmaps, a.before, a.after, rows, header): + for line in render(rows, header): print(line) return exit_code(maps, unmaps) From 2a0b28991e4c2e79acb57283c2280096e888a0ff Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Mon, 16 Nov 2026 08:25:48 -0400 Subject: [PATCH 6/7] diag: on a one-sided capture, a row says which half was never probed The absence branch already refused to claim "never mapped or unmapped" from a one-sided capture, but the rows that DID find an event on the probed side still printed "no map of this iova" for the other half -- which says the iova was never mapped rather than that nobody looked. Two different facts, one phrase. Measured on the 65 MB unmap-only capture: 579,219 unmaps, no map probe, and every row carried "no map of this iova". Now: "no map probe in this capture, so that half is unanswerable (see exit code)". One arm, and its fixture brackets the fault with two events because a single-event capture puts the fault outside the buffered range and the arm would then pass for the wrong reason. --- tools/diag/iommu_map_side.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tools/diag/iommu_map_side.py b/tools/diag/iommu_map_side.py index 7ae97ecb7..72476973d 100644 --- a/tools/diag/iommu_map_side.py +++ b/tools/diag/iommu_map_side.py @@ -177,7 +177,13 @@ def analyse(maps, unmaps, faults, before, after, other=()): parts = [] for label, d, n in (("map", dm, nm), ("unmap", du, nu)): if d is None: - parts.append(f"no {label} of this iova") + # Two different facts print the same word if this is not split: a side that WAS probed and has no + # event for this iova ("no map of this iova"), and a side that was never probed at all ("no map + # probe in this capture"). On a one-sided capture the second is the truth, and writing the first + # makes silence read as absence -- the same error the absence branch above now refuses. + probed = bool(maps if label == "map" else unmaps) + parts.append(f"no {label} of this iova" if probed + else f"no {label} probe in this capture, so that half is unanswerable (see exit code)") else: parts.append(f"nearest {label} {d * 1e6:+.0f} us (of {n})") rows.append((t, iova, " | ".join(parts))) @@ -285,6 +291,17 @@ def chk(cond, what): chk(empty == [] and "says nothing about the device" in hdr2, "an empty capture is refused as evidence rather than reported as a negative") + # 7b. a one-sided capture: the row answers on the side it probed AND says the other half was never asked about. + # Measured: burst.trace has 579,219 unmaps and no map probe, and its rows used to read "no map of this + # iova" -- which says the iova was never mapped rather than that nobody looked. + # two unmap events bracket the fault so it is inside the buffered range: a single event gives lo == hi and + # the row (correctly) reports "OUTSIDE" instead, which would make this arm pass for the wrong reason. + rows4, _h4 = analyse([], [(1000.5, 0x4128000, 0x1000), (1002.0, 0x5128000, 0x1000)], + [(1001.0, 0x4128000)], [], [(1001.0, 0x4128000)]) + txt = " | ".join(r[2] for r in rows4) + chk("no map probe in this capture" in txt and "no map of this iova" not in txt, + "a row on a one-sided capture says the other half was never probed") + # a fault outside the buffered range is named as such, not silently correlated rows3, _ = analyse(maps, unmaps, [(9000.0, 0x4128000)], [], [(9000.0, 0x4128000)]) chk("OUTSIDE the buffered range" in rows3[0][2], "a fault the ring does not cover says so") From 0f57cd1a056ba4369b2414e5fe15492f5a398d56 Mon Sep 17 00:00:00 2001 From: pi-os3 Date: Mon, 16 Nov 2026 08:29:25 -0400 Subject: [PATCH 7/7] diag: iommu_map_side: a widened name is surfaced, not silently counted as a map The substring classifier from the previous commit widens the net, and a wider net catches names that are not IOMMU maps: classify("bitmap_alloc") is "map". Rather than add a denylist -- which would be the "parse by our own convention" defect again -- every function counted by substring but absent from the documented lists is counted separately and named in the header: ...; 3 events counted by substring but not in the documented names (e.g. __iommu_map) So __iommu_map / iommu_map_pages / iommu_map.part.0 are visible as the widening they are, and a name like bitmap_alloc cannot inflate the map side without a reader seeing it. 19 arms -> 22. --- tools/diag/iommu_map_side.py | 37 ++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/tools/diag/iommu_map_side.py b/tools/diag/iommu_map_side.py index 72476973d..25fab255b 100644 --- a/tools/diag/iommu_map_side.py +++ b/tools/diag/iommu_map_side.py @@ -84,8 +84,8 @@ def classify(fn): def parse_events(path): - """([(t, iova, size)] maps, [...] unmaps, Counter of unrecognised functions) -- in log order.""" - maps, unmaps, other = [], [], collections.Counter() + """([(t, iova, size)] maps, [...] unmaps, Counter unrecognised, Counter widened) -- in log order.""" + maps, unmaps, other, widened = [], [], collections.Counter(), collections.Counter() try: fh = open(path, errors="replace") except OSError as e: @@ -106,7 +106,9 @@ def parse_events(path): unmaps.append((t, iova, size)) else: other[fn] += 1 - return maps, unmaps, other + if (kind == "map" and fn not in MAP_FUNCS) or (kind == "unmap" and fn not in UNMAP_FUNCS): + widened[fn] += 1 + return maps, unmaps, other, widened def parse_faults(path): @@ -138,7 +140,7 @@ def _nearest(times, t): return best, len(times) -def analyse(maps, unmaps, faults, before, after, other=()): +def analyse(maps, unmaps, faults, before, after, other=(), widened=()): """[(t, iova, text)] -- one line per fault inside the capture window, in the words the census reads.""" if not maps and not unmaps: return [], "no capture events: the probe recorded nothing, so this file says nothing about the device" @@ -191,6 +193,9 @@ def analyse(maps, unmaps, faults, before, after, other=()): f"{len(rows)} fault(s) in the window") if other: hdr += "; %d events with an unrecognised function (e.g. %s)" % (sum(other.values()), next(iter(other))) + if widened: + hdr += "; %d events counted by substring but not in the documented names (e.g. %s)" % ( + sum(widened.values()), next(iter(widened))) if not maps: hdr += "; no map events in this capture (the map side is unanswerable)" elif not unmaps: @@ -236,7 +241,7 @@ def chk(cond, what): (d / "before.txt").write_text(before) (d / "after.txt").write_text(after) - maps, unmaps, _other = parse_events(str(d / "cap.txt")) + maps, unmaps, _other, _w = parse_events(str(d / "cap.txt")) chk(len(maps) == 2 and len(unmaps) == 1, "a task name containing a space still parses (2 map, 1 unmap)") # 2b. the same events under ANOTHER lane's naming (`unm:`/`map:`/`prs2mp:`) parse identically, because the # class comes from the probed function. A tool that only reads its own event names cannot read the @@ -244,7 +249,7 @@ def chk(cond, what): # are named `unm:`, and the first version of this file refused them. other = pathlib.Path(td) / "other.txt" other.write_text(cap.replace("capmap:", "whatever:").replace("capunm:", "unm:")) - m2, u2, _o2 = parse_events(str(other)) + m2, u2, _o2, _w2 = parse_events(str(other)) chk((m2, u2) == (maps, unmaps), "event names invented by another lane parse by the probed function, not the name") chk(all(e[2] == 0x1000 for e in maps + unmaps), "size parses as 4 KiB") @@ -256,13 +261,25 @@ def chk(cond, what): " -0 [001] ..s1. 1001.000000: p: (iommu_map_pages+0x4/0x80) iova=0x2000 size=0x1000\n" " -0 [001] ..s1. 1002.000000: p: (iommu_map.part.0+0x4/0x80) iova=0x3000 size=0x1000\n" " -0 [001] ..s1. 1003.000000: p: (iommu_tlb_sync+0x4/0x80) iova=0x4000 size=0x1000\n") - m4, u4, o4 = parse_events(str(odd)) + m4, u4, o4, w4 = parse_events(str(odd)) chk(len(m4) == 3, "__iommu_map, iommu_map_pages and iommu_map.part.0 are all counted as maps (3)") chk(u4 == [] and o4 == {"iommu_tlb_sync": 1}, "a function with neither word is counted, not dropped: %s" % dict(o4)) - _r4, hdr4 = analyse(m4, u4, [(1004.0, 0x9999000)], [], [(1004.0, 0x9999000)], o4) + chk(w4 == {"__iommu_map": 1, "iommu_map_pages": 1, "iommu_map.part.0": 1}, + "the three widened names are surfaced with their counts, not just counted as maps: %s" % dict(w4)) + # a name that contains `map` and is NOT an IOMMU map: the substring rule catches it, so the header must + # say so. A denylist would be a guess; a number a reader can see is not. + odd2 = pathlib.Path(td) / "odd2.txt" + odd2.write_text(" -0 [001] ..s1. 1000.000000: p: (bitmap_alloc+0x4/0x80) iova=0x1000 size=0x1000\n") + m6, _u6, _o6, w6 = parse_events(str(odd2)) + chk(len(m6) == 1 and w6 == {"bitmap_alloc": 1}, + "a `map`-containing non-IOMMU name is counted AND surfaced (%s), so the map side cannot be inflated silently" + % dict(w6)) + _r4, hdr4 = analyse(m4, u4, [(1004.0, 0x9999000)], [], [(1004.0, 0x9999000)], o4, w4) chk("1 events with an unrecognised function (e.g. iommu_tlb_sync)" in hdr4, "the unrecognised event is named in the header: %s" % hdr4) + chk("3 events counted by substring but not in the documented names" in hdr4, + "the widened names are surfaced in the header: %s" % hdr4) # 2d. ONE-SIDED captures: an unmap-only trace cannot say a page was never mapped, and says so. rows5, hdr5 = analyse([], [(1000.0, 0x4128000, 0x1000), (1002.0, 0x5128000, 0x1000)], @@ -348,9 +365,9 @@ def main(argv): if not (a.trace and a.before and a.after): ap.print_help() return 2 - maps, unmaps, other = parse_events(a.trace) + maps, unmaps, other, widened = parse_events(a.trace) rows, header = analyse(maps, unmaps, parse_faults(a.after), parse_faults(a.before), parse_faults(a.after), - other) + other, widened) for line in render(rows, header): print(line) return exit_code(maps, unmaps)