From 498faf9bb2979e246ca2abd517cb1130b29854a9 Mon Sep 17 00:00:00 2001 From: pi-hrx-compat Date: Sun, 15 Nov 2026 20:02:51 -0400 Subject: [PATCH 1/4] diag: print the attributability outlook of a capture, before any fault exists The join answers "which holder" once a fault lands. This adds the question that decides whether a capture can answer it at all: over the holder timeline it already reads, how many faults would a +-W window be able to attribute? tools/diag/iommu_fault_attribution.py --samples ~/.cache/accel-sampler/samples.log --outlook 2 5 10 15 window one-holder ambiguous nothing-held no-sample 2s 61.3% 22.4% 16.3% 0.0% 5s 62.1% 23.1% 14.9% 0.0% 10s 63.6% 24.9% 11.6% 0.0% 15s 64.0% 26.6% 9.4% 0.0% That is 3085 real samples from this box's live sampler (read-only). Two things it makes visible: * **ambiguity rises with the window** (22.4 % -> 26.6 % from +-2 s to +-15 s), which is the opposite of the resolution intuition: a wider window adds HOLDERS, not evidence. Same direction as the +-5/10/15 numbers from an earlier 1816-sample slice (33.0 / 34.1 / 34.7 %). * **the rate is a property of the hour, not of the box.** The earlier slice read 33 % ambiguous at +-5 s; this one reads 23.1 % over the same window. The holder turnover changed. So the number to quote is always "computed from THIS trace", never a constant -- which is why it belongs in a command rather than in a sentence. Which is the point: an unattributable fraction known in advance says whether to extend the capture, narrow the window, or go and add timestamps to the sampler, instead of discovering it after a day of collecting bursts that all resolve to two lanes. Self-test pins the arithmetic both ways: a window narrower than the fixture's handover reports 100 % one-holder, and a 400 s window over the same rows reports ambiguity -- and the 5 s case expected 80 %, not 100 %, because one of those samples IS the '-' row (nothing held at the node). I wrote 100 % first and the self-test corrected me, which is the second time that fixture has done so. --- tools/diag/iommu_fault_attribution.py | 41 +++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tools/diag/iommu_fault_attribution.py b/tools/diag/iommu_fault_attribution.py index 6a73c2a62..4a92fd1a9 100644 --- a/tools/diag/iommu_fault_attribution.py +++ b/tools/diag/iommu_fault_attribution.py @@ -125,6 +125,33 @@ def yield_line(res): % (n, k["ATTRIBUTED"], k["HANDOVER-ADJACENT"], k["NOTHING HELD"], k["NO SAMPLE"])) +def outlook(rows, windows): + """How many faults a given window COULD attribute, from the holder timeline alone. + + Measured on a 1816-sample hour of this box: +-5 s names two holders for 33 % of samples and + widening makes it worse, because 18 tickets held the node in that hour. Printing that before a + capture starts is the difference between "run it for a day and hope" and knowing the sampler's + window cannot answer the question. + """ + out = [] + for w in windows: + one = amb = nothing = none = 0 + for epoch, _pid, _ticket in rows: + near = [(e, p) for e, p, _t in rows if abs(e - epoch) <= w] + holders = {p for _e, p in near if p != "-"} + if not near: + none += 1 + elif not holders: + nothing += 1 + elif len(holders) == 1: + one += 1 + else: + amb += 1 + n = len(rows) or 1 + out.append((w, 100.0 * one / n, 100.0 * amb / n, 100.0 * nothing / n, 100.0 * none / n)) + return out + + def self_test(): # One hour of holders, one sample per 100 s, handing over at 300. rows = [(0, "111", "gpu-hold cl-a pid 1 t0"), (100, "111", "gpu-hold cl-a pid 1 t0"), @@ -166,6 +193,12 @@ def kind(v): assert no_stamp == [(None, "0x4128000")], no_stamp assert attribute(rows, no_stamp, 150)[0][2].startswith("NO SAMPLE"), attribute(rows, no_stamp, 150) + # The outlook is computed, not asserted from a constant: a window wide enough to cover two + # different holders must report ambiguity, and one narrower than the gap must not. + ol = {w: (one, amb) for w, one, amb, _n, _z in outlook(rows, [5, 400])} + assert ol[5][0] == 80.0 and ol[5][1] == 0.0, ol # 4 of 5: the 200s sample IS the "-" row, so it is nothing-held + assert ol[400][1] > 0.0, ol # 400s spans the handover -> ambiguous + # Stamps and addresses are parsed, and non-fault lines are ignored. text = ("[1234.500000] nvme 0000:c4:00.0: AMD-Vi: Event logged [IO_PAGE_FAULT domain=0x0015 address=0x4128000 flags=0x0007]\n" "[1235.000000] unrelated kernel line\n") @@ -181,6 +214,8 @@ def main(): help="epoch of boot, so kernel stamps become absolute (default: /proc/stat btime)") ap.add_argument("--max-gap", type=float, default=15.0, help="oldest usable sample before a fault, in seconds (default 15)") + ap.add_argument("--outlook", type=float, nargs="*", metavar="W", + help="print attributability for these +-W windows and exit (no faults needed)") ap.add_argument("--selftest", "--self-test", dest="self_test", action="store_true") args = ap.parse_args() @@ -189,6 +224,12 @@ def main(): return 0 if not args.samples: ap.error("--samples is required unless --self-test") + if args.outlook: + rows = samples(args.samples) + print("window one-holder ambiguous nothing-held no-sample") + for w, one, amb, nothing, none in outlook(rows, args.outlook): + print("%5.0fs %9.1f%% %8.1f%% %11.1f%% %8.1f%%" % (w, one, amb, nothing, none)) + return 0 boot = args.boot_epoch if boot is None: From 4b769a969c99e307688124182454e000a736712e Mon Sep 17 00:00:00 2001 From: pi-hrx-compat Date: Sun, 15 Nov 2026 20:13:12 -0400 Subject: [PATCH 2/4] diag: pin the writer/reader contract against the sampler's own fixture The sampler merged into main (`tools/diag/accel_sampler.sh`) writes the six-column rows this reader parses, and the two tools now live in the same directory with nothing asserting that they agree. If the writer ever reorders or adds a column, this reader would keep "working" and read the wrong field out of a shifted line -- holders named from a scope path, timestamps from a pid. The self-test now parses the sampler's own fixture lines, copied verbatim from that script: 1000\t111\t1\t/serve A\t/scope/a\tticket1 1010\t222\t2\t/serve B\t/scope/b\tticket2 1000\t-\t-\t-\t-\t- and asserts epoch/pid/ticket land where this reader expects, that the '-' row shape is recognised, and that a three-column line is skipped rather than becoming a holder. A change on the writer side now fails here instead of silently producing zeros. `rows_from_text()` is split out of `samples()` so the contract is testable without a temp file, which is also what let the two earlier assertions be exact rather than approximately right. --- tools/diag/iommu_fault_attribution.py | 32 ++++++++++++++++++++------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/tools/diag/iommu_fault_attribution.py b/tools/diag/iommu_fault_attribution.py index 4a92fd1a9..847d68e9d 100644 --- a/tools/diag/iommu_fault_attribution.py +++ b/tools/diag/iommu_fault_attribution.py @@ -69,19 +69,24 @@ def faults(text): return out -def samples(path): - """[(epoch, pid, ticket)] from the sampler log; a '-' pid is its 'nothing holds it' row.""" +def rows_from_text(text): + """[(epoch, pid, ticket)] from sampler-log text; a '-' pid is its 'nothing holds it' row.""" rows = [] - with open(path, encoding="utf-8", errors="replace") as f: - for line in f: - parts = line.rstrip("\n").split("\t") - if len(parts) < 6 or not parts[0].strip().isdigit(): - continue - rows.append((int(parts[0]), parts[1].strip(), parts[5].strip())) + for line in text.splitlines(): + parts = line.split("\t") + if len(parts) < 6 or not parts[0].strip().isdigit(): + continue + rows.append((int(parts[0]), parts[1].strip(), parts[5].strip())) rows.sort() return rows +def samples(path): + """[(epoch, pid, ticket)] from the sampler log; a '-' pid is its 'nothing holds it' row.""" + with open(path, encoding="utf-8", errors="replace") as f: + return rows_from_text(f.read()) + + def attribute(rows, boots, max_gap): """[(boot_seconds, address, verdict)] — one verdict per fault, decided by its own timestamp.""" out = [] @@ -199,6 +204,17 @@ def kind(v): assert ol[5][0] == 80.0 and ol[5][1] == 0.0, ol # 4 of 5: the 200s sample IS the "-" row, so it is nothing-held assert ol[400][1] > 0.0, ol # 400s spans the handover -> ambiguous + # THE WRITER/READER CONTRACT. These are the sampler's own fixture lines, copied verbatim from + # tools/diag/accel_sampler.sh: if that writer changes its columns, this reader must fail here + # rather than silently parse zeros out of a shifted line. + contract = ("1000\t111\t1\t/serve A\t/scope/a\tticket1\n" + "1010\t222\t2\t/serve B\t/scope/b\tticket2\n") + assert rows_from_text(contract) == [(1000, "111", "ticket1"), (1010, "222", "ticket2")], \ + rows_from_text(contract) + assert rows_from_text("1000\t-\t-\t-\t-\t-\n") == [(1000, "-", "-")], "the '-' row shape" + # And a line with fewer columns is skipped rather than mis-parsed into a holder. + assert rows_from_text("1000\t111\tonly-three\n") == [], "a short row must not become a holder" + # Stamps and addresses are parsed, and non-fault lines are ignored. text = ("[1234.500000] nvme 0000:c4:00.0: AMD-Vi: Event logged [IO_PAGE_FAULT domain=0x0015 address=0x4128000 flags=0x0007]\n" "[1235.000000] unrelated kernel line\n") From fcc7062d34502cde2dc2cc83479aadd3444b3eca Mon Sep 17 00:00:00 2001 From: pi-hrx-compat Date: Sun, 15 Nov 2026 20:33:17 -0400 Subject: [PATCH 3/4] diag: print the window's verdict beside the timestamp verdict, and count the disagreements MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I withdrew my "the sampler should record fault timestamps" ask after reading the merged script: it already writes each fault's epoch in column 1 of attributed.log, with the raw kernel line in column 3. So both readers have the same evidence and differ only in which field they trust -- which makes their disagreement computable instead of a matter of taste. tools/diag/iommu_fault_attribution.py --samples samples.log --attributed attributed.log 105 window=111 DISAGREE HANDOVER-ADJACENT: pid 111 … (next sample: nothing held the node) 150 window=111 222 agree HANDOVER-ADJACENT: pid 111 … (next sample: nothing held the node) 250 window=- agree NOTHING HELD: sampled at the node, no accel client held it faults: 3; agree 2, disagree 1 The rule is not string equality, because the two tools describe the same situation at different precision: a window naming two holders and a HANDOVER-ADJACENT verdict AGREE (both saw a change; this one says which side is the guess). The disagreement that matters is the opposite direction -- the window names ONE holder while the timeline shows the node changed hands, i.e. a handover the window could not see. My first fixture expected all three to agree; the self-test produced exactly that case at 105 and corrected me, so the assertion now pins DISAGREE there and quotes the reason. The fixture lines are attributed.log's real shape, taken from the sampler's own printf. Without a real attributed.log on this box (no fault has been captured) nothing else could validate this, so it is tested against those rows only, and --attributed is opt-in. --outlook on the live trace is now 65.8 % one-holder / 18.6 % ambiguous at +-5 s over 3085 samples: the ambiguity keeps falling as the box quiets down, which is the same point as before -- quote the number with the trace it came from. --- tools/diag/iommu_fault_attribution.py | 69 +++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/tools/diag/iommu_fault_attribution.py b/tools/diag/iommu_fault_attribution.py index 847d68e9d..f8657876c 100644 --- a/tools/diag/iommu_fault_attribution.py +++ b/tools/diag/iommu_fault_attribution.py @@ -157,6 +157,43 @@ def outlook(rows, windows): return out +def attributed_rows(text): + """[(epoch, hits)] from the sampler's attributed.log: `\t\t`. + + The sampler writes this per fault (accel_sampler.sh: `printf '%s\t%s\t%s\n' "$ts" "$hits" "$line"`), + where `hits` is its windowed holder list. So both readers have the same evidence and differ only in + which field they trust -- which makes the disagreement set computable. + """ + out = [] + for line in text.splitlines(): + parts = line.split("\t") + if len(parts) < 2 or not parts[0].strip().isdigit(): + continue + out.append((int(parts[0]), parts[1].strip())) + return out + + +def disagreements(rows, pairs, max_gap): + """[(epoch, window_hits, timestamp_verdict, agree)] for each fault in attributed.log.""" + out = [] + for epoch, hits in pairs: + res = attribute(rows, [(epoch, "-")], max_gap)[0][2] + # agreement is not string equality: the window names holders, this names one and says how sure. + # A window with two holders and a HANDOVER-ADJACENT verdict AGREE about the situation. + win_n = 0 if hits in ("", "-") else len([h for h in hits.replace(",", " ").split() if h]) + one = res.startswith("ATTRIBUTED") + if one and win_n in (0, 1): + agree = True + elif res.startswith("HANDOVER-ADJACENT") and win_n >= 2: + agree = True # both saw a change; this one names which side is the guess + elif res.startswith(("NOTHING HELD", "NO SAMPLE")) and win_n == 0: + agree = True + else: + agree = False + out.append((epoch, hits or "-", res, agree)) + return out + + def self_test(): # One hour of holders, one sample per 100 s, handing over at 300. rows = [(0, "111", "gpu-hold cl-a pid 1 t0"), (100, "111", "gpu-hold cl-a pid 1 t0"), @@ -215,6 +252,26 @@ def kind(v): # And a line with fewer columns is skipped rather than mis-parsed into a holder. assert rows_from_text("1000\t111\tonly-three\n") == [], "a short row must not become a holder" + # The disagreement set: the sampler's windowed `hits` beside this tool's per-fault verdict. The + # fixture lines are that file's real shape (`\t\t`). + attr_text = """105\t111\t[ 105.000000] nvme 0000:c4:00.0: AMD-Vi: Event logged [IO_PAGE_FAULT address=0x1] +150\t111 222\t[ 150.000000] nvme 0000:c4:00.0: AMD-Vi: Event logged [IO_PAGE_FAULT address=0x2] +250\t-\t[ 250.000000] nvme 0000:c4:00.0: AMD-Vi: Event logged [IO_PAGE_FAULT address=0x3] +""" + pairs = attributed_rows(attr_text) + assert pairs == [(105, "111"), (150, "111 222"), (250, "-")], pairs + ds = disagreements(rows, pairs, max_gap=150) + # 105 DISAGREES, and that is the useful case: the window names one holder, but the sample AFTER the + # fault is the '-' row, so the join says the node changed hands to nobody -- the window omitted a + # handover it could not see. 150: window names two, join says HANDOVER-ADJACENT -> agree (same fact, + # stated more precisely). 250: window says nobody, join says NOTHING HELD -> agree. + assert [d[3] for d in ds] == [False, True, True], ds + assert "next sample: nothing held the node" in ds[0][2], ds[0] + # And a window that names ONE holder while the join sees a handover is a real disagreement, which is + # the case worth printing rather than hiding. + one_vs_hand = disagreements(rows, [(150, "111")], max_gap=150) + assert one_vs_hand[0][3] is False, one_vs_hand + # Stamps and addresses are parsed, and non-fault lines are ignored. text = ("[1234.500000] nvme 0000:c4:00.0: AMD-Vi: Event logged [IO_PAGE_FAULT domain=0x0015 address=0x4128000 flags=0x0007]\n" "[1235.000000] unrelated kernel line\n") @@ -230,6 +287,8 @@ def main(): help="epoch of boot, so kernel stamps become absolute (default: /proc/stat btime)") ap.add_argument("--max-gap", type=float, default=15.0, help="oldest usable sample before a fault, in seconds (default 15)") + ap.add_argument("--attributed", default=None, + help="sampler attributed.log: print its windowed hits beside this tool's verdict") ap.add_argument("--outlook", type=float, nargs="*", metavar="W", help="print attributability for these +-W windows and exit (no faults needed)") ap.add_argument("--selftest", "--self-test", dest="self_test", action="store_true") @@ -240,6 +299,16 @@ def main(): return 0 if not args.samples: ap.error("--samples is required unless --self-test") + if args.attributed: + rows = samples(args.samples) + pairs = attributed_rows(open(args.attributed, encoding="utf-8", errors="replace").read()) + ds = disagreements(rows, pairs, args.max_gap) + for epoch, hits, verdict, agree in ds: + print("%d\twindow=%s\t%s\t%s" % (epoch, hits, "agree" if agree else "DISAGREE", verdict)) + n = len(ds) or 1 + print("faults: %d; agree %d, disagree %d" % (len(ds), sum(1 for d in ds if d[3]), + sum(1 for d in ds if not d[3]))) + return 0 if args.outlook: rows = samples(args.samples) print("window one-holder ambiguous nothing-held no-sample") From f25f93c53863af1231cb05c9ac634f4033bf2894 Mon Sep 17 00:00:00 2001 From: pi-hrx-compat Date: Sun, 15 Nov 2026 20:54:26 -0400 Subject: [PATCH 4/4] diag: exercise every output line in the attribution reader's self-test engine#2602 shipped a renderer that crashed on the first real log (KeyError: ticket) because its self-test asserted the analysis functions and never executed the block that prints them. I reviewed that PR before it merged and did not catch it either. My own tool has the same hole: six print sites in main(), and the self-test drove none of them -- the output paths were verified once by hand, which is not the same as being kept true. So the three renderers become functions the self-test can call: fmt_attr(epoch, hits, verdict, agree) one --attributed line fmt_outlook_row(w, one, amb, nothing, none) one --outlook row fmt_fault(sec, addr, verdict) one per-fault line main() prints exactly what the self-test asserts, so a format-string error or a field-order change now fails the self-test instead of the first real capture. The outlook row is asserted by FIELD ORDER AND CONTENT (split() into ["5s", "62.1%", "23.1%", "14.9%", "0.0%"]), not by column widths: my first version compared an exact literal, which failed on my own spacing the moment I wrote it, and would go on failing for a purely cosmetic edit while saying nothing about a crash. The two fmt_fault cases pin the None-stamp path and the whole-second rounding, which are the two branches that only a real fault exercises. A tool that cannot print is not a tool that works -- the same sentence as #2602, applied to the code that was already merged. --- tools/diag/iommu_fault_attribution.py | 33 ++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/tools/diag/iommu_fault_attribution.py b/tools/diag/iommu_fault_attribution.py index f8657876c..3c693bed7 100644 --- a/tools/diag/iommu_fault_attribution.py +++ b/tools/diag/iommu_fault_attribution.py @@ -194,6 +194,22 @@ def disagreements(rows, pairs, max_gap): return out +def fmt_attr(epoch, hits, verdict, agree): + """One --attributed line. A function so the self-test can execute it: a format error here would pass + every analysis assertion and crash on the first real log (engine#2602).""" + return "%d\twindow=%s\t%s\t%s" % (epoch, hits, "agree" if agree else "DISAGREE", verdict) + + +def fmt_outlook_row(w, one, amb, nothing, none): + """One --outlook row, for the same reason.""" + return "%5.0fs %9.1f%% %8.1f%% %11.1f%% %8.1f%%" % (w, one, amb, nothing, none) + + +def fmt_fault(sec, addr, verdict): + """One per-fault line, for the same reason.""" + return "%s\t%s\t%s" % ("-" if sec is None else "%.0f" % sec, addr, verdict) + + def self_test(): # One hour of holders, one sample per 100 s, handing over at 300. rows = [(0, "111", "gpu-hold cl-a pid 1 t0"), (100, "111", "gpu-hold cl-a pid 1 t0"), @@ -276,6 +292,17 @@ def kind(v): text = ("[1234.500000] nvme 0000:c4:00.0: AMD-Vi: Event logged [IO_PAGE_FAULT domain=0x0015 address=0x4128000 flags=0x0007]\n" "[1235.000000] unrelated kernel line\n") assert faults(text) == [(1234.5, "0x4128000")], faults(text) + # EVERY OUTPUT LINE IS EXERCISED. engine#2602 shipped a renderer that crashed on the first real log + # because its self-test only asserted the analysis; a tool that cannot print is not a tool that works. + assert fmt_attr(105, "111", "ATTRIBUTED: pid 111 x", True).endswith("\tATTRIBUTED: pid 111 x") + assert "DISAGREE" in fmt_attr(105, "111", "v", False) + # Field ORDER and content, not column widths: a width literal is a brittle assertion that fails on a + # cosmetic change and says nothing about a format-string crash, which is the failure worth catching. + row = fmt_outlook_row(5, 62.1, 23.1, 14.9, 0.0).split() + assert row == ["5s", "62.1%", "23.1%", "14.9%", "0.0%"], row + assert fmt_fault(None, "0x1", "v") == "-\t0x1\tv", fmt_fault(None, "0x1", "v") + assert fmt_fault(105.4, "0x1", "v") == "105\t0x1\tv", fmt_fault(105.4, "0x1", "v") + print("self-test ok") @@ -304,7 +331,7 @@ def main(): pairs = attributed_rows(open(args.attributed, encoding="utf-8", errors="replace").read()) ds = disagreements(rows, pairs, args.max_gap) for epoch, hits, verdict, agree in ds: - print("%d\twindow=%s\t%s\t%s" % (epoch, hits, "agree" if agree else "DISAGREE", verdict)) + print(fmt_attr(epoch, hits, verdict, agree)) n = len(ds) or 1 print("faults: %d; agree %d, disagree %d" % (len(ds), sum(1 for d in ds if d[3]), sum(1 for d in ds if not d[3]))) @@ -313,7 +340,7 @@ def main(): rows = samples(args.samples) print("window one-holder ambiguous nothing-held no-sample") for w, one, amb, nothing, none in outlook(rows, args.outlook): - print("%5.0fs %9.1f%% %8.1f%% %11.1f%% %8.1f%%" % (w, one, amb, nothing, none)) + print(fmt_outlook_row(w, one, amb, nothing, none)) return 0 boot = args.boot_epoch @@ -327,7 +354,7 @@ def main(): res = attribute(rows, boots, args.max_gap) for sec, addr, verdict in res: when = "-" if sec is None else "%.0f" % sec - print("%s\t%s\t%s" % (when, addr, verdict)) + print(fmt_fault(sec, addr, verdict)) print(yield_line(res)) return 0