From d550ded1fe40c2aae2b3b2cdc9201cdc71f32601 Mon Sep 17 00:00:00 2001 From: pi-hrx-compat Date: Mon, 16 Nov 2026 00:44:12 -0400 Subject: [PATCH 1/4] diag: carry the sample distance, and call a far sample weak rather than certain engine#2627 measured what I could not: nearest-sample distances of median 1 s, p90 27 s, max 45 s, and 24 of 119 faults more than 10 s from any sample. My reader had the same information and threw it away in the verdict -- "ATTRIBUTED: pid 1234 gpu-hold cl-a" says nothing about whether that sample was 2 s or 44 s old, and the reader cannot judge an attribution without it. So every holder line now carries its distance: ATTRIBUTED: pid 1234 gpu-hold cl-a pid 9 (sample 2s before) ATTRIBUTED-WEAK: pid 1234 gpu-hold cl-a pid 9 (sample 27s before), the node changes hands every ~200s on this box The cut is 5 s, from my own measurement rather than taste: 18 distinct gpu-hold tickets held the node over 3602 s, i.e. a hand-over about every 200 s, so the chance the holder changed inside the gap is 2.5 % at 5 s, 13.5 % at 27 s and 22.5 % at 45 s. Below 5 s it is noise; past it the attribution is worth reporting and not worth presenting as certain. The tally splits them, so a run cannot look stronger than it is: faults: 5; attributable 2 (weak 1), ambiguous 1, nothing-held 1, no-sample 1, other-device 0 Pinned: a 30 s-old sample is ATTRIBUTED-WEAK, and the summary counts it under weak. The NOT THE ACCEL NODE / padded-stamp / GAP arms are unchanged, and the live capture still reports "faults: 137; ... other-device 137" because those faults are NVMe. (cherry picked from commit a44dc5927030c3fb810fb4b6b3608b1289719cb1) --- tools/diag/iommu_fault_attribution.py | 30 +++++++++++++++++++-------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/tools/diag/iommu_fault_attribution.py b/tools/diag/iommu_fault_attribution.py index dd4d98ae9..c9199bcb5 100644 --- a/tools/diag/iommu_fault_attribution.py +++ b/tools/diag/iommu_fault_attribution.py @@ -58,6 +58,7 @@ ACCEL = "amdxdna" # the driver whose node the sampler's holders describe +WEAK_AFTER_S = 5.0 # nearest-sample distance past which an attribution is reported as weak (engine#2627) def faults(text): @@ -137,14 +138,22 @@ def attribute(rows, boots, max_gap, any_device=False): if pid == "-": out.append((sec, addr, "NOTHING HELD: sampled at the node, no accel client held it")) continue - who = f"pid {pid} {ticket[:64]}" + # How far away the evidence was, on every line: engine#2627 measured nearest-sample distances of median 1 s, + # p90 27 s, max 45 s on ITS capture (this sampler: median 2 s, p90 3 s, max 93 s). With 18 tickets holding the + # node per hour the odds the holder changed inside the gap are 2.5 % at 5 s, 13.5 % at 27 s, 22.5 % at 45 s -- + # so the distance is the difference between strong and weak evidence rather than a detail. + age = sec - epoch + who = f"pid {pid} {ticket[:64]} (sample {age:.0f}s before)" if after and after[0][1] != pid: # Name the other side: a window would name both and stop; this says which one is the guess # and what the next observation saw. '-' there is "nothing held it", not a process named '-'. nxt = "nothing held the node" if after[0][1] == "-" else f"pid {after[0][1]}" out.append((sec, addr, f"HANDOVER-ADJACENT: {who} (next sample: {nxt})")) - else: + elif age <= WEAK_AFTER_S: out.append((sec, addr, f"ATTRIBUTED: {who}")) + else: + # Visible as weak, not dropped and not called strong (engine#2627: report the distance, do not filter on it). + out.append((sec, addr, f"ATTRIBUTED-WEAK: {who}, the node changes hands every ~200s on this box")) return out @@ -160,7 +169,8 @@ def yield_line(res): """The number that decides whether a sampling run can answer the question at all.""" # Every verdict must be counted, or the summary says "0 of everything" about a capture full of faults -- the # silent-drop class this file exists to avoid. "NOT THE ACCEL NODE" is a fifth outcome, not a footnote. - k = {"ATTRIBUTED": 0, "HANDOVER-ADJACENT": 0, "NOTHING HELD": 0, "NO SAMPLE": 0, "NOT THE ACCEL NODE": 0} + k = {"ATTRIBUTED-WEAK": 0, "ATTRIBUTED": 0, "HANDOVER-ADJACENT": 0, "NOTHING HELD": 0, "NO SAMPLE": 0, + "NOT THE ACCEL NODE": 0} # WEAK first: the tally prefixes on these keys for _, _, v in res: for key in k: if v.startswith(key): @@ -168,9 +178,11 @@ def yield_line(res): break n = len(res) total = sum(k.values()) - return ("faults: %d; attributable %d, ambiguous %d, nothing-held %d, no-sample %d, other-device %d" - % (n, k["ATTRIBUTED"], k["HANDOVER-ADJACENT"], k["NOTHING HELD"], k["NO SAMPLE"], - k["NOT THE ACCEL NODE"])) + ("" if total == n else " !! %d fault(s) uncounted" % (n - total)) + total = sum(k.values()) + line = ("faults: %d; attributable %d (weak %d), ambiguous %d, nothing-held %d, no-sample %d, other-device %d" + % (n, k["ATTRIBUTED"] + k["ATTRIBUTED-WEAK"], k["ATTRIBUTED-WEAK"], k["HANDOVER-ADJACENT"], + k["NOTHING HELD"], k["NO SAMPLE"], k["NOT THE ACCEL NODE"])) + return line + ("" if total == n else " !! %d fault(s) uncounted" % (n - total)) + ("" if total == n else " !! %d fault(s) uncounted" % (n - total)) def outlook(rows, windows): @@ -269,7 +281,7 @@ def kind(v): head = v.split(":")[0] return "NO SAMPLE" if head.startswith("NO SAMPLE") else head got = [kind(v) for _, _, v in res] - assert got == ["ATTRIBUTED", "HANDOVER-ADJACENT", "NOTHING HELD", "ATTRIBUTED", "NO SAMPLE"], res + assert got == ["ATTRIBUTED-WEAK", "HANDOVER-ADJACENT", "NOTHING HELD", "ATTRIBUTED", "NO SAMPLE"], res # 50 s is WEAK assert "cl-a" in res[0][2] and "pid 111" in res[0][2], res[0] # The handover verdict must NAME both sides: naming one is the failure mode of a window. Here the # handover is to 'nobody', which must read as words rather than as a process called '-'. @@ -281,7 +293,7 @@ def kind(v): assert hand[0][2].startswith("HANDOVER-ADJACENT"), hand assert "pid 111" in hand[0][2] and "pid 222" in hand[0][2], hand assert "no accel client held it" in res[2][2], res[2] # not "I did not look" - assert yield_line(res) == ("faults: 5; attributable 2, ambiguous 1, nothing-held 1, no-sample 1, other-device 0"), yield_line(res) + assert yield_line(res) == ("faults: 5; attributable 2 (weak 1), ambiguous 1, nothing-held 1, no-sample 1, other-device 0"), yield_line(res) # --max-gap decides freshness, and a smaller gap converts ATTRIBUTED into NO SAMPLE rather than # silently attributing across a long unobserved stretch. @@ -362,7 +374,7 @@ def kind(v): # The summary must account for every fault: a verdict missing from the tally makes "0 of everything" look like # an empty capture. This arm would have printed 0/0/0/0 for 137 NVMe faults before the counter was added. assert yield_line([(1.0, "0x1", "NOT THE ACCEL NODE: other device")]) == ( - "faults: 1; attributable 0, ambiguous 0, nothing-held 0, no-sample 0, other-device 1"), \ + "faults: 1; attributable 0 (weak 0), ambiguous 0, nothing-held 0, no-sample 0, other-device 1"), \ yield_line([(1.0, "0x1", "NOT THE ACCEL NODE: other device")]) # Stamps and addresses are parsed, and non-fault lines are ignored. From e7d8e775aa5cc9a28092e6239019932cdd4dc5b6 Mon Sep 17 00:00:00 2001 From: pi-hrx-compat Date: Mon, 16 Nov 2026 00:54:38 -0400 Subject: [PATCH 2/4] diag: predict the nearest-sample distance from a quiet trace, and restore the boot guard Two changes, both from evidence on this box rather than from taste. --distances prints the statistic engine#2627 measured on real faults (median 1 s, p90 27 s, max 45 s; 24 of 119 beyond 10 s), computed from the sample timeline alone -- a fault lands at a random time, so its nearest-sample distance is distributed like the gaps between samples. Live, on the running sampler: gaps 2867 median 2s p90 3s max 93s nearest sample within 2s: 89.4%, within 5s: 93.1%, within 10s: 99.8% That is the number a capture's owner wants BEFORE any fault: it says whether the sampler is dense enough to attribute one, and how much of a run will be WEAK. The 93 s max is the same lapse #2624 records as a GAP. Second: drop_pre_boot() is restored. It was written when the box cold-rebooted (kernel stamps are boot-relative, so a samples.log straddling a reboot lets the gap test match a fresh fault against a sample taken hours earlier) and it landed on a branch that merged before that commit reached it, so main never got it. Every mode applies it again, with a stderr note naming the count and the boundary, and an unknown --boot-epoch filters nothing rather than guessing. Pinned: the distance distribution on a five-gap fixture (median 1, p90 20, 80 % within 2 s), one sample having no gaps at all, and the three drop_pre_boot cases. (cherry picked from commit f7700ba0a0df31a77e9f19acd2fa689eb9c917d4) --- tools/diag/iommu_fault_attribution.py | 82 +++++++++++++++++++++++++-- 1 file changed, 76 insertions(+), 6 deletions(-) diff --git a/tools/diag/iommu_fault_attribution.py b/tools/diag/iommu_fault_attribution.py index c9199bcb5..56dba5d5b 100644 --- a/tools/diag/iommu_fault_attribution.py +++ b/tools/diag/iommu_fault_attribution.py @@ -265,6 +265,41 @@ def fmt_fault(sec, addr, verdict): return "%s\t%s\t%s" % ("-" if sec is None else "%.0f" % sec, addr, verdict) +def drop_pre_boot(rows, boot_epoch): + """[(epoch, pid, ticket)] with samples from a previous boot removed. + + Kernel stamps are boot-relative, so a fault's absolute time is (this boot's epoch + stamp). A samples.log that + straddles a reboot -- kept across one, or copied from an earlier capture -- holds epochs from BEFORE that boot, + and the gap test is `fault - sample <= max_gap`, so a fault in the first seconds after a reboot falls within + max_gap of a sample taken hours earlier and would be attributed to a holder that cannot have been running. + + This landed on a branch that merged before the commit reached it, so it is restored here (engine#2623). + """ + if boot_epoch is None: + return rows, 0 + kept = [r for r in rows if r[0] >= boot_epoch] + return kept, len(rows) - len(kept) + + +def distances(rows): + """How far a fault would be from its nearest sample, predicted from a trace with no faults in it. + + A fault lands at a random time, so its nearest-sample distance is distributed like the gaps between samples. + engine#2627 measured the real thing (median 1 s, p90 27 s, max 45 s; 24 of 119 beyond 10 s); this prints the same + statistic from the samples alone, which is what tells a capture's owner -- before any fault -- whether the + sampler is dense enough to attribute one, and whether a run will be dominated by WEAK verdicts. + """ + ts = [r[0] for r in rows] + gaps = sorted(b - a for a, b in zip(ts, ts[1:]) if b >= a) + if not gaps: + return None + n = len(gaps) + return {"n": n, "median": gaps[n // 2], "p90": gaps[min(n - 1, n * 9 // 10)], "max": gaps[-1], + "within2": 100.0 * sum(1 for g in gaps if g <= 2) / n, + "within5": 100.0 * sum(1 for g in gaps if g <= 5) / n, + "within10": 100.0 * sum(1 for g in gaps if g <= 10) / n} + + def self_test(): # One hour of holders, one sample per 100 s, handing over at 300. rows = [(0, "111", "gpu-hold cl-a pid 1 t0"), (100, "111", "gpu-hold cl-a pid 1 t0"), @@ -377,6 +412,24 @@ def kind(v): "faults: 1; attributable 0 (weak 0), ambiguous 0, nothing-held 0, no-sample 0, other-device 1"), \ yield_line([(1.0, "0x1", "NOT THE ACCEL NODE: other device")]) + # A far sample is reported as WEAK rather than dropped or called certain (engine#2627), and the tally splits it. + weak = attribute([(1000, "1234", "gpu-hold cl-a pid 9")], [(1030.0, "0x1", True)], 150) + assert weak[0][2].startswith("ATTRIBUTED-WEAK"), weak + assert yield_line(weak).startswith("faults: 1; attributable 1 (weak 1)"), yield_line(weak) + + # The distance distribution must be computable from a quiet trace, and must be the same statistic #2627 measured + # on real faults. 4 gaps of 1 s and one of 20 s: median 1, p90 20, 80 % within 2 s. + dd = distances([(0, "a", ""), (1, "a", ""), (2, "a", ""), (3, "a", ""), (4, "a", ""), (24, "a", "")]) + assert (dd["n"], dd["median"], dd["p90"], dd["max"]) == (5, 1, 20, 20), dd + assert abs(dd["within2"] - 80.0) < 0.01 and dd["within5"] == 80.0, dd + assert distances([(0, "a", "")]) is None, "one sample has no gaps" + + # A samples.log from a previous boot must not be joined to this boot's faults (see drop_pre_boot): the gap test + # would match a fresh fault against a sample taken hours earlier. + assert drop_pre_boot([(100, "111", "old"), (110, "111", "old")], 1000) == ([], 2) + assert drop_pre_boot([(100, "111", "old"), (1200, "222", "new")], 1000) == ([(1200, "222", "new")], 1) + assert drop_pre_boot([(100, "111", "old")], None) == ([(100, "111", "old")], 0), "no boot epoch: filter nothing" + # Stamps and addresses are parsed, and non-fault lines are ignored. text = ("[1234.500000] nvme 0000:c4:00.0: AMD-Vi: Event logged [IO_PAGE_FAULT domain=0x0015 address=0x4128000 flags=0x0007]\n" "[1235.000000] unrelated kernel line\n") @@ -422,6 +475,8 @@ def main(): "holders describe /dev/accel, so attributing another device's fault is a category error)") ap.add_argument("--attributed", default=None, help="sampler attributed.log: print its windowed hits beside this tool's verdict") + ap.add_argument("--distances", action="store_true", + help="print the nearest-sample distance distribution a fault would see (no faults needed)") ap.add_argument("--outlook", type=float, nargs="*", metavar="W", help="print attributability for these +-W windows and exit (no faults needed)") ap.add_argument("--selftest", "--self-test", dest="self_test", action="store_true") @@ -432,8 +487,12 @@ def main(): return 0 if not args.samples: ap.error("--samples is required unless --self-test") + boot = args.boot_epoch + if boot is None: + with open("/proc/stat", encoding="utf-8") as f: + boot = float(next(int(l.split()[1]) for l in f if l.startswith("btime"))) if args.attributed: - rows = samples(args.samples) + rows, _ = drop_pre_boot(samples(args.samples), boot) pairs = attributed_rows(open(args.attributed, encoding="utf-8", errors="replace").read()) ds = disagreements(rows, pairs, args.max_gap) for epoch, hits, verdict, agree in ds: @@ -442,20 +501,31 @@ def main(): print("faults: %d; agree %d, disagree %d" % (len(ds), sum(1 for d in ds if d[3]), sum(1 for d in ds if not d[3]))) return 0 + if args.distances: + rows, _ = drop_pre_boot(samples(args.samples), boot) + d = distances(rows) + if d is None: + print("no sample gaps in this trace: nothing to measure") + return 0 + print("gaps %d median %.0fs p90 %.0fs max %.0fs" % (d["n"], d["median"], d["p90"], d["max"])) + print("nearest sample within 2s: %.1f%%, within 5s: %.1f%%, within 10s: %.1f%%" + % (d["within2"], d["within5"], d["within10"])) + print("an attribution past %.0fs is reported WEAK (ATTRIBUTED-WEAK)" % WEAK_AFTER_S) + return 0 if args.outlook: - rows = samples(args.samples) + rows, _ = drop_pre_boot(samples(args.samples), boot) print("window one-holder ambiguous nothing-held no-sample") for w, one, amb, nothing, none in outlook(rows, args.outlook): print(fmt_outlook_row(w, one, amb, nothing, none)) return 0 - boot = args.boot_epoch - if boot is None: - with open("/proc/stat", encoding="utf-8") as f: - boot = float(next(int(l.split()[1]) for l in f if l.startswith("btime"))) text = sys.stdin.read() if args.dmesg == "-" else open(args.dmesg, encoding="utf-8", errors="replace").read() rows = samples(args.samples) + rows, dropped = drop_pre_boot(rows, boot) + if dropped: + print("note: %d sample(s) predate this boot (btime %d) and were ignored: a fault's absolute time is " + "boot + kernel stamp, so they cannot describe this boot" % (dropped, boot), file=sys.stderr) boots = [(None if s is None else boot + s, a, acc) for s, a, acc in faults(text)] res = attribute(rows, boots, args.max_gap, any_device=args.any_device) for sec, addr, verdict in res: From f15d40412c06646231635852ff2c0cd756ba7fe6 Mon Sep 17 00:00:00 2001 From: pi-hrx-compat Date: Mon, 16 Nov 2026 03:48:56 -0400 Subject: [PATCH 3/4] diag: refuse combining the report flags, and attribute the quoted numbers to their capture Two changes, both from the review at c339f0ec7. 1. The flag bug. `--attributed` ran first and returned, so `--attributed --distances` printed the disagreement report and silently dropped the distance distribution. Re-ordering would only move the silent drop to the other flag, so the modes are now mutually exclusive and say so: error: --attributed and --distances are separate reports: run them one at a time A flag that answers a question must never be answered by a different question. 2. The number in the comment. It said "engine#2627 measured nearest-sample distances of median 1 s, p90 27 s, max 45 s" as though those described this box. They describe that capture. The text now says so and gives the running sampler's own distribution: median 2 s, p90 3 s, max 93 s, with the max being the same lapse the sampler records as a GAP. That is the lesson of #2641 applied to my own prose: an interval measured on one hour is a fact about that hour. Self-test unchanged and green; the live sampler still reads "gaps 7374 median 2s p90 2s max 93s". --- tools/diag/iommu_fault_attribution.py | 32 +++++++++++++++++---------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/tools/diag/iommu_fault_attribution.py b/tools/diag/iommu_fault_attribution.py index 56dba5d5b..c32a40928 100644 --- a/tools/diag/iommu_fault_attribution.py +++ b/tools/diag/iommu_fault_attribution.py @@ -285,8 +285,9 @@ def distances(rows): """How far a fault would be from its nearest sample, predicted from a trace with no faults in it. A fault lands at a random time, so its nearest-sample distance is distributed like the gaps between samples. - engine#2627 measured the real thing (median 1 s, p90 27 s, max 45 s; 24 of 119 beyond 10 s); this prints the same - statistic from the samples alone, which is what tells a capture's owner -- before any fault -- whether the + engine#2627 measured the real thing on ITS capture (median 1 s, p90 27 s, max 45 s; 24 of 119 beyond 10 s). Those + numbers are that hour's load, not this box's: the running sampler here reads median 2 s, p90 3 s, max 93 s, and the + max is the same lapse the sampler records as a GAP. So the same statistic computed from the samples alone, which is what tells a capture's owner -- before any fault -- whether the sampler is dense enough to attribute one, and whether a run will be dominated by WEAK verdicts. """ ts = [r[0] for r in rows] @@ -487,20 +488,17 @@ def main(): return 0 if not args.samples: ap.error("--samples is required unless --self-test") + # The modes all print a report and exit, so combining them silently dropped one (review of #2640: `--attributed` + # ran first and `return 0`d, so `--distances` printed nothing). Refuse the combination instead of picking one: + # a flag that answers a question must never be answered by a different question. + modes = [name for name, on in (("--attributed", args.attributed), ("--distances", args.distances), + ("--outlook", bool(args.outlook))) if on] + if len(modes) > 1: + ap.error("%s are separate reports: run them one at a time" % " and ".join(modes)) boot = args.boot_epoch if boot is None: with open("/proc/stat", encoding="utf-8") as f: boot = float(next(int(l.split()[1]) for l in f if l.startswith("btime"))) - if args.attributed: - rows, _ = drop_pre_boot(samples(args.samples), boot) - pairs = attributed_rows(open(args.attributed, encoding="utf-8", errors="replace").read()) - ds = disagreements(rows, pairs, args.max_gap) - for epoch, hits, verdict, agree in ds: - print(fmt_attr(epoch, hits, verdict, agree)) - n = len(ds) or 1 - print("faults: %d; agree %d, disagree %d" % (len(ds), sum(1 for d in ds if d[3]), - sum(1 for d in ds if not d[3]))) - return 0 if args.distances: rows, _ = drop_pre_boot(samples(args.samples), boot) d = distances(rows) @@ -512,6 +510,16 @@ def main(): % (d["within2"], d["within5"], d["within10"])) print("an attribution past %.0fs is reported WEAK (ATTRIBUTED-WEAK)" % WEAK_AFTER_S) return 0 + if args.attributed: + rows, _ = drop_pre_boot(samples(args.samples), boot) + pairs = attributed_rows(open(args.attributed, encoding="utf-8", errors="replace").read()) + ds = disagreements(rows, pairs, args.max_gap) + for epoch, hits, verdict, agree in ds: + print(fmt_attr(epoch, hits, verdict, agree)) + n = len(ds) or 1 + print("faults: %d; agree %d, disagree %d" % (len(ds), sum(1 for d in ds if d[3]), + sum(1 for d in ds if not d[3]))) + return 0 if args.outlook: rows, _ = drop_pre_boot(samples(args.samples), boot) print("window one-holder ambiguous nothing-held no-sample") From 1da6f9a08251dcbf7c1b0047afe7ad7c372d0728 Mon Sep 17 00:00:00 2001 From: pi-hrx-compat Date: Mon, 16 Nov 2026 04:20:39 -0400 Subject: [PATCH 4/4] diag: say what the agree/disagree column measures The column compares this tool's verdict with the sampler's own `hits` list, and a window naming two holders is DISAGREE by design -- a window listing two lanes is not corroboration. That makes it a measure of HOLDER TURNOVER, not of either rule's correctness, and without saying so a reader concludes the two readers disagree with each other half the time (measured 46 % on one capture, 0.7 % on a stable one). The measured reason the rules coincide when turnover is low: on a live timeline of 281 faults the preceding and following sample name the same holder for 279 of them (99.3 %), so "nearest sample" and "last preceding sample" are the same answer whenever the holder is stable -- the 18.1 % of cases where the NEXT sample is nearer change nothing, because the holder did not change. So the header now prints, on every --attributed run: # agree/disagree compares the sampler's windowed hits with this tool's verdict: a window naming two holders is DISAGREE by design, so the count measures HOLDER TURNOVER, not rule correctness Self-test green; the synthetic attributed run prints the header then the row, as before. --- tools/diag/iommu_fault_attribution.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tools/diag/iommu_fault_attribution.py b/tools/diag/iommu_fault_attribution.py index c32a40928..1d166bce2 100644 --- a/tools/diag/iommu_fault_attribution.py +++ b/tools/diag/iommu_fault_attribution.py @@ -513,6 +513,13 @@ def main(): if args.attributed: rows, _ = drop_pre_boot(samples(args.samples), boot) pairs = attributed_rows(open(args.attributed, encoding="utf-8", errors="replace").read()) + # What agree/disagree MEANS, on every run: it compares this tool's verdict against the sampler's own `hits` + # LIST, so a window naming two holders is DISAGREE by design -- a window listing two lanes is not + # corroboration. That makes the column a measure of HOLDER TURNOVER, not of either rule's correctness: on a + # stable holder the nearest sample (either side) and the last preceding sample name the same holder (measured + # here: 279 of 281 faults, 99.3 %), so the two rules coincide whenever turnover is low. + print("# agree/disagree compares the sampler's windowed hits with this tool's verdict: a window naming two " + "holders is DISAGREE by design, so the count measures HOLDER TURNOVER, not rule correctness") ds = disagreements(rows, pairs, args.max_gap) for epoch, hits, verdict, agree in ds: print(fmt_attr(epoch, hits, verdict, agree))