From 2ce1a40f9531f9cd2bf76b8f32363e9eba0a50a0 Mon Sep 17 00:00:00 2001 From: pi-release Date: Mon, 16 Nov 2026 04:11:19 -0400 Subject: [PATCH] accel_attribution: a fault on another device is named, not attributed to the NPU's holders The sampler watches ONE node (/dev/accel/accel0, amdxdna). A fault from another PCI function says nothing about who holds it, and until now this reader attributed every IO_PAGE_FAULT line to the accel holders regardless. Measured on this box, 2026-11-16: 372 fault records in the live capture, 372 of them 'nvme 0000:c4:00.0 domain=0x0015', and under --any-device 165 of them get a confident holder -- the process that happened to hold the NPU. That is the category error engine#2623 fixed in the sibling reader, and I had reported it there while this file kept producing it. default : 372 'NOT THIS DEVICE: the fault is not amdxdna's, so the holders cannot explain it' --any-device : 165 attributed (the cross-device question, asked deliberately) The fixtures had to change for the fix to be testable: every fault text in the self-test was a bare 'A'/'B'/'C', so none of them named a device and none of them could see this. They now carry a real 'amdxdna 0000:c6:00.1: ...' line, which is what the product always emits. Two traps hit while writing it, both kept in the code's shape rather than in prose: a function-local 'import tempfile' further down made tempfile local to the whole function (UnboundLocalError at the first use), and the module had no tempfile import at all. --- tools/diag/accel_attribution.py | 60 +++++++++++++++++++++++++++++---- 1 file changed, 53 insertions(+), 7 deletions(-) diff --git a/tools/diag/accel_attribution.py b/tools/diag/accel_attribution.py index c20b67d43..8079d30c5 100755 --- a/tools/diag/accel_attribution.py +++ b/tools/diag/accel_attribution.py @@ -38,6 +38,7 @@ import sys import shutil import io +import tempfile def read_samples(path): @@ -58,6 +59,18 @@ def read_samples(path): return out +# The sampler watches ONE device node (/dev/accel/accel0, the amdxdna NPU), so a fault belonging to another PCI +# function says nothing about its holders. Measured 2026-11-16 on this box: 189 of 189 fault records in one capture +# were `nvme 0000:c4:00.0 domain=0x0015`, and every one was attributed to whatever held the NPU at the time -- +# a category error, not a weak answer (the same correction engine#2623 made to the sibling reader). +ACCEL_DEVICE = "amdxdna" + + +def is_accel_fault(line): + """True when the fault line belongs to the device whose holders the sampler records.""" + return ACCEL_DEVICE in line + + def read_attributions(path): """[(epoch, hits, fault_line)] as written by the sampler's watch loop, or None when there is no file yet. @@ -131,10 +144,18 @@ def in_window(samples, ts, window): return [s for s in samples if ts - window <= s[0] <= ts + window] -def explain(samples, attributions, window): - """One block per fault: what was sampled around it, in order, and whether that identifies a single holder.""" +def explain(samples, attributions, window, any_device=False): + """One block per fault: what was sampled around it, in order, and whether that identifies a single holder. + + A fault on another PCI function is reported as NOT THIS DEVICE: attributing an NVMe fault to the process holding + /dev/accel is a category error, and a reader that does it silently produces a confident wrong holder. + """ lines = [] for ts, _hits, fault in attributions: + if not any_device and not is_accel_fault(fault): + lines.append(f" {ts} NOT THIS DEVICE: the fault is not {ACCEL_DEVICE}'s, so the holders cannot explain it") + lines.append(f" {fault[:110]}") + continue w = sorted(in_window(samples, ts, window), key=lambda s: s[0]) if not w: verdict = "NO SAMPLE in the window: nothing can be attributed (this is not 'nobody held it')" @@ -180,13 +201,17 @@ def self_test(): # window 1, fault 112 -> [111,113] -> {'-'} sampled, nothing held the node # window 5, fault 103 -> [98,108] -> {111, 222} AMBIGUOUS: the same fault is unambiguous at 3 # window 3, fault 200 -> [197,203] -> {} nothing sampled: not evidence of absence - narrow, wide = "\n".join(explain(samples, [(100, "", "A"), (106, "", "B"), (109, "", "C"), - (200, "", "D")], 3)), "\n".join(explain(samples, [(103, "", "E")], 5)) + # The fault text names the device, as every real line does -- the fixtures used to carry a bare "A"/"B", which + # is why none of them could see the category error engine#2623/#2542 found: a line with no device cannot be + # classified as ours or not. + A_ = "amdxdna 0000:c6:00.1: AMD-Vi: Event logged [IO_PAGE_FAULT domain=0x0003 address=0x4128000]" + narrow, wide = "\n".join(explain(samples, [(100, "", A_), (106, "", A_), (109, "", A_), + (200, "", A_)], 3)), "\n".join(explain(samples, [(103, "", A_)], 5)) assert narrow.count("one holder across 1 sample(s): pid 111") == 1, narrow assert narrow.count("one holder across 1 sample(s): pid 222") == 2, narrow # faults B and C assert narrow.count("; 1 of 2 samples had nothing holding the node") == 1, narrow # fault C assert "NO SAMPLE in the window" in narrow, narrow - dot = "\n".join(explain(samples, [(112, "", "F")], 1)) + dot = "\n".join(explain(samples, [(112, "", A_)], 1)) assert "sampled 1x, nothing held the node" in dot, dot assert "AMBIGUOUS: 2 holders inside +-5s (111, 222)" in wide, wide assert "not 'nobody held it'" in narrow, narrow @@ -207,8 +232,27 @@ def self_test(): assert "gpu-hold cl-b 2" in rendered, rendered # the LAST span: what the process was relabelled with assert "cl-a" not in rendered, rendered + # engine#2542: a fault on another PCI function must be NAMED, not attributed to the NPU's holders. Measured on + # this box: 189 of 189 records in one capture were `nvme 0000:c4:00.0 domain=0x0015` and every one was given the + # process holding /dev/accel -- a category error that reads as a confident answer. + foreign = ("1794798810\t215633\t/serve\t-\tscope\tgpu-hold cl-a\t" + "[ 321.308031] strixhalo kernel: nvme 0000:c4:00.0: AMD-Vi: Event logged " + "[IO_PAGE_FAULT domain=0x0015 address=0xe4240000 flags=0x0000]\n") + ppath = tempfile.NamedTemporaryFile("w", suffix=".log", delete=False) + ppath.write(foreign); ppath.close() + try: + at = read_attributions(ppath.name) + assert len(at) == 1, at + out = "\n".join(explain(samples, at, 5)) + assert "NOT THIS DEVICE" in out, out + assert "one holder" not in out, out + # ... and the override still lets a reader ask the cross-device question deliberately + out2 = "\n".join(explain(samples, at, 5, any_device=True)) + assert "NOT THIS DEVICE" not in out2, out2 + finally: + os.unlink(ppath.name) + # A partially written row must not kill the reader. - import tempfile with tempfile.NamedTemporaryFile("w", suffix=".log", delete=False) as f: f.write("100\t111\t1\t/a serve\tscope\tticket\n") f.write("106\t22") # torn @@ -254,6 +298,8 @@ def main(): ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument("--dir", default=os.path.expanduser("~/.cache/accel-sampler")) ap.add_argument("--window", type=int, default=5, help="the sampler's WINDOW_S (default 5)") + ap.add_argument("--any-device", action="store_true", + help="also attribute faults from devices the sampler does not watch (default: name and skip them)") ap.add_argument("--self-test", action="store_true") args = ap.parse_args() if args.self_test: @@ -298,7 +344,7 @@ def main(): print(render_holders(t)) print(f"\nattributions: {len(attrs)}") - for line in explain(samples, attrs, args.window): + for line in explain(samples, attrs, args.window, args.any_device): print(line) return 0