diff --git a/tools/diag/iommu_fault_attribution.py b/tools/diag/iommu_fault_attribution.py index 3c693bed7..7e5998ce5 100644 --- a/tools/diag/iommu_fault_attribution.py +++ b/tools/diag/iommu_fault_attribution.py @@ -57,15 +57,24 @@ STAMP = re.compile(r"^\[\s*(\d+\.\d+)\]") +ACCEL = "amdxdna" # the driver whose node the sampler's holders describe + + def faults(text): - """[(boot_seconds, address)] for every IO_PAGE_FAULT line, in log order.""" + """[(boot_seconds, address, is_accel)] for every IO_PAGE_FAULT line, in log order. + + `is_accel` matters because the sampler records who holds `/dev/accel`: a fault on another PCI function + (the NVMe stream is `nvme 0000:c4:00.0 domain=0x0015`) has nothing to do with those holders, and + attributing it to one is a category error rather than a weak answer. Measured on this box: 137 NVMe + faults were "attributed" to a `gpu-hold` ticket that was holding the NPU. + """ out = [] for line in text.splitlines(): if "IO_PAGE_FAULT" not in line: continue m = STAMP.match(line) addr = re.search(r"address=(0x[0-9a-fA-F]+)", line) - out.append((float(m.group(1)) if m else None, addr.group(1) if addr else "?")) + out.append((float(m.group(1)) if m else None, addr.group(1) if addr else "?", ACCEL in line)) return out @@ -76,7 +85,10 @@ def rows_from_text(text): parts = line.split("\t") if len(parts) < 6 or not parts[0].strip().isdigit(): continue - rows.append((int(parts[0]), parts[1].strip(), parts[5].strip())) + # A GAP row (engine#2624) carries its length in column 3 where a sample carries its fds; keep the length as + # the note so the verdict can say how long the sampler was stalled, not just that it was. + pid = parts[1].strip() + rows.append((int(parts[0]), pid, (parts[2].strip() or "?") if pid == "GAP" else parts[5].strip())) rows.sort() return rows @@ -87,10 +99,17 @@ def samples(path): return rows_from_text(f.read()) -def attribute(rows, boots, max_gap): +def attribute(rows, boots, max_gap, any_device=False): """[(boot_seconds, address, verdict)] — one verdict per fault, decided by its own timestamp.""" out = [] - for sec, addr in boots: + for sec, addr, is_accel in boots: + if not is_accel and not any_device: + # The sampler records who holds /dev/accel. A fault on another PCI function has nothing to do with + # those holders: measured on this box, 137 NVMe faults were "attributed" to a gpu-hold ticket that was + # holding the NPU. A category error is not a weak answer, so refuse it unless asked to proceed. + out.append((sec, addr, "NOT THE ACCEL NODE: this fault is on another device, so the sampler's holders " + "do not describe it (pass --any-device to attribute anyway)")) + continue if sec is None: out.append((sec, addr, "NO SAMPLE: the line carries no kernel timestamp")) continue @@ -106,6 +125,11 @@ def attribute(rows, boots, max_gap): if pid == "-": out.append((sec, addr, "NOTHING HELD: sampled at the node, no accel client held it")) continue + if pid == "GAP": + # engine#2624: the sampler writes a GAP row when it slept too long to sample. That is the absence of an + # observation, NOT a process called GAP -- attributing a fault here would invent a holder. + out.append((sec, addr, "NO SAMPLE: the sampler was stalled across this fault (GAP row %s)" % ticket)) + continue who = f"pid {pid} {ticket[:64]}" if after and after[0][1] != pid: # Name the other side: a window would name both and stop; this says which one is the guess @@ -119,15 +143,19 @@ def attribute(rows, boots, max_gap): def yield_line(res): """The number that decides whether a sampling run can answer the question at all.""" - k = {"ATTRIBUTED": 0, "HANDOVER-ADJACENT": 0, "NOTHING HELD": 0, "NO SAMPLE": 0} + # Every verdict must be counted, or the summary says "0 of everything" about a capture full of faults -- the + # silent-drop class this file exists to avoid. "NOT THE ACCEL NODE" is a fifth outcome, not a footnote. + k = {"ATTRIBUTED": 0, "HANDOVER-ADJACENT": 0, "NOTHING HELD": 0, "NO SAMPLE": 0, "NOT THE ACCEL NODE": 0} for _, _, v in res: for key in k: if v.startswith(key): k[key] += 1 break n = len(res) - return ("faults: %d; attributable %d, ambiguous %d, nothing-held %d, no-sample %d" - % (n, k["ATTRIBUTED"], k["HANDOVER-ADJACENT"], k["NOTHING HELD"], k["NO SAMPLE"])) + total = sum(k.values()) + return ("faults: %d; attributable %d, ambiguous %d, nothing-held %d, no-sample %d, other-device %d" + % (n, k["ATTRIBUTED"], k["HANDOVER-ADJACENT"], k["NOTHING HELD"], k["NO SAMPLE"], + k["NOT THE ACCEL NODE"])) + ("" if total == n else " !! %d fault(s) uncounted" % (n - total)) def outlook(rows, windows): @@ -143,7 +171,7 @@ def outlook(rows, windows): one = amb = nothing = none = 0 for epoch, _pid, _ticket in rows: near = [(e, p) for e, p, _t in rows if abs(e - epoch) <= w] - holders = {p for _e, p in near if p != "-"} + holders = {p for _e, p in near if p not in ("-", "GAP")} if not near: none += 1 elif not holders: @@ -177,7 +205,7 @@ def disagreements(rows, pairs, max_gap): """[(epoch, window_hits, timestamp_verdict, agree)] for each fault in attributed.log.""" out = [] for epoch, hits in pairs: - res = attribute(rows, [(epoch, "-")], max_gap)[0][2] + res = attribute(rows, [(epoch, "-", True)], max_gap)[0][2] # attributed.log holds accel faults # agreement is not string equality: the window names holders, this names one and says how sure. # A window with two holders and a HANDOVER-ADJACENT verdict AGREE about the situation. win_n = 0 if hits in ("", "-") else len([h for h in hits.replace(",", " ").split() if h]) @@ -215,11 +243,11 @@ def self_test(): rows = [(0, "111", "gpu-hold cl-a pid 1 t0"), (100, "111", "gpu-hold cl-a pid 1 t0"), (200, "-", "-"), (300, "222", "gpu-hold cl-b pid 2 t1"), (400, "222", "gpu-hold cl-b pid 2 t1")] - boots = [(50.0, "0x1000"), # held by 111, next sample also 111 -> ATTR - (150.0, "0x2000"), # held by 111, next sample is '-' -> HANDOVER-ADJACENT - (250.0, "0x3000"), # last sample before is '-' -> NOTHING HELD - (305.0, "0x4000"), # first sample of 222's tenure -> ATTR - (900.0, "0x5000")] # 500 s after the last sample -> NO SAMPLE + boots = [(50.0, "0x1000", True), # held by 111, next sample also 111 -> ATTR + (150.0, "0x2000", True), # held by 111, next sample is '-' -> HANDOVER-ADJACENT + (250.0, "0x3000", True), # last sample before is '-' -> NOTHING HELD + (305.0, "0x4000", True), # first sample of 222's tenure -> ATTR + (900.0, "0x5000", True)] # 500 s after the last sample -> NO SAMPLE res = attribute(rows, boots, max_gap=150) def kind(v): # the key is the verdict before any detail; "NO SAMPLE within 150s" is still NO SAMPLE @@ -234,21 +262,21 @@ def kind(v): # And the case that motivates not using a window at all: two real holders either side of the fault. hand = attribute([(0, "111", "gpu-hold cl-a pid 1"), (100, "111", "gpu-hold cl-a pid 1"), - (200, "222", "gpu-hold cl-b pid 2")], [(150.0, "0x9000")], max_gap=150) + (200, "222", "gpu-hold cl-b pid 2")], [(150.0, "0x9000", True)], max_gap=150) assert hand[0][2].startswith("HANDOVER-ADJACENT"), hand assert "pid 111" in hand[0][2] and "pid 222" in hand[0][2], hand assert "no accel client held it" in res[2][2], res[2] # not "I did not look" - assert yield_line(res) == ("faults: 5; attributable 2, ambiguous 1, nothing-held 1, no-sample 1"), yield_line(res) + assert yield_line(res) == ("faults: 5; attributable 2, ambiguous 1, nothing-held 1, no-sample 1, other-device 0"), yield_line(res) # --max-gap decides freshness, and a smaller gap converts ATTRIBUTED into NO SAMPLE rather than # silently attributing across a long unobserved stretch. - tight = [kind(v) for _, _, v in attribute(rows, [(50.0, "0x1")], max_gap=5)] + tight = [kind(v) for _, _, v in attribute(rows, [(50.0, "0x1", True)], max_gap=5)] assert tight == ["NO SAMPLE"], tight # The line shape the kernel actually writes: without a leading stamp there is no time to join on, # so the verdict is NO SAMPLE and says why -- it must never default to the nearest holder. - no_stamp = faults("nvme 0000:c4:00.0: AMD-Vi: Event logged [IO_PAGE_FAULT address=0x4128000]") - assert no_stamp == [(None, "0x4128000")], no_stamp + no_stamp = faults("amdxdna 0000:c6:00.1: AMD-Vi: Event logged [IO_PAGE_FAULT address=0x4128000]") + assert no_stamp == [(None, "0x4128000", True)], no_stamp # the device check passes; only the stamp is missing assert attribute(rows, no_stamp, 150)[0][2].startswith("NO SAMPLE"), attribute(rows, no_stamp, 150) # The outlook is computed, not asserted from a constant: a window wide enough to cover two @@ -288,10 +316,47 @@ def kind(v): one_vs_hand = disagreements(rows, [(150, "111")], max_gap=150) assert one_vs_hand[0][3] is False, one_vs_hand + # engine#2624: the sampler writes `\tGAP\t\t-\t-\t-` when it slept too long. Column 2 is the pid + # column, so a reader that trusts it invents a holder called GAP -- which is what this tool did before this arm. + gap_log = "1000\t-\t-\t-\t-\t-\n1010\tGAP\t93\t-\t-\t-\n" + gap_rows = rows_from_text(gap_log) + assert gap_rows == [(1000, "-", "-"), (1010, "GAP", "93")], gap_rows # the note carries the gap length + assert attribute(gap_rows, [(1015.0, "0x1", True)], 150)[0][2].startswith("NO SAMPLE"), attribute(gap_rows, [(1015.0, "0x1", True)], 150) + assert "stalled" in attribute(gap_rows, [(1015.0, "0x1", True)], 150)[0][2] + # ... and it is not a holder for the outlook either: with a real holder after it, the window still sees one. + mixed = gap_rows + [(1020, "1234", "gpu-hold cl-a pid 9")] + assert outlook(mixed, [5])[0][1] > 0.0, outlook(mixed, [5]) # some windows span the real holder + + # THE CATEGORY ERROR, measured: 137 NVMe faults on this box were "attributed" to a gpu-hold ticket that was + # holding the NPU. A fault on another device must be refused, not guessed -- and --any-device must still work. + nvme = [(105.0, "0xe1e18000", False)] + assert attribute(rows, nvme, 150)[0][2].startswith("NOT THE ACCEL NODE"), attribute(rows, nvme, 150) + assert attribute(rows, nvme, 150, any_device=True)[0][2].startswith("HANDOVER-ADJACENT"), attribute(rows, nvme, 150, True) # --any-device proceeds; this fixture is a handover + + # The summary must account for every fault: a verdict missing from the tally makes "0 of everything" look like + # an empty capture. This arm would have printed 0/0/0/0 for 137 NVMe faults before the counter was added. + assert yield_line([(1.0, "0x1", "NOT THE ACCEL NODE: other device")]) == ( + "faults: 1; attributable 0, ambiguous 0, nothing-held 0, no-sample 0, other-device 1"), \ + yield_line([(1.0, "0x1", "NOT THE ACCEL NODE: other device")]) + # Stamps and addresses are parsed, and non-fault lines are ignored. text = ("[1234.500000] nvme 0000:c4:00.0: AMD-Vi: Event logged [IO_PAGE_FAULT domain=0x0015 address=0x4128000 flags=0x0007]\n" "[1235.000000] unrelated kernel line\n") - assert faults(text) == [(1234.5, "0x4128000")], faults(text) + assert faults(text) == [(1234.5, "0x4128000", False)], faults(text) + + # THE KERNEL PADS THE STAMP, and the unpadded form above is not what it writes: two spaces at 321 s, four below + # 10 s (`[ 0.000000]`). engine#2622 shipped a reader whose regex required a digit straight after `[`, matched + # nothing on real captures, and reported "ordering not shown by this capture" -- a negative for input it should + # have matched. `^\[\s*(...)` handles both widths, and this fixture is what keeps that true. + padded = ("[ 321.308031] nvme 0000:c4:00.0: AMD-Vi: Event logged [IO_PAGE_FAULT domain=0x0015 address=0x4128000 flags=0x0000]\n" + "[ 0.000000] amdxdna 0000:c6:00.1: AMD-Vi: Event logged [IO_PAGE_FAULT domain=0x0003 address=0x4130000 flags=0x0007]\n") + assert faults(padded) == [(321.308031, "0x4128000", False), (0.0, "0x4130000", True)], faults(padded) + + # A capture in the human-readable form (`dmesg -T`, journalctl default) carries NO stamp this parser can use. + # The verdict must say it cannot read the file, not that the kernel showed no ordering. + human = "[Sun Nov 15 08:53:39 2026] amdxdna 0000:c6:00.1: AMD-Vi: Event logged [IO_PAGE_FAULT address=0x4128000]\n" + assert faults(human) == [(None, "0x4128000", True)], faults(human) + assert attribute(rows, faults(human), 150)[0][2].startswith("NO SAMPLE"), attribute(rows, faults(human), 150) # EVERY OUTPUT LINE IS EXERCISED. engine#2602 shipped a renderer that crashed on the first real log # because its self-test only asserted the analysis; a tool that cannot print is not a tool that works. assert fmt_attr(105, "111", "ATTRIBUTED: pid 111 x", True).endswith("\tATTRIBUTED: pid 111 x") @@ -314,6 +379,9 @@ def main(): help="epoch of boot, so kernel stamps become absolute (default: /proc/stat btime)") ap.add_argument("--max-gap", type=float, default=15.0, help="oldest usable sample before a fault, in seconds (default 15)") + ap.add_argument("--any-device", action="store_true", + help="attribute faults on devices other than the accel node (off by default: the sampler's " + "holders describe /dev/accel, so attributing another device's fault is a category error)") ap.add_argument("--attributed", default=None, help="sampler attributed.log: print its windowed hits beside this tool's verdict") ap.add_argument("--outlook", type=float, nargs="*", metavar="W", @@ -350,8 +418,8 @@ def main(): text = sys.stdin.read() if args.dmesg == "-" else open(args.dmesg, encoding="utf-8", errors="replace").read() rows = samples(args.samples) - boots = [(None if s is None else boot + s, a) for s, a in faults(text)] - res = attribute(rows, boots, args.max_gap) + boots = [(None if s is None else boot + s, a, acc) for s, a, acc in faults(text)] + res = attribute(rows, boots, args.max_gap, any_device=args.any_device) for sec, addr, verdict in res: when = "-" if sec is None else "%.0f" % sec print(fmt_fault(sec, addr, verdict))