From 6a696102b967784ee923bf60d7fb168e82301e8f Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Mon, 16 Nov 2026 03:21:18 -0400 Subject: [PATCH] diag: a GAP row covers the stall before it, so a fault after a stall is not "stalled" (review of #2623) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit accel_sampler.sh writes a GAP row when it RESUMES (`"$NOW" GAP "$((NOW - LAST))"`), so a row at E with length L describes the stall (E-L, E], before its own timestamp. attribute() treated it as "the last sample before the fault", which is true only for faults AFTER the stall ended, and the handover branch then named the GAP row as the next holder for faults INSIDE one. Measured on main (900 holder 111 | 910 GAP 10 | 912 holder 222): fault 905 (inside) -> HANDOVER-ADJACENT: pid 111 … (next sample: pid GAP) fault 911 (after) -> NO SAMPLE: the sampler was stalled across this fault Now a fault is stalled when a GAP span covers it, and GAP rows are never a sample before or after: fault 905 -> NO SAMPLE: the sampler was stalled across this fault (a 10s gap ending at 910) fault 911 -> HANDOVER-ADJACENT: pid 111 gpu-hold cl-a (next sample: pid 222) The fixture that pinned the old reading (fault 1015 after a GAP at 1010 -> "stalled") now asserts the stall at 1005 and no stall at 1015, and a three-row timeline asserts no verdict names GAP. Mutants: dropping the span check fails (NOTHING HELD at 1005); letting GAP rows back into the before/after search fails ("pid GAP 10" at 911). Co-Authored-By: Claude Opus 5.5 --- tools/diag/iommu_fault_attribution.py | 44 +++++++++++++++++++++------ 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/tools/diag/iommu_fault_attribution.py b/tools/diag/iommu_fault_attribution.py index 7e5998ce5..dd4d98ae9 100644 --- a/tools/diag/iommu_fault_attribution.py +++ b/tools/diag/iommu_fault_attribution.py @@ -113,8 +113,20 @@ def attribute(rows, boots, max_gap, any_device=False): if sec is None: out.append((sec, addr, "NO SAMPLE: the line carries no kernel timestamp")) continue - before = [r for r in rows if r[0] <= sec] - after = [r for r in rows if r[0] > sec] + # A GAP row is written when the sampler RESUMES (accel_sampler.sh: `printf '%s\tGAP\t%s' "$NOW" "$((NOW - LAST))"`), + # so a row at epoch E with length L describes the stall (E-L, E] -- BEFORE its own timestamp. A fault is + # "stalled" when such a span covers it; GAP rows are never a sample to attribute to, before or after + # (review of engine#2623: reading the row as "the last sample before t" called faults AFTER a stall stalled, + # and named "pid GAP" as the next holder for faults INSIDE one). + stall = next((r for r in rows if r[1] == "GAP" and _gap_len(r[2]) is not None + and r[0] - _gap_len(r[2]) < sec <= r[0]), None) + if stall is not None: + out.append((sec, addr, "NO SAMPLE: the sampler was stalled across this fault (a %ss gap ending at %d)" + % (stall[2], stall[0]))) + continue + real = [r for r in rows if r[1] != "GAP"] + before = [r for r in real if r[0] <= sec] + after = [r for r in real if r[0] > sec] if not before: out.append((sec, addr, "NO SAMPLE: no sample at or before the fault")) continue @@ -125,11 +137,6 @@ def attribute(rows, boots, max_gap, any_device=False): if pid == "-": out.append((sec, addr, "NOTHING HELD: sampled at the node, no accel client held it")) continue - if pid == "GAP": - # engine#2624: the sampler writes a GAP row when it slept too long to sample. That is the absence of an - # observation, NOT a process called GAP -- attributing a fault here would invent a holder. - out.append((sec, addr, "NO SAMPLE: the sampler was stalled across this fault (GAP row %s)" % ticket)) - continue who = f"pid {pid} {ticket[:64]}" if after and after[0][1] != pid: # Name the other side: a window would name both and stop; this says which one is the guess @@ -141,6 +148,14 @@ def attribute(rows, boots, max_gap, any_device=False): return out +def _gap_len(field): + """Seconds of a GAP row's length field, or None when it is not a number (a malformed row covers nothing).""" + try: + return int(field) + except (TypeError, ValueError): + return None + + def yield_line(res): """The number that decides whether a sampling run can answer the question at all.""" # Every verdict must be counted, or the summary says "0 of everything" about a capture full of faults -- the @@ -321,8 +336,19 @@ def kind(v): gap_log = "1000\t-\t-\t-\t-\t-\n1010\tGAP\t93\t-\t-\t-\n" gap_rows = rows_from_text(gap_log) assert gap_rows == [(1000, "-", "-"), (1010, "GAP", "93")], gap_rows # the note carries the gap length - assert attribute(gap_rows, [(1015.0, "0x1", True)], 150)[0][2].startswith("NO SAMPLE"), attribute(gap_rows, [(1015.0, "0x1", True)], 150) - assert "stalled" in attribute(gap_rows, [(1015.0, "0x1", True)], 150)[0][2] + # The GAP row at 1010 with length 93 is the stall (917, 1010], written on RESUME: a fault inside it is stalled, + # a fault after it is not (review of engine#2623 -- the first version read it the other way round). + inside = attribute(gap_rows, [(1005.0, "0x1", True)], 150)[0][2] + assert inside.startswith("NO SAMPLE") and "stalled" in inside, inside + after_gap = attribute(gap_rows, [(1015.0, "0x1", True)], 150)[0][2] + assert "stalled" not in after_gap, after_gap + # and a stall between two holders never names a holder called GAP + hand = rows_from_text("900\t111\t1\t/a\ts\tgpu-hold cl-a\n910\tGAP\t10\t-\t-\t-\n912\t222\t1\t/b\ts\tgpu-hold cl-b\n") + for t in (899.5, 905.0, 911.0): + v = attribute(hand, [(t, "0x1", True)], 15)[0][2] + assert "GAP" not in v.replace("a 10s gap", ""), (t, v) + assert "stalled" in attribute(hand, [(905.0, "0x1", True)], 15)[0][2] + assert attribute(hand, [(911.0, "0x1", True)], 15)[0][2].startswith("HANDOVER-ADJACENT: pid 111") # ... and it is not a holder for the outlook either: with a real holder after it, the window still sees one. mixed = gap_rows + [(1020, "1234", "gpu-hold cl-a pid 9")] assert outlook(mixed, [5])[0][1] > 0.0, outlook(mixed, [5]) # some windows span the real holder