From 4b6468787260bdbcdf8024e20f44f27c1f5fc22d Mon Sep 17 00:00:00 2001 From: pi-release Date: Sun, 15 Nov 2026 18:15:53 -0400 Subject: [PATCH 1/2] diag: read the accel sampler's logs without collapsing 'no sample' into 'nothing held' (engine#2542) ORCH split #2542 with pi-pkg: they run the owner-approved 24 h /dev/accel sampler, I do the analysis side. This is the reader. Two properties of the raw logs decide how much a burst can be attributed, and neither is visible in them: - attributed.log records the samples in +-WINDOW_S as of pid,cmd,scope,ticket -- no timestamps, no order. A window that spans a holder change therefore names BOTH holders and cannot say which was live at the fault. The epochs are in samples.log, so this rebuilds the timeline and marks the ambiguous case instead of reporting one holder. - a '-' there means 'no row in the window'; the sampler writes an explicit '-' row for 'sampled, nothing holds the node'. Collapsing them turns 'I did not look' into 'nobody was there'. It also reports the EFFECTIVE interval, which is not SAMPLE_S: on this box the scans measure 6-9 s against a configured 2 s, so a +-5 s window holds one or two samples and the tool says so when the window is shorter than the interval. Self-test pins the window arithmetic with hand-checked cases (window 3 vs 5 on the same samples flips one fault from unambiguous to ambiguous), and both mutations that would hide a defect fail it: collapsing 'no sample' into 'nothing held', and printing one holder when the window holds two. --- CMakeLists.txt | 5 + tools/diag/accel_attribution.py | 220 ++++++++++++++++++++++++++++++++ 2 files changed, 225 insertions(+) create mode 100755 tools/diag/accel_attribution.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 2a99e7833..f73099b58 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -335,6 +335,11 @@ add_test(NAME server_launch_check COMMAND python3 ${CMAKE_SOURCE_DIR}/tools/ci/s # coincidence this row exists to keep apart. add_test(NAME driver_refusal_census_selftest COMMAND python3 ${CMAKE_SOURCE_DIR}/tools/diag/driver_refusal_census.py --self-test) +# engine#2542: the accel sampler's logs (pi-pkg) are written to be *read*, and the reader is where the difference +# between "no sample in the window" and "sampled, nothing held the node" survives. Self-test only: it never reads the +# live logs (a CI runner has no /dev/accel), and it pins the window arithmetic with hand-checked cases. +add_test(NAME accel_attribution_selftest + COMMAND python3 ${CMAKE_SOURCE_DIR}/tools/diag/accel_attribution.py --self-test) add_test(NAME server_launch_check_selftest COMMAND python3 ${CMAKE_SOURCE_DIR}/tools/ci/server_launch_check.py --self-test) # scripts/lemonade-hrx: the PPA lemond's HRX recipe runs our llama-server (docs/lemonade.md): no GPU. add_test(NAME lemonade_hrx COMMAND ${CMAKE_SOURCE_DIR}/tests/lemonade_hrx.sh $) diff --git a/tools/diag/accel_attribution.py b/tools/diag/accel_attribution.py new file mode 100755 index 000000000..335a62cdf --- /dev/null +++ b/tools/diag/accel_attribution.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Read the accel sampler's logs and say what they can and cannot attribute (engine#2542). + +`~/.cache/accel-sampler/accel-sampler.sh` (pi-pkg) samples which process holds /dev/accel/accel0 and, for each +IO_PAGE_FAULT, records the samples within +-WINDOW_S. Two properties of the raw logs decide how much a burst can be +attributed, and neither is visible in them: + + * `attributed.log` records the samples in the window as `sort -u` of `pid,cmd,scope,ticket`, with no timestamps and + no order. When a window spans a holder change it therefore names BOTH holders and cannot say which was live at + the fault. The sample epochs are in `samples.log`, so this tool rebuilds the timeline. + * a `-` in that field means "no row in the window", which is not the same as "sampled, nobody held the node": the + sampler writes an explicit `-`-row for the latter. Collapsing the two turns "I did not look" into "nobody was + there", which is the failure this repository keeps removing. + +Usage: + tools/diag/accel_attribution.py # the live logs + tools/diag/accel_attribution.py --dir DIR + tools/diag/accel_attribution.py --self-test +""" + +import argparse +import collections +import os +import sys + + +def read_samples(path): + """[(epoch, pid, fds, cmd, scope, ticket)]; pid '-' is the sampler's 'nothing holds it' row.""" + out = [] + try: + with open(path, errors="replace") as f: + for i, line in enumerate(f): + f6 = line.rstrip("\n").split("\t") + if len(f6) < 6: + continue # a partially written row: the sampler appends without locking + try: + out.append((int(f6[0]), f6[1], f6[2], f6[3], f6[4], f6[5])) + except ValueError: + print(f" {os.path.basename(path)}:{i + 1}: no epoch, skipped", file=sys.stderr) + except FileNotFoundError: + pass + return out + + +def read_attributions(path): + """[(epoch, hits, fault_line)] as written by the sampler's watch loop.""" + out = [] + try: + with open(path, errors="replace") as f: + for line in f: + f3 = line.rstrip("\n").split("\t", 2) + if len(f3) < 3: + continue + try: + out.append((int(f3[0]), f3[1], f3[2])) + except ValueError: + continue + except FileNotFoundError: + pass + return out + + +def intervals(samples): + """Seconds between consecutive DISTINCT sample epochs -- the effective interval, which is what the window sees. + + It is not SAMPLE_S: the sampler scans every /proc//fd set, which takes seconds on a busy box (measured + 6-9 s against a configured 2 s), so the tuning knob and the observable are two different numbers.""" + ts = sorted({s[0] for s in samples}) + return [b - a for a, b in zip(ts, ts[1:])] + + +def holder_table(samples): + """{pid: {cmd, ticket, first, last, n}} for every real holder (the '-' rows excluded).""" + t = {} + for epoch, pid, _fds, cmd, _scope, ticket in samples: + if pid == "-": + continue + e = t.setdefault(pid, {"cmd": cmd, "ticket": ticket, "first": epoch, "last": epoch, "n": 0}) + e["first"] = min(e["first"], epoch) + e["last"] = max(e["last"], epoch) + e["n"] += 1 + return t + + +def in_window(samples, ts, window): + return [s for s in samples if ts - window <= s[0] <= ts + window] + + +def explain(samples, attributions, window): + """One block per fault: what was sampled around it, in order, and whether that identifies a single holder.""" + lines = [] + for ts, _hits, fault in attributions: + w = sorted(in_window(samples, ts, window), key=lambda s: s[0]) + if not w: + verdict = "NO SAMPLE in the window: nothing can be attributed (this is not 'nobody held it')" + else: + held = [s for s in w if s[1] != "-"] + empty = [s for s in w if s[1] == "-"] + pids = {s[1] for s in held} + if not held: + verdict = f"sampled {len(w)}x, nothing held the node (explicit '-' rows): not an accel client" + elif len(pids) == 1: + s = held[0] + verdict = (f"one holder across {len(held)} sample(s): pid {s[1]} " + f"({s[3].split()[0] if s[3] else '?'}...) ticket={s[5].rstrip()[:40]}") + else: + verdict = (f"AMBIGUOUS: {len(pids)} holders inside +-{window}s " + f"({', '.join(sorted(pids))}) -- the raw 'hits' field names them all and cannot " + f"say which was live at the fault") + if empty: + verdict += f"; {len(empty)} of {len(w)} samples had nothing holding the node" + lines.append(f" {ts} {verdict}") + lines.append(f" {fault[:110]}") + for s in w: + lines.append(f" {s[0]:>11} pid={s[1]:<8} {s[3][:56]} {s[5].rstrip()[:34]}") + return lines + + +def self_test(): + # A window that spans a holder change: the raw record cannot separate them, the timeline can. + samples = [ + (100, "111", "1", "/a serve -m X", "scope-a", "gpu-hold cl-a 1"), + (106, "222", "1", "/b serve -m Y", "scope-b", "gpu-hold cl-b 2"), + (112, "-", "-", "-", "-", "gpu-hold cl-c 3"), # sampled, nobody holds the node + ] + assert intervals(samples) == [6, 6], intervals(samples) + t = holder_table(samples) + assert set(t) == {"111", "222"}, t + assert t["111"]["n"] == 1 and t["222"]["ticket"].startswith("gpu-hold cl-b"), t + + # The window decides the answer, which is the point of printing it. Hand-checked against the samples above: + # window 3, fault 100 -> [97,103] -> {111} one holder + # window 3, fault 106 -> [103,109] -> {222} one holder + # window 3, fault 109 -> [106,112] -> {222, '-'} one holder AND an explicit "nothing holds it" + # window 1, fault 112 -> [111,113] -> {'-'} sampled, nothing held the node + # window 5, fault 103 -> [98,108] -> {111, 222} AMBIGUOUS: the same fault is unambiguous at 3 + # window 3, fault 200 -> [197,203] -> {} nothing sampled: not evidence of absence + narrow, wide = "\n".join(explain(samples, [(100, "", "A"), (106, "", "B"), (109, "", "C"), + (200, "", "D")], 3)), "\n".join(explain(samples, [(103, "", "E")], 5)) + assert narrow.count("one holder across 1 sample(s): pid 111") == 1, narrow + assert narrow.count("one holder across 1 sample(s): pid 222") == 2, narrow # faults B and C + assert narrow.count("; 1 of 2 samples had nothing holding the node") == 1, narrow # fault C + assert "NO SAMPLE in the window" in narrow, narrow + dot = "\n".join(explain(samples, [(112, "", "F")], 1)) + assert "sampled 1x, nothing held the node" in dot, dot + assert "AMBIGUOUS: 2 holders inside +-5s (111, 222)" in wide, wide + assert "not 'nobody held it'" in narrow, narrow + + # A partially written row must not kill the reader. + import tempfile + with tempfile.NamedTemporaryFile("w", suffix=".log", delete=False) as f: + f.write("100\t111\t1\t/a serve\tscope\tticket\n") + f.write("106\t22") # torn + p = f.name + try: + assert [s[0] for s in read_samples(p)] == [100], read_samples(p) + finally: + os.unlink(p) + print("accel_attribution self-test: ok") + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--dir", default=os.path.expanduser("~/.cache/accel-sampler")) + ap.add_argument("--window", type=int, default=5, help="the sampler's WINDOW_S (default 5)") + ap.add_argument("--self-test", action="store_true") + args = ap.parse_args() + if args.self_test: + self_test() + return 0 + + samples = read_samples(os.path.join(args.dir, "samples.log")) + attrs = read_attributions(os.path.join(args.dir, "attributed.log")) + if not samples: + print(f"accel_attribution: no samples in {args.dir}/samples.log", file=sys.stderr) + return 2 + + iv = intervals(samples) + span = max(s[0] for s in samples) - min(s[0] for s in samples) + med = sorted(iv)[len(iv) // 2] if iv else 0 + print(f"samples: {len(samples)} rows over {span} s; effective interval median {med} s, " + f"min {min(iv) if iv else 0}, max {max(iv) if iv else 0}") + gaps = [d for d in iv if med and d > 3 * med] + print(f"gaps > 3x median (sampler down or a restart): {len(gaps)}" + + (f" {gaps[:5]}" if gaps else "")) + if med > 0 and args.window < med: + print(f" note: +-{args.window}s window is shorter than the effective interval ({med}s): some faults " + f"will be sampled zero times") + nobody = sum(1 for s in samples if s[1] == "-") + print(f"'nothing holds the node' rows: {nobody} of {len(samples)}") + + t = holder_table(samples) + if t: + print("\nholders seen (pid, first..last, samples, ticket):") + for pid, e in sorted(t.items(), key=lambda kv: -kv[1]["n"]): + print(f" {pid:<9} {e['first']}..{e['last']} ({e['last'] - e['first']:>5}s) n={e['n']:<4} " + f"{e['cmd'][:46]} {e['ticket'].rstrip()[:30]}") + + print(f"\nattributions: {len(attrs)}") + for line in explain(samples, attrs, args.window): + print(line) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 5ef041671cc7ee1353e3c305a1220394231ac1fe Mon Sep 17 00:00:00 2001 From: pi-release Date: Sun, 15 Nov 2026 18:32:48 -0400 Subject: [PATCH 2/2] diag: keep each holder's ticket SPANS, so a process that outlives its ticket is visible (engine#2542) The sampler records the token as it stood at sample time, not the ticket a process belongs to. A holder that outlives its own ticket is therefore relabelled with whatever ticket is live next, and a single-ticket view launders a leak into 'part of the next ticket'. That is not hypothetical: on 2026-11-15 four /home/bcloud/wt/cl-1077-bigmoe/ build-hip/bin processes held /dev/accel/accel0 from 18:19:40 to ~18:23:00, across THREE ticket boundaries (amdcol-h171-q25-7 released 18:20:16 -> long-h171-r1l8 18:20:17-18:21:34 -> cl-game-w4g-g32h -> long-h171-r1l8 again), with the 'no ticket' intervals visible at each hand-over. My first version of this reader recorded only the first ticket per pid and could not have found it. An fd held across a release is not proof that two AIE2 contexts were live -- a process can hold the node without submitting -- but it is the precondition the fleet's reset guard exists for, so the report says exactly that and nothing more. Self-test: a pid under two tickets must be reported; the plain fixture must not trip it; dropping the detector fails the arm. --- tools/diag/accel_attribution.py | 44 ++++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/tools/diag/accel_attribution.py b/tools/diag/accel_attribution.py index 335a62cdf..e975c3d9f 100755 --- a/tools/diag/accel_attribution.py +++ b/tools/diag/accel_attribution.py @@ -84,18 +84,40 @@ def intervals(samples): def holder_table(samples): - """{pid: {cmd, ticket, first, last, n}} for every real holder (the '-' rows excluded).""" + """{pid: {cmd, spans, first, last, n}} for every real holder (the '-' rows excluded). + + `spans` is the ORDERED list of (ticket, first, last) for that pid. It matters because the sampler records the + token as it stood at sample time, not the ticket the process belongs to: a process that outlives its own ticket + is relabelled with whatever ticket is live next, so a single-ticket view silently launders a leaked holder into + "part of the next ticket". Four cl-1077 PIDs did exactly this on 2026-11-15 18:20 (three boundaries) and were + invisible until the spans were kept. + """ t = {} for epoch, pid, _fds, cmd, _scope, ticket in samples: if pid == "-": continue - e = t.setdefault(pid, {"cmd": cmd, "ticket": ticket, "first": epoch, "last": epoch, "n": 0}) + e = t.setdefault(pid, {"cmd": cmd, "spans": [], "first": epoch, "last": epoch, "n": 0}) e["first"] = min(e["first"], epoch) e["last"] = max(e["last"], epoch) e["n"] += 1 + tk = ticket.rstrip() + if e["spans"] and e["spans"][-1][0] == tk: + e["spans"][-1] = (tk, e["spans"][-1][1], epoch) + else: + e["spans"].append((tk, epoch, epoch)) return t +def spans_ticket_boundaries(table): + """[(pid, spans)] for holders seen under more than one ticket. + + An fd held across a release is not proof that two AIE2 contexts were live -- a process can hold the node open + without submitting -- but it is the precondition the fleet's reset guard exists for (two contexts on one array is + what produced the double TDR on 2026-11-02), so it is reported rather than inferred away. + """ + return [(pid, e["spans"]) for pid, e in table.items() if len(e["spans"]) > 1] + + def in_window(samples, ts, window): return [s for s in samples if ts - window <= s[0] <= ts + window] @@ -140,7 +162,7 @@ def self_test(): assert intervals(samples) == [6, 6], intervals(samples) t = holder_table(samples) assert set(t) == {"111", "222"}, t - assert t["111"]["n"] == 1 and t["222"]["ticket"].startswith("gpu-hold cl-b"), t + assert t["111"]["n"] == 1 and t["222"]["spans"][0][0].startswith("gpu-hold cl-b"), t # The window decides the answer, which is the point of printing it. Hand-checked against the samples above: # window 3, fault 100 -> [97,103] -> {111} one holder @@ -160,6 +182,14 @@ def self_test(): assert "AMBIGUOUS: 2 holders inside +-5s (111, 222)" in wide, wide assert "not 'nobody held it'" in narrow, narrow + # A holder that outlives its ticket is RELABELLED by the sampler (it records the token as it stood), so the + # spans are what make the leak visible: same pid, two tickets. + leaky = [(100, "111", "1", "/a", "s", "gpu-hold cl-a 1"), (106, "111", "1", "/a", "s", "gpu-hold cl-b 2")] + lt = holder_table(leaky) + assert [x[0] for x in lt["111"]["spans"]] == ["gpu-hold cl-a 1", "gpu-hold cl-b 2"], lt + assert [p for p, _ in spans_ticket_boundaries(lt)] == ["111"], spans_ticket_boundaries(lt) + assert spans_ticket_boundaries(holder_table(samples)) == [], "the plain fixture must not trip the arm" + # A partially written row must not kill the reader. import tempfile with tempfile.NamedTemporaryFile("w", suffix=".log", delete=False) as f: @@ -204,6 +234,14 @@ def main(): print(f"'nothing holds the node' rows: {nobody} of {len(samples)}") t = holder_table(samples) + crossed = spans_ticket_boundaries(t) + if crossed: + print(f"\nholders that outlived their ticket ({len(crossed)} pid(s)) -- an fd held across a release is the " + f"precondition for two contexts on one array, not proof of one:") + for pid, spans in crossed: + print(f" {pid:<9} {t[pid]['cmd'][:44]}") + for tk, a, b in spans: + print(f" {a}..{b} ({b - a:>4}s) {tk[:46]}") if t: print("\nholders seen (pid, first..last, samples, ticket):") for pid, e in sorted(t.items(), key=lambda kv: -kv[1]["n"]):