diff --git a/CMakeLists.txt b/CMakeLists.txt index 99ec81ca4..f43a670b8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -325,6 +325,9 @@ add_python_test(tool_selftest_whisper_encoder_baseline ${CMAKE_SOURCE_DIR}/tools # engine#1780: the DFlash draft package's gate before a loader -- tile-grid sizes from the config, tile-header sanity and # the container's scalars against config.json, on a synthetic package with eight negative controls. Stdlib only. add_python_test(tool_selftest_dflash_draft_info ${CMAKE_SOURCE_DIR}/tools/dflash_draft_info.py --selftest) +# engine#1750: what a TDR recovery did, counted from the kernel log (failed context restarts among them). The self-test +# parses lines in the driver's own formats for three module generations. Stdlib only, no device. +add_python_test(tool_selftest_amdxdna_tdr_report ${CMAKE_SOURCE_DIR}/tools/amdxdna_tdr_report.py --self-test) # engine#2013: the qwen35 (gated linear attention) GGUF -> Q4NX converter. Its own selftest (tile round-trips, the # value-head permutation), and a pack + verify of a small qwen35 GGUF the test writes itself, with the three inputs it # has to refuse. numpy only: no gguf package, no model, no device. diff --git a/tests/amdxdna_restart_test.sh b/tests/amdxdna_restart_test.sh index 94fe2ac1e..460a682ce 100755 --- a/tests/amdxdna_restart_test.sh +++ b/tests/amdxdna_restart_test.sh @@ -52,6 +52,22 @@ for p in "$series"/[0-9][0-9][0-9][0-9]-*.patch; do done echo "ok $n patches apply in order on ${pin:0:12}" +# The test-only fault injection (engine#1750) has to keep applying on top of the series, or the one run it exists +# for cannot be built. It is checked, not applied: the harness below is the carried code. And it must stay out of +# the numbered glob above, which is what a package build takes. +inject=$series/test-only/restart-fault-injection.patch +if ! git -C "$scratch/tree" apply --check "$inject" 2> "$scratch/apply.err"; then + echo "FAIL test-only/restart-fault-injection.patch does not apply on top of the $n carried patches:" + sed 's/^/ /' "$scratch/apply.err" | head -n 5 + exit 1 +fi +echo "ok test-only/restart-fault-injection.patch applies on top of them (checked, not applied)" +if ls "$series"/[0-9][0-9][0-9][0-9]-*.patch | xargs grep -l "TEST ONLY" > "$scratch/leak" 2> /dev/null; then + echo "FAIL a numbered patch carries TEST ONLY code: $(tr '\n' ' ' < "$scratch/leak")" + exit 1 +fi +echo "ok no numbered patch carries TEST ONLY code" + # one function definition: from its header line (column 0, not a prototype) to the first closing brace in column 0 for f in aie2_hwctx_job_pause aie2_hwctx_stop aie2_hwctx_restart aie2_hwctx_job_unpause; do awk -v f="$f" '!on && $0 ~ "^[a-z].*[ *]" f "\\(" && $0 !~ /;[ \t]*$/ { on = 1 } on { print } on && /^}/ { exit }' \ diff --git a/third_party/patches/amdxdna/README.md b/third_party/patches/amdxdna/README.md index cadcfa514..5b42e11db 100644 --- a/third_party/patches/amdxdna/README.md +++ b/third_party/patches/amdxdna/README.md @@ -195,6 +195,60 @@ To load the pair (owner's call; strixhalo, 2026-11-06): the module built from th `7.4.0-rc1-1bit` reports `336751493A77C50DE0B50F7`, and the install has to sign it, `depmod`, and **`update-initramfs`** — the initramfs copy is what boots (2026-11-04: three reboots on the old module without that step). +## Reading what a recovery did: `tools/amdxdna_tdr_report.py` + +The driver logs every step of a TDR recovery, and `tools/amdxdna_tdr_report.py` turns the kernel log into the counts +per module load: timeouts detected, power-cycle recoveries, throttled recoveries, skipped contexts, **failed restarts** +(context, error, the step that failed, and whether the module failed the parked jobs, i.e. carries `0013`), whether the +array latched, and refused context creates by error. It reads `journalctl -k -b` or saved logs, opens no device, and +`--fail-on-failed-restart` makes it a check. + +The record on strixhalo with the module the fleet runs (`F87D0BDF248067862F83BA3`, pin + `0001`-`0009`), one boot, +2026-11-08 23:11 to 2026-11-10 17:40 AST, under the fleet's normal load: + +``` +$ python3 tools/amdxdna_tdr_report.py +module load 1: driver 0.18.0 (2636 amdxdna lines) + timeouts detected 26 + power-cycle recoveries 18 + throttled 8 + skipped contexts 4 + failed restarts 0 + latched (reload needed) 0 + context creates refused 767 (ret -19 x767) +``` + +So in 18 real recoveries no context restart failed: the path `0010` and `0013` change (engine#1750) was not reached +once. That says how often it happens here, not what firmware does when it does. + +## The test-only patch, and the run that would measure `0010`/`0013` + +`test-only/restart-fault-injection.patch` is **not part of the series**. It is outside the numbered files, so +`scripts/mk-kernel-packages.sh` (which takes patches by number) and the series loop in `tests/amdxdna_restart_test.sh` +never pick it up; that test checks both that it still applies on top of the series and that no numbered patch carries +`TEST ONLY` code. It adds two module parameters that make `aie2_hwctx_restart()` fail on request, before or after the +firmware context is created, because a real restart failure needs firmware to refuse a message and cannot be arranged +from user space. + +Verified without the device: applies on `0001`-`0013`; the module compiles against `7.4.0-rc1-1bit` +(`amdxdna.ko` 14,045,256 B, `srcversion 57DF58D0185B9D6E0397B6F`, 0 errors, 0 warnings; `modinfo` lists both +parameters). **Never loaded.** + +The run, which needs a module load and is therefore the owner's: + +1. Build pin + `0001`-`0013` + the test-only patch (`make -C drivers/accel/amdxdna ARCH=x86`), load it on a box nobody + else is using the array on. +2. Start one client that keeps a context open and submits (any `npu-hold` serve). +3. `echo 1 > /sys/module/amdxdna/parameters/restart_inject_count`, then `echo 2 > .../restart_inject_step` + (2 = after the create, the case that left a half-built firmware context), and provoke one recovery (the kernel + lane's `tdr_test` hook, or a real timeout). +4. `python3 tools/amdxdna_tdr_report.py`. Expected with `0013`: one failed restart, with + `firmware context taken away (teardown ret 0), N parked job(s) failed with -ENODEV`, and the client's pending + requests returning at once instead of at their own timeout. `teardown ret` is the answer to the question engine#1750 + put first: whether firmware accepts a destroy for a context it has only half of. The array then latches (a failed + restart in a recovery latches `hw_dead` by design), so reload the shipping module afterwards. +5. Repeat with `restart_inject_step` 1 (create failed: no firmware context to take away). + ## What this directory deliberately does not carry The **0010 debugfs test hook** (`debugfs/accel//tdr_test`, root-only, commands `recover` / `throttle` / diff --git a/third_party/patches/amdxdna/test-only/restart-fault-injection.patch b/third_party/patches/amdxdna/test-only/restart-fault-injection.patch new file mode 100644 index 000000000..452da2593 --- /dev/null +++ b/third_party/patches/amdxdna/test-only/restart-fault-injection.patch @@ -0,0 +1,87 @@ +From: bong-water-water-bong +Subject: [PATCH TEST ONLY] accel/amdxdna: fail a context restart on request + +TEST ONLY. Not part of the carried series, never built into a package or an +image: it sits in test-only/ so that nothing that takes the numbered patches +picks it up. + +aie2_hwctx_restart() fails only when firmware refuses one of its four +messages, which cannot be arranged from user space, so what the driver does +with a context whose restart failed (engine#1750, patches 0010 and 0013) +could not be measured on the array. Two module parameters make the next +restarts fail on purpose: + + restart_inject_step 0 = off + 1 = fail before the firmware context is created + (as if aie2_create_context() had failed) + 2 = fail after it was created + (as if aie2_map_host_buf() had failed) + restart_inject_count how many restarts to fail; each one takes one + +Each injected failure logs "TEST ONLY: injecting a restart failure for + at step " at error level, followed by the driver's own lines for a +failed restart. Applies on top of 0001-0013. +--- +diff --git a/drivers/accel/amdxdna/aie2_ctx.c b/drivers/accel/amdxdna/aie2_ctx.c +index 100a07d..55d99a7 100644 +--- a/drivers/accel/amdxdna/aie2_ctx.c ++++ b/drivers/accel/amdxdna/aie2_ctx.c +@@ -34,6 +34,36 @@ uint tdr_timeout_ms = 2000; + module_param(tdr_timeout_ms, uint, 0400); + MODULE_PARM_DESC(tdr_timeout_ms, "TDR (Timeout Detection and Recovery) timeout in milliseconds (0 = disable)"); + ++/* ++ * TEST ONLY (engine#1750), never part of a shipped module: make the next ++ * context restarts fail on purpose, so the failed-restart path can be ++ * measured with one provoked TDR instead of waiting for firmware to ++ * refuse a restart by itself. ++ * ++ * restart_inject_step: 0 = off; 1 = fail before the firmware context is ++ * created (as if aie2_create_context() had failed); 2 = fail after it was ++ * created (as if aie2_map_host_buf() had failed). ++ * restart_inject_count: how many restarts to fail; each injected failure ++ * takes one, and at 0 the injection is off again. ++ */ ++static uint restart_inject_step; ++module_param(restart_inject_step, uint, 0600); ++MODULE_PARM_DESC(restart_inject_step, "TEST ONLY: fail a context restart (1 = before create, 2 = after create)"); ++static uint restart_inject_count; ++module_param(restart_inject_count, uint, 0600); ++MODULE_PARM_DESC(restart_inject_count, "TEST ONLY: number of context restarts restart_inject_step still fails"); ++ ++static bool aie2_restart_inject(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hwctx, uint step) ++{ ++ if (READ_ONCE(restart_inject_step) != step || !READ_ONCE(restart_inject_count)) ++ return false; ++ ++ WRITE_ONCE(restart_inject_count, READ_ONCE(restart_inject_count) - 1); ++ XDNA_ERR(xdna, "TEST ONLY: injecting a restart failure for %s at step %u (%u left)", ++ hwctx->name, step, READ_ONCE(restart_inject_count)); ++ return true; ++} ++ + struct aie2_ctx_health { + struct amdxdna_ctx_health header; + u32 txn_op_idx; +@@ -174,6 +204,12 @@ int aie2_hwctx_restart(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hwctx) + goto out; + } + ++ if (aie2_restart_inject(xdna, hwctx, 1)) { ++ ret = -EIO; ++ XDNA_ERR(xdna, "Create hwctx failed, ret %d", ret); ++ goto out; ++ } ++ + ret = aie2_create_context(xdna->dev_handle, hwctx); + if (ret) { + XDNA_ERR(xdna, "Create hwctx failed, ret %d", ret); +@@ -183,6 +219,8 @@ int aie2_hwctx_restart(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hwctx) + ret = aie2_map_host_buf(xdna->dev_handle, hwctx->fw_ctx_id, + amdxdna_obj_dma_addr(heap), + heap->mem.size); ++ if (!ret && aie2_restart_inject(xdna, hwctx, 2)) ++ ret = -EIO; + if (ret) { + XDNA_ERR(xdna, "Map host buf failed, ret %d", ret); + goto out; diff --git a/tools/amdxdna_tdr_report.py b/tools/amdxdna_tdr_report.py new file mode 100755 index 000000000..386b8948d --- /dev/null +++ b/tools/amdxdna_tdr_report.py @@ -0,0 +1,289 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""amdxdna_tdr_report.py -- what the amdxdna driver's TDR recovery did, read from the kernel log. + + tools/amdxdna_tdr_report.py this boot's kernel log (journalctl -k -b) + tools/amdxdna_tdr_report.py --boot -1 another boot (journalctl's numbering) + tools/amdxdna_tdr_report.py FILE [FILE...] saved logs (dmesg or journalctl text); - is stdin + tools/amdxdna_tdr_report.py --json ... the same numbers as JSON + tools/amdxdna_tdr_report.py --fail-on-failed-restart ... exit 1 when a context restart failed + tools/amdxdna_tdr_report.py --self-test + +Read-only: it opens no device and changes nothing. Stdlib only. + +Why it exists (engine#1750). A timeout (TDR) makes the driver stop every context, power-cycle the array and +restart the contexts. The question that issue asks is what happens to a context whose restart FAILS, and +the first thing anyone needs is how often that has happened on a box. The driver logs every step, but the +answer was a grep one had to know how to write. This prints it per module load: + + timeouts detected every "TDR timeout detected" + power-cycle recoveries the driver reset the array and walked the contexts + throttled inside the minimum interval between resets; no context was touched + skipped contexts being torn down by their owner (-ESRCH), not a failure + failed restarts " restart failed, ret ", with the context, the error, and the step + that failed when the driver logged one + latched "NPU array is unusable until the driver is reloaded" + context creates refused "Init hwctx failed, ret ", by error + +For a failed restart it also says which behaviour the loaded module has: the carried patch 0013 logs +"no firmware context (teardown ret ), failing parked job(s)" from the restart itself; a module +without it logs only the bare line from the TDR walk, and one without 0010 unpauses the context anyway. + +A module load is a "Initialized amdxdna_accel_driver" line. Lines before the first one in the input are +reported as a load of their own, since a saved log often starts mid-boot. +""" +import argparse +import json +import re +import subprocess +import sys + +LINE = re.compile(r"amdxdna [0-9a-f:.]+: (?:\[drm\] )?(?:\*ERROR\* )?(?P[A-Za-z0-9_]+): (?P.*)$") +LOAD = re.compile(r"Initialized amdxdna_accel_driver (?P\S+)") +FAILED = re.compile(r"^(?P\S+) restart failed, ret (?P-?\d+)" + r"(?:: no firmware context \(teardown ret (?P-?\d+)\), failing (?P\d+) parked job)?") +STEP = re.compile(r"^(Create hwctx failed|Map host buf failed|Add heap \d+ failed|Config cu failed),? ?ret (-?\d+)") +THROTTLED = re.compile(r"^TDR recovery throttled|^NPU power-cycle throttled") +SKIPPED = re.compile(r"^TDR: (\d+) context\(s\) skipped") +LATCHED = re.compile(r"^NPU array is unusable until the driver is reloaded") +NOT_RECOVERED = re.compile(r"^NPU power-cycle did not recover the array: (-?\d+)") +CREATE_REFUSED = re.compile(r"^Init hwctx failed, ret (-?\d+)") + + +def new_load(version): + return {"driver": version, "timeouts": 0, "power_cycles": 0, "throttled": 0, "skipped_contexts": 0, + "failed_restarts": [], "latched": 0, "not_recovered": [], "creates_refused": {}, "lines": 0} + + +def parse(lines): + """-> a list of module loads, each a dict of counts. Lines that are not the driver's are ignored.""" + loads, cur, step = [], None, None + for raw in lines: + m = LOAD.search(raw) + if m: + cur = new_load(m.group("ver")) + loads.append(cur) + step = None + continue + m = LINE.search(raw) + if not m: + continue + if cur is None: + cur = new_load(None) + loads.append(cur) + cur["lines"] += 1 + fn, msg = m.group("fn"), m.group("msg").strip() + if msg == "TDR timeout detected": + cur["timeouts"] += 1 + elif "power-cycling NPU" in msg and fn == "aie2_tdr_recover_all": + # aie2_hw_reset() logs its own "power-cycling NPU" line for the same reset: count the walk's + cur["power_cycles"] += 1 + step = None + elif THROTTLED.search(msg): + cur["throttled"] += 1 + elif SKIPPED.search(msg): + cur["skipped_contexts"] += int(SKIPPED.search(msg).group(1)) + elif LATCHED.search(msg): + cur["latched"] += 1 + elif NOT_RECOVERED.search(msg): + cur["not_recovered"].append(int(NOT_RECOVERED.search(msg).group(1))) + elif CREATE_REFUSED.search(msg): + ret = CREATE_REFUSED.search(msg).group(1) + cur["creates_refused"][ret] = cur["creates_refused"].get(ret, 0) + 1 + elif fn == "aie2_hwctx_restart" and STEP.search(msg): + step = STEP.search(msg).group(1) + elif FAILED.search(msg): + f = FAILED.search(msg) + entry = {"context": f.group("ctx"), "ret": int(f.group("ret")), "step": step, + "logged_by": fn, "teardown_ret": None, "parked_jobs": None} + if f.group("td") is not None: + entry["teardown_ret"], entry["parked_jobs"] = int(f.group("td")), int(f.group("parked")) + prev = cur["failed_restarts"][-1] if cur["failed_restarts"] else None + # with 0013 the restart logs the failure and the TDR walk logs it again: one failure, not two + if (prev and prev["context"] == entry["context"] and prev["ret"] == entry["ret"] + and prev["logged_by"] != fn and not prev.get("paired")): + prev["paired"] = True + if entry["teardown_ret"] is not None: + prev["teardown_ret"], prev["parked_jobs"] = entry["teardown_ret"], entry["parked_jobs"] + else: + cur["failed_restarts"].append(entry) + step = None + for load in loads: + for f in load["failed_restarts"]: + f.pop("paired", None) + f.pop("logged_by", None) + f["module_fails_parked_jobs"] = f["teardown_ret"] is not None # the 0013 line + return loads + + +def render(loads): + out = [] + if not loads: + return "no amdxdna lines in the input\n" + for i, l in enumerate(loads): + head = f"driver {l['driver']}" if l["driver"] else "before the first driver load line in the input" + out.append(f"module load {i + 1}: {head} ({l['lines']} amdxdna lines)") + out.append(f" timeouts detected {l['timeouts']}") + out.append(f" power-cycle recoveries {l['power_cycles']}") + out.append(f" throttled {l['throttled']}") + out.append(f" skipped contexts {l['skipped_contexts']}") + out.append(f" failed restarts {len(l['failed_restarts'])}") + for f in l["failed_restarts"]: + where = f" at '{f['step']}'" if f["step"] else "" + if f["module_fails_parked_jobs"]: + tail = (f"; firmware context taken away (teardown ret {f['teardown_ret']}), " + f"{f['parked_jobs']} parked job(s) failed with -ENODEV (patch 0013)") + else: + tail = "; the module does not say what happened to the parked jobs (no patch 0013)" + out.append(f" {f['context']}: ret {f['ret']}{where}{tail}") + out.append(f" latched (reload needed) {l['latched']}") + if l["not_recovered"]: + out.append(f" power-cycle did not recover the array: {len(l['not_recovered'])} " + f"(ret {', '.join(str(r) for r in l['not_recovered'])})") + if l["creates_refused"]: + by = ", ".join(f"ret {k} x{v}" for k, v in sorted(l["creates_refused"].items(), key=lambda kv: -kv[1])) + out.append(f" context creates refused {sum(l['creates_refused'].values())} ({by})") + return "\n".join(out) + "\n" + + +def read_inputs(args): + if args.files: + for p in args.files: + f = sys.stdin if p == "-" else open(p, errors="replace") + with f: + yield from f + return + cmd = ["journalctl", "-k", "--no-pager", "-b", str(args.boot)] + try: + r = subprocess.run(cmd, capture_output=True, text=True, errors="replace") + except OSError as e: + raise SystemExit(f"cannot run {' '.join(cmd)}: {e}; pass a saved log file instead") + if r.returncode: + raise SystemExit(f"{' '.join(cmd)} exited {r.returncode}: {r.stderr.strip()[:300]}; pass a saved log file instead") + yield from r.stdout.splitlines() + + +# Lines as the driver prints them. The first block is from a box running pin + 0001-0009; the failed +# restarts are written from the format strings in the patched source (0010 alone, then 0013). +P = "Nov 09 04:46:22 host kernel: amdxdna 0000:c6:00.1: [drm] " +SELF = [ + "Nov 08 23:11:19 host kernel: [drm] Initialized amdxdna_accel_driver 0.18.0 for 0000:c6:00.1 on minor 0", + P + "*ERROR* aie2_legacy_tdr_detect: TDR timeout detected", + P + "aie2_tdr_work_func: Device isn't making progress", + P + "*ERROR* aie2_set_cmd_timeout: Firmware timeout state capture:", + P + "*ERROR* aie2_set_cmd_timeout: Context ID: 15", + P + "aie2_tdr_recover_all: Array stopped dispatching, power-cycling NPU to recover", + P + "aie2_hw_reset: NPU firmware unhealthy: power-cycling NPU", + P + "*ERROR* aie2_legacy_tdr_detect: TDR timeout detected", + P + "aie2_tdr_recover_all: TDR recovery throttled: last power-cycle 45560 ms ago (< 60000 ms), retrying on " + "the next watchdog tick; no context touched", + P + "*ERROR* aie2_legacy_tdr_detect: TDR timeout detected", + P + "aie2_tdr_recover_all: Array stopped dispatching, power-cycling NPU to recover", + P + "aie2_tdr_recover_all: TDR: 2 context(s) skipped, being torn down", + P + "*ERROR* aie2_hwctx_init: Alloc hw resource failed, ret -19", + P + "*ERROR* amdxdna_drm_create_hwctx_ioctl: Init hwctx failed, ret -19", + P + "*ERROR* amdxdna_drm_create_hwctx_ioctl: Init hwctx failed, ret -16", + P + "*ERROR* amdxdna_drm_create_hwctx_ioctl: Init hwctx failed, ret -19", + "Nov 09 05:00:00 host kernel: usb 1-1: new high-speed USB device number 4", # not ours + # a second load, a module with 0010 only: the TDR walk logs the failure, and the array latches + "[ 812.100000] [drm] Initialized amdxdna_accel_driver 0.18.0 for 0000:c6:00.1 on minor 0", + "[ 900.000000] amdxdna 0000:c6:00.1: [drm] *ERROR* aie2_legacy_tdr_detect: TDR timeout detected", + "[ 900.000100] amdxdna 0000:c6:00.1: [drm] aie2_tdr_recover_all: Firmware unresponsive, power-cycling NPU", + "[ 900.500000] amdxdna 0000:c6:00.1: [drm] *ERROR* aie2_hwctx_restart: Map host buf failed, ret -22", + "[ 900.500100] amdxdna 0000:c6:00.1: [drm] *ERROR* aie2_tdr_resume_hwctx: hwctx.4242.15 restart failed, ret -22", + "[ 900.500200] amdxdna 0000:c6:00.1: [drm] *ERROR* aie2_tdr_recover_all: NPU array is unusable until the driver is " + "reloaded: TDR recovery failed (0, a context could not be restarted); new contexts and submissions are refused " + "with -ENODEV", + # a third load, a module with 0013: the restart logs it (with the jobs), then the walk logs it again + "[ 1500.000000] [drm] Initialized amdxdna_accel_driver 0.18.0 for 0000:c6:00.1 on minor 0", + "[ 1600.000000] amdxdna 0000:c6:00.1: [drm] *ERROR* aie2_legacy_tdr_detect: TDR timeout detected", + "[ 1600.000100] amdxdna 0000:c6:00.1: [drm] aie2_tdr_recover_all: Array stopped dispatching, power-cycling NPU to recover", + "[ 1600.400000] amdxdna 0000:c6:00.1: [drm] *ERROR* aie2_hwctx_restart: Create hwctx failed, ret -5", + "[ 1600.400100] amdxdna 0000:c6:00.1: [drm] *ERROR* aie2_hwctx_restart: hwctx.77.14 restart failed, ret -5: no firmware " + "context (teardown ret 0), failing 3 parked job(s) and later submissions with -ENODEV", + "[ 1600.400200] amdxdna 0000:c6:00.1: [drm] *ERROR* aie2_tdr_resume_hwctx: hwctx.77.14 restart failed, ret -5", + "[ 1600.400300] amdxdna 0000:c6:00.1: [drm] *ERROR* aie2_hwctx_restart: Config cu failed, ret -110", + "[ 1600.400400] amdxdna 0000:c6:00.1: [drm] *ERROR* aie2_hwctx_restart: hwctx.78.13 restart failed, ret -110: no firmware " + "context (teardown ret 0), failing 0 parked job(s) and later submissions with -ENODEV", + "[ 1600.400500] amdxdna 0000:c6:00.1: [drm] *ERROR* aie2_tdr_resume_hwctx: hwctx.78.13 restart failed, ret -110", + "[ 1600.400600] amdxdna 0000:c6:00.1: [drm] *ERROR* aie2_tdr_recover_all: NPU power-cycle did not recover the array: -110", +] + + +def self_test(): + fails = [] + + def check(good, what): + if not good: + fails.append(what) + + loads = parse(SELF) + check(len(loads) == 3, f"three module loads, got {len(loads)}") + a, b, c = (loads + [new_load(None)] * 3)[:3] + check(a["driver"] == "0.18.0", "the driver version is read from the load line") + check(a["timeouts"] == 3, f"load 1: 3 timeouts, got {a['timeouts']}") + check(a["power_cycles"] == 2, f"load 1: 2 power-cycles (aie2_hw_reset's own line is not a second one), got {a['power_cycles']}") + check(a["throttled"] == 1 and a["skipped_contexts"] == 2, "load 1: 1 throttled, 2 skipped contexts") + check(a["failed_restarts"] == [] and a["latched"] == 0, "load 1: no failed restart, not latched") + check(a["creates_refused"] == {"-19": 2, "-16": 1}, f"load 1: creates refused by error, got {a['creates_refused']}") + check(a["lines"] == 15, f"load 1: 15 driver lines (the usb line is not one), got {a['lines']}") + fb = b["failed_restarts"] + check(len(fb) == 1 and fb[0]["context"] == "hwctx.4242.15" and fb[0]["ret"] == -22, "load 2: one failed restart, named") + check(fb and fb[0]["step"] == "Map host buf failed", "load 2: the failing step is the driver's own line before it") + check(fb and fb[0]["module_fails_parked_jobs"] is False, "load 2: a bare line means no patch 0013") + check(b["latched"] == 1 and b["power_cycles"] == 1, "load 2: latched, and 'Firmware unresponsive' is a power-cycle too") + fc = c["failed_restarts"] + check(len(fc) == 2, f"load 3: the restart's line and the walk's line are one failure each, got {len(fc)}") + check(len(fc) == 2 and fc[0]["parked_jobs"] == 3 and fc[0]["teardown_ret"] == 0 and fc[0]["step"] == "Create hwctx failed", + "load 3: the 0013 line gives the parked jobs and the teardown result") + check(len(fc) == 2 and fc[1]["context"] == "hwctx.78.13" and fc[1]["ret"] == -110 and fc[1]["step"] == "Config cu failed" + and fc[1]["parked_jobs"] == 0, "load 3: the second context keeps its own step and count") + check(all(f["module_fails_parked_jobs"] for f in fc), "load 3: the module is one with patch 0013") + check(c["not_recovered"] == [-110], "load 3: a power-cycle that did not recover is reported with its error") + text = render(loads) + check("failed restarts 0" in text and "context creates refused 3 (ret -19 x2, ret -16 x1)" in text, "the text for load 1") + check("hwctx.77.14: ret -5 at 'Create hwctx failed'; firmware context taken away (teardown ret 0), 3 parked job(s) " + "failed with -ENODEV (patch 0013)" in text, "the text for a 0013 failure") + check("hwctx.4242.15: ret -22 at 'Map host buf failed'; the module does not say" in text, "the text for a bare failure") + check(parse(["Nov 09 host kernel: usb 1-1: hello"]) == [] and "no amdxdna lines" in render([]), "an input with none of ours") + mid = parse(SELF[1:6]) + check(len(mid) == 1 and mid[0]["driver"] is None and mid[0]["power_cycles"] == 1, "a log that starts mid-boot is still counted") + json.dumps(loads) + for f in fails: + print("FAIL " + f) + print("FAILED" if fails else "amdxdna_tdr_report: ok") + return 1 if fails else 0 + + +def main(): + p = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + p.add_argument("files", nargs="*", help="saved kernel logs (- is stdin); default: journalctl -k -b") + p.add_argument("--boot", default="0", help="journalctl boot to read when no file is given (default 0, this boot)") + p.add_argument("--json", action="store_true") + p.add_argument("--fail-on-failed-restart", action="store_true", help="exit 1 when any context restart failed") + p.add_argument("--self-test", action="store_true") + a = p.parse_args() + if a.self_test: + return self_test() + loads = parse(read_inputs(a)) + sys.stdout.write(json.dumps(loads, indent=1) + "\n" if a.json else render(loads)) + if a.fail_on_failed_restart and any(l["failed_restarts"] for l in loads): + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main())