From f1ae470c378b9a1d16e510c6529fb4df2c23b808 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 02:26:00 -0400 Subject: [PATCH] gpu-hold: the take-time guard reads the NPU's fault spellings, and it has arms at last `npu-hold` is a wrapper that execs this script with the NPU's own queue and token, so **this take-time check is an NPU ticket's only look at the kernel log before work starts** - and its pattern listed `amdgpu`/`ring`/`MES` only. The during-hold watcher (#2486) already read `amdxdna.*(command timeout|power.?cycle)`; the refusal did not. So the rule "before a run check there is no new amdxdna timeout line" was enforced by nothing on the way in, and a lane could take an NPU ticket minutes after the very line the rule says to stop for. Three changes, each measured: * the pattern now carries `amdxdna.*(command timeout|power.?cycle|fault)`, and the count and the quoted lines use ONE pattern variable instead of two copies that had already drifted (`_reset_re`); * the refusal message says `amdgpu/MES/amdxdna fault line(s)` - it said "reset" while matching three families; * `GPU_HOLD_KLOG_FILE` now overrides the take-time read as it already did for the during-hold watcher. **That is why the guard had no automated arm**: "a reset two minutes ago" cannot be produced against the real journal, so it shipped with my manual arms only. It has four now. Arms added to `tests/gpu_hold_fault_abort_test.sh`, each running the real hold on a private queue with the during-hold watch OFF, so what is tested is the refusal and nothing else: an amdgpu GPU reset begin -> rc 78, token released an amdxdna command timeout -> rc 78, token released (the NPU rule; invisible before this change) an amdxdna power-cycle -> rc 78 a clean log -> rc 0 (a healthy hold still runs) Mutations, each caught: amdxdna spellings dropped -> "FAIL take-time: an amdxdna command timeout -> rc 0, want 78" "FAIL take-time: an amdxdna power-cycle -> rc 0, want 78" the klog override dropped -> all three refusal arms fail (rc 0) rm -f "$TOKEN" dropped -> "FAIL take-time: the token was still held after a refusal" x2 On the real box, through `npu-hold` itself: NPU log with an amdxdna command timeout -> rc 78, "refusing: 1 amdgpu/MES/amdxdna fault line(s)", npu-token released clean log -> rc 0 `ctest -R 'gpu_hold|npu_hold'` -> 100% passed, 0 failed out of 10. copyright rc 0, lane_syntax balanced. --- scripts/gpu-hold | 21 ++++++++++++++---- tests/gpu_hold_fault_abort_test.sh | 35 ++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/scripts/gpu-hold b/scripts/gpu-hold index 07ecc1792..5c2d40e97 100755 --- a/scripts/gpu-hold +++ b/scripts/gpu-hold @@ -289,15 +289,28 @@ while [ "$state" = queued ]; do # kernel log to read and `gpu_hold_*`'s own tests must keep passing, so "cannot read" is a note, not a # refusal - the opposite of a device guard, and worth stating because it is the surprising half. _reset_window=${GPU_HOLD_RESET_WINDOW_MIN:-10} - _klog=$(journalctl -k --since "-${_reset_window} min" --no-pager 2> /dev/null) + # GPU_HOLD_KLOG_FILE is the same override the during-hold watcher uses: without it this refusal had no + # automated arm at all (it shipped with manual arms only), because "a reset two minutes ago" cannot be + # produced in a test that reads the real journal. + if [ -n "${GPU_HOLD_KLOG_FILE:-}" ]; then + _klog=$(cat "$GPU_HOLD_KLOG_FILE" 2> /dev/null) + else + _klog=$(journalctl -k --since "-${_reset_window} min" --no-pager 2> /dev/null) + fi if [ -z "$_klog" ]; then log "note: no kernel log available, so a reset in the last ${_reset_window} min cannot be ruled out (not refusing: CI has no journal)" else - _resets=$(printf '%s\n' "$_klog" | grep -ciE 'amdgpu.*(reset|hang)|ring .*timeout|MES.*(reset|failure)') + # The NPU spellings belong here too: npu-hold execs this script with the NPU's own queue and + # token, so this take-time check is an NPU ticket's ONLY look at the log before work starts, + # while the during-hold watch below already reads amdxdna faults. Without them a lane could + # take an NPU ticket minutes after the command timeout the rules say to stop for (measured + # 2026-11-15: the hold ran on a log whose newest line was an amdxdna command timeout). + _reset_re='amdgpu.*(reset|hang)|ring .*timeout|MES.*(reset|failure)|amdxdna.*(command timeout|power.?cycle|fault)' + _resets=$(printf '%s\n' "$_klog" | grep -ciE "$_reset_re") if [ "${_resets:-0}" -gt 0 ]; then rm -f "$TOKEN"; state=done - log "::error::refusing: ${_resets} amdgpu/MES reset line(s) in the last ${_reset_window} min - the ticket is released unheld. Inspect the kernel log, then retry" - printf '%s\n' "$_klog" | grep -iE 'amdgpu.*(reset|hang)|ring .*timeout|MES.*(reset|failure)' | tail -3 + log "::error::refusing: ${_resets} amdgpu/MES/amdxdna fault line(s) in the last ${_reset_window} min - the ticket is released unheld. Inspect the kernel log, then retry" + printf '%s\n' "$_klog" | grep -iE "$_reset_re" | tail -3 exit 78 fi fi diff --git a/tests/gpu_hold_fault_abort_test.sh b/tests/gpu_hold_fault_abort_test.sh index 52a043ae9..429ad9226 100755 --- a/tests/gpu_hold_fault_abort_test.sh +++ b/tests/gpu_hold_fault_abort_test.sh @@ -65,6 +65,41 @@ run() { # $1 arm $2 payload seconds $3 inject fault (1/0) $4 GPU_HOLD_FAULT echo 40000 > "$scratch/tctl" # 40 C, in millidegrees +# ── the TAKE-TIME guard (engine#2480), which had no automated arm until now ─────────────────────────────────────── +# It shipped with manual arms only, and the NPU spellings were missing from its regex although `npu-hold` execs this +# script: an NPU ticket's only look at the log before work starts is this check, so `amdxdna` faults have to count. +# Each arm runs the real hold against a klog file with a 10-minute take-time window and the during-hold watch OFF, so +# what is being tested is the refusal and nothing else. +take_run() { # $1 arm name $2 the kernel-log line (or "clean") + local q=$scratch/tq$1; mkdir -p "$q"; rm -f "$scratch/TTOKEN$1" + : > "$scratch/tklog$1" + printf '%s\n' "kernel: amdgpu 0000:c5:00.0: [drm] Initialized" > "$scratch/tklog$1" + [ "$2" = clean ] || printf '%s\n' "$2" >> "$scratch/tklog$1" + env GPU_HOLD_KLOG_FILE="$scratch/tklog$1" GPU_QUEUE_DIR="$q" GPU_TOKEN_FILE="$scratch/TTOKEN$1" \ + GPU_HOLD_LOG="$scratch/tlog$1" GPU_HOLD_POLL=1 GPU_HOLD_WAIT=5 GPU_HOLD_RELEASE_WAIT=0 GPU_HOLD_KILL_GRACE=3 \ + GPU_HOLD_RESET_WINDOW_MIN=10 GPU_HOLD_FAULT_ABORT=0 GPU_HOLD_TCTL_START=100 \ + GPU_HOLD_TCTL_FILE="$scratch/tctl" GPU_HOLD_THERMAL_FLAG="$scratch/x" \ + timeout 60 "$hold" -t 1 -l taketest$1 -- true > /dev/null 2>&1 + trc=$? + [ -e "$scratch/TTOKEN$1" ] && ttok=held || ttok=released +} + +take_run amdgpu "kernel: amdgpu 0000:c5:00.0: [drm] *ERROR* amdgpu: GPU reset begin!" +[ "$trc" = 78 ] || { echo "FAIL take-time: an amdgpu reset -> rc $trc, want 78"; fail=1; } +[ "$ttok" = released ] || { echo "FAIL take-time: the token was still held after a refusal"; fail=1; } + +# the NPU spelling: this is the line the rules say to stop for, and npu-hold has no other take-time check +take_run npu "kernel: amdxdna 0000:c6:00.1: [drm] *ERROR* command timeout: opcode 0x2" +[ "$trc" = 78 ] || { echo "FAIL take-time: an amdxdna command timeout -> rc $trc, want 78 (the NPU rule)"; fail=1; } +[ "$ttok" = released ] || { echo "FAIL take-time: the token was still held after an NPU refusal"; fail=1; } + +take_run powercycle "kernel: amdxdna 0000:c6:00.1: [drm] power-cycle" +[ "$trc" = 78 ] || { echo "FAIL take-time: an amdxdna power-cycle -> rc $trc, want 78"; fail=1; } + +take_run clean clean +[ "$trc" = 0 ] || { echo "FAIL take-time: a clean log -> rc $trc, want 0 (a healthy hold must still run)"; fail=1; } + + # arm 1: the fault fires run 1 600 1 1 e1=$elapsed