diff --git a/scripts/gpu-hold b/scripts/gpu-hold index 07ecc1792..5c2d40e97 100755 --- a/scripts/gpu-hold +++ b/scripts/gpu-hold @@ -289,15 +289,28 @@ while [ "$state" = queued ]; do # kernel log to read and `gpu_hold_*`'s own tests must keep passing, so "cannot read" is a note, not a # refusal - the opposite of a device guard, and worth stating because it is the surprising half. _reset_window=${GPU_HOLD_RESET_WINDOW_MIN:-10} - _klog=$(journalctl -k --since "-${_reset_window} min" --no-pager 2> /dev/null) + # GPU_HOLD_KLOG_FILE is the same override the during-hold watcher uses: without it this refusal had no + # automated arm at all (it shipped with manual arms only), because "a reset two minutes ago" cannot be + # produced in a test that reads the real journal. + if [ -n "${GPU_HOLD_KLOG_FILE:-}" ]; then + _klog=$(cat "$GPU_HOLD_KLOG_FILE" 2> /dev/null) + else + _klog=$(journalctl -k --since "-${_reset_window} min" --no-pager 2> /dev/null) + fi if [ -z "$_klog" ]; then log "note: no kernel log available, so a reset in the last ${_reset_window} min cannot be ruled out (not refusing: CI has no journal)" else - _resets=$(printf '%s\n' "$_klog" | grep -ciE 'amdgpu.*(reset|hang)|ring .*timeout|MES.*(reset|failure)') + # The NPU spellings belong here too: npu-hold execs this script with the NPU's own queue and + # token, so this take-time check is an NPU ticket's ONLY look at the log before work starts, + # while the during-hold watch below already reads amdxdna faults. Without them a lane could + # take an NPU ticket minutes after the command timeout the rules say to stop for (measured + # 2026-11-15: the hold ran on a log whose newest line was an amdxdna command timeout). + _reset_re='amdgpu.*(reset|hang)|ring .*timeout|MES.*(reset|failure)|amdxdna.*(command timeout|power.?cycle|fault)' + _resets=$(printf '%s\n' "$_klog" | grep -ciE "$_reset_re") if [ "${_resets:-0}" -gt 0 ]; then rm -f "$TOKEN"; state=done - log "::error::refusing: ${_resets} amdgpu/MES reset line(s) in the last ${_reset_window} min - the ticket is released unheld. Inspect the kernel log, then retry" - printf '%s\n' "$_klog" | grep -iE 'amdgpu.*(reset|hang)|ring .*timeout|MES.*(reset|failure)' | tail -3 + log "::error::refusing: ${_resets} amdgpu/MES/amdxdna fault line(s) in the last ${_reset_window} min - the ticket is released unheld. Inspect the kernel log, then retry" + printf '%s\n' "$_klog" | grep -iE "$_reset_re" | tail -3 exit 78 fi fi diff --git a/tests/gpu_hold_fault_abort_test.sh b/tests/gpu_hold_fault_abort_test.sh index 52a043ae9..429ad9226 100755 --- a/tests/gpu_hold_fault_abort_test.sh +++ b/tests/gpu_hold_fault_abort_test.sh @@ -65,6 +65,41 @@ run() { # $1 arm $2 payload seconds $3 inject fault (1/0) $4 GPU_HOLD_FAULT echo 40000 > "$scratch/tctl" # 40 C, in millidegrees +# ── the TAKE-TIME guard (engine#2480), which had no automated arm until now ─────────────────────────────────────── +# It shipped with manual arms only, and the NPU spellings were missing from its regex although `npu-hold` execs this +# script: an NPU ticket's only look at the log before work starts is this check, so `amdxdna` faults have to count. +# Each arm runs the real hold against a klog file with a 10-minute take-time window and the during-hold watch OFF, so +# what is being tested is the refusal and nothing else. +take_run() { # $1 arm name $2 the kernel-log line (or "clean") + local q=$scratch/tq$1; mkdir -p "$q"; rm -f "$scratch/TTOKEN$1" + : > "$scratch/tklog$1" + printf '%s\n' "kernel: amdgpu 0000:c5:00.0: [drm] Initialized" > "$scratch/tklog$1" + [ "$2" = clean ] || printf '%s\n' "$2" >> "$scratch/tklog$1" + env GPU_HOLD_KLOG_FILE="$scratch/tklog$1" GPU_QUEUE_DIR="$q" GPU_TOKEN_FILE="$scratch/TTOKEN$1" \ + GPU_HOLD_LOG="$scratch/tlog$1" GPU_HOLD_POLL=1 GPU_HOLD_WAIT=5 GPU_HOLD_RELEASE_WAIT=0 GPU_HOLD_KILL_GRACE=3 \ + GPU_HOLD_RESET_WINDOW_MIN=10 GPU_HOLD_FAULT_ABORT=0 GPU_HOLD_TCTL_START=100 \ + GPU_HOLD_TCTL_FILE="$scratch/tctl" GPU_HOLD_THERMAL_FLAG="$scratch/x" \ + timeout 60 "$hold" -t 1 -l taketest$1 -- true > /dev/null 2>&1 + trc=$? + [ -e "$scratch/TTOKEN$1" ] && ttok=held || ttok=released +} + +take_run amdgpu "kernel: amdgpu 0000:c5:00.0: [drm] *ERROR* amdgpu: GPU reset begin!" +[ "$trc" = 78 ] || { echo "FAIL take-time: an amdgpu reset -> rc $trc, want 78"; fail=1; } +[ "$ttok" = released ] || { echo "FAIL take-time: the token was still held after a refusal"; fail=1; } + +# the NPU spelling: this is the line the rules say to stop for, and npu-hold has no other take-time check +take_run npu "kernel: amdxdna 0000:c6:00.1: [drm] *ERROR* command timeout: opcode 0x2" +[ "$trc" = 78 ] || { echo "FAIL take-time: an amdxdna command timeout -> rc $trc, want 78 (the NPU rule)"; fail=1; } +[ "$ttok" = released ] || { echo "FAIL take-time: the token was still held after an NPU refusal"; fail=1; } + +take_run powercycle "kernel: amdxdna 0000:c6:00.1: [drm] power-cycle" +[ "$trc" = 78 ] || { echo "FAIL take-time: an amdxdna power-cycle -> rc $trc, want 78"; fail=1; } + +take_run clean clean +[ "$trc" = 0 ] || { echo "FAIL take-time: a clean log -> rc $trc, want 0 (a healthy hold must still run)"; fail=1; } + + # arm 1: the fault fires run 1 600 1 1 e1=$elapsed