diff --git a/scripts/gpu-hold b/scripts/gpu-hold index 03c263a9c..2ea60db84 100755 --- a/scripts/gpu-hold +++ b/scripts/gpu-hold @@ -316,7 +316,16 @@ while [ "$state" = queued ]; do _resets=$(printf '%s\n' "$_klog" | grep -ciE "$_reset_re") if [ "${_resets:-0}" -gt 0 ]; then rm -f "$TOKEN"; state=done - log "::error::refusing: ${_resets} amdgpu/MES/amdxdna fault line(s) in the last ${_reset_window} min - the ticket is released unheld. Inspect the kernel log, then retry" + # HOW OLD is the newest one? "in the last 10 min" cannot tell a wedged device from another lane's + # teardown nine minutes ago, and on 2026-11-15 that distinction mattered: every amdxdna fault that + # refused a ticket arrived 1.5-8.5 min AFTER the previous ticket released the NPU, i.e. nothing was + # running when it was logged (engine#2542). The age is the datum a reader needs; the epoch comes + # from a second query so the printed lines keep their human-readable form. + _newest=$(journalctl -k --since "-${_reset_window} min" --no-pager -o short-unix -q 2> /dev/null | + grep -iE "$_reset_re" | tail -1 | awk '{print int($1)}') + _age="" + case "${_newest:-}" in ''|*[!0-9]*) ;; *) _age=", newest $(( $(date +%s) - _newest ))s ago" ;; esac + log "::error::refusing: ${_resets} amdgpu/MES/amdxdna fault line(s) in the last ${_reset_window} min${_age} - the ticket is released unheld. Inspect the kernel log, then retry" printf '%s\n' "$_klog" | grep -iE "$_reset_re" | tail -3 exit 78 fi diff --git a/tests/gpu_hold_fault_abort_test.sh b/tests/gpu_hold_fault_abort_test.sh index 523a85a9d..2ce68b6eb 100755 --- a/tests/gpu_hold_fault_abort_test.sh +++ b/tests/gpu_hold_fault_abort_test.sh @@ -115,6 +115,21 @@ take_run ring "kernel: amdgpu 0000:c5:00.0: [drm] *ERROR* ring gfx_0.0.0 timeout take_run hang "kernel: amdgpu 0000:c5:00.0: amdgpu: GPU hang detected on ring gfx" [ "$trc" = 78 ] || { echo "FAIL take-time: an amdgpu hang -> rc $trc, want 78"; fail=1; } +# the refusal must say HOW OLD the newest line is: "in the last 10 min" cannot tell a wedged device from another lane's +# teardown nine minutes ago, and on 2026-11-15 every amdxdna fault that refused a ticket arrived 1.5-8.5 min AFTER the +# previous ticket released the NPU, with nothing running (engine#2542). A stub journalctl supplies the epoch. +mkdir -p "$scratch/agebin" +printf '#!/bin/bash\nprintf "1700000000 kernel: amdgpu 0000:c5:00.0: [drm] *ERROR* amdgpu: GPU reset begin!\\n"\n' > "$scratch/agebin/journalctl" +chmod +x "$scratch/agebin/journalctl" +q=$scratch/tqage; mkdir -p "$q"; rm -f "$scratch/age-token" +env PATH="$scratch/agebin:$PATH" HOME=$scratch ONEBIT_HOLD_DIR="$scratch/agehold" GPU_HOLD_COOLDOWN=0 \ + GPU_HOLD_WAIT=5 GPU_HOLD_MINUTES=1 GPU_HOLD_THERMAL_FLAG="$scratch/x" GPU_HOLD_RESET_WINDOW_MIN=10 \ + timeout 60 "$hold" -l takeage -- true > "$scratch/tlogage" 2>&1 +age_rc=$? +[ "$age_rc" = 78 ] || { echo "FAIL take-time age: rc $age_rc, want 78"; fail=1; } +grep -qE 'newest [0-9]+s ago' "$scratch/tlogage" || { echo "FAIL take-time age: the refusal does not say how old the newest line is"; fail=1; } +grep -qE 'newest [0-9]{9,}s ago' "$scratch/tlogage" && { echo "FAIL take-time age: the age printed in seconds, not an epoch"; fail=1; } + take_run clean clean [ "$trc" = 0 ] || { echo "FAIL take-time: a clean log -> rc $trc, want 0 (a healthy hold must still run)"; fail=1; }