diff --git a/scripts/gpu-hold b/scripts/gpu-hold index 2ea60db84..f3524eaad 100755 --- a/scripts/gpu-hold +++ b/scripts/gpu-hold @@ -99,7 +99,15 @@ COOLDOWN_TCTL=${GPU_HOLD_COOLDOWN_TCTL:-85} # the previous hold with rc 78, and only a MODE2 reset line refused. The during-hold copy lacked the amdxdna "fault" that # #2498 added at take time. The union is the rule's own list: a page fault, any amdgpu/MES reset or failure, a ring # timeout, an amdxdna command timeout, power-cycle or fault. -GPU_FAULT_RE='\[gfxhub\] page fault|amdgpu.*(reset|hang)|ring .*timeout|MES.*(reset|failure|failed to respond)|amdxdna.*(command timeout|power.?cycle|fault)' +# `power.?cycle` missed the driver's own event line: it writes "power-cycling NPU to recover" and "power-cycling +# NPU" (the -ing form does not match `cycle`), while the RECOVERY notice -- "TDR recovery throttled: last +# power-cycle 45560 ms ago ... no context touched" -- did match. Measured 2026-11-15: the notice aborted another +# lane's hold 493 s in (GPU-HOLD.log:134, rc 78), and a real `Array stopped dispatching, power-cycling NPU to +# recover` did not abort at all. The event is `power.?cycl(e|ing)`; the recovery notice is excluded by name. +GPU_FAULT_RE='\[gfxhub\] page fault|amdgpu.*(reset|hang)|ring .*timeout|MES.*(reset|failure|failed to respond)|amdxdna.*(command timeout|power.?cycl(e|ing)|fault)' +# A line that says the driver did NOTHING ("no context touched") is not a fault: it is the throttle that follows +# one, and it is logged for 60 s after every power-cycle, which is exactly when the next ticket is granted. +GPU_FAULT_SKIP_RE='aie2_tdr_recover_all: TDR recovery throttled' while getopts ":t:l:w:q:h" opt; do case $opt in @@ -313,7 +321,7 @@ while [ "$state" = queued ]; do # take an NPU ticket minutes after the command timeout the rules say to stop for (measured # 2026-11-15: the hold ran on a log whose newest line was an amdxdna command timeout). _reset_re=$GPU_FAULT_RE - _resets=$(printf '%s\n' "$_klog" | grep -ciE "$_reset_re") + _resets=$(printf '%s\n' "$_klog" | grep -iE "$_reset_re" | grep -viE "$GPU_FAULT_SKIP_RE" | grep -c .) if [ "${_resets:-0}" -gt 0 ]; then rm -f "$TOKEN"; state=done # HOW OLD is the newest one? "in the last 10 min" cannot tell a wedged device from another lane's @@ -321,12 +329,15 @@ while [ "$state" = queued ]; do # refused a ticket arrived 1.5-8.5 min AFTER the previous ticket released the NPU, i.e. nothing was # running when it was logged (engine#2542). The age is the datum a reader needs; the epoch comes # from a second query so the printed lines keep their human-readable form. + # The skip term applies to the age as well as to the count: the recovery-throttle notice is logged + # for 60 s after every power-cycle and says "no context touched", so without this the refusal could + # count only real faults and then report the notice's age as if it were the newest fault's. _newest=$(journalctl -k --since "-${_reset_window} min" --no-pager -o short-unix -q 2> /dev/null | - grep -iE "$_reset_re" | tail -1 | awk '{print int($1)}') + grep -iE "$_reset_re" | grep -viE "$GPU_FAULT_SKIP_RE" | tail -1 | awk '{print int($1)}') _age="" case "${_newest:-}" in ''|*[!0-9]*) ;; *) _age=", newest $(( $(date +%s) - _newest ))s ago" ;; esac log "::error::refusing: ${_resets} amdgpu/MES/amdxdna fault line(s) in the last ${_reset_window} min${_age} - the ticket is released unheld. Inspect the kernel log, then retry" - printf '%s\n' "$_klog" | grep -iE "$_reset_re" | tail -3 + printf '%s\n' "$_klog" | grep -iE "$_reset_re" | grep -viE "$GPU_FAULT_SKIP_RE" | tail -3 exit 78 fi fi @@ -443,7 +454,7 @@ fault_lines() { cat "$GPU_HOLD_KLOG_FILE" 2> /dev/null else journalctl -k --since "@$started" --no-pager -q 2> /dev/null - fi | grep -iE "$FAULT_RE" + fi | grep -iE "$FAULT_RE" | grep -viE "$GPU_FAULT_SKIP_RE" } watch() { diff --git a/tests/gpu_hold_fault_abort_test.sh b/tests/gpu_hold_fault_abort_test.sh index 2ce68b6eb..6119b788d 100755 --- a/tests/gpu_hold_fault_abort_test.sh +++ b/tests/gpu_hold_fault_abort_test.sh @@ -161,11 +161,23 @@ if [ -n "$deep" ] && kill -0 "$deep" 2> /dev/null; then kill -9 "$deep"; fi # arm 5: an amdxdna fault DURING the hold. #2498 added this spelling at take time only; with one shared pattern the # during-hold watch now reads it too (the NPU rule: on any amdxdna fault, stop). +# engine#2542/11-29Z: the driver's RECOVERY notices contain "power-cycle" and "aie2_tdr_recover_all", so they match +# `amdxdna.*power.?cycle` -- and one aborted a real hold. Measured 2026-11-15 (07:29:38 local, GPU-HOLD.log :134): +# gpu-hold[402117] ::error::ABORT GPU/NPU fault ... (rc 78) after 493s +# | amdxdna ... aie2_tdr_recover_all: TDR recovery throttled: last power-cycle 45560 ms ago (< 60000 ms), +# | retrying on the next watchdog tick; no context touched +# The line says NO CONTEXT TOUCHED: the fault it refers to was 45 s old, and the hold it ended belonged to another +# lane's job. A bookkeeping line must not abort a hold; the events it summarises must still abort. +run 6 600 1 1 "kernel: amdxdna 0000:c6:00.1: [drm] aie2_tdr_recover_all: TDR recovery throttled: last power-cycle 45560 ms ago (< 60000 ms), retrying on the next watchdog tick; no context touched" +if [ "$rc" = 78 ]; then echo "FAIL arm 6: a recovery-throttle notice (\"no context touched\") aborted the hold"; fail=1; fi +run 7 600 1 1 "kernel: amdxdna 0000:c6:00.1: [drm] aie2_tdr_recover_all: Array stopped dispatching, power-cycling NPU to recover" +if [ "$rc" != 78 ]; then echo "FAIL arm 7: a real power-cycle event did not abort (rc $rc)"; fail=1; fi + run 5 600 1 1 "kernel: amdxdna 0000:c6:00.1: [drm] *ERROR* aie2_error_worker: fault at 0x00000000" if [ "$rc" != 78 ]; then echo "FAIL arm 5: rc $rc after an amdxdna fault line, want 78"; fail=1; fi grep -q "newest line is from the NPU" "$scratch/log5" || { echo "FAIL arm 5: the abort does not name the NPU as the faulting device"; fail=1; } grep -q "newest line is from the GPU" "$scratch/log1" || { echo "FAIL arm 1: the abort does not name the GPU as the faulting device"; fail=1; } if [ -n "$deep" ] && kill -0 "$deep" 2> /dev/null; then kill -9 "$deep"; fi -[ "$fail" = 0 ] && echo "gpu_hold_fault_abort: ok (fault at 5 s -> rc 78 after ${e1}s; clean -> 0; opt-out -> 0; default on -> 78; take-time page fault/MES/ring/hang -> 78; NPU fault in hold -> 78)" +[ "$fail" = 0 ] && echo "gpu_hold_fault_abort: ok (fault at 5 s -> rc 78 after ${e1}s; clean -> 0; opt-out -> 0; default on -> 78; take-time page fault/MES/ring/hang -> 78; NPU fault in hold -> 78; a recovery notice does NOT abort; a real power-cycling event does)" exit "$fail"