Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 16 additions & 5 deletions scripts/gpu-hold
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,15 @@ COOLDOWN_TCTL=${GPU_HOLD_COOLDOWN_TCTL:-85}
# the previous hold with rc 78, and only a MODE2 reset line refused. The during-hold copy lacked the amdxdna "fault" that
# #2498 added at take time. The union is the rule's own list: a page fault, any amdgpu/MES reset or failure, a ring
# timeout, an amdxdna command timeout, power-cycle or fault.
GPU_FAULT_RE='\[gfxhub\] page fault|amdgpu.*(reset|hang)|ring .*timeout|MES.*(reset|failure|failed to respond)|amdxdna.*(command timeout|power.?cycle|fault)'
# `power.?cycle` missed the driver's own event line: it writes "power-cycling NPU to recover" and "power-cycling
# NPU" (the -ing form does not match `cycle`), while the RECOVERY notice -- "TDR recovery throttled: last
# power-cycle 45560 ms ago ... no context touched" -- did match. Measured 2026-11-15: the notice aborted another
# lane's hold 493 s in (GPU-HOLD.log:134, rc 78), and a real `Array stopped dispatching, power-cycling NPU to
# recover` did not abort at all. The event is `power.?cycl(e|ing)`; the recovery notice is excluded by name.
GPU_FAULT_RE='\[gfxhub\] page fault|amdgpu.*(reset|hang)|ring .*timeout|MES.*(reset|failure|failed to respond)|amdxdna.*(command timeout|power.?cycl(e|ing)|fault)'
# A line that says the driver did NOTHING ("no context touched") is not a fault: it is the throttle that follows
# one, and it is logged for 60 s after every power-cycle, which is exactly when the next ticket is granted.
GPU_FAULT_SKIP_RE='aie2_tdr_recover_all: TDR recovery throttled'

while getopts ":t:l:w:q:h" opt; do
case $opt in
Expand Down Expand Up @@ -313,20 +321,23 @@ while [ "$state" = queued ]; do
# take an NPU ticket minutes after the command timeout the rules say to stop for (measured
# 2026-11-15: the hold ran on a log whose newest line was an amdxdna command timeout).
_reset_re=$GPU_FAULT_RE
_resets=$(printf '%s\n' "$_klog" | grep -ciE "$_reset_re")
_resets=$(printf '%s\n' "$_klog" | grep -iE "$_reset_re" | grep -viE "$GPU_FAULT_SKIP_RE" | grep -c .)
if [ "${_resets:-0}" -gt 0 ]; then
rm -f "$TOKEN"; state=done
# HOW OLD is the newest one? "in the last 10 min" cannot tell a wedged device from another lane's
# teardown nine minutes ago, and on 2026-11-15 that distinction mattered: every amdxdna fault that
# refused a ticket arrived 1.5-8.5 min AFTER the previous ticket released the NPU, i.e. nothing was
# running when it was logged (engine#2542). The age is the datum a reader needs; the epoch comes
# from a second query so the printed lines keep their human-readable form.
# The skip term applies to the age as well as to the count: the recovery-throttle notice is logged
# for 60 s after every power-cycle and says "no context touched", so without this the refusal could
# count only real faults and then report the notice's age as if it were the newest fault's.
_newest=$(journalctl -k --since "-${_reset_window} min" --no-pager -o short-unix -q 2> /dev/null |
grep -iE "$_reset_re" | tail -1 | awk '{print int($1)}')
grep -iE "$_reset_re" | grep -viE "$GPU_FAULT_SKIP_RE" | tail -1 | awk '{print int($1)}')
_age=""
case "${_newest:-}" in ''|*[!0-9]*) ;; *) _age=", newest $(( $(date +%s) - _newest ))s ago" ;; esac
log "::error::refusing: ${_resets} amdgpu/MES/amdxdna fault line(s) in the last ${_reset_window} min${_age} - the ticket is released unheld. Inspect the kernel log, then retry"
printf '%s\n' "$_klog" | grep -iE "$_reset_re" | tail -3
printf '%s\n' "$_klog" | grep -iE "$_reset_re" | grep -viE "$GPU_FAULT_SKIP_RE" | tail -3
exit 78
fi
fi
Expand Down Expand Up @@ -443,7 +454,7 @@ fault_lines() {
cat "$GPU_HOLD_KLOG_FILE" 2> /dev/null
else
journalctl -k --since "@$started" --no-pager -q 2> /dev/null
fi | grep -iE "$FAULT_RE"
fi | grep -iE "$FAULT_RE" | grep -viE "$GPU_FAULT_SKIP_RE"
}

watch() {
Expand Down
14 changes: 13 additions & 1 deletion tests/gpu_hold_fault_abort_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -161,11 +161,23 @@ if [ -n "$deep" ] && kill -0 "$deep" 2> /dev/null; then kill -9 "$deep"; fi

# arm 5: an amdxdna fault DURING the hold. #2498 added this spelling at take time only; with one shared pattern the
# during-hold watch now reads it too (the NPU rule: on any amdxdna fault, stop).
# engine#2542/11-29Z: the driver's RECOVERY notices contain "power-cycle" and "aie2_tdr_recover_all", so they match
# `amdxdna.*power.?cycle` -- and one aborted a real hold. Measured 2026-11-15 (07:29:38 local, GPU-HOLD.log :134):
# gpu-hold[402117] ::error::ABORT GPU/NPU fault ... (rc 78) after 493s
# | amdxdna ... aie2_tdr_recover_all: TDR recovery throttled: last power-cycle 45560 ms ago (< 60000 ms),
# | retrying on the next watchdog tick; no context touched
# The line says NO CONTEXT TOUCHED: the fault it refers to was 45 s old, and the hold it ended belonged to another
# lane's job. A bookkeeping line must not abort a hold; the events it summarises must still abort.
run 6 600 1 1 "kernel: amdxdna 0000:c6:00.1: [drm] aie2_tdr_recover_all: TDR recovery throttled: last power-cycle 45560 ms ago (< 60000 ms), retrying on the next watchdog tick; no context touched"
if [ "$rc" = 78 ]; then echo "FAIL arm 6: a recovery-throttle notice (\"no context touched\") aborted the hold"; fail=1; fi
run 7 600 1 1 "kernel: amdxdna 0000:c6:00.1: [drm] aie2_tdr_recover_all: Array stopped dispatching, power-cycling NPU to recover"
if [ "$rc" != 78 ]; then echo "FAIL arm 7: a real power-cycle event did not abort (rc $rc)"; fail=1; fi

run 5 600 1 1 "kernel: amdxdna 0000:c6:00.1: [drm] *ERROR* aie2_error_worker: fault at 0x00000000"
if [ "$rc" != 78 ]; then echo "FAIL arm 5: rc $rc after an amdxdna fault line, want 78"; fail=1; fi
grep -q "newest line is from the NPU" "$scratch/log5" || { echo "FAIL arm 5: the abort does not name the NPU as the faulting device"; fail=1; }
grep -q "newest line is from the GPU" "$scratch/log1" || { echo "FAIL arm 1: the abort does not name the GPU as the faulting device"; fail=1; }
if [ -n "$deep" ] && kill -0 "$deep" 2> /dev/null; then kill -9 "$deep"; fi

[ "$fail" = 0 ] && echo "gpu_hold_fault_abort: ok (fault at 5 s -> rc 78 after ${e1}s; clean -> 0; opt-out -> 0; default on -> 78; take-time page fault/MES/ring/hang -> 78; NPU fault in hold -> 78)"
[ "$fail" = 0 ] && echo "gpu_hold_fault_abort: ok (fault at 5 s -> rc 78 after ${e1}s; clean -> 0; opt-out -> 0; default on -> 78; take-time page fault/MES/ring/hang -> 78; NPU fault in hold -> 78; a recovery notice does NOT abort; a real power-cycling event does)"
exit "$fail"
Loading