From 7003acdb8c950e6001a6e449888a93e10add3a85 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 09:12:00 -0400 Subject: [PATCH] gpu-hold: one fault pattern for the take-time refusal and the during-hold watch (#2494) The take-time refusal (#2480) and the during-hold watch (#2486) each had their own copy of the fault regex, and the two had drifted. The take-time copy lacked "[gfxhub] page fault" and "MES ... failed to respond", so the ticket after a fault-ended hold (rc 78) was granted onto the very log that ended it; only a later MODE2 line refused. The during-hold copy lacked the amdxdna "fault" #2498 added at take time. Both now read GPU_FAULT_RE, the union of the two. tests/gpu_hold_fault_abort_test.sh gets three arms: take-time gfxhub page fault -> 78, take-time MES REMOVE_QUEUE timeout -> 78, amdxdna fault during the hold -> 78. Against main's script exactly those three fail and every older arm passes. Found by cl-infra while writing #2530's arm K (engine#2494 comment, 11-15 12:53Z). Co-Authored-By: Claude Opus 5.5 --- scripts/gpu-hold | 11 +++++++++-- tests/gpu_hold_fault_abort_test.sh | 21 +++++++++++++++++++-- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/scripts/gpu-hold b/scripts/gpu-hold index 5c2d40e97..7d71b8a5e 100755 --- a/scripts/gpu-hold +++ b/scripts/gpu-hold @@ -93,6 +93,13 @@ LOG=${GPU_HOLD_LOG:-} THERMAL_FLAG=${GPU_HOLD_THERMAL_FLAG:-$HOME/THERMAL-HOLD} COOLDOWN=${GPU_HOLD_COOLDOWN:-180} COOLDOWN_TCTL=${GPU_HOLD_COOLDOWN_TCTL:-85} +# ONE pattern for a GPU/NPU fault line, read by both the take-time refusal (#2480) and the during-hold watch (#2486). +# They had one each and the two drifted (engine#2494, cl-infra 11-15): the take-time copy lacked the gfxhub page fault +# and the MES "failed to respond", so a ticket was granted onto a log whose newest line was the fault that had just ended +# the previous hold with rc 78, and only a MODE2 reset line refused. The during-hold copy lacked the amdxdna "fault" that +# #2498 added at take time. The union is the rule's own list: a page fault, any amdgpu/MES reset or failure, a ring +# timeout, an amdxdna command timeout, power-cycle or fault. +GPU_FAULT_RE='\[gfxhub\] page fault|amdgpu.*(reset|hang)|ring .*timeout|MES.*(reset|failure|failed to respond)|amdxdna.*(command timeout|power.?cycle|fault)' while getopts ":t:l:w:q:h" opt; do case $opt in @@ -305,7 +312,7 @@ while [ "$state" = queued ]; do # while the during-hold watch below already reads amdxdna faults. Without them a lane could # take an NPU ticket minutes after the command timeout the rules say to stop for (measured # 2026-11-15: the hold ran on a log whose newest line was an amdxdna command timeout). - _reset_re='amdgpu.*(reset|hang)|ring .*timeout|MES.*(reset|failure)|amdxdna.*(command timeout|power.?cycle|fault)' + _reset_re=$GPU_FAULT_RE _resets=$(printf '%s\n' "$_klog" | grep -ciE "$_reset_re") if [ "${_resets:-0}" -gt 0 ]; then rm -f "$TOKEN"; state=done @@ -420,7 +427,7 @@ mem_available_gb() { # GPU_HOLD_FAULT_ABORT=0 turns it off. With no readable kernel log (a CI runner) it is inert, as #2480's check is. # GPU_HOLD_KLOG_FILE replaces the kernel log with a file (tests). FAULT_ABORT=${GPU_HOLD_FAULT_ABORT:-1} -FAULT_RE='\[gfxhub\] page fault|amdgpu.*(reset|hang)|ring .*timeout|MES.*(reset|failure|failed to respond)|amdxdna.*(command timeout|power.?cycle)' +FAULT_RE=$GPU_FAULT_RE FAULT_FLAG="$QUEUE/.fault.$SAFE.$$" fault_lines() { if [ -n "${GPU_HOLD_KLOG_FILE:-}" ]; then diff --git a/tests/gpu_hold_fault_abort_test.sh b/tests/gpu_hold_fault_abort_test.sh index 429ad9226..2a75cc249 100755 --- a/tests/gpu_hold_fault_abort_test.sh +++ b/tests/gpu_hold_fault_abort_test.sh @@ -46,10 +46,12 @@ WRAP chmod +x "$scratch/wrapper.sh" run() { # $1 arm $2 payload seconds $3 inject fault (1/0) $4 GPU_HOLD_FAULT_ABORT, or "unset" to leave it out + # $5 the fault line to inject (default: the 11-13 gfxhub page fault) local q=$scratch/q$1; mkdir -p "$q"; : > "$scratch/klog$1"; rm -f "$scratch/deep.pid" printf '%s\n' "kernel: usb 1-1: new device" > "$scratch/klog$1" + local line=${5:-"kernel: amdgpu 0000:c5:00.0: [gfxhub] page fault (src_id:0 ring:24 vmid:9 pasid:26332)"} if [ "$3" = 1 ]; then - ( sleep 5; printf '%s\n' "kernel: amdgpu 0000:c5:00.0: [gfxhub] page fault (src_id:0 ring:24 vmid:9 pasid:26332)" >> "$scratch/klog$1" ) & + ( sleep 5; printf '%s\n' "$line" >> "$scratch/klog$1" ) & fi local t0; t0=$(date +%s) local abort_env=(GPU_HOLD_FAULT_ABORT="$4"); [ "$4" = unset ] && abort_env=() @@ -96,6 +98,15 @@ take_run npu "kernel: amdxdna 0000:c6:00.1: [drm] *ERROR* command timeout: opcod take_run powercycle "kernel: amdxdna 0000:c6:00.1: [drm] power-cycle" [ "$trc" = 78 ] || { echo "FAIL take-time: an amdxdna power-cycle -> rc $trc, want 78"; fail=1; } +# engine#2494 (cl-infra 11-15): the take-time pattern was a separate copy that lacked these two, so the ticket after a +# fault-ended hold was GRANTED on the very log that had just ended it; only the MODE2 line further down the chain refused. +take_run gfxhub "kernel: amdgpu 0000:c5:00.0: [gfxhub] page fault (src_id:0 ring:24 vmid:9 pasid:26332)" +[ "$trc" = 78 ] || { echo "FAIL take-time: a gfxhub page fault -> rc $trc, want 78 (engine#2494)"; fail=1; } +[ "$ttok" = released ] || { echo "FAIL take-time: the token was still held after a page-fault refusal"; fail=1; } + +take_run mesremove "kernel: amdgpu 0000:c5:00.0: amdgpu: MES failed to respond to msg=REMOVE_QUEUE" +[ "$trc" = 78 ] || { echo "FAIL take-time: an MES REMOVE_QUEUE timeout -> rc $trc, want 78 (engine#2494)"; fail=1; } + take_run clean clean [ "$trc" = 0 ] || { echo "FAIL take-time: a clean log -> rc $trc, want 0 (a healthy hold must still run)"; fail=1; } @@ -125,5 +136,11 @@ run 4 600 1 unset if [ "$rc" != 78 ]; then echo "FAIL arm 4: rc $rc with GPU_HOLD_FAULT_ABORT unset, want 78 (the default must be on)"; fail=1; fi if [ -n "$deep" ] && kill -0 "$deep" 2> /dev/null; then kill -9 "$deep"; fi -[ "$fail" = 0 ] && echo "gpu_hold_fault_abort: ok (fault at 5 s -> rc 78 after ${e1}s; clean -> 0; opt-out -> 0; default on -> 78)" +# arm 5: an amdxdna fault DURING the hold. #2498 added this spelling at take time only; with one shared pattern the +# during-hold watch now reads it too (the NPU rule: on any amdxdna fault, stop). +run 5 600 1 1 "kernel: amdxdna 0000:c6:00.1: [drm] *ERROR* aie2_error_worker: fault at 0x00000000" +if [ "$rc" != 78 ]; then echo "FAIL arm 5: rc $rc after an amdxdna fault line, want 78"; fail=1; fi +if [ -n "$deep" ] && kill -0 "$deep" 2> /dev/null; then kill -9 "$deep"; fi + +[ "$fail" = 0 ] && echo "gpu_hold_fault_abort: ok (fault at 5 s -> rc 78 after ${e1}s; clean -> 0; opt-out -> 0; default on -> 78; take-time page fault/MES -> 78; NPU fault in hold -> 78)" exit "$fail"