Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions scripts/gpu-hold
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,13 @@ LOG=${GPU_HOLD_LOG:-}
THERMAL_FLAG=${GPU_HOLD_THERMAL_FLAG:-$HOME/THERMAL-HOLD}
COOLDOWN=${GPU_HOLD_COOLDOWN:-180}
COOLDOWN_TCTL=${GPU_HOLD_COOLDOWN_TCTL:-85}
# ONE pattern for a GPU/NPU fault line, read by both the take-time refusal (#2480) and the during-hold watch (#2486).
# They had one each and the two drifted (engine#2494, cl-infra 11-15): the take-time copy lacked the gfxhub page fault
# and the MES "failed to respond", so a ticket was granted onto a log whose newest line was the fault that had just ended
# the previous hold with rc 78, and only a MODE2 reset line refused. The during-hold copy lacked the amdxdna "fault" that
# #2498 added at take time. The union is the rule's own list: a page fault, any amdgpu/MES reset or failure, a ring
# timeout, an amdxdna command timeout, power-cycle or fault.
GPU_FAULT_RE='\[gfxhub\] page fault|amdgpu.*(reset|hang)|ring .*timeout|MES.*(reset|failure|failed to respond)|amdxdna.*(command timeout|power.?cycle|fault)'

while getopts ":t:l:w:q:h" opt; do
case $opt in
Expand Down Expand Up @@ -305,7 +312,7 @@ while [ "$state" = queued ]; do
# while the during-hold watch below already reads amdxdna faults. Without them a lane could
# take an NPU ticket minutes after the command timeout the rules say to stop for (measured
# 2026-11-15: the hold ran on a log whose newest line was an amdxdna command timeout).
_reset_re='amdgpu.*(reset|hang)|ring .*timeout|MES.*(reset|failure)|amdxdna.*(command timeout|power.?cycle|fault)'
_reset_re=$GPU_FAULT_RE
_resets=$(printf '%s\n' "$_klog" | grep -ciE "$_reset_re")
if [ "${_resets:-0}" -gt 0 ]; then
rm -f "$TOKEN"; state=done
Expand Down Expand Up @@ -420,7 +427,7 @@ mem_available_gb() {
# GPU_HOLD_FAULT_ABORT=0 turns it off. With no readable kernel log (a CI runner) it is inert, as #2480's check is.
# GPU_HOLD_KLOG_FILE replaces the kernel log with a file (tests).
FAULT_ABORT=${GPU_HOLD_FAULT_ABORT:-1}
FAULT_RE='\[gfxhub\] page fault|amdgpu.*(reset|hang)|ring .*timeout|MES.*(reset|failure|failed to respond)|amdxdna.*(command timeout|power.?cycle)'
FAULT_RE=$GPU_FAULT_RE
FAULT_FLAG="$QUEUE/.fault.$SAFE.$$"
fault_lines() {
if [ -n "${GPU_HOLD_KLOG_FILE:-}" ]; then
Expand Down
21 changes: 19 additions & 2 deletions tests/gpu_hold_fault_abort_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -46,10 +46,12 @@ WRAP
chmod +x "$scratch/wrapper.sh"

run() { # $1 arm $2 payload seconds $3 inject fault (1/0) $4 GPU_HOLD_FAULT_ABORT, or "unset" to leave it out
# $5 the fault line to inject (default: the 11-13 gfxhub page fault)
local q=$scratch/q$1; mkdir -p "$q"; : > "$scratch/klog$1"; rm -f "$scratch/deep.pid"
printf '%s\n' "kernel: usb 1-1: new device" > "$scratch/klog$1"
local line=${5:-"kernel: amdgpu 0000:c5:00.0: [gfxhub] page fault (src_id:0 ring:24 vmid:9 pasid:26332)"}
if [ "$3" = 1 ]; then
( sleep 5; printf '%s\n' "kernel: amdgpu 0000:c5:00.0: [gfxhub] page fault (src_id:0 ring:24 vmid:9 pasid:26332)" >> "$scratch/klog$1" ) &
( sleep 5; printf '%s\n' "$line" >> "$scratch/klog$1" ) &
fi
local t0; t0=$(date +%s)
local abort_env=(GPU_HOLD_FAULT_ABORT="$4"); [ "$4" = unset ] && abort_env=()
Expand Down Expand Up @@ -96,6 +98,15 @@ take_run npu "kernel: amdxdna 0000:c6:00.1: [drm] *ERROR* command timeout: opcod
take_run powercycle "kernel: amdxdna 0000:c6:00.1: [drm] power-cycle"
[ "$trc" = 78 ] || { echo "FAIL take-time: an amdxdna power-cycle -> rc $trc, want 78"; fail=1; }

# engine#2494 (cl-infra 11-15): the take-time pattern was a separate copy that lacked these two, so the ticket after a
# fault-ended hold was GRANTED on the very log that had just ended it; only the MODE2 line further down the chain refused.
take_run gfxhub "kernel: amdgpu 0000:c5:00.0: [gfxhub] page fault (src_id:0 ring:24 vmid:9 pasid:26332)"
[ "$trc" = 78 ] || { echo "FAIL take-time: a gfxhub page fault -> rc $trc, want 78 (engine#2494)"; fail=1; }
[ "$ttok" = released ] || { echo "FAIL take-time: the token was still held after a page-fault refusal"; fail=1; }

take_run mesremove "kernel: amdgpu 0000:c5:00.0: amdgpu: MES failed to respond to msg=REMOVE_QUEUE"
[ "$trc" = 78 ] || { echo "FAIL take-time: an MES REMOVE_QUEUE timeout -> rc $trc, want 78 (engine#2494)"; fail=1; }

take_run clean clean
[ "$trc" = 0 ] || { echo "FAIL take-time: a clean log -> rc $trc, want 0 (a healthy hold must still run)"; fail=1; }

Expand Down Expand Up @@ -125,5 +136,11 @@ run 4 600 1 unset
if [ "$rc" != 78 ]; then echo "FAIL arm 4: rc $rc with GPU_HOLD_FAULT_ABORT unset, want 78 (the default must be on)"; fail=1; fi
if [ -n "$deep" ] && kill -0 "$deep" 2> /dev/null; then kill -9 "$deep"; fi

[ "$fail" = 0 ] && echo "gpu_hold_fault_abort: ok (fault at 5 s -> rc 78 after ${e1}s; clean -> 0; opt-out -> 0; default on -> 78)"
# arm 5: an amdxdna fault DURING the hold. #2498 added this spelling at take time only; with one shared pattern the
# during-hold watch now reads it too (the NPU rule: on any amdxdna fault, stop).
run 5 600 1 1 "kernel: amdxdna 0000:c6:00.1: [drm] *ERROR* aie2_error_worker: fault at 0x00000000"
if [ "$rc" != 78 ]; then echo "FAIL arm 5: rc $rc after an amdxdna fault line, want 78"; fail=1; fi
if [ -n "$deep" ] && kill -0 "$deep" 2> /dev/null; then kill -9 "$deep"; fi

[ "$fail" = 0 ] && echo "gpu_hold_fault_abort: ok (fault at 5 s -> rc 78 after ${e1}s; clean -> 0; opt-out -> 0; default on -> 78; take-time page fault/MES -> 78; NPU fault in hold -> 78)"
exit "$fail"
Loading