From 8413de08cbc0b5e91f2790d791d582f3ebc3689d Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 09:43:10 -0400 Subject: [PATCH] gpu-hold: a fault abort names the faulting device; arms for the ring-timeout and hang terms (#2538 review) Two asks from the #2538 READYs, which arrived after it merged: - The during-hold abort now says whether its newest fault line is from the GPU or the NPU, and that a fault on the other device came from another lane's job. npu-hold execs this script and the watch reads every kernel line, so a GPU ticket can end rc 78 on an NPU fault; the old message sent the lane hunting its own job. - Every term of GPU_FAULT_RE now has a take-time arm: `ring .*timeout` and `amdgpu ... hang` had none, so either could drop out of the pattern with every arm green. Mutants (each run against tests/gpu_hold_fault_abort_test.sh): the ring term deleted fails only the ring arm, `hang` deleted fails only the hang arm, _dev hard-coded to GPU fails only arm 5's device check. All nine gpu_hold_* scripts and coder_llm_ticketed pass. Co-Authored-By: Claude Opus 5.5 --- scripts/gpu-hold | 6 +++++- tests/gpu_hold_fault_abort_test.sh | 12 +++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/scripts/gpu-hold b/scripts/gpu-hold index 7d71b8a5e..03c263a9c 100755 --- a/scripts/gpu-hold +++ b/scripts/gpu-hold @@ -446,7 +446,11 @@ watch() { hits=$(fault_lines) if [ -n "$hits" ]; then : > "$FAULT_FLAG" - log "::error::ABORT GPU/NPU fault in the kernel log during this hold - ending it before the next one (rc 78):" + # Name the device from the newest line: npu-hold execs this script too and the watch reads every kernel + # line, so a GPU ticket can end on another lane's NPU fault (and the other way round). Saying which + # device faulted keeps a lane from searching its own job for a fault it did not cause (#2538 review). + _dev=GPU; printf '%s\n' "$hits" | tail -1 | grep -qi amdxdna && _dev=NPU + log "::error::ABORT GPU/NPU fault in the kernel log during this hold - ending it before the next one (rc 78): newest line is from the ${_dev}; if this ticket is not for the ${_dev}, another lane's job produced it:" printf '%s\n' "$hits" | tail -3 | while IFS= read -r l; do log " $l"; done kill_tree "$target" "${GPU_HOLD_KILL_GRACE:-10}" return diff --git a/tests/gpu_hold_fault_abort_test.sh b/tests/gpu_hold_fault_abort_test.sh index 2a75cc249..523a85a9d 100755 --- a/tests/gpu_hold_fault_abort_test.sh +++ b/tests/gpu_hold_fault_abort_test.sh @@ -107,6 +107,14 @@ take_run gfxhub "kernel: amdgpu 0000:c5:00.0: [gfxhub] page fault (src_id:0 ring take_run mesremove "kernel: amdgpu 0000:c5:00.0: amdgpu: MES failed to respond to msg=REMOVE_QUEUE" [ "$trc" = 78 ] || { echo "FAIL take-time: an MES REMOVE_QUEUE timeout -> rc $trc, want 78 (engine#2494)"; fail=1; } +# every term of GPU_FAULT_RE gets an arm (#2538 review): without one, a term could drop out of the pattern and every +# arm would stay green. These two were ridden only through other spellings. +take_run ring "kernel: amdgpu 0000:c5:00.0: [drm] *ERROR* ring gfx_0.0.0 timeout, signaled seq=1234, emitted seq=1236" +[ "$trc" = 78 ] || { echo "FAIL take-time: a ring timeout -> rc $trc, want 78"; fail=1; } + +take_run hang "kernel: amdgpu 0000:c5:00.0: amdgpu: GPU hang detected on ring gfx" +[ "$trc" = 78 ] || { echo "FAIL take-time: an amdgpu hang -> rc $trc, want 78"; fail=1; } + take_run clean clean [ "$trc" = 0 ] || { echo "FAIL take-time: a clean log -> rc $trc, want 0 (a healthy hold must still run)"; fail=1; } @@ -140,7 +148,9 @@ if [ -n "$deep" ] && kill -0 "$deep" 2> /dev/null; then kill -9 "$deep"; fi # during-hold watch now reads it too (the NPU rule: on any amdxdna fault, stop). run 5 600 1 1 "kernel: amdxdna 0000:c6:00.1: [drm] *ERROR* aie2_error_worker: fault at 0x00000000" if [ "$rc" != 78 ]; then echo "FAIL arm 5: rc $rc after an amdxdna fault line, want 78"; fail=1; fi +grep -q "newest line is from the NPU" "$scratch/log5" || { echo "FAIL arm 5: the abort does not name the NPU as the faulting device"; fail=1; } +grep -q "newest line is from the GPU" "$scratch/log1" || { echo "FAIL arm 1: the abort does not name the GPU as the faulting device"; fail=1; } if [ -n "$deep" ] && kill -0 "$deep" 2> /dev/null; then kill -9 "$deep"; fi -[ "$fail" = 0 ] && echo "gpu_hold_fault_abort: ok (fault at 5 s -> rc 78 after ${e1}s; clean -> 0; opt-out -> 0; default on -> 78; take-time page fault/MES -> 78; NPU fault in hold -> 78)" +[ "$fail" = 0 ] && echo "gpu_hold_fault_abort: ok (fault at 5 s -> rc 78 after ${e1}s; clean -> 0; opt-out -> 0; default on -> 78; take-time page fault/MES/ring/hang -> 78; NPU fault in hold -> 78)" exit "$fail"