Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 17 additions & 4 deletions scripts/gpu-hold
Original file line number Diff line number Diff line change
Expand Up @@ -289,15 +289,28 @@ while [ "$state" = queued ]; do
# kernel log to read and `gpu_hold_*`'s own tests must keep passing, so "cannot read" is a note, not a
# refusal - the opposite of a device guard, and worth stating because it is the surprising half.
_reset_window=${GPU_HOLD_RESET_WINDOW_MIN:-10}
_klog=$(journalctl -k --since "-${_reset_window} min" --no-pager 2> /dev/null)
# GPU_HOLD_KLOG_FILE is the same override the during-hold watcher uses: without it this refusal had no
# automated arm at all (it shipped with manual arms only), because "a reset two minutes ago" cannot be
# produced in a test that reads the real journal.
if [ -n "${GPU_HOLD_KLOG_FILE:-}" ]; then
_klog=$(cat "$GPU_HOLD_KLOG_FILE" 2> /dev/null)
else
_klog=$(journalctl -k --since "-${_reset_window} min" --no-pager 2> /dev/null)
fi
if [ -z "$_klog" ]; then
log "note: no kernel log available, so a reset in the last ${_reset_window} min cannot be ruled out (not refusing: CI has no journal)"
else
_resets=$(printf '%s\n' "$_klog" | grep -ciE 'amdgpu.*(reset|hang)|ring .*timeout|MES.*(reset|failure)')
# The NPU spellings belong here too: npu-hold execs this script with the NPU's own queue and
# token, so this take-time check is an NPU ticket's ONLY look at the log before work starts,
# while the during-hold watch below already reads amdxdna faults. Without them a lane could
# take an NPU ticket minutes after the command timeout the rules say to stop for (measured
# 2026-11-15: the hold ran on a log whose newest line was an amdxdna command timeout).
_reset_re='amdgpu.*(reset|hang)|ring .*timeout|MES.*(reset|failure)|amdxdna.*(command timeout|power.?cycle|fault)'
_resets=$(printf '%s\n' "$_klog" | grep -ciE "$_reset_re")
if [ "${_resets:-0}" -gt 0 ]; then
rm -f "$TOKEN"; state=done
log "::error::refusing: ${_resets} amdgpu/MES reset line(s) in the last ${_reset_window} min - the ticket is released unheld. Inspect the kernel log, then retry"
printf '%s\n' "$_klog" | grep -iE 'amdgpu.*(reset|hang)|ring .*timeout|MES.*(reset|failure)' | tail -3
log "::error::refusing: ${_resets} amdgpu/MES/amdxdna fault line(s) in the last ${_reset_window} min - the ticket is released unheld. Inspect the kernel log, then retry"
printf '%s\n' "$_klog" | grep -iE "$_reset_re" | tail -3
exit 78
fi
fi
Expand Down
35 changes: 35 additions & 0 deletions tests/gpu_hold_fault_abort_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,41 @@ run() { # $1 arm $2 payload seconds $3 inject fault (1/0) $4 GPU_HOLD_FAULT

echo 40000 > "$scratch/tctl" # 40 C, in millidegrees

# ── the TAKE-TIME guard (engine#2480), which had no automated arm until now ───────────────────────────────────────
# It shipped with manual arms only, and the NPU spellings were missing from its regex although `npu-hold` execs this
# script: an NPU ticket's only look at the log before work starts is this check, so `amdxdna` faults have to count.
# Each arm runs the real hold against a klog file with a 10-minute take-time window and the during-hold watch OFF, so
# what is being tested is the refusal and nothing else.
take_run() { # $1 arm name $2 the kernel-log line (or "clean")
local q=$scratch/tq$1; mkdir -p "$q"; rm -f "$scratch/TTOKEN$1"
: > "$scratch/tklog$1"
printf '%s\n' "kernel: amdgpu 0000:c5:00.0: [drm] Initialized" > "$scratch/tklog$1"
[ "$2" = clean ] || printf '%s\n' "$2" >> "$scratch/tklog$1"
env GPU_HOLD_KLOG_FILE="$scratch/tklog$1" GPU_QUEUE_DIR="$q" GPU_TOKEN_FILE="$scratch/TTOKEN$1" \
GPU_HOLD_LOG="$scratch/tlog$1" GPU_HOLD_POLL=1 GPU_HOLD_WAIT=5 GPU_HOLD_RELEASE_WAIT=0 GPU_HOLD_KILL_GRACE=3 \
GPU_HOLD_RESET_WINDOW_MIN=10 GPU_HOLD_FAULT_ABORT=0 GPU_HOLD_TCTL_START=100 \
GPU_HOLD_TCTL_FILE="$scratch/tctl" GPU_HOLD_THERMAL_FLAG="$scratch/x" \
timeout 60 "$hold" -t 1 -l taketest$1 -- true > /dev/null 2>&1
trc=$?
[ -e "$scratch/TTOKEN$1" ] && ttok=held || ttok=released
}

take_run amdgpu "kernel: amdgpu 0000:c5:00.0: [drm] *ERROR* amdgpu: GPU reset begin!"
[ "$trc" = 78 ] || { echo "FAIL take-time: an amdgpu reset -> rc $trc, want 78"; fail=1; }
[ "$ttok" = released ] || { echo "FAIL take-time: the token was still held after a refusal"; fail=1; }

# the NPU spelling: this is the line the rules say to stop for, and npu-hold has no other take-time check
take_run npu "kernel: amdxdna 0000:c6:00.1: [drm] *ERROR* command timeout: opcode 0x2"
[ "$trc" = 78 ] || { echo "FAIL take-time: an amdxdna command timeout -> rc $trc, want 78 (the NPU rule)"; fail=1; }
[ "$ttok" = released ] || { echo "FAIL take-time: the token was still held after an NPU refusal"; fail=1; }

take_run powercycle "kernel: amdxdna 0000:c6:00.1: [drm] power-cycle"
[ "$trc" = 78 ] || { echo "FAIL take-time: an amdxdna power-cycle -> rc $trc, want 78"; fail=1; }

take_run clean clean
[ "$trc" = 0 ] || { echo "FAIL take-time: a clean log -> rc $trc, want 0 (a healthy hold must still run)"; fail=1; }


# arm 1: the fault fires
run 1 600 1 1
e1=$elapsed
Expand Down
Loading