Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions cmake/tests-gpu-hold.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,10 @@ add_test(NAME gpu_hold_tree COMMAND bash ${CMAKE_SOURCE_DIR}/tests/gpu_hold_tree
# failure where an unbudgeted GPU run took the box to 0.7 GiB and the OOM killer chose the PR-agent LLM.
add_test(NAME gpu_hold_mem_abort COMMAND bash ${CMAKE_SOURCE_DIR}/tests/gpu_hold_mem_abort_test.sh ${CMAKE_SOURCE_DIR}/scripts/gpu-hold)

# CRASH-HUNT 2026-11-15: a GPU page fault / MES reset in the kernel log DURING a hold ends it (rc 78). #2480 checks only
# when the token is taken; the 11-13 crash was nine resets inside two tickets. The kernel log is a fixture file.
add_test(NAME gpu_hold_fault_abort COMMAND bash ${CMAKE_SOURCE_DIR}/tests/gpu_hold_fault_abort_test.sh ${CMAKE_SOURCE_DIR}/scripts/gpu-hold)

# engine#1635: a queued ticket that is removed (reaped, or deleted by another lane) used to leave its owner waiting for a
# queue position that no longer existed -- the full -w, 5400 s by default, looking exactly like "queued behind a busy box".
# Drives scripts/gpu-hold on a private queue: the ticket is deleted mid-wait, and separately rewritten to a dead pid so a
Expand Down
34 changes: 33 additions & 1 deletion scripts/gpu-hold
Original file line number Diff line number Diff line change
Expand Up @@ -399,9 +399,39 @@ mem_available_gb() {
awk '/^MemAvailable:/ { printf "%d", $2 / 1048576; exit }' "${GPU_HOLD_MEMINFO:-/proc/meminfo}" 2> /dev/null
}

# A GPU FAULT DURING THE HOLD (CRASH-HUNT 2026-11-15, ~/briefs/CRASH-HUNT-STATE.md): #2480 refuses a ticket after a recent
# reset, but only at the moment the token is taken. The 11-13 crash was nine MES REMOVE_QUEUE -> MODE2 resets inside two
# tickets: each reset followed a gfxhub page fault ~2 s earlier from the ticket's own job, the job's next process faulted
# again, and the GPU wedged after the ninth. So the watchdog also reads the kernel log for lines that appeared SINCE this
# hold started: a page fault, an amdgpu/MES reset or ring timeout, or an amdxdna command timeout ends the hold (rc 78).
# GPU_HOLD_FAULT_ABORT=0 turns it off. With no readable kernel log (a CI runner) it is inert, as #2480's check is.
# GPU_HOLD_KLOG_FILE replaces the kernel log with a file (tests).
FAULT_ABORT=${GPU_HOLD_FAULT_ABORT:-1}
FAULT_RE='\[gfxhub\] page fault|amdgpu.*(reset|hang)|ring .*timeout|MES.*(reset|failure|failed to respond)|amdxdna.*(command timeout|power.?cycle)'
FAULT_FLAG="$QUEUE/.fault.$SAFE.$$"
fault_lines() {
if [ -n "${GPU_HOLD_KLOG_FILE:-}" ]; then
cat "$GPU_HOLD_KLOG_FILE" 2> /dev/null
else
journalctl -k --since "@$started" --no-pager -q 2> /dev/null
fi | grep -iE "$FAULT_RE"
}

watch() {
local target=$1
local target=$1 tick=0
while sleep 2; do
tick=$((tick + 1))
# every 10 s: journalctl over the hold's own window is cheap, but not free every 2 s
if [ "$FAULT_ABORT" != 0 ] && [ $((tick % 5)) = 0 ]; then
hits=$(fault_lines)
if [ -n "$hits" ]; then
: > "$FAULT_FLAG"
log "::error::ABORT GPU/NPU fault in the kernel log during this hold - ending it before the next one (rc 78):"
printf '%s\n' "$hits" | tail -3 | while IFS= read -r l; do log " $l"; done
kill_tree "$target" "${GPU_HOLD_KILL_GRACE:-10}"
return
fi
fi
# The OOM kill this prevents is not the run's own: on 2026-11-08/09 a GPU run took the box to 0.7 GiB
# available and the kernel chose the PR-agent LLM as the victim, which took the fleet's own brains down.
# The temperature gate has the same shape — one sampled number, one threshold, end the hold — so the
Expand Down Expand Up @@ -476,6 +506,8 @@ else
fi
# wait reports 143 for SIGTERM; say 124 for a max hold, as timeout does
if [ "$rc" = 143 ] && [ "$(date +%s)" -ge "$deadline" ]; then rc=124; fi
# a fault abort is its own outcome (the same 78 as #2480's refusal), whatever the killed job returned
if [ -f "$FAULT_FLAG" ]; then rc=78; rm -f "$FAULT_FLAG"; fi
held=$(( $(date +%s) - started ))

# Reap the watchdog BEFORE killing anything it may be in the middle of: it is the process that runs
Expand Down
94 changes: 94 additions & 0 deletions tests/gpu_hold_fault_abort_test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
#!/bin/bash
# Copyright 2026 bong-water-water-bong
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# tests/gpu_hold_fault_abort_test.sh — a hold ends itself when the kernel log shows a GPU fault DURING the hold
# (CRASH-HUNT 2026-11-15). The 11-13 crash was nine MES REMOVE_QUEUE -> MODE2 resets inside two tickets, each after a
# gfxhub page fault from the ticket's own job; #2480 checks for resets only when the token is taken, so a storm inside
# one ticket ran on until the GPU wedged.
#
# Usage: gpu_hold_fault_abort_test.sh [path-to-gpu-hold]
#
# The kernel log is a FILE (GPU_HOLD_KLOG_FILE): the test needs "clean, then a fault", which cannot be produced on
# demand. Three arms:
# 1 a page fault appears 5 s in: the hold ends WELL before its payload would (600 s), with rc 78, the log says why,
# and the job's deepest TERM-ignoring descendant is gone;
# 2 a clean log: the payload runs to completion, rc 0 (the watcher does not fire on nothing);
# 3 the same fault with GPU_HOLD_FAULT_ABORT=0: the payload runs to completion, rc 0 (the opt-out works);
# 4 the same fault with GPU_HOLD_FAULT_ABORT UNSET: rc 78, so the DEFAULT is on (arms 1-3 set it explicitly; a
# mutant that flipped the default to off passed them - review on #2486).
set -u
hold=${1:-./scripts/gpu-hold}
[ -x "$hold" ] || { echo "not executable: $hold"; exit 2; }

scratch=$(mktemp -d)
trap 'rm -rf "$scratch"' EXIT
fail=0

cat > "$scratch/wrapper.sh" <<'WRAP'
#!/bin/bash
( trap '' TERM; exec -a fault-abort-child sleep "$2" ) &
echo $! > "$1/deep.pid"
sleep "$2"
WRAP
chmod +x "$scratch/wrapper.sh"

run() { # $1 arm $2 payload seconds $3 inject fault (1/0) $4 GPU_HOLD_FAULT_ABORT, or "unset" to leave it out
local q=$scratch/q$1; mkdir -p "$q"; : > "$scratch/klog$1"; rm -f "$scratch/deep.pid"
printf '%s\n' "kernel: usb 1-1: new device" > "$scratch/klog$1"
if [ "$3" = 1 ]; then
( sleep 5; printf '%s\n' "kernel: amdgpu 0000:c5:00.0: [gfxhub] page fault (src_id:0 ring:24 vmid:9 pasid:26332)" >> "$scratch/klog$1" ) &
fi
local t0; t0=$(date +%s)
local abort_env=(GPU_HOLD_FAULT_ABORT="$4"); [ "$4" = unset ] && abort_env=()
env -u GPU_HOLD_FAULT_ABORT "${abort_env[@]}" GPU_HOLD_KLOG_FILE="$scratch/klog$1" \
GPU_QUEUE_DIR="$q" GPU_TOKEN_FILE="$scratch/TOKEN$1" GPU_HOLD_LOG="$scratch/log$1" \
GPU_HOLD_POLL=1 GPU_HOLD_WAIT=5 GPU_HOLD_RELEASE_WAIT=0 GPU_HOLD_KILL_GRACE=3 GPU_HOLD_RESET_WINDOW_MIN=0 \
GPU_HOLD_TCTL_START=100 GPU_HOLD_TCTL_FILE="$scratch/tctl" GPU_HOLD_THERMAL_FLAG="$scratch/x" \
timeout 120 "$hold" -t 10 -l faulttest$1 -- "$scratch/wrapper.sh" "$scratch" "$2" > /dev/null 2>&1
rc=$?
elapsed=$(( $(date +%s) - t0 ))
deep=$(cat "$scratch/deep.pid" 2> /dev/null)
}

echo 40000 > "$scratch/tctl" # 40 C, in millidegrees

# arm 1: the fault fires
run 1 600 1 1
e1=$elapsed
if [ "$rc" != 78 ]; then echo "FAIL arm 1: rc $rc, want 78 (the fault abort)"; fail=1; fi
if [ "$elapsed" -ge 60 ]; then echo "FAIL arm 1: the hold took ${elapsed}s; the fault was at 5 s and the payload sleeps 600 s"; fail=1; fi
grep -q "ABORT GPU/NPU fault" "$scratch/log1" || { echo "FAIL arm 1: the log does not say the fault ended the hold"; fail=1; }
grep -q "gfxhub" "$scratch/log1" || { echo "FAIL arm 1: the log does not quote the fault line"; fail=1; }
if [ -n "$deep" ] && kill -0 "$deep" 2> /dev/null; then echo "FAIL arm 1: the TERM-ignoring grandchild $deep survived"; kill -9 "$deep"; fail=1; fi
[ -e "$scratch/TOKEN1" ] && { echo "FAIL arm 1: the token is still held"; fail=1; }

# arm 2: a clean log
run 2 12 0 1
if [ "$rc" != 0 ]; then echo "FAIL arm 2: rc $rc with a clean kernel log, want 0"; fail=1; fi
grep -q "ABORT GPU/NPU fault" "$scratch/log2" && { echo "FAIL arm 2: the watcher fired on a clean log"; fail=1; }

# arm 3: the opt-out
run 3 12 1 0
if [ "$rc" != 0 ]; then echo "FAIL arm 3: rc $rc with GPU_HOLD_FAULT_ABORT=0, want 0"; fail=1; fi
grep -q "ABORT GPU/NPU fault" "$scratch/log3" && { echo "FAIL arm 3: the watcher fired with GPU_HOLD_FAULT_ABORT=0"; fail=1; }

# arm 4: the default (variable unset) must be ON
run 4 600 1 unset
if [ "$rc" != 78 ]; then echo "FAIL arm 4: rc $rc with GPU_HOLD_FAULT_ABORT unset, want 78 (the default must be on)"; fail=1; fi
if [ -n "$deep" ] && kill -0 "$deep" 2> /dev/null; then kill -9 "$deep"; fi

[ "$fail" = 0 ] && echo "gpu_hold_fault_abort: ok (fault at 5 s -> rc 78 after ${e1}s; clean -> 0; opt-out -> 0; default on -> 78)"
exit "$fail"
Loading