From dc6034d9261055f4aa02a4acc37dd6ec4e04e357 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 00:41:50 -0400 Subject: [PATCH 1/2] gpu-hold: a GPU page fault or reset in the kernel log DURING the hold ends it (rc 78) #2480 refuses a ticket after a recent amdgpu/MES reset, but only when the token is taken. The 11-13 crash (~/briefs/CRASH-HUNT-STATE.md) was nine MES REMOVE_QUEUE -> MODE2 resets inside two tickets, each one ~2 s after a gfxhub page fault from the ticket's own job; the job's next process faulted again and the GPU wedged after the ninth. The watchdog now reads the kernel log every 10 s for lines since the hold started (gfxhub page fault, amdgpu/MES reset or ring timeout, amdxdna command timeout). On a hit it logs the lines, ends the job tree and returns 78, the same code as #2480's refusal. GPU_HOLD_FAULT_ABORT=0 turns it off; with no readable kernel log (CI) it is inert. tests/gpu_hold_fault_abort_test.sh drives it with a fixture log: fault at 5 s -> rc 78 well inside a 600 s payload with the TERM-ignoring grandchild gone, clean log -> 0, opt-out -> 0. A mutant with the watcher off fails arm 1. Refs CRASH-HUNT (orchestrator 11-15) Co-Authored-By: Claude Opus 5.5 --- cmake/tests-gpu-hold.cmake | 4 ++ scripts/gpu-hold | 34 +++++++++++- tests/gpu_hold_fault_abort_test.sh | 86 ++++++++++++++++++++++++++++++ 3 files changed, 123 insertions(+), 1 deletion(-) create mode 100755 tests/gpu_hold_fault_abort_test.sh diff --git a/cmake/tests-gpu-hold.cmake b/cmake/tests-gpu-hold.cmake index e1f63f6f7..d0418a3be 100644 --- a/cmake/tests-gpu-hold.cmake +++ b/cmake/tests-gpu-hold.cmake @@ -34,6 +34,10 @@ add_test(NAME gpu_hold_tree COMMAND bash ${CMAKE_SOURCE_DIR}/tests/gpu_hold_tree # failure where an unbudgeted GPU run took the box to 0.7 GiB and the OOM killer chose the PR-agent LLM. add_test(NAME gpu_hold_mem_abort COMMAND bash ${CMAKE_SOURCE_DIR}/tests/gpu_hold_mem_abort_test.sh ${CMAKE_SOURCE_DIR}/scripts/gpu-hold) +# CRASH-HUNT 2026-11-15: a GPU page fault / MES reset in the kernel log DURING a hold ends it (rc 78). #2480 checks only +# when the token is taken; the 11-13 crash was nine resets inside two tickets. The kernel log is a fixture file. +add_test(NAME gpu_hold_fault_abort COMMAND bash ${CMAKE_SOURCE_DIR}/tests/gpu_hold_fault_abort_test.sh ${CMAKE_SOURCE_DIR}/scripts/gpu-hold) + # engine#1635: a queued ticket that is removed (reaped, or deleted by another lane) used to leave its owner waiting for a # queue position that no longer existed -- the full -w, 5400 s by default, looking exactly like "queued behind a busy box". # Drives scripts/gpu-hold on a private queue: the ticket is deleted mid-wait, and separately rewritten to a dead pid so a diff --git a/scripts/gpu-hold b/scripts/gpu-hold index a25faca12..07ecc1792 100755 --- a/scripts/gpu-hold +++ b/scripts/gpu-hold @@ -399,9 +399,39 @@ mem_available_gb() { awk '/^MemAvailable:/ { printf "%d", $2 / 1048576; exit }' "${GPU_HOLD_MEMINFO:-/proc/meminfo}" 2> /dev/null } +# A GPU FAULT DURING THE HOLD (CRASH-HUNT 2026-11-15, ~/briefs/CRASH-HUNT-STATE.md): #2480 refuses a ticket after a recent +# reset, but only at the moment the token is taken. The 11-13 crash was nine MES REMOVE_QUEUE -> MODE2 resets inside two +# tickets: each reset followed a gfxhub page fault ~2 s earlier from the ticket's own job, the job's next process faulted +# again, and the GPU wedged after the ninth. So the watchdog also reads the kernel log for lines that appeared SINCE this +# hold started: a page fault, an amdgpu/MES reset or ring timeout, or an amdxdna command timeout ends the hold (rc 78). +# GPU_HOLD_FAULT_ABORT=0 turns it off. With no readable kernel log (a CI runner) it is inert, as #2480's check is. +# GPU_HOLD_KLOG_FILE replaces the kernel log with a file (tests). +FAULT_ABORT=${GPU_HOLD_FAULT_ABORT:-1} +FAULT_RE='\[gfxhub\] page fault|amdgpu.*(reset|hang)|ring .*timeout|MES.*(reset|failure|failed to respond)|amdxdna.*(command timeout|power.?cycle)' +FAULT_FLAG="$QUEUE/.fault.$SAFE.$$" +fault_lines() { + if [ -n "${GPU_HOLD_KLOG_FILE:-}" ]; then + cat "$GPU_HOLD_KLOG_FILE" 2> /dev/null + else + journalctl -k --since "@$started" --no-pager -q 2> /dev/null + fi | grep -iE "$FAULT_RE" +} + watch() { - local target=$1 + local target=$1 tick=0 while sleep 2; do + tick=$((tick + 1)) + # every 10 s: journalctl over the hold's own window is cheap, but not free every 2 s + if [ "$FAULT_ABORT" != 0 ] && [ $((tick % 5)) = 0 ]; then + hits=$(fault_lines) + if [ -n "$hits" ]; then + : > "$FAULT_FLAG" + log "::error::ABORT GPU/NPU fault in the kernel log during this hold - ending it before the next one (rc 78):" + printf '%s\n' "$hits" | tail -3 | while IFS= read -r l; do log " $l"; done + kill_tree "$target" "${GPU_HOLD_KILL_GRACE:-10}" + return + fi + fi # The OOM kill this prevents is not the run's own: on 2026-11-08/09 a GPU run took the box to 0.7 GiB # available and the kernel chose the PR-agent LLM as the victim, which took the fleet's own brains down. # The temperature gate has the same shape — one sampled number, one threshold, end the hold — so the @@ -476,6 +506,8 @@ else fi # wait reports 143 for SIGTERM; say 124 for a max hold, as timeout does if [ "$rc" = 143 ] && [ "$(date +%s)" -ge "$deadline" ]; then rc=124; fi +# a fault abort is its own outcome (the same 78 as #2480's refusal), whatever the killed job returned +if [ -f "$FAULT_FLAG" ]; then rc=78; rm -f "$FAULT_FLAG"; fi held=$(( $(date +%s) - started )) # Reap the watchdog BEFORE killing anything it may be in the middle of: it is the process that runs diff --git a/tests/gpu_hold_fault_abort_test.sh b/tests/gpu_hold_fault_abort_test.sh new file mode 100755 index 000000000..a32d26d59 --- /dev/null +++ b/tests/gpu_hold_fault_abort_test.sh @@ -0,0 +1,86 @@ +#!/bin/bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# tests/gpu_hold_fault_abort_test.sh — a hold ends itself when the kernel log shows a GPU fault DURING the hold +# (CRASH-HUNT 2026-11-15). The 11-13 crash was nine MES REMOVE_QUEUE -> MODE2 resets inside two tickets, each after a +# gfxhub page fault from the ticket's own job; #2480 checks for resets only when the token is taken, so a storm inside +# one ticket ran on until the GPU wedged. +# +# Usage: gpu_hold_fault_abort_test.sh [path-to-gpu-hold] +# +# The kernel log is a FILE (GPU_HOLD_KLOG_FILE): the test needs "clean, then a fault", which cannot be produced on +# demand. Three arms: +# 1 a page fault appears 5 s in: the hold ends WELL before its payload would (600 s), with rc 78, the log says why, +# and the job's deepest TERM-ignoring descendant is gone; +# 2 a clean log: the payload runs to completion, rc 0 (the watcher does not fire on nothing); +# 3 the same fault with GPU_HOLD_FAULT_ABORT=0: the payload runs to completion, rc 0 (the opt-out works). +set -u +hold=${1:-./scripts/gpu-hold} +[ -x "$hold" ] || { echo "not executable: $hold"; exit 2; } + +scratch=$(mktemp -d) +trap 'rm -rf "$scratch"' EXIT +fail=0 + +cat > "$scratch/wrapper.sh" <<'WRAP' +#!/bin/bash +( trap '' TERM; exec -a fault-abort-child sleep "$2" ) & +echo $! > "$1/deep.pid" +sleep "$2" +WRAP +chmod +x "$scratch/wrapper.sh" + +run() { # $1 arm $2 payload seconds $3 inject fault (1/0) $4 GPU_HOLD_FAULT_ABORT + local q=$scratch/q$1; mkdir -p "$q"; : > "$scratch/klog$1"; rm -f "$scratch/deep.pid" + printf '%s\n' "kernel: usb 1-1: new device" > "$scratch/klog$1" + if [ "$3" = 1 ]; then + ( sleep 5; printf '%s\n' "kernel: amdgpu 0000:c5:00.0: [gfxhub] page fault (src_id:0 ring:24 vmid:9 pasid:26332)" >> "$scratch/klog$1" ) & + fi + local t0; t0=$(date +%s) + GPU_HOLD_KLOG_FILE="$scratch/klog$1" GPU_HOLD_FAULT_ABORT="$4" \ + GPU_QUEUE_DIR="$q" GPU_TOKEN_FILE="$scratch/TOKEN$1" GPU_HOLD_LOG="$scratch/log$1" \ + GPU_HOLD_POLL=1 GPU_HOLD_WAIT=5 GPU_HOLD_RELEASE_WAIT=0 GPU_HOLD_KILL_GRACE=3 GPU_HOLD_RESET_WINDOW_MIN=0 \ + GPU_HOLD_TCTL_START=100 GPU_HOLD_TCTL_FILE="$scratch/tctl" GPU_HOLD_THERMAL_FLAG="$scratch/x" \ + timeout 120 "$hold" -t 10 -l faulttest$1 -- "$scratch/wrapper.sh" "$scratch" "$2" > /dev/null 2>&1 + rc=$? + elapsed=$(( $(date +%s) - t0 )) + deep=$(cat "$scratch/deep.pid" 2> /dev/null) +} + +echo 40000 > "$scratch/tctl" # 40 C, in millidegrees + +# arm 1: the fault fires +run 1 600 1 1 +e1=$elapsed +if [ "$rc" != 78 ]; then echo "FAIL arm 1: rc $rc, want 78 (the fault abort)"; fail=1; fi +if [ "$elapsed" -ge 60 ]; then echo "FAIL arm 1: the hold took ${elapsed}s; the fault was at 5 s and the payload sleeps 600 s"; fail=1; fi +grep -q "ABORT GPU/NPU fault" "$scratch/log1" || { echo "FAIL arm 1: the log does not say the fault ended the hold"; fail=1; } +grep -q "gfxhub" "$scratch/log1" || { echo "FAIL arm 1: the log does not quote the fault line"; fail=1; } +if [ -n "$deep" ] && kill -0 "$deep" 2> /dev/null; then echo "FAIL arm 1: the TERM-ignoring grandchild $deep survived"; kill -9 "$deep"; fail=1; fi +[ -e "$scratch/TOKEN1" ] && { echo "FAIL arm 1: the token is still held"; fail=1; } + +# arm 2: a clean log +run 2 12 0 1 +if [ "$rc" != 0 ]; then echo "FAIL arm 2: rc $rc with a clean kernel log, want 0"; fail=1; fi +grep -q "ABORT GPU/NPU fault" "$scratch/log2" && { echo "FAIL arm 2: the watcher fired on a clean log"; fail=1; } + +# arm 3: the opt-out +run 3 12 1 0 +if [ "$rc" != 0 ]; then echo "FAIL arm 3: rc $rc with GPU_HOLD_FAULT_ABORT=0, want 0"; fail=1; fi +grep -q "ABORT GPU/NPU fault" "$scratch/log3" && { echo "FAIL arm 3: the watcher fired with GPU_HOLD_FAULT_ABORT=0"; fail=1; } + +[ "$fail" = 0 ] && echo "gpu_hold_fault_abort: ok (fault at 5 s -> rc 78 after ${e1}s; clean -> 0; opt-out -> 0)" +exit "$fail" From ded6fb97927215fb1d85d4ce43bca2c47b9fc80f Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 01:08:24 -0400 Subject: [PATCH 2/2] tests: gpu_hold_fault_abort checks the DEFAULT is on (an arm with GPU_HOLD_FAULT_ABORT unset; review on #2486) Arms 1-3 set GPU_HOLD_FAULT_ABORT explicitly, so a mutant that flipped the default to off passed them. Arm 4 leaves it unset and needs rc 78; the same mutant now fails it (rc 124). Co-Authored-By: Claude Opus 5.5 --- tests/gpu_hold_fault_abort_test.sh | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/gpu_hold_fault_abort_test.sh b/tests/gpu_hold_fault_abort_test.sh index a32d26d59..52a043ae9 100755 --- a/tests/gpu_hold_fault_abort_test.sh +++ b/tests/gpu_hold_fault_abort_test.sh @@ -26,7 +26,9 @@ # 1 a page fault appears 5 s in: the hold ends WELL before its payload would (600 s), with rc 78, the log says why, # and the job's deepest TERM-ignoring descendant is gone; # 2 a clean log: the payload runs to completion, rc 0 (the watcher does not fire on nothing); -# 3 the same fault with GPU_HOLD_FAULT_ABORT=0: the payload runs to completion, rc 0 (the opt-out works). +# 3 the same fault with GPU_HOLD_FAULT_ABORT=0: the payload runs to completion, rc 0 (the opt-out works); +# 4 the same fault with GPU_HOLD_FAULT_ABORT UNSET: rc 78, so the DEFAULT is on (arms 1-3 set it explicitly; a +# mutant that flipped the default to off passed them - review on #2486). set -u hold=${1:-./scripts/gpu-hold} [ -x "$hold" ] || { echo "not executable: $hold"; exit 2; } @@ -43,14 +45,15 @@ sleep "$2" WRAP chmod +x "$scratch/wrapper.sh" -run() { # $1 arm $2 payload seconds $3 inject fault (1/0) $4 GPU_HOLD_FAULT_ABORT +run() { # $1 arm $2 payload seconds $3 inject fault (1/0) $4 GPU_HOLD_FAULT_ABORT, or "unset" to leave it out local q=$scratch/q$1; mkdir -p "$q"; : > "$scratch/klog$1"; rm -f "$scratch/deep.pid" printf '%s\n' "kernel: usb 1-1: new device" > "$scratch/klog$1" if [ "$3" = 1 ]; then ( sleep 5; printf '%s\n' "kernel: amdgpu 0000:c5:00.0: [gfxhub] page fault (src_id:0 ring:24 vmid:9 pasid:26332)" >> "$scratch/klog$1" ) & fi local t0; t0=$(date +%s) - GPU_HOLD_KLOG_FILE="$scratch/klog$1" GPU_HOLD_FAULT_ABORT="$4" \ + local abort_env=(GPU_HOLD_FAULT_ABORT="$4"); [ "$4" = unset ] && abort_env=() + env -u GPU_HOLD_FAULT_ABORT "${abort_env[@]}" GPU_HOLD_KLOG_FILE="$scratch/klog$1" \ GPU_QUEUE_DIR="$q" GPU_TOKEN_FILE="$scratch/TOKEN$1" GPU_HOLD_LOG="$scratch/log$1" \ GPU_HOLD_POLL=1 GPU_HOLD_WAIT=5 GPU_HOLD_RELEASE_WAIT=0 GPU_HOLD_KILL_GRACE=3 GPU_HOLD_RESET_WINDOW_MIN=0 \ GPU_HOLD_TCTL_START=100 GPU_HOLD_TCTL_FILE="$scratch/tctl" GPU_HOLD_THERMAL_FLAG="$scratch/x" \ @@ -82,5 +85,10 @@ run 3 12 1 0 if [ "$rc" != 0 ]; then echo "FAIL arm 3: rc $rc with GPU_HOLD_FAULT_ABORT=0, want 0"; fail=1; fi grep -q "ABORT GPU/NPU fault" "$scratch/log3" && { echo "FAIL arm 3: the watcher fired with GPU_HOLD_FAULT_ABORT=0"; fail=1; } -[ "$fail" = 0 ] && echo "gpu_hold_fault_abort: ok (fault at 5 s -> rc 78 after ${e1}s; clean -> 0; opt-out -> 0)" +# arm 4: the default (variable unset) must be ON +run 4 600 1 unset +if [ "$rc" != 78 ]; then echo "FAIL arm 4: rc $rc with GPU_HOLD_FAULT_ABORT unset, want 78 (the default must be on)"; fail=1; fi +if [ -n "$deep" ] && kill -0 "$deep" 2> /dev/null; then kill -9 "$deep"; fi + +[ "$fail" = 0 ] && echo "gpu_hold_fault_abort: ok (fault at 5 s -> rc 78 after ${e1}s; clean -> 0; opt-out -> 0; default on -> 78)" exit "$fail"