From 69719511b95e89adbff1088d6a8fbd5e23124f1b Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 08:21:04 -0400 Subject: [PATCH] coder-llm-ticketed: rc 78 says which of its two causes it was; every outcome line says how long the window was open Review of engine#2527 (12:02Z): rc 78 is one gpu-hold code for two causes with different responses. The journal line now tells them apart by whether the window had opened: "fault during the window after Ns" (a GPU/NPU fault line while it served: report it) vs "ticket refused before the window" (#2480's take-time refusal on a recent fault/reset line: nothing ran). Every outcome line (0/124/130/137/other) carries "open Ns of M min" or "the window never opened", so the 124/137 boundary at the max hold can be checked from the log. Test: arm D asserts the fault-during wording; new arm K puts a MODE2 reset line in the kernel-log fixture so gpu-hold refuses the grant, and asserts rc 78, the refused wording, and no server; arm J asserts the open time on the 137 line. Mutants: the 78 split always "during" -> FAIL K; no open time on the 137 line -> FAIL J. Co-Authored-By: Claude Opus 5.5 --- scripts/coder-llm-ticketed | 26 ++++++++++++++++++-------- tests/coder_llm_ticketed_test.sh | 20 ++++++++++++++++++-- 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/scripts/coder-llm-ticketed b/scripts/coder-llm-ticketed index 4a058c78d..4e973c627 100755 --- a/scripts/coder-llm-ticketed +++ b/scripts/coder-llm-ticketed @@ -72,7 +72,10 @@ # (a 137 once the window has been open for the max hold, timed from the window's own start). Planned, not a crash # 130 a stop was requested (gpu-hold's stop stamp) and the window refused to die; gpu-hold released the token anyway # 137 something KILLed the window or the server BEFORE the max hold (the OOM killer, a manual kill -9): a failure -# 78 gpu-hold ended the hold on a GPU/NPU fault in the kernel log, or refused the ticket; backed off before exiting +# 78 one code from gpu-hold for two causes, told apart in the journal line by whether the window had opened: +# "fault during the window after Ns" = the kernel logged a GPU/NPU fault while it served (report it); +# "ticket refused before the window" = a recent fault/reset line made gpu-hold refuse the grant (#2480; +# nothing ran). Either way it backs off CODER_LLM_FAULT_BACKOFF_S before exiting # 75 the queue never came round within CODER_LLM_QUEUE_WAIT_S # any other: the server's own exit status (it exited by itself) set -u @@ -211,17 +214,24 @@ rc=$? # a KILL once the window has been open for the max hold is still the max hold (as 143 is, which gpu-hold maps to # 124); timed from the window's own start, so a long queue wait followed by some other KILL is not relabelled opened=$(cat "$CODER_LLM_WINDOW_STAMP" 2> /dev/null); rm -f "$CODER_LLM_WINDOW_STAMP" +# printed on every outcome line, so the 124/137 boundary at the max hold can be checked from the log, not trusted +if [ -n "$opened" ]; then open_for="open $(( $(date +%s) - opened ))s of ${MAX_HOLD} min"; else open_for="the window never opened"; fi if [ "$rc" = 137 ] && [ -n "$opened" ] && awk -v e=$(( $(date +%s) - opened )) -v m="$MAX_HOLD" 'BEGIN { exit !(e >= m * 60) }'; then - log "the window was KILLed after the max hold (rc 137; raise CODER_LLM_STOP_WAIT_S if the server needs longer to exit)" + log "the window was KILLed after the max hold (rc 137, $open_for; raise CODER_LLM_STOP_WAIT_S if the server needs longer to exit)" rc=124 fi case "$rc" in - 0) log "window closed (idle)" ;; - 124) log "window closed (max hold ${MAX_HOLD} min); re-queueing" ;; - 130) log "window stop requested and refused; gpu-hold released the token anyway" ;; - 137) log "window KILLed before its max hold (rc 137: the OOM killer or a kill -9, not a planned close)" ;; + 0) log "window closed (idle; $open_for)" ;; + 124) log "window closed (max hold ${MAX_HOLD} min; $open_for); re-queueing" ;; + 130) log "window stop requested and refused ($open_for); gpu-hold released the token anyway" ;; + 137) log "window KILLed before its max hold (rc 137, $open_for: the OOM killer or a kill -9, not a planned close)" ;; 75) log "the queue never came round in ${QUEUE_WAIT}s" ;; - 78) log "gpu-hold rc 78 (a GPU fault in the kernel log, or a refused ticket): backing off ${BACKOFF}s"; sleep "$BACKOFF" ;; - *) log "window ended rc $rc" ;; + 78) if [ -n "$opened" ]; then + log "gpu-hold rc 78: fault during the window after $(( $(date +%s) - opened ))s (a GPU/NPU fault line in the kernel log; report it): backing off ${BACKOFF}s" + else + log "gpu-hold rc 78: ticket refused before the window (a recent fault/reset line in the kernel log; nothing ran): backing off ${BACKOFF}s" + fi + sleep "$BACKOFF" ;; + *) log "window ended rc $rc ($open_for)" ;; esac exit "$rc" diff --git a/tests/coder_llm_ticketed_test.sh b/tests/coder_llm_ticketed_test.sh index 7f7e03913..2ff125dd3 100755 --- a/tests/coder_llm_ticketed_test.sh +++ b/tests/coder_llm_ticketed_test.sh @@ -35,7 +35,10 @@ # as the max hold, rc 124 (the first live window, 11-15 10:56Z, ended rc 137); the teardown time is logged, and a # grace too short for the stop wait is warned about; # J a server KILLed 1 s into a window that queued 9 s behind a lane stays rc 137: the relabel is timed from the -# window's start, not the queue's, so a real KILL is not reported as a max hold. +# window's start, not the queue's, so a real KILL is not reported as a max hold; every outcome line says how long +# the window was open; +# K a fault line already in the kernel log makes gpu-hold refuse the grant (rc 78): the journal says "ticket refused +# before the window", not a fault during one (arm D says "fault during the window"), and no server starts. set -u root=$(cd "$(dirname "$0")/.." && pwd) # the defaults work from any directory, not only the repo root tick=${1:-$root/scripts/coder-llm-ticketed} @@ -119,6 +122,7 @@ env "${envs[@]}" GPU_HOLD_FAULT_ABORT=1 CODER_LLM_IDLE_S=60 CODER_LLM_MAX_HOLD_M rc=$? [ "$rc" = 78 ] || { echo "FAIL D: rc $rc after a gfxhub fault, want 78"; fail=1; } grep -q "backing off" "$scratch/d.log" || { echo "FAIL D: no back-off after rc 78"; fail=1; } +grep -Eq "rc 78: fault during the window after [0-9]+s" "$scratch/d.log" || { echo "FAIL D: rc 78 not reported as a fault during the window"; fail=1; } server_alive && { echo "FAIL D: the server survived the fault abort"; fail=1; } [ -e "$scratch/TOKEN" ] && { echo "FAIL D: the token is still held"; fail=1; } : > "$scratch/klog" @@ -210,10 +214,22 @@ rc=$? wait "$lj" [ "$rc" = 137 ] || { echo "FAIL J: rc $rc for a server KILLed 1 s into its window after a 9 s queue, want 137 (relabelled as a max hold?)"; fail=1; } grep -q "KILLed before its max hold" "$scratch/j.log" || { echo "FAIL J: the log does not call it a KILL before the max hold"; fail=1; } +grep -Eq "KILLed before its max hold \(rc 137, open [0-9]s of 0.1 min" "$scratch/j.log" || { echo "FAIL J: the 137 line does not say how long the window was open"; fail=1; } + +# ---- K: a grant refused on a recent fault line is not a fault during a window ---------------------------------------- +echo busy > "$scratch/state"; rm -f "$scratch/server.pid"; touch "$scratch/demand" +echo "$(date '+%b %d %H:%M:%S') strixhalo kernel: amdgpu 0000:c5:00.0: amdgpu: MODE2 reset" > "$scratch/klog" +nowin=(); for e in "${envs[@]}"; do case "$e" in GPU_HOLD_RESET_WINDOW_MIN=*) ;; *) nowin+=("$e") ;; esac; done +env "${nowin[@]}" GPU_HOLD_RESET_WINDOW_MIN=10 CODER_LLM_IDLE_S=60 CODER_LLM_MAX_HOLD_MIN=1 timeout 60 "$tick" -- "$scratch/fake" > "$scratch/k.log" 2>&1 +rc=$? +: > "$scratch/klog" +[ "$rc" = 78 ] || { echo "FAIL K: rc $rc for a refused grant, want 78 ($(grep -m1 'refus' "$scratch/k.log"))"; fail=1; } +grep -q "rc 78: ticket refused before the window" "$scratch/k.log" || { echo "FAIL K: a refused grant is not reported as such ($(grep -m1 'rc 78' "$scratch/k.log"))"; fail=1; } +[ -s "$scratch/server.pid" ] && { echo "FAIL K: a server started on a refused grant"; fail=1; } # ---- G: the default queue wait keeps the ticket's place on a busy box (a give-up re-queues at the back) ------------ sed -n 's/^QUEUE_WAIT=\${CODER_LLM_QUEUE_WAIT_S:-\([0-9]*\)}$/\1/p' "$tick" > "$scratch/qw" [ "$(cat "$scratch/qw")" -ge 14400 ] 2> /dev/null || { echo "FAIL G: default queue wait $(cat "$scratch/qw")s; under 4 h a busy queue makes it re-queue at the back and starve"; fail=1; } -[ "$fail" = 0 ] && echo "coder_llm_ticketed: ok (no demand -> no ticket; idle -> rc 0; busy -> 124 at max hold; fault -> 78 + back-off; FIFO: coder-llm, lane, coder-llm; a failing gh poll is logged once, no window; default queue wait keeps its place; a stale grant releases without a load; a slow-exiting server gets its grace, a KILL after the max hold is 124, one after a queue wait stays 137)" +[ "$fail" = 0 ] && echo "coder_llm_ticketed: ok (no demand -> no ticket; idle -> rc 0; busy -> 124 at max hold; fault -> 78 + back-off; FIFO: coder-llm, lane, coder-llm; a failing gh poll is logged once, no window; default queue wait keeps its place; a stale grant releases without a load; a slow-exiting server gets its grace, a KILL after the max hold is 124, one after a queue wait stays 137; a refused grant is told from a fault during a window)" exit "$fail"