From 70cc51abeb65d799c60f74b75f87cfada8f64740 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 07:42:37 -0400 Subject: [PATCH] coder-llm-ticketed: one exit-status table, a warning for a grace shorter than the stop wait, the teardown time logged The review asks on engine#2525 (merged before they could be answered): - the header lists what each exit status of one invocation means: 0 idle/stale grant, 124 the max hold (including a KILL after it), 130 a refused stop, 137 a KILL BEFORE the max hold (a failure, not a planned close), 78 fault or refused ticket, 75 queue timeout, otherwise the server's own status; 130 and 137 get their own journal lines; - an explicit GPU_HOLD_KILL_GRACE that is not at least 5 s longer than CODER_LLM_STOP_WAIT_S is warned about (it undoes the nesting #2525 added); it is not clamped, the operator's value stands; - stop_server logs how long the teardown took ("server stopped in Ns after TERM"), or that it had to KILL. Test arm J: a server KILLed 1 s into a window that queued 9 s behind a lane stays rc 137, so the relabel is pinned to the window's start, not the queue's. Arm I now also asserts the teardown line and the warning (absent at a fitting grace, present at grace 3 s / stop wait 30 s). Mutants: relabel timed from the queue start -> FAIL J (rc 124); no warning -> FAIL I; no teardown line -> FAIL I. Co-Authored-By: Claude Opus 5.5 --- scripts/coder-llm-ticketed | 26 ++++++++++++++++++++++++-- tests/coder_llm_ticketed_test.sh | 27 +++++++++++++++++++++++++-- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/scripts/coder-llm-ticketed b/scripts/coder-llm-ticketed index 18324ad74..4a058c78d 100755 --- a/scripts/coder-llm-ticketed +++ b/scripts/coder-llm-ticketed @@ -65,6 +65,16 @@ # CODER_LLM_ALWAYS=1 skip the demand wait (a resident window, still ticketed and still max-hold bounded) # # Internal: "coder-llm-ticketed --window SERVER_CMD..." is what gpu-hold runs; it is the idle-aware window itself. +# +# Exit status of one invocation (the journal line after "window" says which; review of engine#2525): +# 0 the window closed idle, or a grant whose demand went stale was released without starting the server +# 124 the max hold ended the window: gpu-hold's deadline, including a window gpu-hold had to KILL after the deadline +# (a 137 once the window has been open for the max hold, timed from the window's own start). Planned, not a crash +# 130 a stop was requested (gpu-hold's stop stamp) and the window refused to die; gpu-hold released the token anyway +# 137 something KILLed the window or the server BEFORE the max hold (the OOM killer, a manual kill -9): a failure +# 78 gpu-hold ended the hold on a GPU/NPU fault in the kernel log, or refused the ticket; backed off before exiting +# 75 the queue never came round within CODER_LLM_QUEUE_WAIT_S +# any other: the server's own exit status (it exited by itself) set -u DEMAND=${CODER_LLM_DEMAND:-$HOME/.cache/coder-llm/demand} @@ -135,10 +145,16 @@ if [ "${1:-}" = "--window" ]; then [ -n "${CODER_LLM_WINDOW_STAMP:-}" ] && date +%s > "$CODER_LLM_WINDOW_STAMP" "$@" & server=$! - stop_server() { + stop_server() { # logs how long the teardown took: the number that says how close the next one is to the grace + local t0 n + t0=$(date +%s) kill -TERM "$server" 2> /dev/null - for _ in $(seq 1 "$STOP_WAIT"); do kill -0 "$server" 2> /dev/null || return; sleep 1; done + for n in $(seq 1 "$STOP_WAIT"); do + kill -0 "$server" 2> /dev/null || { log "server stopped in $(( $(date +%s) - t0 ))s after TERM (stop wait ${STOP_WAIT}s)"; return; } + sleep 1 + done kill -KILL "$server" 2> /dev/null + log "server did not stop in ${STOP_WAIT}s after TERM: KILLed (raise CODER_LLM_STOP_WAIT_S)" } trap 'stop_server; exit 143' TERM INT HUP log "window open: server pid $server" @@ -181,6 +197,10 @@ if [ "${CODER_LLM_ALWAYS:-0}" != 1 ]; then done fi holder=$(head -c 120 "${GPU_TOKEN_FILE:-$HOME/GPU-TOKEN}" 2> /dev/null | head -1) +# an explicit GPU_HOLD_KILL_GRACE wins, so it can undo the nesting (gpu-hold KILLing the window while the server is still +# inside its stop wait): say so, do not override the operator's choice +[ "$KILL_GRACE" -ge $(( STOP_WAIT + 5 )) ] 2> /dev/null \ + || log "WARNING: the kill grace ${KILL_GRACE}s is not at least 5 s longer than CODER_LLM_STOP_WAIT_S=${STOP_WAIT}s: gpu-hold will KILL the window before the server can exit" log "demand: queueing for the GPU (gpu-hold -l coder-llm -t $MAX_HOLD -w $QUEUE_WAIT)${holder:+; queued behind $holder}" # gpu-hold's default 10 s grace KILLed the window at the first live max hold while the server was still exiting: # rc 137 and an unclean stop instead of 124 (11-15 10:56Z) @@ -198,6 +218,8 @@ fi case "$rc" in 0) log "window closed (idle)" ;; 124) log "window closed (max hold ${MAX_HOLD} min); re-queueing" ;; + 130) log "window stop requested and refused; gpu-hold released the token anyway" ;; + 137) log "window KILLed before its max hold (rc 137: the OOM killer or a kill -9, not a planned close)" ;; 75) log "the queue never came round in ${QUEUE_WAIT}s" ;; 78) log "gpu-hold rc 78 (a GPU fault in the kernel log, or a refused ticket): backing off ${BACKOFF}s"; sleep "$BACKOFF" ;; *) log "window ended rc $rc" ;; diff --git a/tests/coder_llm_ticketed_test.sh b/tests/coder_llm_ticketed_test.sh index 7bff2e997..7f7e03913 100755 --- a/tests/coder_llm_ticketed_test.sh +++ b/tests/coder_llm_ticketed_test.sh @@ -32,7 +32,10 @@ # without starting the server; # I a server that needs 12 s to exit after TERM at the max hold: the window's own kill grace (not gpu-hold's 10 s # default) lets it exit cleanly, rc 124; with a grace too short for it, gpu-hold's KILL (rc 137) is still reported -# as the max hold, rc 124 (the first live window, 11-15 10:56Z, ended rc 137). +# as the max hold, rc 124 (the first live window, 11-15 10:56Z, ended rc 137); the teardown time is logged, and a +# grace too short for the stop wait is warned about; +# J a server KILLed 1 s into a window that queued 9 s behind a lane stays rc 137: the relabel is timed from the +# window's start, not the queue's, so a real KILL is not reported as a max hold. set -u root=$(cd "$(dirname "$0")/.." && pwd) # the defaults work from any directory, not only the repo root tick=${1:-$root/scripts/coder-llm-ticketed} @@ -179,18 +182,38 @@ rc=$? [ "$rc" = 124 ] || { echo "FAIL I: rc $rc for a slow-exiting server at the max hold, want 124"; fail=1; } [ -e "$scratch/slow.exit" ] || { echo "FAIL I: the slow server was KILLed before it could exit (the window's kill grace did not reach gpu-hold)"; fail=1; } grep -q "window closed (max hold" "$scratch/i1.log" || { echo "FAIL I: the log does not say max hold"; fail=1; } +grep -Eq "server stopped in 1[1-4]s after TERM" "$scratch/i1.log" || { echo "FAIL I: the teardown time is not logged ($(grep -m1 'server stopped\|did not stop' "$scratch/i1.log"))"; fail=1; } +grep -q "WARNING: the kill grace" "$scratch/i1.log" && { echo "FAIL I: warned about a grace that fits the stop wait"; fail=1; } [ -e "$scratch/TOKEN" ] && { echo "FAIL I: the token is still held"; fail=1; } echo busy > "$scratch/state"; rm -f "$scratch/server.pid" "$scratch/slow.exit"; touch "$scratch/demand" env "${envs[@]}" CODER_LLM_IDLE_S=60 CODER_LLM_MAX_HOLD_MIN=0.1 timeout 120 "$tick" -- "$scratch/slow" > "$scratch/i2.log" 2>&1 rc=$? [ "$rc" = 124 ] || { echo "FAIL I: rc $rc when gpu-hold KILLs the window after the max hold, want 124"; fail=1; } grep -q "KILLed after the max hold" "$scratch/i2.log" || { echo "FAIL I: the KILL after the max hold was not reported as such"; fail=1; } +grep -q "WARNING: the kill grace 3s is not at least 5 s longer than CODER_LLM_STOP_WAIT_S=30s" "$scratch/i2.log" || { echo "FAIL I: an explicit grace shorter than the stop wait was not warned about"; fail=1; } [ -e "$scratch/TOKEN" ] && { echo "FAIL I: the token is still held after the KILL"; fail=1; } pkill -9 -f "$scratch/slow" 2> /dev/null +# ---- J: a KILL shortly after a long queue wait is a real 137 ---------------------------------------------------------- +cat > "$scratch/selfkill" < /dev/null; kill -9 \$\$ +SK +chmod +x "$scratch/selfkill" +echo busy > "$scratch/state"; rm -f "$scratch/server.pid"; : > "$scratch/klog"; touch "$scratch/demand" +env "${envs[@]}" GPU_HOLD_WAIT=60 "$hold" -t 1 -l lanehold -- sleep 9 > /dev/null 2>&1 & +lj=$!; pids+=($lj) +wait_for 10 test -e "$scratch/TOKEN" || { echo "FAIL J: the lane did not take the token"; fail=1; } +env "${envs[@]}" CODER_LLM_IDLE_S=60 CODER_LLM_MAX_HOLD_MIN=0.1 timeout 60 "$tick" -- "$scratch/selfkill" > "$scratch/j.log" 2>&1 +rc=$? +wait "$lj" +[ "$rc" = 137 ] || { echo "FAIL J: rc $rc for a server KILLed 1 s into its window after a 9 s queue, want 137 (relabelled as a max hold?)"; fail=1; } +grep -q "KILLed before its max hold" "$scratch/j.log" || { echo "FAIL J: the log does not call it a KILL before the max hold"; fail=1; } + # ---- G: the default queue wait keeps the ticket's place on a busy box (a give-up re-queues at the back) ------------ sed -n 's/^QUEUE_WAIT=\${CODER_LLM_QUEUE_WAIT_S:-\([0-9]*\)}$/\1/p' "$tick" > "$scratch/qw" [ "$(cat "$scratch/qw")" -ge 14400 ] 2> /dev/null || { echo "FAIL G: default queue wait $(cat "$scratch/qw")s; under 4 h a busy queue makes it re-queue at the back and starve"; fail=1; } -[ "$fail" = 0 ] && echo "coder_llm_ticketed: ok (no demand -> no ticket; idle -> rc 0; busy -> 124 at max hold; fault -> 78 + back-off; FIFO: coder-llm, lane, coder-llm; a failing gh poll is logged once, no window; default queue wait keeps its place; a stale grant releases without a load; a slow-exiting server gets its grace, a KILL after the max hold is 124)" +[ "$fail" = 0 ] && echo "coder_llm_ticketed: ok (no demand -> no ticket; idle -> rc 0; busy -> 124 at max hold; fault -> 78 + back-off; FIFO: coder-llm, lane, coder-llm; a failing gh poll is logged once, no window; default queue wait keeps its place; a stale grant releases without a load; a slow-exiting server gets its grace, a KILL after the max hold is 124, one after a queue wait stays 137)" exit "$fail"