From 9cc7642505ca5508d1db4a2377424205aa0a39d4 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 07:05:02 -0400 Subject: [PATCH] coder-llm-ticketed: the window's kill grace fits the 30B server's exit; a KILL after the max hold is rc 124 The first live window (2026-11-15, 10:31:46Z-10:56:57Z) was cut at its 25 min max hold. The server got TERM and logged "stopping" at once, but its HRX teardown took longer than gpu-hold's default 10 s GPU_HOLD_KILL_GRACE, so gpu-hold KILLed the window: rc 137, logged "window ended rc 137", an unclean stop instead of the max-hold close. The token release and the FIFO were not affected (the next lane took the GPU 1 s later). - CODER_LLM_STOP_WAIT_S (default 30, was a fixed 30) is how long the window waits for its server after TERM; - gpu-hold runs with GPU_HOLD_KILL_GRACE = STOP_WAIT + 10 (CODER_LLM_KILL_GRACE_S, or an explicit GPU_HOLD_KILL_GRACE, overrides), so the window can end its own server before gpu-hold KILLs it; - a rc 137 once the window has been open for the max hold (timed from the window's own start, not the queue wait) is reported as the max hold, rc 124. Test arm I: a server that takes 12 s to exit gets its grace and exits cleanly (rc 124); with a grace too short, the KILL is still rc 124 and logged as such. Mutants: dropping the grace -> FAIL I (the server is KILLed); dropping the relabel -> FAIL I (rc 137). Co-Authored-By: Claude Opus 5.5 --- docs/ci-runners.md | 5 ++++- scripts/coder-llm-ticketed | 23 +++++++++++++++++++++-- tests/coder_llm_ticketed_test.sh | 32 ++++++++++++++++++++++++++++++-- 3 files changed, 55 insertions(+), 5 deletions(-) diff --git a/docs/ci-runners.md b/docs/ci-runners.md index ba7e266a7..92d4c89a0 100644 --- a/docs/ci-runners.md +++ b/docs/ci-runners.md @@ -284,7 +284,10 @@ ticket, in on-demand windows, through `scripts/coder-llm-ticketed` (deployed to 2. **It queues for the GPU like any other job**: `gpu-hold -l coder-llm -t 25`. The FIFO queue, the temperature, memory and kernel-fault watchdogs, and the release wait that lets the next label in all apply. 3. **Inside the ticket it serves**, and ends the window when no slot in `/slots` has been processing **and** the demand - file has not been touched for 5 minutes, or when the 25-minute max hold ends it. + file has not been touched for 5 minutes, or when the 25-minute max hold ends it. A server that is stopping gets 30 s + (`CODER_LLM_STOP_WAIT_S`), and gpu-hold's TERM-to-KILL grace for the window is set 10 s longer. That way the window + ends its own server before gpu-hold KILLs it. The 30B server needed more than gpu-hold's default 10 s at the first + live max hold (2026-11-15 10:56Z), which ended rc 137. A KILL after the max hold is still reported as rc 124. 4. **It exits**, and the unit's `Restart=always` starts the next invocation, which waits for demand and queues again at the back of the line. After a kernel fault (gpu-hold rc 78) it backs off 10 minutes first. It queues with a long `gpu-hold -w` (24 h): a ticket that gave up would be replaced at the back, so a short wait on a busy box starves it. diff --git a/scripts/coder-llm-ticketed b/scripts/coder-llm-ticketed index 8b0fde146..18324ad74 100755 --- a/scripts/coder-llm-ticketed +++ b/scripts/coder-llm-ticketed @@ -56,6 +56,11 @@ # touched the demand file; 0 = the demand file only # CODER_LLM_GH_REPO default 1bit-MONSTER/engine # CODER_LLM_FAULT_BACKOFF_S back-off after gpu-hold rc 78 (a fault, or a refused ticket), default 600 +# CODER_LLM_STOP_WAIT_S how long a stopping server gets after TERM before this KILLs it, default 30. The 30B +# HRX server needed more than 10 s to exit at the first live max hold (11-15 10:56Z) +# CODER_LLM_KILL_GRACE_S gpu-hold's TERM->KILL grace for the window (GPU_HOLD_KILL_GRACE), default STOP_WAIT+10, +# so the window can end its own server before gpu-hold KILLs the window; an explicit +# GPU_HOLD_KILL_GRACE in the environment wins # CODER_LLM_GPU_HOLD gpu-hold to use, default the one on PATH # CODER_LLM_ALWAYS=1 skip the demand wait (a resident window, still ticketed and still max-hold bounded) # @@ -73,6 +78,8 @@ GH_POLL=${CODER_LLM_GH_POLL:-1} GH_POLL_S=${CODER_LLM_GH_POLL_S:-60} GH_REPO=${CODER_LLM_GH_REPO:-1bit-MONSTER/engine} BACKOFF=${CODER_LLM_FAULT_BACKOFF_S:-600} +STOP_WAIT=${CODER_LLM_STOP_WAIT_S:-30} +KILL_GRACE=${GPU_HOLD_KILL_GRACE:-${CODER_LLM_KILL_GRACE_S:-$(( STOP_WAIT + 10 ))}} GPU_HOLD=${CODER_LLM_GPU_HOLD:-gpu-hold} log() { echo "coder-llm-ticketed[$$] $*"; } @@ -125,11 +132,12 @@ if [ "${1:-}" = "--window" ]; then log "ticket granted with no fresh demand: releasing without starting the server" exit 0 fi + [ -n "${CODER_LLM_WINDOW_STAMP:-}" ] && date +%s > "$CODER_LLM_WINDOW_STAMP" "$@" & server=$! stop_server() { kill -TERM "$server" 2> /dev/null - for _ in $(seq 1 30); do kill -0 "$server" 2> /dev/null || return; sleep 1; done + for _ in $(seq 1 "$STOP_WAIT"); do kill -0 "$server" 2> /dev/null || return; sleep 1; done kill -KILL "$server" 2> /dev/null } trap 'stop_server; exit 143' TERM INT HUP @@ -174,8 +182,19 @@ if [ "${CODER_LLM_ALWAYS:-0}" != 1 ]; then fi holder=$(head -c 120 "${GPU_TOKEN_FILE:-$HOME/GPU-TOKEN}" 2> /dev/null | head -1) log "demand: queueing for the GPU (gpu-hold -l coder-llm -t $MAX_HOLD -w $QUEUE_WAIT)${holder:+; queued behind $holder}" -"$GPU_HOLD" -t "$MAX_HOLD" -l coder-llm -w "$QUEUE_WAIT" -- "$0" --window "$@" +# gpu-hold's default 10 s grace KILLed the window at the first live max hold while the server was still exiting: +# rc 137 and an unclean stop instead of 124 (11-15 10:56Z) +export CODER_LLM_WINDOW_STAMP="${TMPDIR:-/tmp}/coder-llm-window.$$" +rm -f "$CODER_LLM_WINDOW_STAMP" +GPU_HOLD_KILL_GRACE=$KILL_GRACE "$GPU_HOLD" -t "$MAX_HOLD" -l coder-llm -w "$QUEUE_WAIT" -- "$0" --window "$@" rc=$? +# a KILL once the window has been open for the max hold is still the max hold (as 143 is, which gpu-hold maps to +# 124); timed from the window's own start, so a long queue wait followed by some other KILL is not relabelled +opened=$(cat "$CODER_LLM_WINDOW_STAMP" 2> /dev/null); rm -f "$CODER_LLM_WINDOW_STAMP" +if [ "$rc" = 137 ] && [ -n "$opened" ] && awk -v e=$(( $(date +%s) - opened )) -v m="$MAX_HOLD" 'BEGIN { exit !(e >= m * 60) }'; then + log "the window was KILLed after the max hold (rc 137; raise CODER_LLM_STOP_WAIT_S if the server needs longer to exit)" + rc=124 +fi case "$rc" in 0) log "window closed (idle)" ;; 124) log "window closed (max hold ${MAX_HOLD} min); re-queueing" ;; diff --git a/tests/coder_llm_ticketed_test.sh b/tests/coder_llm_ticketed_test.sh index c46daafb0..7bff2e997 100755 --- a/tests/coder_llm_ticketed_test.sh +++ b/tests/coder_llm_ticketed_test.sh @@ -29,7 +29,10 @@ # E a lane that queues during a window gets the GPU before the next coder-llm window (FIFO; the restart re-queues); # F a failing gh (the pr-agent-run fallback) is logged once per wait with its reason, and opens no window; # G the default queue wait is long enough to keep the ticket's place; H a grant whose demand went stale releases -# without starting the server. +# without starting the server; +# I a server that needs 12 s to exit after TERM at the max hold: the window's own kill grace (not gpu-hold's 10 s +# default) lets it exit cleanly, rc 124; with a grace too short for it, gpu-hold's KILL (rc 137) is still reported +# as the max hold, rc 124 (the first live window, 11-15 10:56Z, ended rc 137). set -u root=$(cd "$(dirname "$0")/.." && pwd) # the defaults work from any directory, not only the repo root tick=${1:-$root/scripts/coder-llm-ticketed} @@ -160,9 +163,34 @@ wait "$lh" grep -q "ticket granted with no fresh demand" "$scratch/h.log" || { echo "FAIL H: the stale grant did not say it released"; fail=1; } [ -s "$scratch/server.pid" ] && { echo "FAIL H: the server was started for a stale grant"; fail=1; } +# ---- I: a slow-exiting server at the max hold --------------------------------------------------------------------- +cat > "$scratch/slow" < "$scratch/slow.exit"; kill \$p 2> /dev/null; exit 0' TERM +"$scratch/fake" & p=\$! +while :; do sleep 1 & wait \$!; done +SLOW +chmod +x "$scratch/slow" +nograce=(); for e in "${envs[@]}"; do case "$e" in GPU_HOLD_KILL_GRACE=*) ;; *) nograce+=("$e") ;; esac; done +echo busy > "$scratch/state"; rm -f "$scratch/server.pid" "$scratch/slow.exit"; touch "$scratch/demand" +env "${nograce[@]}" CODER_LLM_STOP_WAIT_S=20 CODER_LLM_IDLE_S=60 CODER_LLM_MAX_HOLD_MIN=0.1 timeout 120 "$tick" -- "$scratch/slow" > "$scratch/i1.log" 2>&1 +rc=$? +[ "$rc" = 124 ] || { echo "FAIL I: rc $rc for a slow-exiting server at the max hold, want 124"; fail=1; } +[ -e "$scratch/slow.exit" ] || { echo "FAIL I: the slow server was KILLed before it could exit (the window's kill grace did not reach gpu-hold)"; fail=1; } +grep -q "window closed (max hold" "$scratch/i1.log" || { echo "FAIL I: the log does not say max hold"; fail=1; } +[ -e "$scratch/TOKEN" ] && { echo "FAIL I: the token is still held"; fail=1; } +echo busy > "$scratch/state"; rm -f "$scratch/server.pid" "$scratch/slow.exit"; touch "$scratch/demand" +env "${envs[@]}" CODER_LLM_IDLE_S=60 CODER_LLM_MAX_HOLD_MIN=0.1 timeout 120 "$tick" -- "$scratch/slow" > "$scratch/i2.log" 2>&1 +rc=$? +[ "$rc" = 124 ] || { echo "FAIL I: rc $rc when gpu-hold KILLs the window after the max hold, want 124"; fail=1; } +grep -q "KILLed after the max hold" "$scratch/i2.log" || { echo "FAIL I: the KILL after the max hold was not reported as such"; fail=1; } +[ -e "$scratch/TOKEN" ] && { echo "FAIL I: the token is still held after the KILL"; fail=1; } +pkill -9 -f "$scratch/slow" 2> /dev/null + # ---- G: the default queue wait keeps the ticket's place on a busy box (a give-up re-queues at the back) ------------ sed -n 's/^QUEUE_WAIT=\${CODER_LLM_QUEUE_WAIT_S:-\([0-9]*\)}$/\1/p' "$tick" > "$scratch/qw" [ "$(cat "$scratch/qw")" -ge 14400 ] 2> /dev/null || { echo "FAIL G: default queue wait $(cat "$scratch/qw")s; under 4 h a busy queue makes it re-queue at the back and starve"; fail=1; } -[ "$fail" = 0 ] && echo "coder_llm_ticketed: ok (no demand -> no ticket; idle -> rc 0; busy -> 124 at max hold; fault -> 78 + back-off; FIFO: coder-llm, lane, coder-llm; a failing gh poll is logged once, no window; default queue wait keeps its place; a stale grant releases without a load)" +[ "$fail" = 0 ] && echo "coder_llm_ticketed: ok (no demand -> no ticket; idle -> rc 0; busy -> 124 at max hold; fault -> 78 + back-off; FIFO: coder-llm, lane, coder-llm; a failing gh poll is logged once, no window; default queue wait keeps its place; a stale grant releases without a load; a slow-exiting server gets its grace, a KILL after the max hold is 124)" exit "$fail"