diff --git a/docs/ci-runners.md b/docs/ci-runners.md index ba7e266a7..92d4c89a0 100644 --- a/docs/ci-runners.md +++ b/docs/ci-runners.md @@ -284,7 +284,10 @@ ticket, in on-demand windows, through `scripts/coder-llm-ticketed` (deployed to 2. **It queues for the GPU like any other job**: `gpu-hold -l coder-llm -t 25`. The FIFO queue, the temperature, memory and kernel-fault watchdogs, and the release wait that lets the next label in all apply. 3. **Inside the ticket it serves**, and ends the window when no slot in `/slots` has been processing **and** the demand - file has not been touched for 5 minutes, or when the 25-minute max hold ends it. + file has not been touched for 5 minutes, or when the 25-minute max hold ends it. A server that is stopping gets 30 s + (`CODER_LLM_STOP_WAIT_S`), and gpu-hold's TERM-to-KILL grace for the window is set 10 s longer. That way the window + ends its own server before gpu-hold KILLs it. The 30B server needed more than gpu-hold's default 10 s at the first + live max hold (2026-11-15 10:56Z), which ended rc 137. A KILL after the max hold is still reported as rc 124. 4. **It exits**, and the unit's `Restart=always` starts the next invocation, which waits for demand and queues again at the back of the line. After a kernel fault (gpu-hold rc 78) it backs off 10 minutes first. It queues with a long `gpu-hold -w` (24 h): a ticket that gave up would be replaced at the back, so a short wait on a busy box starves it. diff --git a/scripts/coder-llm-ticketed b/scripts/coder-llm-ticketed index 8b0fde146..18324ad74 100755 --- a/scripts/coder-llm-ticketed +++ b/scripts/coder-llm-ticketed @@ -56,6 +56,11 @@ # touched the demand file; 0 = the demand file only # CODER_LLM_GH_REPO default 1bit-MONSTER/engine # CODER_LLM_FAULT_BACKOFF_S back-off after gpu-hold rc 78 (a fault, or a refused ticket), default 600 +# CODER_LLM_STOP_WAIT_S how long a stopping server gets after TERM before this KILLs it, default 30. The 30B +# HRX server needed more than 10 s to exit at the first live max hold (11-15 10:56Z) +# CODER_LLM_KILL_GRACE_S gpu-hold's TERM->KILL grace for the window (GPU_HOLD_KILL_GRACE), default STOP_WAIT+10, +# so the window can end its own server before gpu-hold KILLs the window; an explicit +# GPU_HOLD_KILL_GRACE in the environment wins # CODER_LLM_GPU_HOLD gpu-hold to use, default the one on PATH # CODER_LLM_ALWAYS=1 skip the demand wait (a resident window, still ticketed and still max-hold bounded) # @@ -73,6 +78,8 @@ GH_POLL=${CODER_LLM_GH_POLL:-1} GH_POLL_S=${CODER_LLM_GH_POLL_S:-60} GH_REPO=${CODER_LLM_GH_REPO:-1bit-MONSTER/engine} BACKOFF=${CODER_LLM_FAULT_BACKOFF_S:-600} +STOP_WAIT=${CODER_LLM_STOP_WAIT_S:-30} +KILL_GRACE=${GPU_HOLD_KILL_GRACE:-${CODER_LLM_KILL_GRACE_S:-$(( STOP_WAIT + 10 ))}} GPU_HOLD=${CODER_LLM_GPU_HOLD:-gpu-hold} log() { echo "coder-llm-ticketed[$$] $*"; } @@ -125,11 +132,12 @@ if [ "${1:-}" = "--window" ]; then log "ticket granted with no fresh demand: releasing without starting the server" exit 0 fi + [ -n "${CODER_LLM_WINDOW_STAMP:-}" ] && date +%s > "$CODER_LLM_WINDOW_STAMP" "$@" & server=$! stop_server() { kill -TERM "$server" 2> /dev/null - for _ in $(seq 1 30); do kill -0 "$server" 2> /dev/null || return; sleep 1; done + for _ in $(seq 1 "$STOP_WAIT"); do kill -0 "$server" 2> /dev/null || return; sleep 1; done kill -KILL "$server" 2> /dev/null } trap 'stop_server; exit 143' TERM INT HUP @@ -174,8 +182,19 @@ if [ "${CODER_LLM_ALWAYS:-0}" != 1 ]; then fi holder=$(head -c 120 "${GPU_TOKEN_FILE:-$HOME/GPU-TOKEN}" 2> /dev/null | head -1) log "demand: queueing for the GPU (gpu-hold -l coder-llm -t $MAX_HOLD -w $QUEUE_WAIT)${holder:+; queued behind $holder}" -"$GPU_HOLD" -t "$MAX_HOLD" -l coder-llm -w "$QUEUE_WAIT" -- "$0" --window "$@" +# gpu-hold's default 10 s grace KILLed the window at the first live max hold while the server was still exiting: +# rc 137 and an unclean stop instead of 124 (11-15 10:56Z) +export CODER_LLM_WINDOW_STAMP="${TMPDIR:-/tmp}/coder-llm-window.$$" +rm -f "$CODER_LLM_WINDOW_STAMP" +GPU_HOLD_KILL_GRACE=$KILL_GRACE "$GPU_HOLD" -t "$MAX_HOLD" -l coder-llm -w "$QUEUE_WAIT" -- "$0" --window "$@" rc=$? +# a KILL once the window has been open for the max hold is still the max hold (as 143 is, which gpu-hold maps to +# 124); timed from the window's own start, so a long queue wait followed by some other KILL is not relabelled +opened=$(cat "$CODER_LLM_WINDOW_STAMP" 2> /dev/null); rm -f "$CODER_LLM_WINDOW_STAMP" +if [ "$rc" = 137 ] && [ -n "$opened" ] && awk -v e=$(( $(date +%s) - opened )) -v m="$MAX_HOLD" 'BEGIN { exit !(e >= m * 60) }'; then + log "the window was KILLed after the max hold (rc 137; raise CODER_LLM_STOP_WAIT_S if the server needs longer to exit)" + rc=124 +fi case "$rc" in 0) log "window closed (idle)" ;; 124) log "window closed (max hold ${MAX_HOLD} min); re-queueing" ;; diff --git a/tests/coder_llm_ticketed_test.sh b/tests/coder_llm_ticketed_test.sh index c46daafb0..7bff2e997 100755 --- a/tests/coder_llm_ticketed_test.sh +++ b/tests/coder_llm_ticketed_test.sh @@ -29,7 +29,10 @@ # E a lane that queues during a window gets the GPU before the next coder-llm window (FIFO; the restart re-queues); # F a failing gh (the pr-agent-run fallback) is logged once per wait with its reason, and opens no window; # G the default queue wait is long enough to keep the ticket's place; H a grant whose demand went stale releases -# without starting the server. +# without starting the server; +# I a server that needs 12 s to exit after TERM at the max hold: the window's own kill grace (not gpu-hold's 10 s +# default) lets it exit cleanly, rc 124; with a grace too short for it, gpu-hold's KILL (rc 137) is still reported +# as the max hold, rc 124 (the first live window, 11-15 10:56Z, ended rc 137). set -u root=$(cd "$(dirname "$0")/.." && pwd) # the defaults work from any directory, not only the repo root tick=${1:-$root/scripts/coder-llm-ticketed} @@ -160,9 +163,34 @@ wait "$lh" grep -q "ticket granted with no fresh demand" "$scratch/h.log" || { echo "FAIL H: the stale grant did not say it released"; fail=1; } [ -s "$scratch/server.pid" ] && { echo "FAIL H: the server was started for a stale grant"; fail=1; } +# ---- I: a slow-exiting server at the max hold --------------------------------------------------------------------- +cat > "$scratch/slow" < "$scratch/slow.exit"; kill \$p 2> /dev/null; exit 0' TERM +"$scratch/fake" & p=\$! +while :; do sleep 1 & wait \$!; done +SLOW +chmod +x "$scratch/slow" +nograce=(); for e in "${envs[@]}"; do case "$e" in GPU_HOLD_KILL_GRACE=*) ;; *) nograce+=("$e") ;; esac; done +echo busy > "$scratch/state"; rm -f "$scratch/server.pid" "$scratch/slow.exit"; touch "$scratch/demand" +env "${nograce[@]}" CODER_LLM_STOP_WAIT_S=20 CODER_LLM_IDLE_S=60 CODER_LLM_MAX_HOLD_MIN=0.1 timeout 120 "$tick" -- "$scratch/slow" > "$scratch/i1.log" 2>&1 +rc=$? +[ "$rc" = 124 ] || { echo "FAIL I: rc $rc for a slow-exiting server at the max hold, want 124"; fail=1; } +[ -e "$scratch/slow.exit" ] || { echo "FAIL I: the slow server was KILLed before it could exit (the window's kill grace did not reach gpu-hold)"; fail=1; } +grep -q "window closed (max hold" "$scratch/i1.log" || { echo "FAIL I: the log does not say max hold"; fail=1; } +[ -e "$scratch/TOKEN" ] && { echo "FAIL I: the token is still held"; fail=1; } +echo busy > "$scratch/state"; rm -f "$scratch/server.pid" "$scratch/slow.exit"; touch "$scratch/demand" +env "${envs[@]}" CODER_LLM_IDLE_S=60 CODER_LLM_MAX_HOLD_MIN=0.1 timeout 120 "$tick" -- "$scratch/slow" > "$scratch/i2.log" 2>&1 +rc=$? +[ "$rc" = 124 ] || { echo "FAIL I: rc $rc when gpu-hold KILLs the window after the max hold, want 124"; fail=1; } +grep -q "KILLed after the max hold" "$scratch/i2.log" || { echo "FAIL I: the KILL after the max hold was not reported as such"; fail=1; } +[ -e "$scratch/TOKEN" ] && { echo "FAIL I: the token is still held after the KILL"; fail=1; } +pkill -9 -f "$scratch/slow" 2> /dev/null + # ---- G: the default queue wait keeps the ticket's place on a busy box (a give-up re-queues at the back) ------------ sed -n 's/^QUEUE_WAIT=\${CODER_LLM_QUEUE_WAIT_S:-\([0-9]*\)}$/\1/p' "$tick" > "$scratch/qw" [ "$(cat "$scratch/qw")" -ge 14400 ] 2> /dev/null || { echo "FAIL G: default queue wait $(cat "$scratch/qw")s; under 4 h a busy queue makes it re-queue at the back and starve"; fail=1; } -[ "$fail" = 0 ] && echo "coder_llm_ticketed: ok (no demand -> no ticket; idle -> rc 0; busy -> 124 at max hold; fault -> 78 + back-off; FIFO: coder-llm, lane, coder-llm; a failing gh poll is logged once, no window; default queue wait keeps its place; a stale grant releases without a load)" +[ "$fail" = 0 ] && echo "coder_llm_ticketed: ok (no demand -> no ticket; idle -> rc 0; busy -> 124 at max hold; fault -> 78 + back-off; FIFO: coder-llm, lane, coder-llm; a failing gh poll is logged once, no window; default queue wait keeps its place; a stale grant releases without a load; a slow-exiting server gets its grace, a KILL after the max hold is 124)" exit "$fail"