Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 24 additions & 2 deletions scripts/coder-llm-ticketed
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,16 @@
# CODER_LLM_ALWAYS=1 skip the demand wait (a resident window, still ticketed and still max-hold bounded)
#
# Internal: "coder-llm-ticketed --window SERVER_CMD..." is what gpu-hold runs; it is the idle-aware window itself.
#
# Exit status of one invocation (the journal line after "window" says which; review of engine#2525):
# 0 the window closed idle, or a grant whose demand went stale was released without starting the server
# 124 the max hold ended the window: gpu-hold's deadline, including a window gpu-hold had to KILL after the deadline
# (a 137 once the window has been open for the max hold, timed from the window's own start). Planned, not a crash
# 130 a stop was requested (gpu-hold's stop stamp) and the window refused to die; gpu-hold released the token anyway
# 137 something KILLed the window or the server BEFORE the max hold (the OOM killer, a manual kill -9): a failure
# 78 gpu-hold ended the hold on a GPU/NPU fault in the kernel log, or refused the ticket; backed off before exiting
# 75 the queue never came round within CODER_LLM_QUEUE_WAIT_S
# any other: the server's own exit status (it exited by itself)
set -u

DEMAND=${CODER_LLM_DEMAND:-$HOME/.cache/coder-llm/demand}
Expand Down Expand Up @@ -135,10 +145,16 @@ if [ "${1:-}" = "--window" ]; then
[ -n "${CODER_LLM_WINDOW_STAMP:-}" ] && date +%s > "$CODER_LLM_WINDOW_STAMP"
"$@" &
server=$!
stop_server() {
stop_server() { # logs how long the teardown took: the number that says how close the next one is to the grace
local t0 n
t0=$(date +%s)
kill -TERM "$server" 2> /dev/null
for _ in $(seq 1 "$STOP_WAIT"); do kill -0 "$server" 2> /dev/null || return; sleep 1; done
for n in $(seq 1 "$STOP_WAIT"); do
kill -0 "$server" 2> /dev/null || { log "server stopped in $(( $(date +%s) - t0 ))s after TERM (stop wait ${STOP_WAIT}s)"; return; }
sleep 1
done
kill -KILL "$server" 2> /dev/null
log "server did not stop in ${STOP_WAIT}s after TERM: KILLed (raise CODER_LLM_STOP_WAIT_S)"
}
trap 'stop_server; exit 143' TERM INT HUP
log "window open: server pid $server"
Expand Down Expand Up @@ -181,6 +197,10 @@ if [ "${CODER_LLM_ALWAYS:-0}" != 1 ]; then
done
fi
holder=$(head -c 120 "${GPU_TOKEN_FILE:-$HOME/GPU-TOKEN}" 2> /dev/null | head -1)
# an explicit GPU_HOLD_KILL_GRACE wins, so it can undo the nesting (gpu-hold KILLing the window while the server is still
# inside its stop wait): say so, do not override the operator's choice
[ "$KILL_GRACE" -ge $(( STOP_WAIT + 5 )) ] 2> /dev/null \
|| log "WARNING: the kill grace ${KILL_GRACE}s is not at least 5 s longer than CODER_LLM_STOP_WAIT_S=${STOP_WAIT}s: gpu-hold will KILL the window before the server can exit"
log "demand: queueing for the GPU (gpu-hold -l coder-llm -t $MAX_HOLD -w $QUEUE_WAIT)${holder:+; queued behind $holder}"
# gpu-hold's default 10 s grace KILLed the window at the first live max hold while the server was still exiting:
# rc 137 and an unclean stop instead of 124 (11-15 10:56Z)
Expand All @@ -198,6 +218,8 @@ fi
case "$rc" in
0) log "window closed (idle)" ;;
124) log "window closed (max hold ${MAX_HOLD} min); re-queueing" ;;
130) log "window stop requested and refused; gpu-hold released the token anyway" ;;
137) log "window KILLed before its max hold (rc 137: the OOM killer or a kill -9, not a planned close)" ;;
75) log "the queue never came round in ${QUEUE_WAIT}s" ;;
78) log "gpu-hold rc 78 (a GPU fault in the kernel log, or a refused ticket): backing off ${BACKOFF}s"; sleep "$BACKOFF" ;;
*) log "window ended rc $rc" ;;
Expand Down
27 changes: 25 additions & 2 deletions tests/coder_llm_ticketed_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,10 @@
# without starting the server;
# I a server that needs 12 s to exit after TERM at the max hold: the window's own kill grace (not gpu-hold's 10 s
# default) lets it exit cleanly, rc 124; with a grace too short for it, gpu-hold's KILL (rc 137) is still reported
# as the max hold, rc 124 (the first live window, 11-15 10:56Z, ended rc 137).
# as the max hold, rc 124 (the first live window, 11-15 10:56Z, ended rc 137); the teardown time is logged, and a
# grace too short for the stop wait is warned about;
# J a server KILLed 1 s into a window that queued 9 s behind a lane stays rc 137: the relabel is timed from the
# window's start, not the queue's, so a real KILL is not reported as a max hold.
set -u
root=$(cd "$(dirname "$0")/.." && pwd) # the defaults work from any directory, not only the repo root
tick=${1:-$root/scripts/coder-llm-ticketed}
Expand Down Expand Up @@ -179,18 +182,38 @@ rc=$?
[ "$rc" = 124 ] || { echo "FAIL I: rc $rc for a slow-exiting server at the max hold, want 124"; fail=1; }
[ -e "$scratch/slow.exit" ] || { echo "FAIL I: the slow server was KILLed before it could exit (the window's kill grace did not reach gpu-hold)"; fail=1; }
grep -q "window closed (max hold" "$scratch/i1.log" || { echo "FAIL I: the log does not say max hold"; fail=1; }
grep -Eq "server stopped in 1[1-4]s after TERM" "$scratch/i1.log" || { echo "FAIL I: the teardown time is not logged ($(grep -m1 'server stopped\|did not stop' "$scratch/i1.log"))"; fail=1; }
grep -q "WARNING: the kill grace" "$scratch/i1.log" && { echo "FAIL I: warned about a grace that fits the stop wait"; fail=1; }
[ -e "$scratch/TOKEN" ] && { echo "FAIL I: the token is still held"; fail=1; }
echo busy > "$scratch/state"; rm -f "$scratch/server.pid" "$scratch/slow.exit"; touch "$scratch/demand"
env "${envs[@]}" CODER_LLM_IDLE_S=60 CODER_LLM_MAX_HOLD_MIN=0.1 timeout 120 "$tick" -- "$scratch/slow" > "$scratch/i2.log" 2>&1
rc=$?
[ "$rc" = 124 ] || { echo "FAIL I: rc $rc when gpu-hold KILLs the window after the max hold, want 124"; fail=1; }
grep -q "KILLed after the max hold" "$scratch/i2.log" || { echo "FAIL I: the KILL after the max hold was not reported as such"; fail=1; }
grep -q "WARNING: the kill grace 3s is not at least 5 s longer than CODER_LLM_STOP_WAIT_S=30s" "$scratch/i2.log" || { echo "FAIL I: an explicit grace shorter than the stop wait was not warned about"; fail=1; }
[ -e "$scratch/TOKEN" ] && { echo "FAIL I: the token is still held after the KILL"; fail=1; }
pkill -9 -f "$scratch/slow" 2> /dev/null

# ---- J: a KILL shortly after a long queue wait is a real 137 ----------------------------------------------------------
cat > "$scratch/selfkill" <<SK
#!/bin/bash
# the fake, then a SIGKILL of the server 1 s in (the OOM killer's shape)
"$scratch/fake" & sleep 1; kill -9 \$! 2> /dev/null; kill -9 \$\$
SK
chmod +x "$scratch/selfkill"
echo busy > "$scratch/state"; rm -f "$scratch/server.pid"; : > "$scratch/klog"; touch "$scratch/demand"
env "${envs[@]}" GPU_HOLD_WAIT=60 "$hold" -t 1 -l lanehold -- sleep 9 > /dev/null 2>&1 &
lj=$!; pids+=($lj)
wait_for 10 test -e "$scratch/TOKEN" || { echo "FAIL J: the lane did not take the token"; fail=1; }
env "${envs[@]}" CODER_LLM_IDLE_S=60 CODER_LLM_MAX_HOLD_MIN=0.1 timeout 60 "$tick" -- "$scratch/selfkill" > "$scratch/j.log" 2>&1
rc=$?
wait "$lj"
[ "$rc" = 137 ] || { echo "FAIL J: rc $rc for a server KILLed 1 s into its window after a 9 s queue, want 137 (relabelled as a max hold?)"; fail=1; }
grep -q "KILLed before its max hold" "$scratch/j.log" || { echo "FAIL J: the log does not call it a KILL before the max hold"; fail=1; }

# ---- G: the default queue wait keeps the ticket's place on a busy box (a give-up re-queues at the back) ------------
sed -n 's/^QUEUE_WAIT=\${CODER_LLM_QUEUE_WAIT_S:-\([0-9]*\)}$/\1/p' "$tick" > "$scratch/qw"
[ "$(cat "$scratch/qw")" -ge 14400 ] 2> /dev/null || { echo "FAIL G: default queue wait $(cat "$scratch/qw")s; under 4 h a busy queue makes it re-queue at the back and starve"; fail=1; }

[ "$fail" = 0 ] && echo "coder_llm_ticketed: ok (no demand -> no ticket; idle -> rc 0; busy -> 124 at max hold; fault -> 78 + back-off; FIFO: coder-llm, lane, coder-llm; a failing gh poll is logged once, no window; default queue wait keeps its place; a stale grant releases without a load; a slow-exiting server gets its grace, a KILL after the max hold is 124)"
[ "$fail" = 0 ] && echo "coder_llm_ticketed: ok (no demand -> no ticket; idle -> rc 0; busy -> 124 at max hold; fault -> 78 + back-off; FIFO: coder-llm, lane, coder-llm; a failing gh poll is logged once, no window; default queue wait keeps its place; a stale grant releases without a load; a slow-exiting server gets its grace, a KILL after the max hold is 124, one after a queue wait stays 137)"
exit "$fail"
Loading