diff --git a/docs/ci-runners.md b/docs/ci-runners.md index e32e494ec..fe15c07d7 100644 --- a/docs/ci-runners.md +++ b/docs/ci-runners.md @@ -286,7 +286,8 @@ ticket, in on-demand windows, through `scripts/coder-llm-ticketed` (deployed to 3. **Inside the ticket it serves**, and ends the window when no slot in `/slots` has been processing **and** the demand file has not been touched for 5 minutes, or when the 25-minute max hold ends it. 4. **It exits**, and the unit's `Restart=always` starts the next invocation, which waits for demand and queues again at - the back of the line. After a kernel fault (gpu-hold rc 78) it backs off 10 minutes first. + the back of the line. After a kernel fault (gpu-hold rc 78) it backs off 10 minutes first. It queues with a long + `gpu-hold -w` (24 h): a ticket that gave up would be replaced at the back, so a short wait on a busy box starves it. Anything else that needs the model asks the same way: touch `~/.cache/coder-llm/demand`, keep touching it while it works, and wait for `:8090` to answer a real completion. Until a window opens, the port refuses. diff --git a/scripts/coder-llm-ticketed b/scripts/coder-llm-ticketed index cba5110ee..8b0fde146 100755 --- a/scripts/coder-llm-ticketed +++ b/scripts/coder-llm-ticketed @@ -45,7 +45,10 @@ # CODER_LLM_DEMAND_TTL_MIN a touch this recent is demand, default 30 (CODER_LLM_DEMAND_TTL_S in seconds wins) # CODER_LLM_IDLE_MIN idle minutes that end a window, default 5 (CODER_LLM_IDLE_S in seconds wins) # CODER_LLM_MAX_HOLD_MIN max hold per window (gpu-hold -t), default 25 -# CODER_LLM_QUEUE_WAIT_S how long one window queues for the GPU (gpu-hold -w), default 5400 +# CODER_LLM_QUEUE_WAIT_S how long one window queues for the GPU (gpu-hold -w), default 86400. Long on purpose: +# a ticket that gives up (rc 75) is replaced by a NEW one at the BACK of the FIFO, so a +# short wait on a busy box (10+ lane tickets) would lose its place every time and starve +# reviews; systemd's Restart= makes the wait unbounded anyway, so length only keeps place # CODER_LLM_URL the server's base URL for /slots, default http://127.0.0.1:8090 # CODER_LLM_POLL_S poll interval, default 15 # CODER_LLM_GH_POLL 1 (the default) = also treat a queued/in-progress pr-agent run as demand (REST, @@ -63,7 +66,7 @@ DEMAND=${CODER_LLM_DEMAND:-$HOME/.cache/coder-llm/demand} DEMAND_TTL=${CODER_LLM_DEMAND_TTL_S:-$(( ${CODER_LLM_DEMAND_TTL_MIN:-30} * 60 ))} IDLE=${CODER_LLM_IDLE_S:-$(( ${CODER_LLM_IDLE_MIN:-5} * 60 ))} MAX_HOLD=${CODER_LLM_MAX_HOLD_MIN:-25} -QUEUE_WAIT=${CODER_LLM_QUEUE_WAIT_S:-5400} +QUEUE_WAIT=${CODER_LLM_QUEUE_WAIT_S:-86400} URL=${CODER_LLM_URL:-http://127.0.0.1:8090} POLL=${CODER_LLM_POLL_S:-15} GH_POLL=${CODER_LLM_GH_POLL:-1} @@ -116,6 +119,12 @@ if [ "${1:-}" = "--window" ]; then shift [ "${1:-}" = "--" ] && shift [ $# -gt 0 ] || { log "--window needs a server command"; exit 2; } + # a ticket granted after its demand went stale (a long queue) releases at once instead of loading a 30B model and + # idling for IDLE seconds: on a deep queue that is a load per stale entry, about an hour of GPU (review of #2514) + if [ "${CODER_LLM_ALWAYS:-0}" != 1 ] && ! demand_fresh "$DEMAND_TTL" && ! gh_demand; then + log "ticket granted with no fresh demand: releasing without starting the server" + exit 0 + fi "$@" & server=$! stop_server() { diff --git a/tests/coder_llm_ticketed_test.sh b/tests/coder_llm_ticketed_test.sh index 6577d9021..c46daafb0 100755 --- a/tests/coder_llm_ticketed_test.sh +++ b/tests/coder_llm_ticketed_test.sh @@ -27,7 +27,9 @@ # C a busy server is cut at the max hold (rc 124), the server is gone; # D a gfxhub page fault in the kernel log during the window ends it (rc 78) and the server dies with it; # E a lane that queues during a window gets the GPU before the next coder-llm window (FIFO; the restart re-queues); -# F a failing gh (the pr-agent-run fallback) is logged once per wait with its reason, and opens no window. +# F a failing gh (the pr-agent-run fallback) is logged once per wait with its reason, and opens no window; +# G the default queue wait is long enough to keep the ticket's place; H a grant whose demand went stale releases +# without starting the server. set -u root=$(cd "$(dirname "$0")/.." && pwd) # the defaults work from any directory, not only the repo root tick=${1:-$root/scripts/coder-llm-ticketed} @@ -145,5 +147,22 @@ env "${envs[@]}" CODER_LLM_GH_POLL=1 CODER_LLM_GH_POLL_S=1 PATH="$scratch/ghbin: grep -q 'gh is not logged in' "$scratch/f.log" || { echo "FAIL F: no warning that gh is not logged in"; fail=1; } [ -e "$scratch/TOKEN" ] && { echo "FAIL F: a failing poll opened a window"; fail=1; } -[ "$fail" = 0 ] && echo "coder_llm_ticketed: ok (no demand -> no ticket; idle -> rc 0; busy -> 124 at max hold; fault -> 78 + back-off; FIFO: coder-llm, lane, coder-llm; a failing gh poll is logged once, no window)" +# ---- H: a ticket granted after its demand went stale releases without starting the server --------------------------- +echo busy > "$scratch/state"; rm -f "$scratch/server.pid"; : > "$scratch/klog" +env "${envs[@]}" GPU_HOLD_WAIT=60 "$hold" -t 1 -l lanehold -- sleep 7 > /dev/null 2>&1 & +lh=$!; pids+=($lh) +wait_for 10 test -e "$scratch/TOKEN" || { echo "FAIL H: the lane did not take the token"; fail=1; } +touch "$scratch/demand" # fresh when coder-llm queues, stale (TTL 3 s) by the time the lane releases +env "${envs[@]}" CODER_LLM_DEMAND_TTL_S=3 CODER_LLM_IDLE_S=60 CODER_LLM_MAX_HOLD_MIN=5 timeout 60 "$tick" -- "$scratch/fake" > "$scratch/h.log" 2>&1 +rc=$? +wait "$lh" +[ "$rc" = 0 ] || { echo "FAIL H: rc $rc for a stale-demand grant, want 0"; fail=1; } +grep -q "ticket granted with no fresh demand" "$scratch/h.log" || { echo "FAIL H: the stale grant did not say it released"; fail=1; } +[ -s "$scratch/server.pid" ] && { echo "FAIL H: the server was started for a stale grant"; fail=1; } + +# ---- G: the default queue wait keeps the ticket's place on a busy box (a give-up re-queues at the back) ------------ +sed -n 's/^QUEUE_WAIT=\${CODER_LLM_QUEUE_WAIT_S:-\([0-9]*\)}$/\1/p' "$tick" > "$scratch/qw" +[ "$(cat "$scratch/qw")" -ge 14400 ] 2> /dev/null || { echo "FAIL G: default queue wait $(cat "$scratch/qw")s; under 4 h a busy queue makes it re-queue at the back and starve"; fail=1; } + +[ "$fail" = 0 ] && echo "coder_llm_ticketed: ok (no demand -> no ticket; idle -> rc 0; busy -> 124 at max hold; fault -> 78 + back-off; FIFO: coder-llm, lane, coder-llm; a failing gh poll is logged once, no window; default queue wait keeps its place; a stale grant releases without a load)" exit "$fail"