Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 17 additions & 14 deletions scripts/ticketed-wait
Original file line number Diff line number Diff line change
Expand Up @@ -42,31 +42,34 @@ unit=${1:-}; url=${2:-}; limit=${3:-900}
demand=${TICKETED_WAIT_DEMAND:-$HOME/.cache/$unit/demand}
poll=${TICKETED_WAIT_POLL_S:-5}
model=${TICKETED_WAIT_MODEL:-m} # 1bit serve answers any model name with its loaded model
# $1 = the most seconds this probe may take: SECONDS is a hard bound, so a server that accepts and then hangs cannot
# stretch the wait by a probe's own timeout (review of #2590)
# One deadline for the whole call, set before the first probe, and each curl gets min(its cap, deadline - now) at the
# moment it starts: SECONDS is a hard bound (reviews of #2590 and #2593). A server that accepts and hangs, or one whose
# models answer while its completion hangs, cannot stretch the wait by a probe's own timeout, and the two curls in
# one probe cannot spend the time left twice.
start=$(date +%s); deadline=$(( start + limit ))
left() { echo $(( deadline - $(date +%s) )); }
up() {
local cap=${1:-30} m1 m2
m1=$(( cap < 5 ? cap : 5 )); m2=$(( cap < 30 ? cap : 30 ))
[ "$m1" -ge 1 ] || return 1
[ "$(curl -s -o /dev/null -m "$m1" -w '%{http_code}' "${url%/}/v1/models" 2> /dev/null)" = 200 ] || return 1
[ "$(curl -s -o /dev/null -m "$m2" -w '%{http_code}' -H 'Content-Type: application/json' \
local m
m=$(left); m=$(( m < 5 ? m : 5 )); [ "$m" -ge 1 ] || return 1
[ "$(curl -s -o /dev/null -m "$m" -w '%{http_code}' "${url%/}/v1/models" 2> /dev/null)" = 200 ] || return 1
m=$(left); m=$(( m < 30 ? m : 30 )); [ "$m" -ge 1 ] || return 1
[ "$(curl -s -o /dev/null -m "$m" -w '%{http_code}' -H 'Content-Type: application/json' \
-d '{"model":"'"$model"'","messages":[{"role":"user","content":"ok"}],"max_tokens":1,"temperature":0}' \
"${url%/}/v1/chat/completions" 2> /dev/null)" = 200 ]
}
up "$(( limit < 30 ? (limit < 1 ? 1 : limit) : 30 ))" && exit 0
up && exit 0
mkdir -p "$(dirname "$demand")" && touch "$demand" || { echo "ticketed-wait: cannot touch $demand" >&2; exit 2; }
start=$(date +%s); touched=$start
touched=$(date +%s)
echo "ticketed-wait: asked for $unit (touched $demand); waiting up to ${limit}s for $url; GPU token: $(head -c 80 "${GPU_TOKEN_FILE:-$HOME/GPU-TOKEN}" 2> /dev/null | head -1)" >&2
while :; do
now=$(date +%s)
if [ $(( now - start )) -ge "$limit" ]; then
rem=$(left)
if [ "$rem" -le 0 ]; then
echo "ticketed-wait: $unit did not answer in ${limit}s (still queued for the GPU?): not run; demand withdrawn" >&2
rm -f "$demand"
exit 75
fi
now=$(date +%s)
[ $(( now - touched )) -ge 30 ] && { touch "$demand"; touched=$now; }
rem=$(( limit - (now - start) ))
sleep "$(( poll < rem ? poll : rem ))"
rem=$(( limit - ($(date +%s) - start) ))
[ "$rem" -ge 1 ] && up "$rem" && { echo "ticketed-wait: $unit answers after $(( $(date +%s) - start ))s" >&2; touch "$demand"; exit 0; }
up && { echo "ticketed-wait: $unit answers after $(( $(date +%s) - start ))s" >&2; touch "$demand"; exit 0; }
done
34 changes: 26 additions & 8 deletions tests/coder_llm_ticketed_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,10 @@
# N ticketed-wait on a model that never answers gives up with 75 after its limit and WITHDRAWS the demand it made,
# so the unit does not take a ticket for a caller that has gone (review of #2588);
# O ticketed-wait on a model that already answers returns 0 at once and creates no demand at all.
# Q SECONDS is a hard bound (review of #2590): against a port that accepts and never answers (each probe would wait for
# its own curl timeout), a 3 s limit still ends in 75 within a few seconds.
# Q SECONDS is a hard bound (reviews of #2590, #2593): against a port that accepts and never answers, a limit ends in 75
# within limit + 1 s (3 and 10 s);
# R the same for the half-hung shape: /v1/models answers 200 and the completion hangs (the probe's two curls must not
# each spend the time left), 3 and 10 s.
# P readiness, not liveness (review of #2588): a server that lists /v1/models (200 from bind) but cannot complete yet
# (503 while "loading") does not count as answering: ticketed-wait waits, and returns 0 only once a 1-token
# completion is answered.
Expand Down Expand Up @@ -80,6 +82,8 @@ class H(http.server.BaseHTTPRequestHandler):
# a completion answers only once "loaded": a $scratch/loading file makes it 503 (weights still loading), while
# GET /v1/models above keeps answering 200 from bind, as 1bit serve's static listing does
self.rfile.read(int(self.headers.get("Content-Length", 0)))
if os.path.exists("$scratch/hang"): # the half-hung shape: models answer, the completion never does
import time; time.sleep(60); return
if os.path.exists("$scratch/loading"):
self.send_response(503); self.end_headers(); return
body = json.dumps({"choices": [{"message": {"role": "assistant", "content": "ok"}}]}).encode()
Expand Down Expand Up @@ -305,16 +309,30 @@ kill "$pf" 2> /dev/null; wait "$pf" 2> /dev/null; rm -f "$scratch/loading"
# ---- Q: the limit holds against a server that accepts and hangs -------------------------------------------------------
hang=$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",0)); print(s.getsockname()[1])')
python3 -c "import socket,time; s=socket.socket(); s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1); s.bind(('127.0.0.1',$hang)); s.listen(8); time.sleep(120)" & hq=$!; pids+=($hq)
sleep 1; t0=$(date +%s)
env TICKETED_WAIT_DEMAND="$scratch/q.demand" TICKETED_WAIT_POLL_S=1 "$waiter" test-unit "http://127.0.0.1:$hang" 3 2> "$scratch/q.wait"; rc=$?
el=$(( $(date +%s) - t0 ))
[ "$rc" = 75 ] || { echo "FAIL Q: rc $rc against a hanging server, want 75"; fail=1; }
[ "$el" -le 7 ] || { echo "FAIL Q: a 3 s limit took ${el}s against a server that accepts and hangs (the probe's own timeout stretched it)"; fail=1; }
sleep 1
bound() { # $1 arm, $2 url, $3 limit, $4 poll: rc 75 within limit + 1 s
local t0 rc el
t0=$(date +%s)
env TICKETED_WAIT_DEMAND="$scratch/$1.demand" TICKETED_WAIT_POLL_S="$4" "$waiter" test-unit "$2" "$3" 2> "$scratch/$1.wait"; rc=$?
el=$(( $(date +%s) - t0 ))
[ "$rc" = 75 ] || { echo "FAIL $1: rc $rc for a ${3} s limit, want 75"; fail=1; }
[ "$el" -le $(( $3 + 1 )) ] || { echo "FAIL $1: a ${3} s limit took ${el}s (a probe's own timeout stretched it)"; fail=1; }
}
bound Q "http://127.0.0.1:$hang" 3 1
bound Q "http://127.0.0.1:$hang" 10 5
kill "$hq" 2> /dev/null; wait "$hq" 2> /dev/null

# ---- R: the half-hung shape: models 200, the completion hangs ------------------------------------------------------
echo idle > "$scratch/state"; touch "$scratch/hang"
"$scratch/fake" > /dev/null 2>&1 & rf=$!; pids+=($rf)
wait_for 10 curl -sf -o /dev/null "http://127.0.0.1:$port/v1/models" || { echo "FAIL R: the fake did not come up"; fail=1; }
bound R "http://127.0.0.1:$port" 3 1
bound R "http://127.0.0.1:$port" 10 5
kill "$rf" 2> /dev/null; wait "$rf" 2> /dev/null; rm -f "$scratch/hang"

# ---- G: the default queue wait keeps the ticket's place on a busy box (a give-up re-queues at the back) ------------
sed -n 's/^QUEUE_WAIT=\${CODER_LLM_QUEUE_WAIT_S:-\([0-9]*\)}$/\1/p' "$tick" > "$scratch/qw"
[ "$(cat "$scratch/qw")" -ge 14400 ] 2> /dev/null || { echo "FAIL G: default queue wait $(cat "$scratch/qw")s; under 4 h a busy queue makes it re-queue at the back and starve"; fail=1; }

[ "$fail" = 0 ] && echo "coder_llm_ticketed: ok (no demand -> no ticket; idle -> rc 0; busy -> 124 at max hold; fault -> 78 + back-off; FIFO: coder-llm, lane, coder-llm; a failing gh poll is logged once, no window; default queue wait keeps its place; a stale grant releases without a load; a slow-exiting server gets its grace, a KILL after the max hold is 124, one after a queue wait stays 137; a refused grant is told from a fault during a window; a window takes its label; ticketed-wait asks, waits, gives up with 75 and withdraws its demand, and asks for nothing when the model is up; bound-but-loading is not an answer; the limit is hard)"
[ "$fail" = 0 ] && echo "coder_llm_ticketed: ok (no demand -> no ticket; idle -> rc 0; busy -> 124 at max hold; fault -> 78 + back-off; FIFO: coder-llm, lane, coder-llm; a failing gh poll is logged once, no window; default queue wait keeps its place; a stale grant releases without a load; a slow-exiting server gets its grace, a KILL after the max hold is 124, one after a queue wait stays 137; a refused grant is told from a fault during a window; a window takes its label; ticketed-wait asks, waits, gives up with 75 and withdraws its demand, and asks for nothing when the model is up; bound-but-loading is not an answer; the limit is hard, half-hung included)"
exit "$fail"
Loading