From ce42367682e522d34774b299c16c35e4942aa17b Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 19:49:23 -0400 Subject: [PATCH] ticketed-wait: one deadline for the whole call; each curl gets only the time that is left The note on #2593 (23:25Z, after four READYs): SECONDS was still not a hard bound. The first probe ran before the clock started, and the probe's two curls were each capped by the time left but run one after the other. Against a half-hung server (models 200, the completion hangs) a 40 s limit took 70 s. Now a single deadline is set before the first probe, and each curl gets min(its cap, deadline - now) when it starts. Tests: arm Q (a socket that accepts and never answers) now requires rc 75 within limit + 1 s, at 3 s and 10 s. New arm R does the same for the half-hung shape (the fake's completion hangs while a "hang" file exists). main's ticketed-wait fails all four: 3 s took 6 s and 10 s took 15 s (Q), 3 s took 6 s and 10 s took 20 s (R), matching the reviewer's numbers. Co-Authored-By: Claude Opus 5.5 --- scripts/ticketed-wait | 31 ++++++++++++++++------------- tests/coder_llm_ticketed_test.sh | 34 ++++++++++++++++++++++++-------- 2 files changed, 43 insertions(+), 22 deletions(-) diff --git a/scripts/ticketed-wait b/scripts/ticketed-wait index 92c705f06..badca4fa6 100755 --- a/scripts/ticketed-wait +++ b/scripts/ticketed-wait @@ -42,31 +42,34 @@ unit=${1:-}; url=${2:-}; limit=${3:-900} demand=${TICKETED_WAIT_DEMAND:-$HOME/.cache/$unit/demand} poll=${TICKETED_WAIT_POLL_S:-5} model=${TICKETED_WAIT_MODEL:-m} # 1bit serve answers any model name with its loaded model -# $1 = the most seconds this probe may take: SECONDS is a hard bound, so a server that accepts and then hangs cannot -# stretch the wait by a probe's own timeout (review of #2590) +# One deadline for the whole call, set before the first probe, and each curl gets min(its cap, deadline - now) at the +# moment it starts: SECONDS is a hard bound (reviews of #2590 and #2593). A server that accepts and hangs, or one whose +# models answer while its completion hangs, cannot stretch the wait by a probe's own timeout, and the two curls in +# one probe cannot spend the time left twice. +start=$(date +%s); deadline=$(( start + limit )) +left() { echo $(( deadline - $(date +%s) )); } up() { - local cap=${1:-30} m1 m2 - m1=$(( cap < 5 ? cap : 5 )); m2=$(( cap < 30 ? cap : 30 )) - [ "$m1" -ge 1 ] || return 1 - [ "$(curl -s -o /dev/null -m "$m1" -w '%{http_code}' "${url%/}/v1/models" 2> /dev/null)" = 200 ] || return 1 - [ "$(curl -s -o /dev/null -m "$m2" -w '%{http_code}' -H 'Content-Type: application/json' \ + local m + m=$(left); m=$(( m < 5 ? m : 5 )); [ "$m" -ge 1 ] || return 1 + [ "$(curl -s -o /dev/null -m "$m" -w '%{http_code}' "${url%/}/v1/models" 2> /dev/null)" = 200 ] || return 1 + m=$(left); m=$(( m < 30 ? m : 30 )); [ "$m" -ge 1 ] || return 1 + [ "$(curl -s -o /dev/null -m "$m" -w '%{http_code}' -H 'Content-Type: application/json' \ -d '{"model":"'"$model"'","messages":[{"role":"user","content":"ok"}],"max_tokens":1,"temperature":0}' \ "${url%/}/v1/chat/completions" 2> /dev/null)" = 200 ] } -up "$(( limit < 30 ? (limit < 1 ? 1 : limit) : 30 ))" && exit 0 +up && exit 0 mkdir -p "$(dirname "$demand")" && touch "$demand" || { echo "ticketed-wait: cannot touch $demand" >&2; exit 2; } -start=$(date +%s); touched=$start +touched=$(date +%s) echo "ticketed-wait: asked for $unit (touched $demand); waiting up to ${limit}s for $url; GPU token: $(head -c 80 "${GPU_TOKEN_FILE:-$HOME/GPU-TOKEN}" 2> /dev/null | head -1)" >&2 while :; do - now=$(date +%s) - if [ $(( now - start )) -ge "$limit" ]; then + rem=$(left) + if [ "$rem" -le 0 ]; then echo "ticketed-wait: $unit did not answer in ${limit}s (still queued for the GPU?): not run; demand withdrawn" >&2 rm -f "$demand" exit 75 fi + now=$(date +%s) [ $(( now - touched )) -ge 30 ] && { touch "$demand"; touched=$now; } - rem=$(( limit - (now - start) )) sleep "$(( poll < rem ? poll : rem ))" - rem=$(( limit - ($(date +%s) - start) )) - [ "$rem" -ge 1 ] && up "$rem" && { echo "ticketed-wait: $unit answers after $(( $(date +%s) - start ))s" >&2; touch "$demand"; exit 0; } + up && { echo "ticketed-wait: $unit answers after $(( $(date +%s) - start ))s" >&2; touch "$demand"; exit 0; } done diff --git a/tests/coder_llm_ticketed_test.sh b/tests/coder_llm_ticketed_test.sh index a9e98d6f7..f7d69c82d 100755 --- a/tests/coder_llm_ticketed_test.sh +++ b/tests/coder_llm_ticketed_test.sh @@ -46,8 +46,10 @@ # N ticketed-wait on a model that never answers gives up with 75 after its limit and WITHDRAWS the demand it made, # so the unit does not take a ticket for a caller that has gone (review of #2588); # O ticketed-wait on a model that already answers returns 0 at once and creates no demand at all. -# Q SECONDS is a hard bound (review of #2590): against a port that accepts and never answers (each probe would wait for -# its own curl timeout), a 3 s limit still ends in 75 within a few seconds. +# Q SECONDS is a hard bound (reviews of #2590, #2593): against a port that accepts and never answers, a limit ends in 75 +# within limit + 1 s (3 and 10 s); +# R the same for the half-hung shape: /v1/models answers 200 and the completion hangs (the probe's two curls must not +# each spend the time left), 3 and 10 s. # P readiness, not liveness (review of #2588): a server that lists /v1/models (200 from bind) but cannot complete yet # (503 while "loading") does not count as answering: ticketed-wait waits, and returns 0 only once a 1-token # completion is answered. @@ -80,6 +82,8 @@ class H(http.server.BaseHTTPRequestHandler): # a completion answers only once "loaded": a $scratch/loading file makes it 503 (weights still loading), while # GET /v1/models above keeps answering 200 from bind, as 1bit serve's static listing does self.rfile.read(int(self.headers.get("Content-Length", 0))) + if os.path.exists("$scratch/hang"): # the half-hung shape: models answer, the completion never does + import time; time.sleep(60); return if os.path.exists("$scratch/loading"): self.send_response(503); self.end_headers(); return body = json.dumps({"choices": [{"message": {"role": "assistant", "content": "ok"}}]}).encode() @@ -305,16 +309,30 @@ kill "$pf" 2> /dev/null; wait "$pf" 2> /dev/null; rm -f "$scratch/loading" # ---- Q: the limit holds against a server that accepts and hangs ------------------------------------------------------- hang=$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",0)); print(s.getsockname()[1])') python3 -c "import socket,time; s=socket.socket(); s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1); s.bind(('127.0.0.1',$hang)); s.listen(8); time.sleep(120)" & hq=$!; pids+=($hq) -sleep 1; t0=$(date +%s) -env TICKETED_WAIT_DEMAND="$scratch/q.demand" TICKETED_WAIT_POLL_S=1 "$waiter" test-unit "http://127.0.0.1:$hang" 3 2> "$scratch/q.wait"; rc=$? -el=$(( $(date +%s) - t0 )) -[ "$rc" = 75 ] || { echo "FAIL Q: rc $rc against a hanging server, want 75"; fail=1; } -[ "$el" -le 7 ] || { echo "FAIL Q: a 3 s limit took ${el}s against a server that accepts and hangs (the probe's own timeout stretched it)"; fail=1; } +sleep 1 +bound() { # $1 arm, $2 url, $3 limit, $4 poll: rc 75 within limit + 1 s + local t0 rc el + t0=$(date +%s) + env TICKETED_WAIT_DEMAND="$scratch/$1.demand" TICKETED_WAIT_POLL_S="$4" "$waiter" test-unit "$2" "$3" 2> "$scratch/$1.wait"; rc=$? + el=$(( $(date +%s) - t0 )) + [ "$rc" = 75 ] || { echo "FAIL $1: rc $rc for a ${3} s limit, want 75"; fail=1; } + [ "$el" -le $(( $3 + 1 )) ] || { echo "FAIL $1: a ${3} s limit took ${el}s (a probe's own timeout stretched it)"; fail=1; } +} +bound Q "http://127.0.0.1:$hang" 3 1 +bound Q "http://127.0.0.1:$hang" 10 5 kill "$hq" 2> /dev/null; wait "$hq" 2> /dev/null +# ---- R: the half-hung shape: models 200, the completion hangs ------------------------------------------------------ +echo idle > "$scratch/state"; touch "$scratch/hang" +"$scratch/fake" > /dev/null 2>&1 & rf=$!; pids+=($rf) +wait_for 10 curl -sf -o /dev/null "http://127.0.0.1:$port/v1/models" || { echo "FAIL R: the fake did not come up"; fail=1; } +bound R "http://127.0.0.1:$port" 3 1 +bound R "http://127.0.0.1:$port" 10 5 +kill "$rf" 2> /dev/null; wait "$rf" 2> /dev/null; rm -f "$scratch/hang" + # ---- G: the default queue wait keeps the ticket's place on a busy box (a give-up re-queues at the back) ------------ sed -n 's/^QUEUE_WAIT=\${CODER_LLM_QUEUE_WAIT_S:-\([0-9]*\)}$/\1/p' "$tick" > "$scratch/qw" [ "$(cat "$scratch/qw")" -ge 14400 ] 2> /dev/null || { echo "FAIL G: default queue wait $(cat "$scratch/qw")s; under 4 h a busy queue makes it re-queue at the back and starve"; fail=1; } -[ "$fail" = 0 ] && echo "coder_llm_ticketed: ok (no demand -> no ticket; idle -> rc 0; busy -> 124 at max hold; fault -> 78 + back-off; FIFO: coder-llm, lane, coder-llm; a failing gh poll is logged once, no window; default queue wait keeps its place; a stale grant releases without a load; a slow-exiting server gets its grace, a KILL after the max hold is 124, one after a queue wait stays 137; a refused grant is told from a fault during a window; a window takes its label; ticketed-wait asks, waits, gives up with 75 and withdraws its demand, and asks for nothing when the model is up; bound-but-loading is not an answer; the limit is hard)" +[ "$fail" = 0 ] && echo "coder_llm_ticketed: ok (no demand -> no ticket; idle -> rc 0; busy -> 124 at max hold; fault -> 78 + back-off; FIFO: coder-llm, lane, coder-llm; a failing gh poll is logged once, no window; default queue wait keeps its place; a stale grant releases without a load; a slow-exiting server gets its grace, a KILL after the max hold is 124, one after a queue wait stays 137; a refused grant is told from a fault during a window; a window takes its label; ticketed-wait asks, waits, gives up with 75 and withdraws its demand, and asks for nothing when the model is up; bound-but-loading is not an answer; the limit is hard, half-hung included)" exit "$fail"