From 01cc78fcc1a4c6fc2ae4ae0bb942113facd63429 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 06:05:35 -0400 Subject: [PATCH] window-run: leave a ticketed coder-llm alone; stopping it only drops its queued ticket (engine#2496) Seen live on 2026-11-15 at 10:00:21Z, 11 min after coder-llm was switched to coder-llm-ticketed: a lane's memory window ran `systemctl --user stop coder-llm`. The unit was only queued, 11th in the GPU FIFO and not resident. The stop threw its ticket away, and the restart after the window queues it again at the back. A ticketed unit cannot be serving during a window (the window holds the token), so there is nothing to free. window-run now checks the unit's ExecStart: when it runs coder-llm-ticketed it skips the stop (and so the restart), and the start line says "(coder-llm ticketed: not stopped)". The old always-on unit is stopped as before. Self-test arm 3i: a fake systemctl reports a ticketed ExecStart and an active unit, and a real window through a pass-through gpu-hold makes no stop call and logs the note. A mutant without the check fails both. The self-test passes. Co-Authored-By: Claude Opus 5.5 --- docs/ci-runners.md | 3 +++ scripts/window-run.sh | 21 +++++++++++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/docs/ci-runners.md b/docs/ci-runners.md index e32e494ec..12411378e 100644 --- a/docs/ci-runners.md +++ b/docs/ci-runners.md @@ -296,6 +296,9 @@ wrapper that restarts the unit from inside its own gpu-hold ticket and then poll behind that very ticket. The server can answer only after the caller's ticket is released, so such a poll must not count a silent `:8090` as "the fleet LLM did not come back". The unit's journal says `queued behind ` when it queues. With the model ticketed, a memory window no longer needs to stop it: it is not resident outside its own windows. +`scripts/window-run.sh` therefore leaves a ticketed unit alone (its window log says `coder-llm ticketed: not stopped`). +Stopping it would only drop its queued ticket. That happened on 2026-11-15 at 10:00Z, when a window stopped the +just-switched unit and its place in the queue was lost. `scripts/install-coder-llm-ticketed.sh` does the switch: diff --git a/scripts/window-run.sh b/scripts/window-run.sh index 42f564b27..111aa2fd0 100755 --- a/scripts/window-run.sh +++ b/scripts/window-run.sh @@ -158,6 +158,17 @@ JLOG chk "a window that ended while queued refuses the ticket (75)" "$?" 75 chk " ... coder-llm is never stopped" "$(grep -c 'stop coder-llm' "$tmp/systemctl.calls")" 0 chk " ... and the log holds only the other lane's end line" "$(wc -l < "$tmp/r.log")" 1 + # 3i. a TICKETED coder-llm is never stopped by a window (it is queued, not resident): the fake systemctl reports an + # ExecStart that runs coder-llm-ticketed and an active unit; the window runs for real through a pass-through + # gpu-hold, and must make no stop call and say why in its start line. + mkdir -p "$tmp/tbin" + printf '#!/bin/bash\necho "$*" >> %s/t.calls\ncase "$*" in *"show coder-llm -p ExecStart"*) echo "{ path=/x/coder-llm-ticketed ; argv[]=/x/coder-llm-ticketed -- serve }";; esac\nexit 0\n' "$tmp" > "$tmp/tbin/systemctl" + printf '#!/bin/bash\n[ "${1:-}" = --status ] && exit 0\nwhile [ $# -gt 0 ] && [ "$1" != -- ]; do shift; done; shift\nexec "$@"\n' > "$tmp/tbin/gpu-hold" + chmod +x "$tmp/tbin/systemctl" "$tmp/tbin/gpu-hold"; : > "$tmp/t.calls" + printf '%s end lane=t issue=0 label=t coder-llm=active :8090=200 rc=0\n' "$(date -u -d '90 minutes ago' +%FT%TZ)" > "$tmp/t.log" + PATH=$tmp/tbin:$PATH GPU_HOLD=$tmp/tbin/gpu-hold WINDOWS_LOG=$tmp/t.log "$0" -m 1 -l x -c true >/dev/null 2>&1 + chk "a ticketed coder-llm is never stopped by a window" "$(grep -c 'stop coder-llm' "$tmp/t.calls")" 0 + chk " ... and the window's start line says so" "$(grep -c 'coder-llm ticketed: not stopped' "$tmp/t.log")" 1 # 4. an old enough end opens the gate again (the refusal above is about the gap, not about the file) printf '%s end lane=t issue=0 label=t coder-llm=active :8090=200 rc=0\n' "$(date -u -d '90 minutes ago' +%FT%TZ)" > "$tmp/w.log" out=$(WINDOWS_LOG=$tmp/w.log "$0" --dry-run -m 5 -l x -c true 2>&1); rc=$? @@ -382,8 +393,14 @@ trap 'exit 143' TERM trap 'exit 130' INT setsid "$GPU_HOLD" -t "$MINS" -l "$LABEL" -- bash -c ' "$WR_SELF" --gap-ok || exit 75 - if systemctl --user is-active --quiet coder-llm; then systemctl --user stop coder-llm && touch "$WR_STOPPED"; fi - printf "%s start lane=%s issue=%s label=%s mins=%s cmd=%s\n" "$(date -u +%FT%TZ)" "$WR_LANE" "$WR_ISSUE" "$WR_LABEL" "$WR_MINS" "$WR_CMD" >> "$WR_LOG" + # a TICKETED coder-llm (engine#2496) is not resident outside its own gpu-hold windows, and this window holds the + # token, so it cannot be serving now; stopping it would only throw away its queued ticket (seen 2026-11-15 10:00Z: + # a window stopped it and its place in the queue was lost). Stop the old always-on unit only. + note="" + if systemctl --user show coder-llm -p ExecStart --value 2>/dev/null | grep -q coder-llm-ticketed; then + note=" (coder-llm ticketed: not stopped)" + elif systemctl --user is-active --quiet coder-llm; then systemctl --user stop coder-llm && touch "$WR_STOPPED"; fi + printf "%s start lane=%s issue=%s label=%s mins=%s cmd=%s%s\n" "$(date -u +%FT%TZ)" "$WR_LANE" "$WR_ISSUE" "$WR_LABEL" "$WR_MINS" "$WR_CMD" "$note" >> "$WR_LOG" touch "$WR_STARTED" holder=$(sed -n "s/.*pid \([0-9]*\).*/\1/p" <<<"$("$GPU_HOLD" --status 2>/dev/null | head -1)" | head -1) [ -n "$holder" ] && export GPU_HOLD_TICKET="${GPU_HOLD_TICKET:-$holder}"