From e86991a13fda36243027b9e3e67e1c089809ba6a Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 02:03:28 -0400 Subject: [PATCH 1/5] coder-llm-ticketed: the PR-Agent model runs only inside gpu-hold tickets, in on-demand windows (engine#2494) Owner, 2026-11-15 (CODER-LLM-TICKETED, option b). coder-llm served Qwen3-Coder-30B on HRX around the clock, outside any gpu-hold ticket, so "one GPU job at a time" never held. Its model is also the one whose prefill faults the GPU (engine#2494). scripts/coder-llm-ticketed wraps the unchanged server command: - it waits for demand: ~/.cache/coder-llm/demand touched within 30 min, or, with CODER_LLM_GH_POLL=1, a queued or in-progress pr-agent run; - it queues with `gpu-hold -l coder-llm -t 25`, so FIFO and every gpu-hold watchdog apply, the kernel-fault abort included; - inside the ticket it serves, and closes the window when /slots shows nothing processing and demand has gone stale for 5 min, or at the max hold; - it exits, and the unit's Restart=always re-queues it. After rc 78 it backs off 10 min first. tests/coder_llm_ticketed_test.sh (ctest coder_llm_ticketed) uses the real scripts/gpu-hold on a private queue, a kernel-log fixture and a fake /slots server. Arms: no demand means no ticket; idle means rc 0, with the server gone and the token released; busy means rc 124 at the max hold; a gfxhub fault means rc 78 plus the back-off; a lane queued during a window runs before the next window (coder-llm, lane, coder-llm). Mutants fail it: running the window without gpu-hold, and dropping the back-off. docs/ci-runners.md: a new section on the windows, the demand file, and the unit drop-in. The live unit and the pr-agent workflow change (which needs the workflow scope) are not in this commit. Co-Authored-By: Claude Opus 5.5 --- cmake/tests-gpu-hold.cmake | 6 ++ docs/ci-runners.md | 34 +++++++ scripts/coder-llm-ticketed | 152 +++++++++++++++++++++++++++++++ tests/coder_llm_ticketed_test.sh | 135 +++++++++++++++++++++++++++ 4 files changed, 327 insertions(+) create mode 100755 scripts/coder-llm-ticketed create mode 100755 tests/coder_llm_ticketed_test.sh diff --git a/cmake/tests-gpu-hold.cmake b/cmake/tests-gpu-hold.cmake index d0418a3be..f850ba18d 100644 --- a/cmake/tests-gpu-hold.cmake +++ b/cmake/tests-gpu-hold.cmake @@ -38,6 +38,12 @@ add_test(NAME gpu_hold_mem_abort COMMAND bash ${CMAKE_SOURCE_DIR}/tests/gpu_hold # when the token is taken; the 11-13 crash was nine resets inside two tickets. The kernel log is a fixture file. add_test(NAME gpu_hold_fault_abort COMMAND bash ${CMAKE_SOURCE_DIR}/tests/gpu_hold_fault_abort_test.sh ${CMAKE_SOURCE_DIR}/scripts/gpu-hold) +# CODER-LLM-TICKETED 2026-11-15 (engine#2494): the PR-Agent model (coder-llm, :8090) runs only inside a gpu-hold ticket, +# in on-demand windows. Drives scripts/coder-llm-ticketed with the real scripts/gpu-hold on a private queue and a fake +# /slots server: no demand -> no ticket, idle -> window closes, busy -> max hold, kernel fault -> rc 78, FIFO with a lane. +add_test(NAME coder_llm_ticketed COMMAND bash ${CMAKE_SOURCE_DIR}/tests/coder_llm_ticketed_test.sh + ${CMAKE_SOURCE_DIR}/scripts/coder-llm-ticketed ${CMAKE_SOURCE_DIR}/scripts/gpu-hold) + # engine#1635: a queued ticket that is removed (reaped, or deleted by another lane) used to leave its owner waiting for a # queue position that no longer existed -- the full -w, 5400 s by default, looking exactly like "queued behind a busy box". # Drives scripts/gpu-hold on a private queue: the ticket is deleted mid-wait, and separately rewritten to a dead pid so a diff --git a/docs/ci-runners.md b/docs/ci-runners.md index 107760817..5fb77a18e 100644 --- a/docs/ci-runners.md +++ b/docs/ci-runners.md @@ -269,6 +269,40 @@ allocation failed as an exception rather than as an OOM kill, so that experiment directive is the documented mechanism and it is what the fresh install's single end-to-end run should confirm, together with the rest of #1504's acceptance. +## The PR-Agent model runs in gpu-hold windows (engine#2494) + +The `pr-agent` job talks to a local model, Qwen3-Coder-30B-A3B served by `1bit serve` on HRX at `127.0.0.1:8090` (the +`coder-llm` user service). Until 2026-11-15 that server ran around the clock **outside any gpu-hold ticket**, so "one GPU +job at a time" never held, and its model is the one whose prefill faults the GPU (engine#2494). It now runs only inside a +ticket, in on-demand windows, through `scripts/coder-llm-ticketed` (deployed to `~/.local/bin`): + +1. **It waits for demand.** Demand is the file `~/.cache/coder-llm/demand` touched within the last 30 minutes, or + (`CODER_LLM_GH_POLL=1`) a `pr-agent` workflow run that is queued or in progress. +2. **It queues for the GPU like any other job**: `gpu-hold -l coder-llm -t 25`. The FIFO queue, the temperature, memory + and kernel-fault watchdogs, and the release wait that lets the next label in all apply. +3. **Inside the ticket it serves**, and ends the window when no slot in `/slots` has been processing **and** the demand + file has not been touched for 5 minutes, or when the 25-minute max hold ends it. +4. **It exits**, and the unit's `Restart=always` starts the next invocation, which waits for demand and queues again at + the back of the line. After a kernel fault (gpu-hold rc 78) it backs off 10 minutes first. + +Anything else that needs the model asks the same way: touch `~/.cache/coder-llm/demand`, keep touching it while it +works, and wait for `:8090` to answer a real completion. Until a window opens, the port refuses. + +The unit's drop-in runs the same server command as before, behind the wrapper: + +```ini +[Service] +ExecStart= +ExecStart=%h/.local/bin/coder-llm-ticketed -- +Environment=CODER_LLM_GH_POLL=1 +Restart=always +RestartSec=15 +TimeoutStopSec=90 +``` + +`tests/coder_llm_ticketed_test.sh` (ctest `coder_llm_ticketed`) drives the wrapper with the real `scripts/gpu-hold` on a +private queue and a fake `/slots` server. No device is used. + ## What a restart policy cannot do It cannot notice that a runner is *not* coming back (a version mismatch, a revoked token, a full disk under diff --git a/scripts/coder-llm-ticketed b/scripts/coder-llm-ticketed new file mode 100755 index 000000000..fe6054fba --- /dev/null +++ b/scripts/coder-llm-ticketed @@ -0,0 +1,152 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# coder-llm-ticketed [--] SERVER_CMD [ARGS...] +# +# Runs the PR-Agent reviewer model (the coder-llm user service, :8090) ONLY inside a gpu-hold ticket, in on-demand +# windows (owner 2026-11-15, CODER-LLM-TICKETED; engine#2494). It used to run 24/7 outside any ticket, so "one GPU job +# at a time" never held, and the model it serves (Qwen3-Coder-30B on HRX) is the one whose prefill faults the GPU. +# +# One invocation is ONE window, and systemd's Restart= makes the next one: +# 1. wait for demand: the demand file was touched within CODER_LLM_DEMAND_TTL_MIN, or (CODER_LLM_GH_POLL=1) a +# pr-agent workflow run is queued or in progress; +# 2. queue for the GPU like any other job: gpu-hold -l coder-llm -t CODER_LLM_MAX_HOLD_MIN (FIFO; gpu-hold's +# temperature, memory and kernel-fault watchdogs apply, and its release wait lets the next label in); +# 3. inside the ticket, start SERVER_CMD and end the window once the server has been idle (no slot in /slots is +# processing) AND the demand file has not been touched for CODER_LLM_IDLE_MIN - or when the max hold ends it; +# 4. exit, so systemd starts the next invocation, which goes back to waiting for demand and re-queues. +# A window cut short by a kernel fault (gpu-hold rc 78) or a refused ticket backs off CODER_LLM_FAULT_BACKOFF_S +# before exiting, so a faulting GPU does not turn into a restart loop. +# +# Anything else that needs the model asks for it the same way the workflow does: touch "$CODER_LLM_DEMAND" (and +# keep touching it while working), then wait for :8090 to answer. +# +# Environment (all optional): +# CODER_LLM_DEMAND demand file, default ~/.cache/coder-llm/demand +# CODER_LLM_DEMAND_TTL_MIN a touch this recent is demand, default 30 (CODER_LLM_DEMAND_TTL_S in seconds wins) +# CODER_LLM_IDLE_MIN idle minutes that end a window, default 5 (CODER_LLM_IDLE_S in seconds wins) +# CODER_LLM_MAX_HOLD_MIN max hold per window (gpu-hold -t), default 25 +# CODER_LLM_QUEUE_WAIT_S how long one window queues for the GPU (gpu-hold -w), default 5400 +# CODER_LLM_URL the server's base URL for /slots, default http://127.0.0.1:8090 +# CODER_LLM_POLL_S poll interval, default 15 +# CODER_LLM_GH_POLL 1 = also treat a queued/in-progress pr-agent run as demand (REST, every +# CODER_LLM_GH_POLL_S, default 60), default 0 +# CODER_LLM_GH_REPO default 1bit-MONSTER/engine +# CODER_LLM_FAULT_BACKOFF_S back-off after gpu-hold rc 78 (a fault, or a refused ticket), default 600 +# CODER_LLM_GPU_HOLD gpu-hold to use, default the one on PATH +# CODER_LLM_ALWAYS=1 skip the demand wait (a resident window, still ticketed and still max-hold bounded) +# +# Internal: "coder-llm-ticketed --window SERVER_CMD..." is what gpu-hold runs; it is the idle-aware window itself. +set -u + +DEMAND=${CODER_LLM_DEMAND:-$HOME/.cache/coder-llm/demand} +DEMAND_TTL=${CODER_LLM_DEMAND_TTL_S:-$(( ${CODER_LLM_DEMAND_TTL_MIN:-30} * 60 ))} +IDLE=${CODER_LLM_IDLE_S:-$(( ${CODER_LLM_IDLE_MIN:-5} * 60 ))} +MAX_HOLD=${CODER_LLM_MAX_HOLD_MIN:-25} +QUEUE_WAIT=${CODER_LLM_QUEUE_WAIT_S:-5400} +URL=${CODER_LLM_URL:-http://127.0.0.1:8090} +POLL=${CODER_LLM_POLL_S:-15} +GH_POLL=${CODER_LLM_GH_POLL:-0} +GH_POLL_S=${CODER_LLM_GH_POLL_S:-60} +GH_REPO=${CODER_LLM_GH_REPO:-1bit-MONSTER/engine} +BACKOFF=${CODER_LLM_FAULT_BACKOFF_S:-600} +GPU_HOLD=${CODER_LLM_GPU_HOLD:-gpu-hold} + +log() { echo "coder-llm-ticketed[$$] $*"; } + +mtime() { stat -c %Y "$1" 2> /dev/null || echo 0; } + +demand_fresh() { # $1 = window in seconds + [ $(( $(date +%s) - $(mtime "$DEMAND") )) -lt "$1" ] +} + +gh_demand() { + [ "$GH_POLL" = 1 ] || return 1 + local s n + for s in queued in_progress; do + n=$(gh api "repos/$GH_REPO/actions/workflows/pr-agent.yml/runs?status=$s&per_page=1" -q .total_count 2> /dev/null) + [ "${n:-0}" -gt 0 ] 2> /dev/null && return 0 + done + return 1 +} + +slots_busy() { # 0 = some slot is processing, or the server cannot say (not up yet counts as busy) + local j + j=$(curl -sf --max-time 5 "$URL/slots" 2> /dev/null) || return 0 + grep -Eq '"is_processing"[[:space:]]*:[[:space:]]*true' <<< "$j" && return 0 + return 1 +} + +# ---- the window: runs INSIDE the gpu-hold ticket -------------------------------------------------------------------- +if [ "${1:-}" = "--window" ]; then + shift + [ "${1:-}" = "--" ] && shift + [ $# -gt 0 ] || { log "--window needs a server command"; exit 2; } + "$@" & + server=$! + stop_server() { + kill -TERM "$server" 2> /dev/null + for _ in $(seq 1 30); do kill -0 "$server" 2> /dev/null || return; sleep 1; done + kill -KILL "$server" 2> /dev/null + } + trap 'stop_server; exit 143' TERM INT HUP + log "window open: server pid $server" + last_busy=$(date +%s) + while kill -0 "$server" 2> /dev/null; do + sleep "$POLL" + now=$(date +%s) + if slots_busy || demand_fresh "$IDLE"; then + last_busy=$now + elif [ $(( now - last_busy )) -ge "$IDLE" ]; then + log "idle ${IDLE}s and no fresh demand: closing the window" + stop_server + exit 0 + fi + done + wait "$server" + rc=$? + log "server exited by itself, rc $rc" + exit "$rc" +fi + +# ---- one service invocation: wait for demand, then one ticketed window ---------------------------------------------- +[ "${1:-}" = "--" ] && shift +[ $# -gt 0 ] || { echo "usage: coder-llm-ticketed [--] SERVER_CMD [ARGS...]" >&2; exit 2; } +mkdir -p "$(dirname "$DEMAND")" + +if [ "${CODER_LLM_ALWAYS:-0}" != 1 ]; then + log "waiting for demand ($DEMAND touched within ${DEMAND_TTL}s$([ "$GH_POLL" = 1 ] && echo ", or a pr-agent run on $GH_REPO"))" + next_gh=0 + while :; do + demand_fresh "$DEMAND_TTL" && break + if [ "$GH_POLL" = 1 ] && [ "$(date +%s)" -ge "$next_gh" ]; then + gh_demand && { touch "$DEMAND"; break; } + next_gh=$(( $(date +%s) + GH_POLL_S )) + fi + sleep "$POLL" + done +fi +log "demand: queueing for the GPU (gpu-hold -l coder-llm -t $MAX_HOLD -w $QUEUE_WAIT)" +"$GPU_HOLD" -t "$MAX_HOLD" -l coder-llm -w "$QUEUE_WAIT" -- "$0" --window "$@" +rc=$? +case "$rc" in + 0) log "window closed (idle)" ;; + 124) log "window closed (max hold ${MAX_HOLD} min); re-queueing" ;; + 75) log "the queue never came round in ${QUEUE_WAIT}s" ;; + 78) log "gpu-hold rc 78 (a GPU fault in the kernel log, or a refused ticket): backing off ${BACKOFF}s"; sleep "$BACKOFF" ;; + *) log "window ended rc $rc" ;; +esac +exit "$rc" diff --git a/tests/coder_llm_ticketed_test.sh b/tests/coder_llm_ticketed_test.sh new file mode 100755 index 000000000..bbca31080 --- /dev/null +++ b/tests/coder_llm_ticketed_test.sh @@ -0,0 +1,135 @@ +#!/bin/bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# tests/coder_llm_ticketed_test.sh — the PR-Agent model runs only inside a gpu-hold ticket, in on-demand windows +# (CODER-LLM-TICKETED, 2026-11-15; engine#2494). No GPU: the real scripts/gpu-hold with a private queue, token, die +# temperature and kernel-log fixture, and a fake server whose /slots says busy or idle from a state file. +# +# Usage: coder_llm_ticketed_test.sh [path-to-coder-llm-ticketed] [path-to-gpu-hold] +# +# Arms: +# A no demand: no window opens, no token is taken; +# B demand opens a window under the label coder-llm; an idle server and stale demand close it (rc 0), the server is +# gone and the token released; +# C a busy server is cut at the max hold (rc 124), the server is gone; +# D a gfxhub page fault in the kernel log during the window ends it (rc 78) and the server dies with it; +# E a lane that queues during a window gets the GPU before the next coder-llm window (FIFO; the restart re-queues). +set -u +tick=${1:-./scripts/coder-llm-ticketed} +hold=${2:-./scripts/gpu-hold} +for f in "$tick" "$hold"; do [ -x "$f" ] || { echo "not executable: $f"; exit 2; }; done +tick=$(readlink -f "$tick"); hold=$(readlink -f "$hold") + +scratch=$(mktemp -d) +pids=() +cleanup() { for p in "${pids[@]}"; do kill -9 "$p" 2> /dev/null; done; pkill -9 -f "$scratch/fake" 2> /dev/null; rm -rf "$scratch"; } +trap cleanup EXIT +fail=0 +echo 40000 > "$scratch/tctl" +port=$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",0)); print(s.getsockname()[1])') + +cat > "$scratch/fake" < "$scratch/klog" + +wait_for() { # $1 seconds, $2.. a test command + local n=$1; shift + for _ in $(seq 1 "$n"); do "$@" && return 0; sleep 1; done + return 1 +} +server_alive() { [ -s "$scratch/server.pid" ] && kill -0 "$(cat "$scratch/server.pid")" 2> /dev/null; } +token_is() { [ "$(cat "$scratch/TOKEN" 2> /dev/null)" = "$1" ] || grep -qs "^$1" "$scratch/TOKEN"; } + +# ---- A + B: no demand, then demand; idle closes the window ---------------------------------------------------------- +echo idle > "$scratch/state"; rm -f "$scratch/server.pid" "$scratch/demand" +touch -d '-10 min' "$scratch/demand" # stale demand +env "${envs[@]}" CODER_LLM_IDLE_S=4 CODER_LLM_MAX_HOLD_MIN=5 timeout 120 "$tick" -- "$scratch/fake" > "$scratch/ab.log" 2>&1 & +ab=$!; pids+=($ab) +sleep 6 +if [ -e "$scratch/TOKEN" ] || server_alive; then echo "FAIL A: a window opened with only stale demand"; fail=1; fi +touch "$scratch/demand" +if ! wait_for 15 server_alive; then echo "FAIL B: demand did not open a window"; fail=1; fi +grep -qs coder-llm "$scratch/TOKEN" || { echo "FAIL B: the window does not hold the token as coder-llm ($(cat "$scratch/TOKEN" 2> /dev/null))"; fail=1; } +srv=$(cat "$scratch/server.pid" 2> /dev/null) +wait "$ab"; rc=$? +[ "$rc" = 0 ] || { echo "FAIL B: rc $rc after an idle close, want 0"; fail=1; } +grep -q "idle 4s and no fresh demand" "$scratch/ab.log" || { echo "FAIL B: the log does not say the window closed idle"; fail=1; } +[ -n "$srv" ] && kill -0 "$srv" 2> /dev/null && { echo "FAIL B: the server $srv outlived its window"; fail=1; } +[ -e "$scratch/TOKEN" ] && { echo "FAIL B: the token is still held"; fail=1; } + +# ---- C: busy, cut at the max hold ---------------------------------------------------------------------------------- +echo busy > "$scratch/state"; rm -f "$scratch/server.pid"; touch "$scratch/demand" +t0=$(date +%s) +env "${envs[@]}" CODER_LLM_IDLE_S=2 CODER_LLM_MAX_HOLD_MIN=0.1 timeout 120 "$tick" -- "$scratch/fake" > "$scratch/c.log" 2>&1 +rc=$?; el=$(( $(date +%s) - t0 )) +[ "$rc" = 124 ] || { echo "FAIL C: rc $rc for a busy server at the max hold, want 124"; fail=1; } +[ "$el" -lt 40 ] || { echo "FAIL C: took ${el}s for a 6 s max hold"; fail=1; } +server_alive && { echo "FAIL C: the server outlived the max hold"; fail=1; } +[ -e "$scratch/TOKEN" ] && { echo "FAIL C: the token is still held"; fail=1; } + +# ---- D: a kernel fault during the window --------------------------------------------------------------------------- +echo busy > "$scratch/state"; rm -f "$scratch/server.pid"; touch "$scratch/demand"; : > "$scratch/klog" +( sleep 5; echo "kernel: amdgpu 0000:c5:00.0: [gfxhub] page fault (src_id:0 ring:24 vmid:9 pasid:26332)" >> "$scratch/klog" ) & +pids+=($!) +env "${envs[@]}" GPU_HOLD_FAULT_ABORT=1 CODER_LLM_IDLE_S=60 CODER_LLM_MAX_HOLD_MIN=5 timeout 120 "$tick" -- "$scratch/fake" > "$scratch/d.log" 2>&1 +rc=$? +[ "$rc" = 78 ] || { echo "FAIL D: rc $rc after a gfxhub fault, want 78"; fail=1; } +grep -q "backing off" "$scratch/d.log" || { echo "FAIL D: no back-off after rc 78"; fail=1; } +server_alive && { echo "FAIL D: the server survived the fault abort"; fail=1; } +[ -e "$scratch/TOKEN" ] && { echo "FAIL D: the token is still held"; fail=1; } +: > "$scratch/klog" + +# ---- E: a lane queued during a window runs before the next window -------------------------------------------------- +echo busy > "$scratch/state"; rm -f "$scratch/server.pid"; touch "$scratch/demand"; : > "$scratch/hold.log" +( env "${envs[@]}" CODER_LLM_IDLE_S=60 CODER_LLM_MAX_HOLD_MIN=0.1 timeout 120 "$tick" -- "$scratch/fake" > "$scratch/e1.log" 2>&1 + # what systemd's Restart= does: the next invocation, which queues again + env "${envs[@]}" CODER_LLM_IDLE_S=60 CODER_LLM_MAX_HOLD_MIN=0.05 timeout 120 "$tick" -- "$scratch/fake" > "$scratch/e2.log" 2>&1 ) & +e=$!; pids+=($e) +wait_for 15 server_alive || { echo "FAIL E: the first window did not open"; fail=1; } +env "${envs[@]}" GPU_HOLD_WAIT=60 timeout 120 "$hold" -t 0.05 -l lanejob -- sleep 1 > /dev/null 2>&1 +lrc=$? +[ "$lrc" = 0 ] || { echo "FAIL E: the lane ticket did not run (rc $lrc)"; fail=1; } +wait "$e" +order=$(grep -oE "took .*|queued '[a-z-]+'" "$scratch/hold.log" | tr '\n' ' ') +lane_took=$(grep -n "queued 'lanejob'" "$scratch/hold.log" | head -1 | cut -d: -f1) +took_lines=$(grep -n "took " "$scratch/hold.log" | cut -d: -f1 | tr '\n' ' ') +# three holds, in order: coder-llm, lanejob, coder-llm +holders=$(grep -oE "^gpu-hold\[[0-9]+\] took" "$scratch/hold.log" | wc -l) +seq_labels=$(awk '/queued/ {match($0, /queued .[a-z-]+./); q[$1]=substr($0, RSTART+8, RLENGTH-9)} / took / {print q[$1]}' "$scratch/hold.log" | tr '\n' ' ') +[ "$seq_labels" = "coder-llm lanejob coder-llm " ] || { echo "FAIL E: hold order '$seq_labels', want 'coder-llm lanejob coder-llm '"; fail=1; } + +[ "$fail" = 0 ] && echo "coder_llm_ticketed: ok (no demand -> no ticket; idle -> rc 0; busy -> 124 at max hold; fault -> 78 + back-off; FIFO: coder-llm, lane, coder-llm)" +exit "$fail" From 94744cc8551e8cc721fbb13979b33ce1ff628794 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 02:27:21 -0400 Subject: [PATCH 2/5] coder-llm-ticketed: an installer with --status, the "start means queued" contract, and tests that run from anywhere Review of #2496: - scripts/install-coder-llm-ticketed.sh: --diff (the default) prints the drop-in change; --status says whether the wrapper is installed and the RUNNING unit executes it; --install backs up, keeps the server command byte for byte behind the wrapper, installs it, and does daemon-reload and restart. It is idempotent. tests/install_coder_llm_ticketed_test.sh (ctest install_coder_llm_ticketed) runs it on a fake unit dir and a stub systemctl. On the box today --status says NOT ticketed: the unit serves while another lane holds the token. - The contract is stated (wrapper header and docs/ci-runners.md): `systemctl --user start coder-llm` now means queued, not ":8090 answers". A window wrapper that polls :8090 inside its own ticket is polling a server queued behind it. The wrapper logs "queued behind " so waiting is told apart from dead. - tests/coder_llm_ticketed_test.sh resolves its defaults from its own location, not ./scripts. Co-Authored-By: Claude Opus 5.5 --- cmake/tests-gpu-hold.cmake | 3 + docs/ci-runners.md | 14 ++- scripts/coder-llm-ticketed | 8 +- scripts/install-coder-llm-ticketed.sh | 105 +++++++++++++++++++++++ tests/coder_llm_ticketed_test.sh | 5 +- tests/install_coder_llm_ticketed_test.sh | 77 +++++++++++++++++ 6 files changed, 208 insertions(+), 4 deletions(-) create mode 100755 scripts/install-coder-llm-ticketed.sh create mode 100755 tests/install_coder_llm_ticketed_test.sh diff --git a/cmake/tests-gpu-hold.cmake b/cmake/tests-gpu-hold.cmake index f850ba18d..cb68e67a4 100644 --- a/cmake/tests-gpu-hold.cmake +++ b/cmake/tests-gpu-hold.cmake @@ -43,6 +43,9 @@ add_test(NAME gpu_hold_fault_abort COMMAND bash ${CMAKE_SOURCE_DIR}/tests/gpu_ho # /slots server: no demand -> no ticket, idle -> window closes, busy -> max hold, kernel fault -> rc 78, FIFO with a lane. add_test(NAME coder_llm_ticketed COMMAND bash ${CMAKE_SOURCE_DIR}/tests/coder_llm_ticketed_test.sh ${CMAKE_SOURCE_DIR}/scripts/coder-llm-ticketed ${CMAKE_SOURCE_DIR}/scripts/gpu-hold) +# ...and scripts/install-coder-llm-ticketed.sh, on a fake unit dir and a stub systemctl: --diff changes nothing, --install +# backs up and keeps the server command byte for byte behind the wrapper, --status tells ticketed from not, idempotent. +add_test(NAME install_coder_llm_ticketed COMMAND bash ${CMAKE_SOURCE_DIR}/tests/install_coder_llm_ticketed_test.sh) # engine#1635: a queued ticket that is removed (reaped, or deleted by another lane) used to leave its owner waiting for a # queue position that no longer existed -- the full -w, 5400 s by default, looking exactly like "queued behind a busy box". diff --git a/docs/ci-runners.md b/docs/ci-runners.md index 5fb77a18e..d7e755926 100644 --- a/docs/ci-runners.md +++ b/docs/ci-runners.md @@ -288,7 +288,19 @@ ticket, in on-demand windows, through `scripts/coder-llm-ticketed` (deployed to Anything else that needs the model asks the same way: touch `~/.cache/coder-llm/demand`, keep touching it while it works, and wait for `:8090` to answer a real completion. Until a window opens, the port refuses. -The unit's drop-in runs the same server command as before, behind the wrapper: +**The contract this changes: `systemctl --user start coder-llm` now means *queued*, not ":8090 answers".** A window +wrapper that restarts the unit from inside its own gpu-hold ticket and then polls `:8090` is polling a server that is queued +behind that very ticket. The server can answer only after the caller's ticket is released, so such a poll must not count a +silent `:8090` as "the fleet LLM did not come back". The unit's journal says `queued behind ` when it queues. With +the model ticketed, a memory window no longer needs to stop it: it is not resident outside its own windows. + +`scripts/install-coder-llm-ticketed.sh` does the switch: + +- with no argument, or `--diff`, it prints the drop-in change; +- `--status` says whether the wrapper is installed and the running unit executes it (exit 0 only when both are true); +- `--install` backs the drop-in up as `.bak-`, rewrites it, installs the wrapper, reloads and restarts. + +The server command stays byte for byte; the wrapper goes in front of it: ```ini [Service] diff --git a/scripts/coder-llm-ticketed b/scripts/coder-llm-ticketed index fe6054fba..098e1f313 100755 --- a/scripts/coder-llm-ticketed +++ b/scripts/coder-llm-ticketed @@ -34,6 +34,11 @@ # Anything else that needs the model asks for it the same way the workflow does: touch "$CODER_LLM_DEMAND" (and # keep touching it while working), then wait for :8090 to answer. # +# The contract this changes: `systemctl --user start coder-llm` now means QUEUED, not ":8090 answers". A window +# wrapper that restarts the unit from inside its own gpu-hold ticket and then polls :8090 is polling a server that is +# queued behind that very ticket; it answers only after the caller's ticket is released. The journal line +# "queued behind " says which state it is in. +# # Environment (all optional): # CODER_LLM_DEMAND demand file, default ~/.cache/coder-llm/demand # CODER_LLM_DEMAND_TTL_MIN a touch this recent is demand, default 30 (CODER_LLM_DEMAND_TTL_S in seconds wins) @@ -139,7 +144,8 @@ if [ "${CODER_LLM_ALWAYS:-0}" != 1 ]; then sleep "$POLL" done fi -log "demand: queueing for the GPU (gpu-hold -l coder-llm -t $MAX_HOLD -w $QUEUE_WAIT)" +holder=$(head -c 120 "${GPU_TOKEN_FILE:-$HOME/GPU-TOKEN}" 2> /dev/null | head -1) +log "demand: queueing for the GPU (gpu-hold -l coder-llm -t $MAX_HOLD -w $QUEUE_WAIT)${holder:+; queued behind $holder}" "$GPU_HOLD" -t "$MAX_HOLD" -l coder-llm -w "$QUEUE_WAIT" -- "$0" --window "$@" rc=$? case "$rc" in diff --git a/scripts/install-coder-llm-ticketed.sh b/scripts/install-coder-llm-ticketed.sh new file mode 100755 index 000000000..7c956af78 --- /dev/null +++ b/scripts/install-coder-llm-ticketed.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# install-coder-llm-ticketed.sh [--diff | --status | --install] +# +# Switches the coder-llm user unit (the PR-Agent model, :8090) to scripts/coder-llm-ticketed: the server command is +# kept exactly as the live drop-in has it, and only gets the wrapper in front of it (docs/ci-runners.md, "The PR-Agent +# model runs in gpu-hold windows"; engine#2494). +# +# --diff (default) print the drop-in change, apply nothing +# --status is the wrapper installed, and does the RUNNING unit execute it? exit 0 when both, 1 otherwise +# --install back up the drop-in as .bak-, write the new one, install the wrapper into ~/.local/bin, +# daemon-reload, restart the unit, print --status. Already ticketed: nothing to do, exit 0. +# +# Environment (tests): CODER_LLM_UNIT_DIR (default ~/.config/systemd/user/coder-llm.service.d), CODER_LLM_DROPIN +# (default zz-c22-selfcontained.conf), CODER_LLM_BIN_DIR (default ~/.local/bin), CODER_LLM_SYSTEMCTL (default +# "systemctl --user"). +set -u +here=$(cd "$(dirname "$0")" && pwd) +UNIT_DIR=${CODER_LLM_UNIT_DIR:-$HOME/.config/systemd/user/coder-llm.service.d} +DROPIN=$UNIT_DIR/${CODER_LLM_DROPIN:-zz-c22-selfcontained.conf} +BIN_DIR=${CODER_LLM_BIN_DIR:-$HOME/.local/bin} +read -r -a SYSTEMCTL <<< "${CODER_LLM_SYSTEMCTL:-systemctl --user}" +WRAPPER=$BIN_DIR/coder-llm-ticketed +mode=${1:---diff} + +# the new drop-in, from the live one: env lines before the final "ExecStart=" reset, wrapper in front of the command +render() { + awk -v w="$WRAPPER" -v b="$BIN_DIR" ' + /^ExecStart=$/ && !done { + print "# 2026-11-15 CODER-LLM-TICKETED (engine#2494): the model runs ONLY inside gpu-hold tickets, in on-demand windows" + print "# (label coder-llm, max hold 25 min, re-queued FIFO by Restart=always); the server command below is unchanged." + print "# `systemctl --user start coder-llm` now means QUEUED, not \":8090 answers\" (docs/ci-runners.md)." + print "Environment=CODER_LLM_GH_POLL=1" + print "Environment=CODER_LLM_GPU_HOLD=" b "/gpu-hold" + print "Environment=PATH=" b ":/usr/local/bin:/usr/bin:/bin" + print "Environment=CODER_LLM_MAX_HOLD_MIN=25" + print "Environment=CODER_LLM_IDLE_MIN=5" + print "Environment=GPU_HOLD_LOG=" ENVIRON["HOME"] "/GPU-HOLD.log" + print "Restart=always" + print "RestartSec=15" + print "TimeoutStopSec=90" + print; reset = 1; done = 1; next + } + reset && /^ExecStart=./ { sub(/^ExecStart=/, "ExecStart=" w " -- "); print; reset = 0; next } + { print }' "$DROPIN" +} + +is_ticketed_file() { grep -q '^ExecStart=.*coder-llm-ticketed' "$DROPIN" 2> /dev/null; } + +status() { + local ok=0 exec_line + if [ -x "$WRAPPER" ] && cmp -s "$WRAPPER" "$here/coder-llm-ticketed"; then + echo "wrapper: $WRAPPER installed, same as the tree" + elif [ -x "$WRAPPER" ]; then + echo "wrapper: $WRAPPER installed, DIFFERS from $here/coder-llm-ticketed"; ok=1 + else + echo "wrapper: NOT installed at $WRAPPER"; ok=1 + fi + if is_ticketed_file; then echo "drop-in: $DROPIN runs the wrapper"; else echo "drop-in: $DROPIN does NOT run the wrapper"; ok=1; fi + exec_line=$("${SYSTEMCTL[@]}" show coder-llm -p ExecStart --value 2> /dev/null | head -1) + case "$exec_line" in + *coder-llm-ticketed*) echo "running unit: executes coder-llm-ticketed" ;; + *) echo "running unit: does NOT execute coder-llm-ticketed (${exec_line:0:120})"; ok=1 ;; + esac + echo "token: $(head -c 80 "${GPU_TOKEN_FILE:-$HOME/GPU-TOKEN}" 2> /dev/null | head -1 || true)" + [ "$ok" = 0 ] && echo "STATUS: ticketed" || echo "STATUS: NOT ticketed" + return "$ok" +} + +[ -f "$DROPIN" ] || { echo "no drop-in at $DROPIN" >&2; exit 2; } +case "$mode" in + --diff) + if is_ticketed_file; then echo "already ticketed: $DROPIN"; exit 0; fi + diff -u "$DROPIN" <(render) | sed "2s|/dev/fd/[0-9]*|$DROPIN (new)|" + exit 0 ;; + --status) + status; exit $? ;; + --install) + if is_ticketed_file; then echo "already ticketed: $DROPIN"; status; exit 0; fi + grep -q '^ExecStart=$' "$DROPIN" || { echo "no 'ExecStart=' reset line in $DROPIN: refusing to guess" >&2; exit 2; } + bak=$DROPIN.bak-$(date +%Y%m%d) + [ -e "$bak" ] && bak=$bak-$(date +%H%M%S) + cp -p "$DROPIN" "$bak" || exit 1 + render > "$DROPIN.new" && mv "$DROPIN.new" "$DROPIN" || exit 1 + mkdir -p "$BIN_DIR" && install -m 0755 "$here/coder-llm-ticketed" "$WRAPPER" || exit 1 + echo "backed up: $bak" + "${SYSTEMCTL[@]}" daemon-reload && "${SYSTEMCTL[@]}" restart coder-llm || { echo "systemctl failed; roll back with: cp -p $bak $DROPIN" >&2; exit 1; } + status; exit $? ;; + *) + echo "usage: $0 [--diff | --status | --install]" >&2; exit 2 ;; +esac diff --git a/tests/coder_llm_ticketed_test.sh b/tests/coder_llm_ticketed_test.sh index bbca31080..4d095d002 100755 --- a/tests/coder_llm_ticketed_test.sh +++ b/tests/coder_llm_ticketed_test.sh @@ -28,8 +28,9 @@ # D a gfxhub page fault in the kernel log during the window ends it (rc 78) and the server dies with it; # E a lane that queues during a window gets the GPU before the next coder-llm window (FIFO; the restart re-queues). set -u -tick=${1:-./scripts/coder-llm-ticketed} -hold=${2:-./scripts/gpu-hold} +root=$(cd "$(dirname "$0")/.." && pwd) # the defaults work from any directory, not only the repo root +tick=${1:-$root/scripts/coder-llm-ticketed} +hold=${2:-$root/scripts/gpu-hold} for f in "$tick" "$hold"; do [ -x "$f" ] || { echo "not executable: $f"; exit 2; }; done tick=$(readlink -f "$tick"); hold=$(readlink -f "$hold") diff --git a/tests/install_coder_llm_ticketed_test.sh b/tests/install_coder_llm_ticketed_test.sh new file mode 100755 index 000000000..5ab8d33c7 --- /dev/null +++ b/tests/install_coder_llm_ticketed_test.sh @@ -0,0 +1,77 @@ +#!/bin/bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# tests/install_coder_llm_ticketed_test.sh — scripts/install-coder-llm-ticketed.sh on a fake unit directory and a stub +# systemctl (no real unit is touched): +# 1 --diff prints the change and modifies nothing; +# 2 --status says NOT ticketed (exit 1) before the install; +# 3 --install backs up the drop-in, keeps the server command byte for byte behind the wrapper, installs the wrapper, +# reloads and restarts (the stub records both), and --status then says ticketed (exit 0); +# 4 a second --install changes nothing (idempotent). +set -u +root=$(cd "$(dirname "$0")/.." && pwd) +inst=${1:-$root/scripts/install-coder-llm-ticketed.sh} +[ -x "$inst" ] || { echo "not executable: $inst"; exit 2; } +scratch=$(mktemp -d) +trap 'rm -rf "$scratch"' EXIT +fail=0 +mkdir -p "$scratch/unit" "$scratch/bin" +serve='/opt/x/1bit serve -m /models/m.gguf --device hrx --host 127.0.0.1 --port 8090 --ctx-size 131072 --alias qwen3-coder --parallel 4' +cat > "$scratch/unit/zz.conf" < "$scratch/systemctl" <> "$scratch/calls" +if [ "\$1" = show ]; then grep '^ExecStart=.' "$scratch/unit/zz.conf" | tail -1 | sed 's/^ExecStart=//'; fi +EOF +chmod +x "$scratch/systemctl" +envs=(CODER_LLM_UNIT_DIR="$scratch/unit" CODER_LLM_DROPIN=zz.conf CODER_LLM_BIN_DIR="$scratch/bin" + CODER_LLM_SYSTEMCTL="$scratch/systemctl" GPU_TOKEN_FILE="$scratch/none") + +out=$(env "${envs[@]}" "$inst" --diff) +cmp -s "$scratch/unit/zz.conf" "$scratch/orig.conf" || { echo "FAIL 1: --diff modified the drop-in"; fail=1; } +grep -q "^+ExecStart=$scratch/bin/coder-llm-ticketed -- $serve\$" <<< "$out" || { echo "FAIL 1: --diff does not show the wrapped command"; fail=1; } +[ -e "$scratch/bin/coder-llm-ticketed" ] && { echo "FAIL 1: --diff installed the wrapper"; fail=1; } + +env "${envs[@]}" "$inst" --status > "$scratch/s1" 2>&1; rc=$? +[ "$rc" = 1 ] && grep -q "STATUS: NOT ticketed" "$scratch/s1" || { echo "FAIL 2: --status before install: rc $rc, $(tail -1 "$scratch/s1")"; fail=1; } + +env "${envs[@]}" "$inst" --install > "$scratch/i1" 2>&1; rc=$? +[ "$rc" = 0 ] || { echo "FAIL 3: --install rc $rc: $(tail -3 "$scratch/i1")"; fail=1; } +bak=$(ls "$scratch/unit"/zz.conf.bak-* 2> /dev/null | head -1) +[ -n "$bak" ] && cmp -s "$bak" "$scratch/orig.conf" || { echo "FAIL 3: no faithful backup ($bak)"; fail=1; } +[ "$(grep -c '^ExecStart=.' "$scratch/unit/zz.conf")" = 1 ] || { echo "FAIL 3: not exactly one command line"; fail=1; } +grep -qxF "ExecStart=$scratch/bin/coder-llm-ticketed -- $serve" "$scratch/unit/zz.conf" || { echo "FAIL 3: the server command did not survive byte for byte"; fail=1; } +grep -qx 'ExecStart=' "$scratch/unit/zz.conf" || { echo "FAIL 3: the ExecStart= reset line is gone"; fail=1; } +grep -qx 'Environment=HSA_USE_SVM=0' "$scratch/unit/zz.conf" || { echo "FAIL 3: an existing line was lost"; fail=1; } +cmp -s "$scratch/bin/coder-llm-ticketed" "$root/scripts/coder-llm-ticketed" || { echo "FAIL 3: the wrapper was not installed"; fail=1; } +grep -qx 'daemon-reload' "$scratch/calls" && grep -qx 'restart coder-llm' "$scratch/calls" || { echo "FAIL 3: no daemon-reload + restart ($(tr '\n' ';' < "$scratch/calls"))"; fail=1; } +grep -q "STATUS: ticketed" "$scratch/i1" || { echo "FAIL 3: --status after install: $(tail -1 "$scratch/i1")"; fail=1; } + +cp "$scratch/unit/zz.conf" "$scratch/after.conf" +env "${envs[@]}" "$inst" --install > "$scratch/i2" 2>&1; rc=$? +[ "$rc" = 0 ] && cmp -s "$scratch/unit/zz.conf" "$scratch/after.conf" && grep -q "already ticketed" "$scratch/i2" \ + || { echo "FAIL 4: a second --install was not a no-op (rc $rc)"; fail=1; } +[ "$(ls "$scratch/unit"/zz.conf.bak-* | wc -l)" = 1 ] || { echo "FAIL 4: a second backup was written"; fail=1; } + +[ "$fail" = 0 ] && echo "install_coder_llm_ticketed: ok (--diff changes nothing; status before = NOT; install keeps the command byte for byte, backs up, reloads+restarts; idempotent)" +exit "$fail" From d86214abfcc1a03ac06788e97299e38549171096 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 03:16:37 -0400 Subject: [PATCH 3/5] tests: coder_llm_ticketed takes the route_ports lock (its fake /slots server binds a loopback port) CI on 94744cc failed only route_port_lock: "these tests start a loopback server without RESOURCE_LOCK route_ports: coder_llm_ticketed". The 338 other tests passed, coder_llm_ticketed included. The check is right: the test's fake server binds a loopback port, so it takes the lock like every other route test (#1070). `ctest -N --show-only=json-v1` now lists RESOURCE_LOCK route_ports on it. Co-Authored-By: Claude Opus 5.5 --- cmake/tests-gpu-hold.cmake | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmake/tests-gpu-hold.cmake b/cmake/tests-gpu-hold.cmake index cb68e67a4..83083a74d 100644 --- a/cmake/tests-gpu-hold.cmake +++ b/cmake/tests-gpu-hold.cmake @@ -43,6 +43,8 @@ add_test(NAME gpu_hold_fault_abort COMMAND bash ${CMAKE_SOURCE_DIR}/tests/gpu_ho # /slots server: no demand -> no ticket, idle -> window closes, busy -> max hold, kernel fault -> rc 78, FIFO with a lane. add_test(NAME coder_llm_ticketed COMMAND bash ${CMAKE_SOURCE_DIR}/tests/coder_llm_ticketed_test.sh ${CMAKE_SOURCE_DIR}/scripts/coder-llm-ticketed ${CMAKE_SOURCE_DIR}/scripts/gpu-hold) +# its fake /slots server binds a loopback port, so it takes the route-port lock like every other route test (#1070) +set_tests_properties(coder_llm_ticketed PROPERTIES RESOURCE_LOCK route_ports) # ...and scripts/install-coder-llm-ticketed.sh, on a fake unit dir and a stub systemctl: --diff changes nothing, --install # backs up and keeps the server command byte for byte behind the wrapper, --status tells ticketed from not, idempotent. add_test(NAME install_coder_llm_ticketed COMMAND bash ${CMAKE_SOURCE_DIR}/tests/install_coder_llm_ticketed_test.sh) From 000f5cb5e1e0679f4681d619044ff302f7b9adf5 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 04:19:31 -0400 Subject: [PATCH 4/5] coder-llm-ticketed: the pr-agent-run poll is on by default, so a review cannot starve (review of #2496) The CHANGES at d86214a: CODER_LLM_GH_POLL defaulted to 0, and nothing in this PR produces the demand file. The producer is engine#2509: the pr-agent workflow's reviewer job touches ~/.cache/coder-llm/demand. As the fallback, a queued or in-progress pr-agent run now counts as demand by default (CODER_LLM_GH_POLL=1), so a workflow that forgot to ask still gets a window. The tests pin CODER_LLM_GH_POLL=0 so CI never calls gh. The header and docs/ci-runners.md say who touches the file. Co-Authored-By: Claude Opus 5.5 --- docs/ci-runners.md | 5 +++-- scripts/coder-llm-ticketed | 12 +++++++----- tests/coder_llm_ticketed_test.sh | 1 + 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/ci-runners.md b/docs/ci-runners.md index d7e755926..d531d19db 100644 --- a/docs/ci-runners.md +++ b/docs/ci-runners.md @@ -276,8 +276,9 @@ The `pr-agent` job talks to a local model, Qwen3-Coder-30B-A3B served by `1bit s job at a time" never held, and its model is the one whose prefill faults the GPU (engine#2494). It now runs only inside a ticket, in on-demand windows, through `scripts/coder-llm-ticketed` (deployed to `~/.local/bin`): -1. **It waits for demand.** Demand is the file `~/.cache/coder-llm/demand` touched within the last 30 minutes, or - (`CODER_LLM_GH_POLL=1`) a `pr-agent` workflow run that is queued or in progress. +1. **It waits for demand.** Demand is the file `~/.cache/coder-llm/demand` touched within the last 30 minutes (the + `pr-agent` workflow's `reviewer` job touches it), or, as the fallback that is on by default (`CODER_LLM_GH_POLL=1`), + a `pr-agent` workflow run that is queued or in progress. So a review is never starved by a workflow that forgot to ask. 2. **It queues for the GPU like any other job**: `gpu-hold -l coder-llm -t 25`. The FIFO queue, the temperature, memory and kernel-fault watchdogs, and the release wait that lets the next label in all apply. 3. **Inside the ticket it serves**, and ends the window when no slot in `/slots` has been processing **and** the demand diff --git a/scripts/coder-llm-ticketed b/scripts/coder-llm-ticketed index 098e1f313..0dbc27ce7 100755 --- a/scripts/coder-llm-ticketed +++ b/scripts/coder-llm-ticketed @@ -21,8 +21,9 @@ # at a time" never held, and the model it serves (Qwen3-Coder-30B on HRX) is the one whose prefill faults the GPU. # # One invocation is ONE window, and systemd's Restart= makes the next one: -# 1. wait for demand: the demand file was touched within CODER_LLM_DEMAND_TTL_MIN, or (CODER_LLM_GH_POLL=1) a -# pr-agent workflow run is queued or in progress; +# 1. wait for demand: the demand file was touched within CODER_LLM_DEMAND_TTL_MIN (the pr-agent workflow's reviewer +# job touches it, engine#2509), or a pr-agent workflow run is queued or in progress (CODER_LLM_GH_POLL, on by +# default as the fallback); # 2. queue for the GPU like any other job: gpu-hold -l coder-llm -t CODER_LLM_MAX_HOLD_MIN (FIFO; gpu-hold's # temperature, memory and kernel-fault watchdogs apply, and its release wait lets the next label in); # 3. inside the ticket, start SERVER_CMD and end the window once the server has been idle (no slot in /slots is @@ -47,8 +48,9 @@ # CODER_LLM_QUEUE_WAIT_S how long one window queues for the GPU (gpu-hold -w), default 5400 # CODER_LLM_URL the server's base URL for /slots, default http://127.0.0.1:8090 # CODER_LLM_POLL_S poll interval, default 15 -# CODER_LLM_GH_POLL 1 = also treat a queued/in-progress pr-agent run as demand (REST, every -# CODER_LLM_GH_POLL_S, default 60), default 0 +# CODER_LLM_GH_POLL 1 (the default) = also treat a queued/in-progress pr-agent run as demand (REST, +# every CODER_LLM_GH_POLL_S, default 60), so a review is never starved even if nothing +# touched the demand file; 0 = the demand file only # CODER_LLM_GH_REPO default 1bit-MONSTER/engine # CODER_LLM_FAULT_BACKOFF_S back-off after gpu-hold rc 78 (a fault, or a refused ticket), default 600 # CODER_LLM_GPU_HOLD gpu-hold to use, default the one on PATH @@ -64,7 +66,7 @@ MAX_HOLD=${CODER_LLM_MAX_HOLD_MIN:-25} QUEUE_WAIT=${CODER_LLM_QUEUE_WAIT_S:-5400} URL=${CODER_LLM_URL:-http://127.0.0.1:8090} POLL=${CODER_LLM_POLL_S:-15} -GH_POLL=${CODER_LLM_GH_POLL:-0} +GH_POLL=${CODER_LLM_GH_POLL:-1} GH_POLL_S=${CODER_LLM_GH_POLL_S:-60} GH_REPO=${CODER_LLM_GH_REPO:-1bit-MONSTER/engine} BACKOFF=${CODER_LLM_FAULT_BACKOFF_S:-600} diff --git a/tests/coder_llm_ticketed_test.sh b/tests/coder_llm_ticketed_test.sh index 4d095d002..d5e112c0b 100755 --- a/tests/coder_llm_ticketed_test.sh +++ b/tests/coder_llm_ticketed_test.sh @@ -63,6 +63,7 @@ envs=(GPU_QUEUE_DIR="$scratch/q" GPU_TOKEN_FILE="$scratch/TOKEN" GPU_HOLD_LOG="$ GPU_HOLD_TCTL_START=100 GPU_HOLD_TCTL_FILE="$scratch/tctl" GPU_HOLD_THERMAL_FLAG="$scratch/x" GPU_HOLD_KLOG_FILE="$scratch/klog" PATH="$(dirname "$hold"):$PATH" CODER_LLM_GPU_HOLD="$hold" CODER_LLM_DEMAND="$scratch/demand" CODER_LLM_URL="http://127.0.0.1:$port" + CODER_LLM_GH_POLL=0 CODER_LLM_POLL_S=1 CODER_LLM_DEMAND_TTL_S=60 CODER_LLM_FAULT_BACKOFF_S=1 CODER_LLM_QUEUE_WAIT_S=60) mkdir -p "$scratch/q"; : > "$scratch/klog" From 77415b2b38067421829bee34db0c41d4fb5e8975 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 04:47:42 -0400 Subject: [PATCH 5/5] coder-llm-ticketed: the fallback poll counts pull_request runs only, and says so when it cannot ask (review of #2496) Two asks at 000f5cb: - CHANGES: the poll counted every pr-agent run. Comments create runs too (80 issue_comment vs 20 pull_request runs since 06:00Z, as the reviewer measured), so on by default it would keep a 30B window open on chatter. It now asks for event=pull_request only. Command comments still reach the model through the demand file the reviewer job touches (engine#2509). - A failed `gh api` (expired token, rate limit, empty answer) used to read silently as "no demand". It is still no demand, the safe side, but it is now logged once per wait with its reason and the command to check. The wait also warns at start when gh is not logged in. New test arm F: a gh stub that answers 401 gives exactly one "gh poll failed (HTTP 401" line, the not-logged-in warning, and no window. Co-Authored-By: Claude Opus 5.5 --- docs/ci-runners.md | 4 +++- scripts/coder-llm-ticketed | 25 +++++++++++++++++++++---- tests/coder_llm_ticketed_test.sh | 16 ++++++++++++++-- 3 files changed, 38 insertions(+), 7 deletions(-) diff --git a/docs/ci-runners.md b/docs/ci-runners.md index d531d19db..e32e494ec 100644 --- a/docs/ci-runners.md +++ b/docs/ci-runners.md @@ -278,7 +278,9 @@ ticket, in on-demand windows, through `scripts/coder-llm-ticketed` (deployed to 1. **It waits for demand.** Demand is the file `~/.cache/coder-llm/demand` touched within the last 30 minutes (the `pr-agent` workflow's `reviewer` job touches it), or, as the fallback that is on by default (`CODER_LLM_GH_POLL=1`), - a `pr-agent` workflow run that is queued or in progress. So a review is never starved by a workflow that forgot to ask. + a `pr-agent` **pull_request** run that is queued or in progress (comment runs are not counted: most never review). So + a review is never starved by a workflow that forgot to ask. A poll that cannot ask (an expired token, a rate limit) is + logged once per wait as `gh poll failed ()` and treated as no demand. 2. **It queues for the GPU like any other job**: `gpu-hold -l coder-llm -t 25`. The FIFO queue, the temperature, memory and kernel-fault watchdogs, and the release wait that lets the next label in all apply. 3. **Inside the ticket it serves**, and ends the window when no slot in `/slots` has been processing **and** the demand diff --git a/scripts/coder-llm-ticketed b/scripts/coder-llm-ticketed index 0dbc27ce7..cba5110ee 100755 --- a/scripts/coder-llm-ticketed +++ b/scripts/coder-llm-ticketed @@ -80,12 +80,26 @@ demand_fresh() { # $1 = window in seconds [ $(( $(date +%s) - $(mtime "$DEMAND") )) -lt "$1" ] } +# Only pull_request runs count: every comment on a PR also creates a pr-agent run (about one every two minutes on a +# busy night) and most never review, so counting them would keep a 30B window open on chatter. Command comments +# (/review, /ask ...) reach the model through the demand file the workflow's reviewer job touches (engine#2509). +GH_FAIL_LOGGED=0 gh_demand() { [ "$GH_POLL" = 1 ] || return 1 - local s n + local s n err errf="${TMPDIR:-/tmp}/coder-llm-gh.$$" for s in queued in_progress; do - n=$(gh api "repos/$GH_REPO/actions/workflows/pr-agent.yml/runs?status=$s&per_page=1" -q .total_count 2> /dev/null) - [ "${n:-0}" -gt 0 ] 2> /dev/null && return 0 + n=$(gh api "repos/$GH_REPO/actions/workflows/pr-agent.yml/runs?status=$s&event=pull_request&per_page=1" \ + -q .total_count 2> "$errf") + if [ $? -ne 0 ] || ! [ "$n" -ge 0 ] 2> /dev/null; then + # an absent answer is not "no demand": say why, once per wait (the behaviour stays "no demand", the safe side) + err=$(head -c 200 "$errf" 2> /dev/null | tr '\n' ' ') + [ "$GH_FAIL_LOGGED" = 1 ] || log "gh poll failed (${err:-answer '$n'}): treating it as no demand; check with: gh api repos/$GH_REPO/actions/workflows/pr-agent.yml/runs" + GH_FAIL_LOGGED=1 + rm -f "$errf" + return 1 + fi + rm -f "$errf" + [ "$n" -gt 0 ] && return 0 done return 1 } @@ -135,7 +149,10 @@ fi mkdir -p "$(dirname "$DEMAND")" if [ "${CODER_LLM_ALWAYS:-0}" != 1 ]; then - log "waiting for demand ($DEMAND touched within ${DEMAND_TTL}s$([ "$GH_POLL" = 1 ] && echo ", or a pr-agent run on $GH_REPO"))" + log "waiting for demand ($DEMAND touched within ${DEMAND_TTL}s$([ "$GH_POLL" = 1 ] && echo ", or a pr-agent pull_request run on $GH_REPO"))" + if [ "$GH_POLL" = 1 ] && ! gh auth status > /dev/null 2>&1; then + log "WARNING: gh is not logged in, so the pr-agent-run fallback cannot see demand; only the demand file will wake the model" + fi next_gh=0 while :; do demand_fresh "$DEMAND_TTL" && break diff --git a/tests/coder_llm_ticketed_test.sh b/tests/coder_llm_ticketed_test.sh index d5e112c0b..6577d9021 100755 --- a/tests/coder_llm_ticketed_test.sh +++ b/tests/coder_llm_ticketed_test.sh @@ -26,7 +26,8 @@ # gone and the token released; # C a busy server is cut at the max hold (rc 124), the server is gone; # D a gfxhub page fault in the kernel log during the window ends it (rc 78) and the server dies with it; -# E a lane that queues during a window gets the GPU before the next coder-llm window (FIFO; the restart re-queues). +# E a lane that queues during a window gets the GPU before the next coder-llm window (FIFO; the restart re-queues); +# F a failing gh (the pr-agent-run fallback) is logged once per wait with its reason, and opens no window. set -u root=$(cd "$(dirname "$0")/.." && pwd) # the defaults work from any directory, not only the repo root tick=${1:-$root/scripts/coder-llm-ticketed} @@ -133,5 +134,16 @@ holders=$(grep -oE "^gpu-hold\[[0-9]+\] took" "$scratch/hold.log" | wc -l) seq_labels=$(awk '/queued/ {match($0, /queued .[a-z-]+./); q[$1]=substr($0, RSTART+8, RLENGTH-9)} / took / {print q[$1]}' "$scratch/hold.log" | tr '\n' ' ') [ "$seq_labels" = "coder-llm lanejob coder-llm " ] || { echo "FAIL E: hold order '$seq_labels', want 'coder-llm lanejob coder-llm '"; fail=1; } -[ "$fail" = 0 ] && echo "coder_llm_ticketed: ok (no demand -> no ticket; idle -> rc 0; busy -> 124 at max hold; fault -> 78 + back-off; FIFO: coder-llm, lane, coder-llm)" +# ---- F: the pr-agent-run fallback says so when it cannot ask (a failing gh), once per wait ------------------------- +mkdir -p "$scratch/ghbin" +printf '#!/bin/bash\nif [ "$1" = auth ]; then exit 1; fi\necho "HTTP 401: Bad credentials" >&2; exit 1\n' > "$scratch/ghbin/gh" +chmod +x "$scratch/ghbin/gh" +touch -d '-2 hours' "$scratch/demand" +env "${envs[@]}" CODER_LLM_GH_POLL=1 CODER_LLM_GH_POLL_S=1 PATH="$scratch/ghbin:$PATH" \ + timeout 6 "$tick" -- "$scratch/fake" > "$scratch/f.log" 2>&1 +[ "$(grep -c 'gh poll failed (HTTP 401' "$scratch/f.log")" = 1 ] || { echo "FAIL F: want exactly one 'gh poll failed' line, got $(grep -c 'gh poll failed' "$scratch/f.log")"; fail=1; } +grep -q 'gh is not logged in' "$scratch/f.log" || { echo "FAIL F: no warning that gh is not logged in"; fail=1; } +[ -e "$scratch/TOKEN" ] && { echo "FAIL F: a failing poll opened a window"; fail=1; } + +[ "$fail" = 0 ] && echo "coder_llm_ticketed: ok (no demand -> no ticket; idle -> rc 0; busy -> 124 at max hold; fault -> 78 + back-off; FIFO: coder-llm, lane, coder-llm; a failing gh poll is logged once, no window)" exit "$fail"