diff --git a/cmake/tests-gpu-hold.cmake b/cmake/tests-gpu-hold.cmake index d0418a3be..83083a74d 100644 --- a/cmake/tests-gpu-hold.cmake +++ b/cmake/tests-gpu-hold.cmake @@ -38,6 +38,17 @@ add_test(NAME gpu_hold_mem_abort COMMAND bash ${CMAKE_SOURCE_DIR}/tests/gpu_hold # when the token is taken; the 11-13 crash was nine resets inside two tickets. The kernel log is a fixture file. add_test(NAME gpu_hold_fault_abort COMMAND bash ${CMAKE_SOURCE_DIR}/tests/gpu_hold_fault_abort_test.sh ${CMAKE_SOURCE_DIR}/scripts/gpu-hold) +# CODER-LLM-TICKETED 2026-11-15 (engine#2494): the PR-Agent model (coder-llm, :8090) runs only inside a gpu-hold ticket, +# in on-demand windows. Drives scripts/coder-llm-ticketed with the real scripts/gpu-hold on a private queue and a fake +# /slots server: no demand -> no ticket, idle -> window closes, busy -> max hold, kernel fault -> rc 78, FIFO with a lane. +add_test(NAME coder_llm_ticketed COMMAND bash ${CMAKE_SOURCE_DIR}/tests/coder_llm_ticketed_test.sh + ${CMAKE_SOURCE_DIR}/scripts/coder-llm-ticketed ${CMAKE_SOURCE_DIR}/scripts/gpu-hold) +# its fake /slots server binds a loopback port, so it takes the route-port lock like every other route test (#1070) +set_tests_properties(coder_llm_ticketed PROPERTIES RESOURCE_LOCK route_ports) +# ...and scripts/install-coder-llm-ticketed.sh, on a fake unit dir and a stub systemctl: --diff changes nothing, --install +# backs up and keeps the server command byte for byte behind the wrapper, --status tells ticketed from not, idempotent. +add_test(NAME install_coder_llm_ticketed COMMAND bash ${CMAKE_SOURCE_DIR}/tests/install_coder_llm_ticketed_test.sh) + # engine#1635: a queued ticket that is removed (reaped, or deleted by another lane) used to leave its owner waiting for a # queue position that no longer existed -- the full -w, 5400 s by default, looking exactly like "queued behind a busy box". # Drives scripts/gpu-hold on a private queue: the ticket is deleted mid-wait, and separately rewritten to a dead pid so a diff --git a/docs/ci-runners.md b/docs/ci-runners.md index 107760817..e32e494ec 100644 --- a/docs/ci-runners.md +++ b/docs/ci-runners.md @@ -269,6 +269,55 @@ allocation failed as an exception rather than as an OOM kill, so that experiment directive is the documented mechanism and it is what the fresh install's single end-to-end run should confirm, together with the rest of #1504's acceptance. +## The PR-Agent model runs in gpu-hold windows (engine#2494) + +The `pr-agent` job talks to a local model, Qwen3-Coder-30B-A3B served by `1bit serve` on HRX at `127.0.0.1:8090` (the +`coder-llm` user service). Until 2026-11-15 that server ran around the clock **outside any gpu-hold ticket**, so "one GPU +job at a time" never held, and its model is the one whose prefill faults the GPU (engine#2494). It now runs only inside a +ticket, in on-demand windows, through `scripts/coder-llm-ticketed` (deployed to `~/.local/bin`): + +1. **It waits for demand.** Demand is the file `~/.cache/coder-llm/demand` touched within the last 30 minutes (the + `pr-agent` workflow's `reviewer` job touches it), or, as the fallback that is on by default (`CODER_LLM_GH_POLL=1`), + a `pr-agent` **pull_request** run that is queued or in progress (comment runs are not counted: most never review). So + a review is never starved by a workflow that forgot to ask. A poll that cannot ask (an expired token, a rate limit) is + logged once per wait as `gh poll failed ()` and treated as no demand. +2. **It queues for the GPU like any other job**: `gpu-hold -l coder-llm -t 25`. The FIFO queue, the temperature, memory + and kernel-fault watchdogs, and the release wait that lets the next label in all apply. +3. **Inside the ticket it serves**, and ends the window when no slot in `/slots` has been processing **and** the demand + file has not been touched for 5 minutes, or when the 25-minute max hold ends it. +4. **It exits**, and the unit's `Restart=always` starts the next invocation, which waits for demand and queues again at + the back of the line. After a kernel fault (gpu-hold rc 78) it backs off 10 minutes first. + +Anything else that needs the model asks the same way: touch `~/.cache/coder-llm/demand`, keep touching it while it +works, and wait for `:8090` to answer a real completion. Until a window opens, the port refuses. + +**The contract this changes: `systemctl --user start coder-llm` now means *queued*, not ":8090 answers".** A window +wrapper that restarts the unit from inside its own gpu-hold ticket and then polls `:8090` is polling a server that is queued +behind that very ticket. The server can answer only after the caller's ticket is released, so such a poll must not count a +silent `:8090` as "the fleet LLM did not come back". The unit's journal says `queued behind ` when it queues. With +the model ticketed, a memory window no longer needs to stop it: it is not resident outside its own windows. + +`scripts/install-coder-llm-ticketed.sh` does the switch: + +- with no argument, or `--diff`, it prints the drop-in change; +- `--status` says whether the wrapper is installed and the running unit executes it (exit 0 only when both are true); +- `--install` backs the drop-in up as `.bak-`, rewrites it, installs the wrapper, reloads and restarts. + +The server command stays byte for byte; the wrapper goes in front of it: + +```ini +[Service] +ExecStart= +ExecStart=%h/.local/bin/coder-llm-ticketed -- +Environment=CODER_LLM_GH_POLL=1 +Restart=always +RestartSec=15 +TimeoutStopSec=90 +``` + +`tests/coder_llm_ticketed_test.sh` (ctest `coder_llm_ticketed`) drives the wrapper with the real `scripts/gpu-hold` on a +private queue and a fake `/slots` server. No device is used. + ## What a restart policy cannot do It cannot notice that a runner is *not* coming back (a version mismatch, a revoked token, a full disk under diff --git a/scripts/coder-llm-ticketed b/scripts/coder-llm-ticketed new file mode 100755 index 000000000..cba5110ee --- /dev/null +++ b/scripts/coder-llm-ticketed @@ -0,0 +1,177 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# coder-llm-ticketed [--] SERVER_CMD [ARGS...] +# +# Runs the PR-Agent reviewer model (the coder-llm user service, :8090) ONLY inside a gpu-hold ticket, in on-demand +# windows (owner 2026-11-15, CODER-LLM-TICKETED; engine#2494). It used to run 24/7 outside any ticket, so "one GPU job +# at a time" never held, and the model it serves (Qwen3-Coder-30B on HRX) is the one whose prefill faults the GPU. +# +# One invocation is ONE window, and systemd's Restart= makes the next one: +# 1. wait for demand: the demand file was touched within CODER_LLM_DEMAND_TTL_MIN (the pr-agent workflow's reviewer +# job touches it, engine#2509), or a pr-agent workflow run is queued or in progress (CODER_LLM_GH_POLL, on by +# default as the fallback); +# 2. queue for the GPU like any other job: gpu-hold -l coder-llm -t CODER_LLM_MAX_HOLD_MIN (FIFO; gpu-hold's +# temperature, memory and kernel-fault watchdogs apply, and its release wait lets the next label in); +# 3. inside the ticket, start SERVER_CMD and end the window once the server has been idle (no slot in /slots is +# processing) AND the demand file has not been touched for CODER_LLM_IDLE_MIN - or when the max hold ends it; +# 4. exit, so systemd starts the next invocation, which goes back to waiting for demand and re-queues. +# A window cut short by a kernel fault (gpu-hold rc 78) or a refused ticket backs off CODER_LLM_FAULT_BACKOFF_S +# before exiting, so a faulting GPU does not turn into a restart loop. +# +# Anything else that needs the model asks for it the same way the workflow does: touch "$CODER_LLM_DEMAND" (and +# keep touching it while working), then wait for :8090 to answer. +# +# The contract this changes: `systemctl --user start coder-llm` now means QUEUED, not ":8090 answers". A window +# wrapper that restarts the unit from inside its own gpu-hold ticket and then polls :8090 is polling a server that is +# queued behind that very ticket; it answers only after the caller's ticket is released. The journal line +# "queued behind " says which state it is in. +# +# Environment (all optional): +# CODER_LLM_DEMAND demand file, default ~/.cache/coder-llm/demand +# CODER_LLM_DEMAND_TTL_MIN a touch this recent is demand, default 30 (CODER_LLM_DEMAND_TTL_S in seconds wins) +# CODER_LLM_IDLE_MIN idle minutes that end a window, default 5 (CODER_LLM_IDLE_S in seconds wins) +# CODER_LLM_MAX_HOLD_MIN max hold per window (gpu-hold -t), default 25 +# CODER_LLM_QUEUE_WAIT_S how long one window queues for the GPU (gpu-hold -w), default 5400 +# CODER_LLM_URL the server's base URL for /slots, default http://127.0.0.1:8090 +# CODER_LLM_POLL_S poll interval, default 15 +# CODER_LLM_GH_POLL 1 (the default) = also treat a queued/in-progress pr-agent run as demand (REST, +# every CODER_LLM_GH_POLL_S, default 60), so a review is never starved even if nothing +# touched the demand file; 0 = the demand file only +# CODER_LLM_GH_REPO default 1bit-MONSTER/engine +# CODER_LLM_FAULT_BACKOFF_S back-off after gpu-hold rc 78 (a fault, or a refused ticket), default 600 +# CODER_LLM_GPU_HOLD gpu-hold to use, default the one on PATH +# CODER_LLM_ALWAYS=1 skip the demand wait (a resident window, still ticketed and still max-hold bounded) +# +# Internal: "coder-llm-ticketed --window SERVER_CMD..." is what gpu-hold runs; it is the idle-aware window itself. +set -u + +DEMAND=${CODER_LLM_DEMAND:-$HOME/.cache/coder-llm/demand} +DEMAND_TTL=${CODER_LLM_DEMAND_TTL_S:-$(( ${CODER_LLM_DEMAND_TTL_MIN:-30} * 60 ))} +IDLE=${CODER_LLM_IDLE_S:-$(( ${CODER_LLM_IDLE_MIN:-5} * 60 ))} +MAX_HOLD=${CODER_LLM_MAX_HOLD_MIN:-25} +QUEUE_WAIT=${CODER_LLM_QUEUE_WAIT_S:-5400} +URL=${CODER_LLM_URL:-http://127.0.0.1:8090} +POLL=${CODER_LLM_POLL_S:-15} +GH_POLL=${CODER_LLM_GH_POLL:-1} +GH_POLL_S=${CODER_LLM_GH_POLL_S:-60} +GH_REPO=${CODER_LLM_GH_REPO:-1bit-MONSTER/engine} +BACKOFF=${CODER_LLM_FAULT_BACKOFF_S:-600} +GPU_HOLD=${CODER_LLM_GPU_HOLD:-gpu-hold} + +log() { echo "coder-llm-ticketed[$$] $*"; } + +mtime() { stat -c %Y "$1" 2> /dev/null || echo 0; } + +demand_fresh() { # $1 = window in seconds + [ $(( $(date +%s) - $(mtime "$DEMAND") )) -lt "$1" ] +} + +# Only pull_request runs count: every comment on a PR also creates a pr-agent run (about one every two minutes on a +# busy night) and most never review, so counting them would keep a 30B window open on chatter. Command comments +# (/review, /ask ...) reach the model through the demand file the workflow's reviewer job touches (engine#2509). +GH_FAIL_LOGGED=0 +gh_demand() { + [ "$GH_POLL" = 1 ] || return 1 + local s n err errf="${TMPDIR:-/tmp}/coder-llm-gh.$$" + for s in queued in_progress; do + n=$(gh api "repos/$GH_REPO/actions/workflows/pr-agent.yml/runs?status=$s&event=pull_request&per_page=1" \ + -q .total_count 2> "$errf") + if [ $? -ne 0 ] || ! [ "$n" -ge 0 ] 2> /dev/null; then + # an absent answer is not "no demand": say why, once per wait (the behaviour stays "no demand", the safe side) + err=$(head -c 200 "$errf" 2> /dev/null | tr '\n' ' ') + [ "$GH_FAIL_LOGGED" = 1 ] || log "gh poll failed (${err:-answer '$n'}): treating it as no demand; check with: gh api repos/$GH_REPO/actions/workflows/pr-agent.yml/runs" + GH_FAIL_LOGGED=1 + rm -f "$errf" + return 1 + fi + rm -f "$errf" + [ "$n" -gt 0 ] && return 0 + done + return 1 +} + +slots_busy() { # 0 = some slot is processing, or the server cannot say (not up yet counts as busy) + local j + j=$(curl -sf --max-time 5 "$URL/slots" 2> /dev/null) || return 0 + grep -Eq '"is_processing"[[:space:]]*:[[:space:]]*true' <<< "$j" && return 0 + return 1 +} + +# ---- the window: runs INSIDE the gpu-hold ticket -------------------------------------------------------------------- +if [ "${1:-}" = "--window" ]; then + shift + [ "${1:-}" = "--" ] && shift + [ $# -gt 0 ] || { log "--window needs a server command"; exit 2; } + "$@" & + server=$! + stop_server() { + kill -TERM "$server" 2> /dev/null + for _ in $(seq 1 30); do kill -0 "$server" 2> /dev/null || return; sleep 1; done + kill -KILL "$server" 2> /dev/null + } + trap 'stop_server; exit 143' TERM INT HUP + log "window open: server pid $server" + last_busy=$(date +%s) + while kill -0 "$server" 2> /dev/null; do + sleep "$POLL" + now=$(date +%s) + if slots_busy || demand_fresh "$IDLE"; then + last_busy=$now + elif [ $(( now - last_busy )) -ge "$IDLE" ]; then + log "idle ${IDLE}s and no fresh demand: closing the window" + stop_server + exit 0 + fi + done + wait "$server" + rc=$? + log "server exited by itself, rc $rc" + exit "$rc" +fi + +# ---- one service invocation: wait for demand, then one ticketed window ---------------------------------------------- +[ "${1:-}" = "--" ] && shift +[ $# -gt 0 ] || { echo "usage: coder-llm-ticketed [--] SERVER_CMD [ARGS...]" >&2; exit 2; } +mkdir -p "$(dirname "$DEMAND")" + +if [ "${CODER_LLM_ALWAYS:-0}" != 1 ]; then + log "waiting for demand ($DEMAND touched within ${DEMAND_TTL}s$([ "$GH_POLL" = 1 ] && echo ", or a pr-agent pull_request run on $GH_REPO"))" + if [ "$GH_POLL" = 1 ] && ! gh auth status > /dev/null 2>&1; then + log "WARNING: gh is not logged in, so the pr-agent-run fallback cannot see demand; only the demand file will wake the model" + fi + next_gh=0 + while :; do + demand_fresh "$DEMAND_TTL" && break + if [ "$GH_POLL" = 1 ] && [ "$(date +%s)" -ge "$next_gh" ]; then + gh_demand && { touch "$DEMAND"; break; } + next_gh=$(( $(date +%s) + GH_POLL_S )) + fi + sleep "$POLL" + done +fi +holder=$(head -c 120 "${GPU_TOKEN_FILE:-$HOME/GPU-TOKEN}" 2> /dev/null | head -1) +log "demand: queueing for the GPU (gpu-hold -l coder-llm -t $MAX_HOLD -w $QUEUE_WAIT)${holder:+; queued behind $holder}" +"$GPU_HOLD" -t "$MAX_HOLD" -l coder-llm -w "$QUEUE_WAIT" -- "$0" --window "$@" +rc=$? +case "$rc" in + 0) log "window closed (idle)" ;; + 124) log "window closed (max hold ${MAX_HOLD} min); re-queueing" ;; + 75) log "the queue never came round in ${QUEUE_WAIT}s" ;; + 78) log "gpu-hold rc 78 (a GPU fault in the kernel log, or a refused ticket): backing off ${BACKOFF}s"; sleep "$BACKOFF" ;; + *) log "window ended rc $rc" ;; +esac +exit "$rc" diff --git a/scripts/install-coder-llm-ticketed.sh b/scripts/install-coder-llm-ticketed.sh new file mode 100755 index 000000000..7c956af78 --- /dev/null +++ b/scripts/install-coder-llm-ticketed.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# install-coder-llm-ticketed.sh [--diff | --status | --install] +# +# Switches the coder-llm user unit (the PR-Agent model, :8090) to scripts/coder-llm-ticketed: the server command is +# kept exactly as the live drop-in has it, and only gets the wrapper in front of it (docs/ci-runners.md, "The PR-Agent +# model runs in gpu-hold windows"; engine#2494). +# +# --diff (default) print the drop-in change, apply nothing +# --status is the wrapper installed, and does the RUNNING unit execute it? exit 0 when both, 1 otherwise +# --install back up the drop-in as .bak-, write the new one, install the wrapper into ~/.local/bin, +# daemon-reload, restart the unit, print --status. Already ticketed: nothing to do, exit 0. +# +# Environment (tests): CODER_LLM_UNIT_DIR (default ~/.config/systemd/user/coder-llm.service.d), CODER_LLM_DROPIN +# (default zz-c22-selfcontained.conf), CODER_LLM_BIN_DIR (default ~/.local/bin), CODER_LLM_SYSTEMCTL (default +# "systemctl --user"). +set -u +here=$(cd "$(dirname "$0")" && pwd) +UNIT_DIR=${CODER_LLM_UNIT_DIR:-$HOME/.config/systemd/user/coder-llm.service.d} +DROPIN=$UNIT_DIR/${CODER_LLM_DROPIN:-zz-c22-selfcontained.conf} +BIN_DIR=${CODER_LLM_BIN_DIR:-$HOME/.local/bin} +read -r -a SYSTEMCTL <<< "${CODER_LLM_SYSTEMCTL:-systemctl --user}" +WRAPPER=$BIN_DIR/coder-llm-ticketed +mode=${1:---diff} + +# the new drop-in, from the live one: env lines before the final "ExecStart=" reset, wrapper in front of the command +render() { + awk -v w="$WRAPPER" -v b="$BIN_DIR" ' + /^ExecStart=$/ && !done { + print "# 2026-11-15 CODER-LLM-TICKETED (engine#2494): the model runs ONLY inside gpu-hold tickets, in on-demand windows" + print "# (label coder-llm, max hold 25 min, re-queued FIFO by Restart=always); the server command below is unchanged." + print "# `systemctl --user start coder-llm` now means QUEUED, not \":8090 answers\" (docs/ci-runners.md)." + print "Environment=CODER_LLM_GH_POLL=1" + print "Environment=CODER_LLM_GPU_HOLD=" b "/gpu-hold" + print "Environment=PATH=" b ":/usr/local/bin:/usr/bin:/bin" + print "Environment=CODER_LLM_MAX_HOLD_MIN=25" + print "Environment=CODER_LLM_IDLE_MIN=5" + print "Environment=GPU_HOLD_LOG=" ENVIRON["HOME"] "/GPU-HOLD.log" + print "Restart=always" + print "RestartSec=15" + print "TimeoutStopSec=90" + print; reset = 1; done = 1; next + } + reset && /^ExecStart=./ { sub(/^ExecStart=/, "ExecStart=" w " -- "); print; reset = 0; next } + { print }' "$DROPIN" +} + +is_ticketed_file() { grep -q '^ExecStart=.*coder-llm-ticketed' "$DROPIN" 2> /dev/null; } + +status() { + local ok=0 exec_line + if [ -x "$WRAPPER" ] && cmp -s "$WRAPPER" "$here/coder-llm-ticketed"; then + echo "wrapper: $WRAPPER installed, same as the tree" + elif [ -x "$WRAPPER" ]; then + echo "wrapper: $WRAPPER installed, DIFFERS from $here/coder-llm-ticketed"; ok=1 + else + echo "wrapper: NOT installed at $WRAPPER"; ok=1 + fi + if is_ticketed_file; then echo "drop-in: $DROPIN runs the wrapper"; else echo "drop-in: $DROPIN does NOT run the wrapper"; ok=1; fi + exec_line=$("${SYSTEMCTL[@]}" show coder-llm -p ExecStart --value 2> /dev/null | head -1) + case "$exec_line" in + *coder-llm-ticketed*) echo "running unit: executes coder-llm-ticketed" ;; + *) echo "running unit: does NOT execute coder-llm-ticketed (${exec_line:0:120})"; ok=1 ;; + esac + echo "token: $(head -c 80 "${GPU_TOKEN_FILE:-$HOME/GPU-TOKEN}" 2> /dev/null | head -1 || true)" + [ "$ok" = 0 ] && echo "STATUS: ticketed" || echo "STATUS: NOT ticketed" + return "$ok" +} + +[ -f "$DROPIN" ] || { echo "no drop-in at $DROPIN" >&2; exit 2; } +case "$mode" in + --diff) + if is_ticketed_file; then echo "already ticketed: $DROPIN"; exit 0; fi + diff -u "$DROPIN" <(render) | sed "2s|/dev/fd/[0-9]*|$DROPIN (new)|" + exit 0 ;; + --status) + status; exit $? ;; + --install) + if is_ticketed_file; then echo "already ticketed: $DROPIN"; status; exit 0; fi + grep -q '^ExecStart=$' "$DROPIN" || { echo "no 'ExecStart=' reset line in $DROPIN: refusing to guess" >&2; exit 2; } + bak=$DROPIN.bak-$(date +%Y%m%d) + [ -e "$bak" ] && bak=$bak-$(date +%H%M%S) + cp -p "$DROPIN" "$bak" || exit 1 + render > "$DROPIN.new" && mv "$DROPIN.new" "$DROPIN" || exit 1 + mkdir -p "$BIN_DIR" && install -m 0755 "$here/coder-llm-ticketed" "$WRAPPER" || exit 1 + echo "backed up: $bak" + "${SYSTEMCTL[@]}" daemon-reload && "${SYSTEMCTL[@]}" restart coder-llm || { echo "systemctl failed; roll back with: cp -p $bak $DROPIN" >&2; exit 1; } + status; exit $? ;; + *) + echo "usage: $0 [--diff | --status | --install]" >&2; exit 2 ;; +esac diff --git a/tests/coder_llm_ticketed_test.sh b/tests/coder_llm_ticketed_test.sh new file mode 100755 index 000000000..6577d9021 --- /dev/null +++ b/tests/coder_llm_ticketed_test.sh @@ -0,0 +1,149 @@ +#!/bin/bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# tests/coder_llm_ticketed_test.sh — the PR-Agent model runs only inside a gpu-hold ticket, in on-demand windows +# (CODER-LLM-TICKETED, 2026-11-15; engine#2494). No GPU: the real scripts/gpu-hold with a private queue, token, die +# temperature and kernel-log fixture, and a fake server whose /slots says busy or idle from a state file. +# +# Usage: coder_llm_ticketed_test.sh [path-to-coder-llm-ticketed] [path-to-gpu-hold] +# +# Arms: +# A no demand: no window opens, no token is taken; +# B demand opens a window under the label coder-llm; an idle server and stale demand close it (rc 0), the server is +# gone and the token released; +# C a busy server is cut at the max hold (rc 124), the server is gone; +# D a gfxhub page fault in the kernel log during the window ends it (rc 78) and the server dies with it; +# E a lane that queues during a window gets the GPU before the next coder-llm window (FIFO; the restart re-queues); +# F a failing gh (the pr-agent-run fallback) is logged once per wait with its reason, and opens no window. +set -u +root=$(cd "$(dirname "$0")/.." && pwd) # the defaults work from any directory, not only the repo root +tick=${1:-$root/scripts/coder-llm-ticketed} +hold=${2:-$root/scripts/gpu-hold} +for f in "$tick" "$hold"; do [ -x "$f" ] || { echo "not executable: $f"; exit 2; }; done +tick=$(readlink -f "$tick"); hold=$(readlink -f "$hold") + +scratch=$(mktemp -d) +pids=() +cleanup() { for p in "${pids[@]}"; do kill -9 "$p" 2> /dev/null; done; pkill -9 -f "$scratch/fake" 2> /dev/null; rm -rf "$scratch"; } +trap cleanup EXIT +fail=0 +echo 40000 > "$scratch/tctl" +port=$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",0)); print(s.getsockname()[1])') + +cat > "$scratch/fake" < "$scratch/klog" + +wait_for() { # $1 seconds, $2.. a test command + local n=$1; shift + for _ in $(seq 1 "$n"); do "$@" && return 0; sleep 1; done + return 1 +} +server_alive() { [ -s "$scratch/server.pid" ] && kill -0 "$(cat "$scratch/server.pid")" 2> /dev/null; } +token_is() { [ "$(cat "$scratch/TOKEN" 2> /dev/null)" = "$1" ] || grep -qs "^$1" "$scratch/TOKEN"; } + +# ---- A + B: no demand, then demand; idle closes the window ---------------------------------------------------------- +echo idle > "$scratch/state"; rm -f "$scratch/server.pid" "$scratch/demand" +touch -d '-10 min' "$scratch/demand" # stale demand +env "${envs[@]}" CODER_LLM_IDLE_S=4 CODER_LLM_MAX_HOLD_MIN=5 timeout 120 "$tick" -- "$scratch/fake" > "$scratch/ab.log" 2>&1 & +ab=$!; pids+=($ab) +sleep 6 +if [ -e "$scratch/TOKEN" ] || server_alive; then echo "FAIL A: a window opened with only stale demand"; fail=1; fi +touch "$scratch/demand" +if ! wait_for 15 server_alive; then echo "FAIL B: demand did not open a window"; fail=1; fi +grep -qs coder-llm "$scratch/TOKEN" || { echo "FAIL B: the window does not hold the token as coder-llm ($(cat "$scratch/TOKEN" 2> /dev/null))"; fail=1; } +srv=$(cat "$scratch/server.pid" 2> /dev/null) +wait "$ab"; rc=$? +[ "$rc" = 0 ] || { echo "FAIL B: rc $rc after an idle close, want 0"; fail=1; } +grep -q "idle 4s and no fresh demand" "$scratch/ab.log" || { echo "FAIL B: the log does not say the window closed idle"; fail=1; } +[ -n "$srv" ] && kill -0 "$srv" 2> /dev/null && { echo "FAIL B: the server $srv outlived its window"; fail=1; } +[ -e "$scratch/TOKEN" ] && { echo "FAIL B: the token is still held"; fail=1; } + +# ---- C: busy, cut at the max hold ---------------------------------------------------------------------------------- +echo busy > "$scratch/state"; rm -f "$scratch/server.pid"; touch "$scratch/demand" +t0=$(date +%s) +env "${envs[@]}" CODER_LLM_IDLE_S=2 CODER_LLM_MAX_HOLD_MIN=0.1 timeout 120 "$tick" -- "$scratch/fake" > "$scratch/c.log" 2>&1 +rc=$?; el=$(( $(date +%s) - t0 )) +[ "$rc" = 124 ] || { echo "FAIL C: rc $rc for a busy server at the max hold, want 124"; fail=1; } +[ "$el" -lt 40 ] || { echo "FAIL C: took ${el}s for a 6 s max hold"; fail=1; } +server_alive && { echo "FAIL C: the server outlived the max hold"; fail=1; } +[ -e "$scratch/TOKEN" ] && { echo "FAIL C: the token is still held"; fail=1; } + +# ---- D: a kernel fault during the window --------------------------------------------------------------------------- +echo busy > "$scratch/state"; rm -f "$scratch/server.pid"; touch "$scratch/demand"; : > "$scratch/klog" +( sleep 5; echo "kernel: amdgpu 0000:c5:00.0: [gfxhub] page fault (src_id:0 ring:24 vmid:9 pasid:26332)" >> "$scratch/klog" ) & +pids+=($!) +env "${envs[@]}" GPU_HOLD_FAULT_ABORT=1 CODER_LLM_IDLE_S=60 CODER_LLM_MAX_HOLD_MIN=5 timeout 120 "$tick" -- "$scratch/fake" > "$scratch/d.log" 2>&1 +rc=$? +[ "$rc" = 78 ] || { echo "FAIL D: rc $rc after a gfxhub fault, want 78"; fail=1; } +grep -q "backing off" "$scratch/d.log" || { echo "FAIL D: no back-off after rc 78"; fail=1; } +server_alive && { echo "FAIL D: the server survived the fault abort"; fail=1; } +[ -e "$scratch/TOKEN" ] && { echo "FAIL D: the token is still held"; fail=1; } +: > "$scratch/klog" + +# ---- E: a lane queued during a window runs before the next window -------------------------------------------------- +echo busy > "$scratch/state"; rm -f "$scratch/server.pid"; touch "$scratch/demand"; : > "$scratch/hold.log" +( env "${envs[@]}" CODER_LLM_IDLE_S=60 CODER_LLM_MAX_HOLD_MIN=0.1 timeout 120 "$tick" -- "$scratch/fake" > "$scratch/e1.log" 2>&1 + # what systemd's Restart= does: the next invocation, which queues again + env "${envs[@]}" CODER_LLM_IDLE_S=60 CODER_LLM_MAX_HOLD_MIN=0.05 timeout 120 "$tick" -- "$scratch/fake" > "$scratch/e2.log" 2>&1 ) & +e=$!; pids+=($e) +wait_for 15 server_alive || { echo "FAIL E: the first window did not open"; fail=1; } +env "${envs[@]}" GPU_HOLD_WAIT=60 timeout 120 "$hold" -t 0.05 -l lanejob -- sleep 1 > /dev/null 2>&1 +lrc=$? +[ "$lrc" = 0 ] || { echo "FAIL E: the lane ticket did not run (rc $lrc)"; fail=1; } +wait "$e" +order=$(grep -oE "took .*|queued '[a-z-]+'" "$scratch/hold.log" | tr '\n' ' ') +lane_took=$(grep -n "queued 'lanejob'" "$scratch/hold.log" | head -1 | cut -d: -f1) +took_lines=$(grep -n "took " "$scratch/hold.log" | cut -d: -f1 | tr '\n' ' ') +# three holds, in order: coder-llm, lanejob, coder-llm +holders=$(grep -oE "^gpu-hold\[[0-9]+\] took" "$scratch/hold.log" | wc -l) +seq_labels=$(awk '/queued/ {match($0, /queued .[a-z-]+./); q[$1]=substr($0, RSTART+8, RLENGTH-9)} / took / {print q[$1]}' "$scratch/hold.log" | tr '\n' ' ') +[ "$seq_labels" = "coder-llm lanejob coder-llm " ] || { echo "FAIL E: hold order '$seq_labels', want 'coder-llm lanejob coder-llm '"; fail=1; } + +# ---- F: the pr-agent-run fallback says so when it cannot ask (a failing gh), once per wait ------------------------- +mkdir -p "$scratch/ghbin" +printf '#!/bin/bash\nif [ "$1" = auth ]; then exit 1; fi\necho "HTTP 401: Bad credentials" >&2; exit 1\n' > "$scratch/ghbin/gh" +chmod +x "$scratch/ghbin/gh" +touch -d '-2 hours' "$scratch/demand" +env "${envs[@]}" CODER_LLM_GH_POLL=1 CODER_LLM_GH_POLL_S=1 PATH="$scratch/ghbin:$PATH" \ + timeout 6 "$tick" -- "$scratch/fake" > "$scratch/f.log" 2>&1 +[ "$(grep -c 'gh poll failed (HTTP 401' "$scratch/f.log")" = 1 ] || { echo "FAIL F: want exactly one 'gh poll failed' line, got $(grep -c 'gh poll failed' "$scratch/f.log")"; fail=1; } +grep -q 'gh is not logged in' "$scratch/f.log" || { echo "FAIL F: no warning that gh is not logged in"; fail=1; } +[ -e "$scratch/TOKEN" ] && { echo "FAIL F: a failing poll opened a window"; fail=1; } + +[ "$fail" = 0 ] && echo "coder_llm_ticketed: ok (no demand -> no ticket; idle -> rc 0; busy -> 124 at max hold; fault -> 78 + back-off; FIFO: coder-llm, lane, coder-llm; a failing gh poll is logged once, no window)" +exit "$fail" diff --git a/tests/install_coder_llm_ticketed_test.sh b/tests/install_coder_llm_ticketed_test.sh new file mode 100755 index 000000000..5ab8d33c7 --- /dev/null +++ b/tests/install_coder_llm_ticketed_test.sh @@ -0,0 +1,77 @@ +#!/bin/bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# tests/install_coder_llm_ticketed_test.sh — scripts/install-coder-llm-ticketed.sh on a fake unit directory and a stub +# systemctl (no real unit is touched): +# 1 --diff prints the change and modifies nothing; +# 2 --status says NOT ticketed (exit 1) before the install; +# 3 --install backs up the drop-in, keeps the server command byte for byte behind the wrapper, installs the wrapper, +# reloads and restarts (the stub records both), and --status then says ticketed (exit 0); +# 4 a second --install changes nothing (idempotent). +set -u +root=$(cd "$(dirname "$0")/.." && pwd) +inst=${1:-$root/scripts/install-coder-llm-ticketed.sh} +[ -x "$inst" ] || { echo "not executable: $inst"; exit 2; } +scratch=$(mktemp -d) +trap 'rm -rf "$scratch"' EXIT +fail=0 +mkdir -p "$scratch/unit" "$scratch/bin" +serve='/opt/x/1bit serve -m /models/m.gguf --device hrx --host 127.0.0.1 --port 8090 --ctx-size 131072 --alias qwen3-coder --parallel 4' +cat > "$scratch/unit/zz.conf" < "$scratch/systemctl" <> "$scratch/calls" +if [ "\$1" = show ]; then grep '^ExecStart=.' "$scratch/unit/zz.conf" | tail -1 | sed 's/^ExecStart=//'; fi +EOF +chmod +x "$scratch/systemctl" +envs=(CODER_LLM_UNIT_DIR="$scratch/unit" CODER_LLM_DROPIN=zz.conf CODER_LLM_BIN_DIR="$scratch/bin" + CODER_LLM_SYSTEMCTL="$scratch/systemctl" GPU_TOKEN_FILE="$scratch/none") + +out=$(env "${envs[@]}" "$inst" --diff) +cmp -s "$scratch/unit/zz.conf" "$scratch/orig.conf" || { echo "FAIL 1: --diff modified the drop-in"; fail=1; } +grep -q "^+ExecStart=$scratch/bin/coder-llm-ticketed -- $serve\$" <<< "$out" || { echo "FAIL 1: --diff does not show the wrapped command"; fail=1; } +[ -e "$scratch/bin/coder-llm-ticketed" ] && { echo "FAIL 1: --diff installed the wrapper"; fail=1; } + +env "${envs[@]}" "$inst" --status > "$scratch/s1" 2>&1; rc=$? +[ "$rc" = 1 ] && grep -q "STATUS: NOT ticketed" "$scratch/s1" || { echo "FAIL 2: --status before install: rc $rc, $(tail -1 "$scratch/s1")"; fail=1; } + +env "${envs[@]}" "$inst" --install > "$scratch/i1" 2>&1; rc=$? +[ "$rc" = 0 ] || { echo "FAIL 3: --install rc $rc: $(tail -3 "$scratch/i1")"; fail=1; } +bak=$(ls "$scratch/unit"/zz.conf.bak-* 2> /dev/null | head -1) +[ -n "$bak" ] && cmp -s "$bak" "$scratch/orig.conf" || { echo "FAIL 3: no faithful backup ($bak)"; fail=1; } +[ "$(grep -c '^ExecStart=.' "$scratch/unit/zz.conf")" = 1 ] || { echo "FAIL 3: not exactly one command line"; fail=1; } +grep -qxF "ExecStart=$scratch/bin/coder-llm-ticketed -- $serve" "$scratch/unit/zz.conf" || { echo "FAIL 3: the server command did not survive byte for byte"; fail=1; } +grep -qx 'ExecStart=' "$scratch/unit/zz.conf" || { echo "FAIL 3: the ExecStart= reset line is gone"; fail=1; } +grep -qx 'Environment=HSA_USE_SVM=0' "$scratch/unit/zz.conf" || { echo "FAIL 3: an existing line was lost"; fail=1; } +cmp -s "$scratch/bin/coder-llm-ticketed" "$root/scripts/coder-llm-ticketed" || { echo "FAIL 3: the wrapper was not installed"; fail=1; } +grep -qx 'daemon-reload' "$scratch/calls" && grep -qx 'restart coder-llm' "$scratch/calls" || { echo "FAIL 3: no daemon-reload + restart ($(tr '\n' ';' < "$scratch/calls"))"; fail=1; } +grep -q "STATUS: ticketed" "$scratch/i1" || { echo "FAIL 3: --status after install: $(tail -1 "$scratch/i1")"; fail=1; } + +cp "$scratch/unit/zz.conf" "$scratch/after.conf" +env "${envs[@]}" "$inst" --install > "$scratch/i2" 2>&1; rc=$? +[ "$rc" = 0 ] && cmp -s "$scratch/unit/zz.conf" "$scratch/after.conf" && grep -q "already ticketed" "$scratch/i2" \ + || { echo "FAIL 4: a second --install was not a no-op (rc $rc)"; fail=1; } +[ "$(ls "$scratch/unit"/zz.conf.bak-* | wc -l)" = 1 ] || { echo "FAIL 4: a second backup was written"; fail=1; } + +[ "$fail" = 0 ] && echo "install_coder_llm_ticketed: ok (--diff changes nothing; status before = NOT; install keeps the command byte for byte, backs up, reloads+restarts; idempotent)" +exit "$fail"