From c0cefd557fc428da32a9b43a37b4cd4910ae5639 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 01:19:53 -0400 Subject: [PATCH 1/2] window-run: a window that leaves coder-llm down fails with 69, not a WARNING line Follow-up from #2483's review: when the restart does not bring :8090 back, the end line carried the job's rc (often 0) and stderr said WARNING, so a window that left the fleet LLM down looked like a successful one to every caller and to WINDOWS.log. final_rc() now turns a successful job into 69 (EX_UNAVAILABLE) when :8090 is not 200; a job that already failed keeps its own rc. The end line adds " restart=FAILED" and stderr says ERROR. Self-test arms via --final-rc: 200/0 -> 0, 000/0 -> 69, 503/3 -> 3. Co-Authored-By: Claude Opus 5.5 --- scripts/window-run.sh | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/scripts/window-run.sh b/scripts/window-run.sh index b48363cb8..db0f3d43f 100755 --- a/scripts/window-run.sh +++ b/scripts/window-run.sh @@ -89,6 +89,10 @@ self_test() { chk "a TERM'd runner exits 143" "$rrc" 143 kill -0 "$(cat "$tmp/fake.pid" 2>/dev/null)" 2>/dev/null; chk " ... and its queued job is gone" "$?" 1 chk " ... and nothing is logged" "$(wc -l < "$tmp/q.log")" 0 + # 3e. coder-llm not back after the restart fails the run (69) unless the job already failed + chk "--final-rc 200 0 -> 0" "$("$0" --final-rc 200 0)" 0 + chk "--final-rc 000 0 -> 69 (coder-llm did not come back)" "$("$0" --final-rc 000 0)" 69 + chk "--final-rc 503 3 -> 3 (the job's own failure is kept)" "$("$0" --final-rc 503 3)" 3 # 4. an old enough end opens the gate again (the refusal above is about the gap, not about the file) printf '%s end lane=t issue=0 label=t coder-llm=active :8090=200 rc=0\n' "$(date -u -d '90 minutes ago' +%FT%TZ)" > "$tmp/w.log" out=$(WINDOWS_LOG=$tmp/w.log "$0" --dry-run -m 5 -l x -c true 2>&1); rc=$? @@ -121,6 +125,10 @@ self_test() { } [ "${1:-}" = --self-test ] && { self_test; exit $?; } +# The run's final status once coder-llm was restarted: the job's rc when :8090 answers 200, else 69 (EX_UNAVAILABLE) for +# a job that itself succeeded. A window that leaves the fleet LLM down is a failed window, not a WARNING line. +final_rc() { if [ "$1" = 200 ] || [ "$2" != 0 ]; then echo "$2"; else echo 69; fi; } +if [ "${1:-}" = "--final-rc" ]; then final_rc "${2:-}" "${3:-0}"; exit 0; fi # --gap-ok: rc 0 when the 45-minute gap is open now, rc 1 (with the reason) when it is not. The ticket's child runs it # right before it stops coder-llm, so a window another lane ran while this one queued still counts. if [ "${1:-}" = "--gap-ok" ]; then @@ -221,9 +229,11 @@ restore() { code=$(curl -s -m 2 -o /dev/null -w '%{http_code}' http://127.0.0.1:8090/v1/models 2>/dev/null || true) [ "$code" = 200 ] && break; sleep 3 done - printf '%s end lane=%s issue=%s label=%s coder-llm=%s :8090=%s rc=%s\n' "$(date -u +%FT%TZ)" \ - "$WR_LANE" "$ISSUE" "$LABEL" "$(systemctl --user is-active coder-llm)" "$code" "$rc" >> "$LOG" - [ "$code" = 200 ] || printf '%s: WARNING coder-llm restarted but :8090 answers %s\n' "$SELF" "$code" >&2 + rc=$(final_rc "$code" "$rc") + printf '%s end lane=%s issue=%s label=%s coder-llm=%s :8090=%s rc=%s%s\n' "$(date -u +%FT%TZ)" \ + "$WR_LANE" "$ISSUE" "$LABEL" "$(systemctl --user is-active coder-llm)" "$code" "$rc" \ + "$([ "$code" = 200 ] || echo ' restart=FAILED')" >> "$LOG" + [ "$code" = 200 ] || printf '%s: ERROR coder-llm was restarted but :8090 answers %s: exit %s\n' "$SELF" "$code" "$rc" >&2 else printf '%s end lane=%s issue=%s label=%s coder-llm=was-not-running :8090=n/a rc=%s\n' "$(date -u +%FT%TZ)" \ "$WR_LANE" "$ISSUE" "$LABEL" "$rc" >> "$LOG" From 79e7b2e7f8c6ff295161cc5a1ae028636dc24760 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 01:46:11 -0400 Subject: [PATCH 2/2] window-run: pin the child's gap recheck with a behavioural arm; exit-code table; temp dirs off /tmp; setsid check Review of #2492 (CHANGES at c0cefd5): deleting the child's `"$WR_SELF" --gap-ok || exit 75` left the self-test green. New arm 3f: a fake gpu-hold appends another lane's end line (a window that ended while this one queued) and then runs the real child, with a recording fake systemctl first on PATH -> rc 75, no `stop coder-llm` call, the log holds only that end line. With the recheck line deleted all three fail (got 0 / 1 / 3), and the real coder-llm cannot be touched either way. Also from the reviews: an exit-status table in the usage (0, the job's N, 2, 69, 75, 130/143; 69 only when THIS window stopped coder-llm and it did not come back); markers and self-test temp dirs under ${WR_TMPDIR:-~/.cache}, not /tmp; a missing setsid is refused before anything is logged. Co-Authored-By: Claude Opus 5.5 --- scripts/window-run.sh | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/scripts/window-run.sh b/scripts/window-run.sh index db0f3d43f..82af417ee 100755 --- a/scripts/window-run.sh +++ b/scripts/window-run.sh @@ -34,6 +34,17 @@ # command and the rc -- the command is what makes a window auditable afterwards # the child gets its own ticket pid in GPU_HOLD_TICKET, so a measurement script can prove # ownership instead of guessing it from the global token file +# +# Exit status (what a caller should do with it): +# 0 the window ran, the job succeeded, and coder-llm is back (:8090 answered 200) +# N the job's own rc, whenever the job failed (a restart failure never masks it) +# 2 refused before anything happened: bad arguments, inside the 45-min gap without -w, no gpu-hold / setsid +# 69 the job succeeded but THIS window stopped coder-llm and :8090 did not answer 200 after the restart: the fleet +# LLM is down because of this run; look now (a window that found coder-llm already stopped never returns 69, +# its end line says coder-llm=was-not-running and it does not start it) +# 75 the ticket came, but another lane's window had ended less than 45 min before: nothing stopped, nothing logged; +# queue again later (-w waits only before the ticket, not after a refusal) +# 130 / 143 INT / TERM: the job group was killed, coder-llm restored if this window stopped it set -uo pipefail SELF=${0##*/} @@ -57,7 +68,7 @@ die() { printf '%s: %s\n' "$SELF" "$*" >&2; exit 2; } self_test() { local fail=0 chk() { local label=$1 got=$2 want=$3; if [ "$got" = "$want" ]; then echo "ok $label"; else echo "FAIL $label (got $got, want $want)"; fail=$((fail+1)); fi; } - tmp=$(mktemp -d); trap 'rm -rf "$tmp"' RETURN + mkdir -p "${WR_TMPDIR:-$HOME/.cache}"; tmp=$(mktemp -d "${WR_TMPDIR:-$HOME/.cache}/window-run-test.XXXXXX"); trap 'rm -rf "$tmp"' RETURN # 1. a duration outside 1..15 is refused, with the reason out=$(WINDOWS_LOG=$tmp/w.log "$0" --dry-run -m 0 -l x -c true 2>&1); rc=$? chk "a zero-minute window is refused" "$rc" 2 @@ -93,6 +104,17 @@ self_test() { chk "--final-rc 200 0 -> 0" "$("$0" --final-rc 200 0)" 0 chk "--final-rc 000 0 -> 69 (coder-llm did not come back)" "$("$0" --final-rc 000 0)" 69 chk "--final-rc 503 3 -> 3 (the job's own failure is kept)" "$("$0" --final-rc 503 3)" 3 + # 3f. the ticket's child really re-checks the gap: another window ends while this one is queued (the fake gpu-hold + # appends that end line, then runs the real child) -> rc 75, coder-llm never stopped, nothing else logged. + # systemctl is a recording fake first on PATH, so even a regression here cannot stop the real coder-llm. + mkdir -p "$tmp/fakebin" + printf '#!/bin/bash\necho "$*" >> %s/systemctl.calls\n[ "$*" = "--user is-active --quiet coder-llm" ] && exit 0\nexit 0\n' "$tmp" > "$tmp/fakebin/systemctl" + printf '#!/bin/bash\n[ "${1:-}" = --status ] && exit 0\nwhile [ $# -gt 0 ] && [ "$1" != -- ]; do shift; done; shift\nprintf "%%s end lane=other issue=0 label=other coder-llm=active :8090=200 rc=0\\n" "$(date -u +%%FT%%TZ)" >> "$WR_LOG"\nexec "$@"\n' > "$tmp/fakebin/gpu-hold" + chmod +x "$tmp/fakebin/systemctl" "$tmp/fakebin/gpu-hold"; : > "$tmp/r.log"; : > "$tmp/systemctl.calls" + PATH=$tmp/fakebin:$PATH GPU_HOLD=$tmp/fakebin/gpu-hold WINDOWS_LOG=$tmp/r.log "$0" -m 1 -l x -c true >/dev/null 2>&1 + chk "a window that ended while queued refuses the ticket (75)" "$?" 75 + chk " ... coder-llm is never stopped" "$(grep -c 'stop coder-llm' "$tmp/systemctl.calls")" 0 + chk " ... and the log holds only the other lane's end line" "$(wc -l < "$tmp/r.log")" 1 # 4. an old enough end opens the gate again (the refusal above is about the gap, not about the file) printf '%s end lane=t issue=0 label=t coder-llm=active :8090=200 rc=0\n' "$(date -u -d '90 minutes ago' +%FT%TZ)" > "$tmp/w.log" out=$(WINDOWS_LOG=$tmp/w.log "$0" --dry-run -m 5 -l x -c true 2>&1); rc=$? @@ -200,6 +222,8 @@ if [ "$NOW" = 1 ]; then fi [ -x "$GPU_HOLD" ] || die "no executable gpu-hold at $GPU_HOLD (set GPU_HOLD): refusing before anything is logged" +# setsid (util-linux) puts the job in its own process group, which is what lets a TERM kill the whole tree +command -v setsid >/dev/null 2>&1 || die "setsid not found (util-linux): the job could not be killed as one group; refusing" if [ "$DRY" = 1 ]; then printf '%s: would take gpu-hold -t %s -l %s, stop coder-llm, run: %s\n' "$SELF" "$MINS" "$LABEL" "$CMD" exit 0 @@ -207,7 +231,8 @@ fi [ -d "$(dirname "$LOG")" ] || mkdir -p "$(dirname "$LOG")" # Markers the ticket's child writes: STARTED once it logged the start line, STOPPED once it stopped coder-llm. A run that # never got that far (refused, killed in the queue, gpu-hold missing) logs nothing and restarts nothing. -WR_DIR=$(mktemp -d); export WR_STARTED=$WR_DIR/started WR_STOPPED=$WR_DIR/stopped +mkdir -p "${WR_TMPDIR:-$HOME/.cache}" # /tmp is tmpfs on strixhalo and the lane rules keep new files off it +WR_DIR=$(mktemp -d "${WR_TMPDIR:-$HOME/.cache}/window-run.XXXXXX"); export WR_STARTED=$WR_DIR/started WR_STOPPED=$WR_DIR/stopped export WR_SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/${BASH_SOURCE[0]##*/}" WR_LOG=$LOG WR_LANE=${LANE:-$SELF} \ WR_ISSUE=$ISSUE WR_LABEL=$LABEL WR_MINS=$MINS WR_CMD=$CMD WINDOWS_LOG=$LOG JOB=""