From 9fa4f3285e37a210d42ba2dc251248252b2f5c58 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 02:57:12 -0400 Subject: [PATCH 1/2] window-run: a ticketed coder-llm asleep on purpose is not a failed restart The restart probe returns 69 - "the fleet LLM is down because of this run; look now" - whenever `:8090` did not answer 200 after the restart. That was right when coder-llm was resident. It stops being right the moment `coder-llm` runs inside a ticket (`scripts/coder-llm-ticketed`, engine#2496): the unit **waits for the demand file before it takes a ticket**, so with stale demand it never serves and never exits - `systemctl --user is-active` stays `active` while the port is silent. Every window that ran more than `CODER_LLM_DEMAND_TTL_MIN` (30) after the last PR-Agent call would then report a false failure, in the one message that says to go and look now. restart_rc 200 -> the job's rc (the model is back) the job itself failed -> the job's rc (a restart failure never masks it) active AND "waiting for demand" -> the job's rc (asleep ON PURPOSE) anything else -> 69 (we stopped it, it did not come back) The end line keeps its `Z end ` shape for the gap parser and now names which case it took: `asleep-no-demand` or `restart=FAILED`, so a reader does not have to infer it from a port number. stderr says the same in words. Seven arms in `--self-test` drive all four rows through a hidden `--restart-rc` mode (no service, no port): --restart-rc 200 0 1 1 -> 0 --restart-rc 000 0 1 1 -> 0 (asleep on purpose; was 69) --restart-rc 200 0 1 0 -> 0 --restart-rc 000 0 1 0 -> 69 (a real failed restart) --restart-rc 503 0 1 1 -> 0 --restart-rc 000 0 0 1 -> 69 (inactive is never healthy) --restart-rc 000 3 1 1 -> 3 (the job's failure is still kept) Mutations, each caught: restart_rc back to final_rc -> "FAIL ... active waiting -> 0 (asleep on purpose) (got 69, want 0)" x2 the unit-state term dropped -> "FAIL ... inactive -> 69 (inactive is never healthy) (got 0, want 69)" Self-test ok, copyright rc 0, lane_syntax balanced. --- scripts/window-run.sh | 46 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/scripts/window-run.sh b/scripts/window-run.sh index dcb7ab2df..40e980e69 100755 --- a/scripts/window-run.sh +++ b/scripts/window-run.sh @@ -113,6 +113,15 @@ self_test() { chk "--final-rc 200 0 -> 0" "$("$0" --final-rc 200 0)" 0 chk "--final-rc 000 0 -> 69 (coder-llm did not come back)" "$("$0" --final-rc 000 0)" 69 chk "--final-rc 503 3 -> 3 (the job's own failure is kept)" "$("$0" --final-rc 503 3)" 3 + # 3g. the restart verdict, four rows (engine#2496: a ticketed coder-llm is `active` while :8090 is silent, waiting + # for demand). The third row is the one that was missing: asleep-on-purpose is not a failed restart. + chk "--restart-rc 200 0 active waiting -> 0" "$("$0" --restart-rc 200 0 1 1)" 0 + chk "--restart-rc 200 0 active not-waiting -> 0" "$("$0" --restart-rc 200 0 1 0)" 0 + chk "--restart-rc 000 0 active waiting -> 0 (asleep on purpose)" "$("$0" --restart-rc 000 0 1 1)" 0 + chk "--restart-rc 000 0 active NOT waiting -> 69 (a real failed restart)" "$("$0" --restart-rc 000 0 1 0)" 69 + chk "--restart-rc 000 0 inactive waiting -> 69 (inactive is never healthy)" "$("$0" --restart-rc 000 0 0 1)" 69 + chk "--restart-rc 000 3 active waiting -> 3 (the job's failure is still kept)" "$("$0" --restart-rc 000 3 1 1)" 3 + chk "--restart-rc 503 0 active waiting -> 0 (a 503 with a waiting unit is still asleep)" "$("$0" --restart-rc 503 0 1 1)" 0 # 3f. the ticket's child really re-checks the gap: another window ends while this one is queued (the fake gpu-hold # appends that end line, then runs the real child) -> rc 75, coder-llm never stopped, nothing else logged. # systemctl is a recording fake first on PATH, so even a regression here cannot stop the real coder-llm. @@ -159,7 +168,24 @@ self_test() { # The run's final status once coder-llm was restarted: the job's rc when :8090 answers 200, else 69 (EX_UNAVAILABLE) for # a job that itself succeeded. A window that leaves the fleet LLM down is a failed window, not a WARNING line. final_rc() { if [ "$1" = 200 ] || [ "$2" != 0 ]; then echo "$2"; else echo 69; fi; } +# The restart's verdict, once the port did not answer (engine#2496's contract, and the reason final_rc is not enough): +# +# code=200 -> the job's rc (the model is back) +# the job itself failed -> the job's rc (a restart failure never masks it) +# the unit is ACTIVE and its own journal says "waiting for demand" +# -> the job's rc (the model is asleep ON PURPOSE; this run did not break it) +# anything else -> 69 (we stopped it and it did not come back) +# +# The middle case is why this exists: a ticketed coder-llm (`scripts/coder-llm-ticketed`, engine#2496) waits for the +# demand file BEFORE it takes a ticket, so with stale demand it never serves and never exits - `is-active` stays +# `active` while `:8090` is silent. Without this, every window that ran more than CODER_LLM_DEMAND_TTL_MIN after the +# last PR-Agent call would return 69 and print "the fleet LLM is down because of this run", which is false and is the +# alert that gets ignored by the third occurrence. +restart_rc() { # + if [ "$1" = 200 ] || [ "$3" != 1 ] || [ "$4" != 1 ]; then final_rc "$1" "$2"; else echo "$2"; fi +} if [ "${1:-}" = "--final-rc" ]; then final_rc "${2:-}" "${3:-0}"; exit 0; fi +if [ "${1:-}" = "--restart-rc" ]; then restart_rc "${2:-000}" "${3:-0}" "${4:-0}" "${5:-0}"; exit 0; fi # --gap-ok: rc 0 when the 45-minute gap is open now, rc 1 (with the reason) when it is not. The ticket's child runs it # right before it stops coder-llm, so a window another lane ran while this one queued still counts. if [ "${1:-}" = "--gap-ok" ]; then @@ -269,11 +295,23 @@ restore() { code=$(curl -s -m 2 -o /dev/null -w '%{http_code}' http://127.0.0.1:8090/v1/health 2>/dev/null || true) [ "$code" = 200 ] && break; sleep 3 done - rc=$(final_rc "$code" "$rc") + _state=$(systemctl --user is-active coder-llm 2>/dev/null) + _waiting=0 + if [ "$code" != 200 ] && [ "$_state" = active ] && + journalctl --user -u coder-llm -n 60 --no-pager 2>/dev/null | grep -q 'waiting for demand'; then + _waiting=1 + fi + rc=$(restart_rc "$code" "$rc" "$([ "$_state" = active ] && echo 1 || echo 0)" "$_waiting") + # The end line keeps its shape for the gap parser (`Z end ` prefix) and names which case it took, so a reader does + # not have to infer it from a port number: asleep-no-demand is not a failure, restart=FAILED is. printf '%s end lane=%s issue=%s label=%s coder-llm=%s :8090=%s rc=%s%s\n' "$(date -u +%FT%TZ)" \ - "$WR_LANE" "$ISSUE" "$LABEL" "$(systemctl --user is-active coder-llm)" "$code" "$rc" \ - "$([ "$code" = 200 ] || echo ' restart=FAILED')" >> "$LOG" - [ "$code" = 200 ] || printf '%s: ERROR coder-llm was restarted but :8090 answers %s: exit %s\n' "$SELF" "$code" "$rc" >&2 + "$WR_LANE" "$ISSUE" "$LABEL" "$_state" "$code" "$rc" \ + "$([ "$code" = 200 ] && echo '' || { [ "$_waiting" = 1 ] && echo ' asleep-no-demand' || echo ' restart=FAILED'; })" >> "$LOG" + if [ "$code" != 200 ] && [ "$_waiting" != 1 ]; then + printf '%s: ERROR coder-llm was restarted but :8090 answers %s: exit %s\n' "$SELF" "$code" "$rc" >&2 + elif [ "$_waiting" = 1 ]; then + printf '%s: coder-llm is waiting for demand (ticketed, :8090 silent by design): keeping rc %s\n' "$SELF" "$rc" >&2 + fi else printf '%s end lane=%s issue=%s label=%s coder-llm=was-not-running :8090=n/a rc=%s\n' "$(date -u +%FT%TZ)" \ "$WR_LANE" "$ISSUE" "$LABEL" "$rc" >> "$LOG" From 6472fed907dbcc228614e30fb73b2190a2f21c3a Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 03:16:01 -0400 Subject: [PATCH 2/2] window-run: the "asleep" answer is scoped to THIS invocation and read from its newest line (review on #2503) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reviewer built the exact sequence my first version misread, out of coder-llm-ticketed's own log lines: coder-llm-ticketed[100] waiting for demand (...) <- the PREVIOUS invocation Stopping/Started coder-llm.service coder-llm-ticketed[200] demand: queueing for the GPU (...) coder-llm-ticketed[201] window open: server pid 202 E hrx: device lost coder-llm-ticketed[201] server exited by itself, rc 1 coder-llm-ticketed[200] window ended rc 1 (38 lines in all, under 60) `journalctl -n 60 | grep -q 'waiting for demand'` is true for that, so a model that crashed on load right after this window was reported as healthy-by-design and the run exited 0 - masking precisely the failure 69 exists for. Two corrections, both in `coder_llm_asleep()`: * **scoped to the current invocation**: `systemctl --user show -p InvocationID --value coder-llm` then `journalctl --user _SYSTEMD_INVOCATION_ID=`, because the same unit's previous run logs the same sentence; * **the LAST wrapper state line decides**, not any matching one: `grep -F 'coder-llm-ticketed[' | tail -1`, and only that line may say `waiting for demand`. An empty or unreadable journal answers **not asleep**, so an unknown state fails toward 69 rather than toward a false all-clear. The journal is read through a seam (`coder_llm_asleep [journal-file]`, plus `WR_JOURNAL_FILE` and a hidden `--asleep` mode) so a test can feed fixtures. Five arms, built from the reviewer's own sequence: crash-on-load after a stale 'waiting for demand' -> not asleep, and the restart verdict is 69 the newest wrapper line is 'waiting for demand' -> asleep (1) empty journal -> not asleep (the safe direction) journal with no wrapper line at all -> not asleep the four --restart-rc rows -> unchanged Mutations, each caught: back to "any matching line" (the reviewer's bug) -> FAIL a crash-on-load … is NOT asleep (got 1, want 0) FAIL … so the restart verdict for it is 69 (got 0, want 69) empty journal flipped to 1 -> FAIL an empty journal is not asleep (got 1, want 0) Self-test ok, copyright rc 0. Not armed: dropping the invocation scoping on its own (a fixture cannot exercise the live journal), which is why the tail-1 line is the belt and the scoping is the braces. --- scripts/window-run.sh | 53 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 3 deletions(-) diff --git a/scripts/window-run.sh b/scripts/window-run.sh index 40e980e69..6dd708844 100755 --- a/scripts/window-run.sh +++ b/scripts/window-run.sh @@ -122,6 +122,26 @@ self_test() { chk "--restart-rc 000 0 inactive waiting -> 69 (inactive is never healthy)" "$("$0" --restart-rc 000 0 0 1)" 69 chk "--restart-rc 000 3 active waiting -> 3 (the job's failure is still kept)" "$("$0" --restart-rc 000 3 1 1)" 3 chk "--restart-rc 503 0 active waiting -> 0 (a 503 with a waiting unit is still asleep)" "$("$0" --restart-rc 503 0 1 1)" 0 + # 3h. the journal reader itself (review on this PR): the crash-on-load sequence must NOT read as "waiting for demand", + # even though a stale "waiting for demand" from the PREVIOUS invocation is still in the tail. + cat > "$tmp/j-crash.log" <<'JLOG' +coder-llm-ticketed[100] waiting for demand (demand file touched within 1800s) +Stopping coder-llm.service... +Started coder-llm.service... +coder-llm-ticketed[200] demand: queueing for the GPU (gpu-hold -l coder-llm -t 25 -w 900) +coder-llm-ticketed[201] window open: server pid 202 +E hrx: device lost +coder-llm-ticketed[201] server exited by itself, rc 1 +coder-llm-ticketed[200] window ended rc 1 +JLOG + chk "a crash-on-load after a stale 'waiting for demand' is NOT asleep" "$("$0" --asleep "$tmp/j-crash.log")" 0 + chk " ... so the restart verdict for it is 69" "$("$0" --restart-rc 000 0 1 "$("$0" --asleep "$tmp/j-crash.log")")" 69 + printf '%s\n' 'coder-llm-ticketed[300] waiting for demand (demand file touched within 1800s)' > "$tmp/j-wait.log" + chk "the newest wrapper line saying 'waiting for demand' IS asleep" "$("$0" --asleep "$tmp/j-wait.log")" 1 + : > "$tmp/j-empty.log" + chk "an empty journal is not asleep (the safe direction)" "$("$0" --asleep "$tmp/j-empty.log")" 0 + printf '%s\n' 'no wrapper lines here at all' > "$tmp/j-none.log" + chk "a journal with no wrapper line at all is not asleep" "$("$0" --asleep "$tmp/j-none.log")" 0 # 3f. the ticket's child really re-checks the gap: another window ends while this one is queued (the fake gpu-hold # appends that end line, then runs the real child) -> rc 75, coder-llm never stopped, nothing else logged. # systemctl is a recording fake first on PATH, so even a regression here cannot stop the real coder-llm. @@ -181,11 +201,39 @@ final_rc() { if [ "$1" = 200 ] || [ "$2" != 0 ]; then echo "$2"; else echo 69; f # `active` while `:8090` is silent. Without this, every window that ran more than CODER_LLM_DEMAND_TTL_MIN after the # last PR-Agent call would return 69 and print "the fleet LLM is down because of this run", which is false and is the # alert that gets ignored by the third occurrence. +# "Waiting for demand" is only the answer if it is what THIS invocation is doing NOW. Two things make that true, and my +# first version got both wrong (review on this PR): +# * scope to the current invocation -- `journalctl --user _SYSTEMD_INVOCATION_ID=` -- because the same unit's +# PREVIOUS run also logged "waiting for demand", and the last 60 lines can easily still hold it; +# * read the LAST wrapper state line, not any matching one. A crash on load right after this window logs +# `demand: queueing …` -> `window open: server pid …` -> `device lost` -> `server exited by itself, rc 1`, all after +# that stale "waiting for demand", and calling that "asleep on purpose" is exactly the failure 69 exists for. +# The journal is read through a seam so a test can feed a fixture: coder_llm_asleep [journal-file]. +coder_llm_asleep() { # [journal file] -> prints 1 when the unit is waiting for demand on THIS invocation, else 0 + local jid src + if [ -n "${1:-}" ]; then + src=$(cat "$1" 2>/dev/null) + else + jid=$(systemctl --user show -p InvocationID --value coder-llm 2>/dev/null) + [ -n "$jid" ] || { echo 0; return; } + src=$(journalctl --user --no-pager -q _SYSTEMD_INVOCATION_ID="$jid" 2>/dev/null) + fi + [ -n "$src" ] || { echo 0; return; } + # the wrapper's own lines, newest last; only the newest one is a statement about now + local last + last=$(printf '%s\n' "$src" | grep -F 'coder-llm-ticketed[' | tail -1) + case "$last" in + *'waiting for demand'*) echo 1 ;; + *) echo 0 ;; + esac +} + restart_rc() { # if [ "$1" = 200 ] || [ "$3" != 1 ] || [ "$4" != 1 ]; then final_rc "$1" "$2"; else echo "$2"; fi } if [ "${1:-}" = "--final-rc" ]; then final_rc "${2:-}" "${3:-0}"; exit 0; fi if [ "${1:-}" = "--restart-rc" ]; then restart_rc "${2:-000}" "${3:-0}" "${4:-0}" "${5:-0}"; exit 0; fi +if [ "${1:-}" = "--asleep" ]; then coder_llm_asleep "${2:-}"; exit 0; fi # --gap-ok: rc 0 when the 45-minute gap is open now, rc 1 (with the reason) when it is not. The ticket's child runs it # right before it stops coder-llm, so a window another lane ran while this one queued still counts. if [ "${1:-}" = "--gap-ok" ]; then @@ -297,9 +345,8 @@ restore() { done _state=$(systemctl --user is-active coder-llm 2>/dev/null) _waiting=0 - if [ "$code" != 200 ] && [ "$_state" = active ] && - journalctl --user -u coder-llm -n 60 --no-pager 2>/dev/null | grep -q 'waiting for demand'; then - _waiting=1 + if [ "$code" != 200 ] && [ "$_state" = active ]; then + _waiting=$(coder_llm_asleep ${WR_JOURNAL_FILE:+"$WR_JOURNAL_FILE"}) fi rc=$(restart_rc "$code" "$rc" "$([ "$_state" = active ] && echo 1 || echo 0)" "$_waiting") # The end line keeps its shape for the gap parser (`Z end ` prefix) and names which case it took, so a reader does