From 86c692259588bb29228b6aa5b7e4dda2eb1d356c Mon Sep 17 00:00:00 2001 From: t Date: Sun, 15 Nov 2026 13:22:17 -0400 Subject: [PATCH 1/4] window-run: a ticketed coder-llm that is QUEUED is not a failed restart engine#2496 merged, so `coder-llm` now runs inside `coder-llm-ticketed` and `systemctl start` means "queue for a GPU ticket". With lanes ahead of it the unit is `active` and `:8090` stays empty for minutes -- so the restore's poll expires, restarts it twice, and records the window as a failed restart. Measured on this box an hour after #2496 landed (window 17:07:40-17:08:59Z): the unit ran coder-llm-ticketed, `coder-llm` was 4th in ~/GPU-QUEUE, and the end line read ':8090=000 restart=failed' for a window whose job had succeeded. That is a false signal on EVERY window, which is exactly how a lane learns to ignore the one that matters. The restore now has the third state it was missing: token held by another lane -> restart=deferred (already, #2547) unit active + in the queue -> restart=queued <- this commit unit active + not queued -> the existing asleep/loading/wedged logic 2 arms: the queued case writes restart=queued(waiting for a GPU ticket) and does not write restart=FAILED. Mutation-checked: disabling the branch turns 2 arms red; restoring gives rc 0. --- scripts/window-run.sh | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/scripts/window-run.sh b/scripts/window-run.sh index 55265f2ba..e6dff2bee 100755 --- a/scripts/window-run.sh +++ b/scripts/window-run.sh @@ -186,6 +186,13 @@ JLOG chk " ... and the deferral is not reported as a failed restart" "$(grep -c 'restart=FAILED' "$tmp/t.log")" 0 # the loading-vs-wedged check needs a live child growing in CPU, which a self-test cannot fake without testing its # own fake; pin the reader instead so it cannot be deleted silently (the same technique the probe arm uses). + # 3k. engine#2496's contract: a ticketed coder-llm that is QUEUED is not a failed restart + : > "$tmp/t.calls"; mkdir -p "$tmp/q"; : > "$tmp/q/0179.coder-llm"; printf 'label=coder-llm\npid=999999\n' > "$tmp/q/0179.coder-llm" + printf '%s end lane=t issue=0 label=t coder-llm=active :8090=200 rc=0\n' "$(date -u -d '90 minutes ago' +%FT%TZ)" > "$tmp/t.log" + PATH=$tmp/tbin2:$tmp/tbin:$PATH GPU_HOLD=$tmp/tbin/gpu-hold GPU_TOKEN_FILE=$tmp/none-token GPU_QUEUE_DIR=$tmp/q \ + WINDOWS_LOG=$tmp/t.log "$0" -m 1 -l x -c true >/dev/null 2>&1 + chk "a queued coder-llm is recorded as queued, not failed" "$(grep -c 'restart=queued(waiting for a GPU ticket)' "$tmp/t.log")" 1 + chk " ... and it is not reported as a failed restart" "$(grep -c 'restart=FAILED' "$tmp/t.log")" 0 chk "the poll reads the largest child's CPU, not the idle embed child" "$(grep -c 'sort -k2 -n | tai[l] -1' "$me")" 1 # 4. an old enough end opens the gate again (the refusal above is about the gap, not about the file) printf '%s end lane=t issue=0 label=t coder-llm=active :8090=200 rc=0\n' "$(date -u -d '90 minutes ago' +%FT%TZ)" > "$tmp/w.log" @@ -379,6 +386,20 @@ restore() { printf '%s: coder-llm restarts when the next lane releases the GPU (holder %s); not a failed restart\n' "$SELF" "${_held:-?}" >&2 rm -rf "$WR_DIR"; exit "$rc" fi + # QUEUED IS NOT DEAD EITHER (engine#2496's contract, live 2026-11-15): once `coder-llm` runs inside + # `coder-llm-ticketed`, `systemctl start` means "queue for a GPU ticket" -- so with lanes ahead of it the unit + # is `active` and `:8090` stays empty for minutes. A restart-then-fail there is a false signal on EVERY window, + # which is how lanes learn to ignore the failure. Measured: the unit ran `coder-llm-ticketed`, `coder-llm` was + # 4th in ~/GPU-QUEUE, and this restore reported failure for a window whose job had succeeded. + coder_llm_queued() { + local q=${GPU_QUEUE_DIR:-$HOME/GPU-QUEUE} f + [ -d "$q" ] || return 1 + for f in "$q"/*; do + case "${f##*/}" in .*) continue ;; esac + grep -qx 'label=coder-llm' "$f" 2>/dev/null && return 0 + done + return 1 + } # LOADING IS NOT WEDGED (same windows): a child loading a 30B answers 503 while its CPU time advances; # restarting it there throws the load away. Only a child whose CPU time has not moved for ~30 s is dead. # The LARGEST llama-server is the one loading -- the embed service's idle child is ~15 MB. @@ -408,6 +429,12 @@ restore() { sleep "$_poll_s" done _state=$(systemctl --user is-active coder-llm 2>/dev/null) + if [ "$code" != 200 ] && [ "$_state" = active ] && coder_llm_queued; then + printf '%s end lane=%s issue=%s label=%s coder-llm=active :8090=queued restart=queued(waiting for a GPU ticket) rc=%s\n' \ + "$(date -u +%FT%TZ)" "$WR_LANE" "$ISSUE" "$LABEL" "$rc" >> "$LOG" + printf '%s: coder-llm is queued for a GPU ticket (engine#2496), not dead: :8090 answers when it gets one\n' "$SELF" >&2 + rm -rf "$WR_DIR"; exit "$rc" + fi _waiting=0 if [ "$code" != 200 ] && [ "$_state" = active ]; then _waiting=$(coder_llm_asleep ${WR_JOURNAL_FILE:+"$WR_JOURNAL_FILE"}) From 67543ad1f3d83993b8d717d522c32e3fa37d8c87 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 15 Nov 2026 14:03:55 -0400 Subject: [PATCH 2/4] window-run: the queued test asks for LIVENESS, not existence (review of #2566) Two review findings, both mine: 1. coder_llm_queued() matched the label alone, so a queue entry left behind by a killed process licensed 'queued' for every later window -- a permanent false calm, worse than the false alarm this PR removes. gpu-hold itself calls such an entry stale and drops it (stale(): no pid, or whose pid is gone), so the reader was using a weaker rule than the writer. Now: read pid= and require kill -0. The arm that used to write pid=999999 (dead) and assert 'queued' is the bug in fixture form -- it is now the STALE arm, asserting the opposite, and the live arm uses this test's own live shell pid. 2. The queued arm passed only while :8090 happened to be down; with the fleet LLM serving, its poll returned 200 and the arm could not fire. The readiness URL is now WR_HEALTH_URL (default unchanged) and the arms force it to a refusing port, so the arm tests the branch and not the box. Mutation-checked: dropping kill -0 turns the stale arm red; restoring gives rc 0. Self-test green with the live box serving. --- scripts/window-run.sh | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/scripts/window-run.sh b/scripts/window-run.sh index e6dff2bee..75c8cfa20 100755 --- a/scripts/window-run.sh +++ b/scripts/window-run.sh @@ -187,12 +187,20 @@ JLOG # the loading-vs-wedged check needs a live child growing in CPU, which a self-test cannot fake without testing its # own fake; pin the reader instead so it cannot be deleted silently (the same technique the probe arm uses). # 3k. engine#2496's contract: a ticketed coder-llm that is QUEUED is not a failed restart - : > "$tmp/t.calls"; mkdir -p "$tmp/q"; : > "$tmp/q/0179.coder-llm"; printf 'label=coder-llm\npid=999999\n' > "$tmp/q/0179.coder-llm" + : > "$tmp/t.calls"; mkdir -p "$tmp/q" + # a LIVE pid: the arm used to write pid=999999, which does not exist -- so it demonstrated a STALE entry and + # asserted "queued" as correct (review of this PR). $$ is this test's own live shell. + printf 'label=coder-llm\npid=%s\n' "$$" > "$tmp/q/0179.coder-llm" printf '%s end lane=t issue=0 label=t coder-llm=active :8090=200 rc=0\n' "$(date -u -d '90 minutes ago' +%FT%TZ)" > "$tmp/t.log" - PATH=$tmp/tbin2:$tmp/tbin:$PATH GPU_HOLD=$tmp/tbin/gpu-hold GPU_TOKEN_FILE=$tmp/none-token GPU_QUEUE_DIR=$tmp/q \ + PATH=$tmp/tbin2:$tmp/tbin:$PATH GPU_HOLD=$tmp/tbin/gpu-hold GPU_TOKEN_FILE=$tmp/none-token GPU_QUEUE_DIR=$tmp/q WR_HEALTH_URL=http://127.0.0.1:9/x \ WINDOWS_LOG=$tmp/t.log "$0" -m 1 -l x -c true >/dev/null 2>&1 - chk "a queued coder-llm is recorded as queued, not failed" "$(grep -c 'restart=queued(waiting for a GPU ticket)' "$tmp/t.log")" 1 + chk "a queued coder-llm is recorded as queued, not failed" "$(grep -c 'restart=queued(WAITING for a GPU ticket when this window ended, not restored)' "$tmp/t.log")" 1 chk " ... and it is not reported as a failed restart" "$(grep -c 'restart=FAILED' "$tmp/t.log")" 0 + # a DEAD pid is a stale entry, which gpu-hold drops -- it must not license "queued" forever + : > "$tmp/t.log"; printf 'label=coder-llm\npid=999999\n' > "$tmp/q/0179.coder-llm" + PATH=$tmp/tbin2:$tmp/tbin:$PATH GPU_HOLD=$tmp/tbin/gpu-hold GPU_TOKEN_FILE=$tmp/none-token GPU_QUEUE_DIR=$tmp/q WR_HEALTH_URL=http://127.0.0.1:9/x \ + WINDOWS_LOG=$tmp/t.log "$0" -m 1 -l x -c true >/dev/null 2>&1 + chk "a STALE queue entry (dead pid) does not count as queued" "$(grep -c 'restart=queued' "$tmp/t.log")" 0 chk "the poll reads the largest child's CPU, not the idle embed child" "$(grep -c 'sort -k2 -n | tai[l] -1' "$me")" 1 # 4. an old enough end opens the gate again (the refusal above is about the gap, not about the file) printf '%s end lane=t issue=0 label=t coder-llm=active :8090=200 rc=0\n' "$(date -u -d '90 minutes ago' +%FT%TZ)" > "$tmp/w.log" @@ -391,12 +399,19 @@ restore() { # is `active` and `:8090` stays empty for minutes. A restart-then-fail there is a false signal on EVERY window, # which is how lanes learn to ignore the failure. Measured: the unit ran `coder-llm-ticketed`, `coder-llm` was # 4th in ~/GPU-QUEUE, and this restore reported failure for a window whose job had succeeded. + # LIVENESS, not existence (review of this PR, and the same mistake #2547 made with the token file): an entry + # left behind by a killed process outlives it, and `gpu-hold` itself calls such an entry stale and DROPS it + # (stale(), nineteen lines in: "no pid, or whose pid is gone"). Matching on the label alone makes a dead + # ticket licence a `queued` verdict for every later window -- a permanent false calm, which is worse than the + # false alarm this PR removes. coder_llm_queued() { - local q=${GPU_QUEUE_DIR:-$HOME/GPU-QUEUE} f + local q=${GPU_QUEUE_DIR:-$HOME/GPU-QUEUE} f pid [ -d "$q" ] || return 1 for f in "$q"/*; do case "${f##*/}" in .*) continue ;; esac - grep -qx 'label=coder-llm' "$f" 2>/dev/null && return 0 + grep -qx 'label=coder-llm' "$f" 2>/dev/null || continue + pid=$(sed -n 's/^pid=\([0-9][0-9]*\)$/\1/p' "$f" 2>/dev/null | head -1) + [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null && return 0 done return 1 } @@ -420,7 +435,7 @@ restore() { # check asks for: slower than a status code, and the only probe that cannot be right while the LLM is unusable. code=$(curl -s -m 20 -o /dev/null -w '%{http_code}' -H 'Content-Type: application/json' \ -d '{"model":"qwen3-coder","messages":[{"role":"user","content":"ok"}],"max_tokens":1,"temperature":0}' \ - http://127.0.0.1:8090/v1/chat/completions 2>/dev/null || true) + "${WR_HEALTH_URL:-http://127.0.0.1:8090/v1/chat/completions}" 2>/dev/null || true) [ "$code" = 200 ] && break _cpu=$(cpu_of) if [ -n "$_cpu" ] && [ "$_cpu" = "$prev" ]; then froze=$((froze + 1)); else froze=0; fi @@ -430,7 +445,7 @@ restore() { done _state=$(systemctl --user is-active coder-llm 2>/dev/null) if [ "$code" != 200 ] && [ "$_state" = active ] && coder_llm_queued; then - printf '%s end lane=%s issue=%s label=%s coder-llm=active :8090=queued restart=queued(waiting for a GPU ticket) rc=%s\n' \ + printf '%s end lane=%s issue=%s label=%s coder-llm=active :8090=queued restart=queued(WAITING for a GPU ticket when this window ended, not restored) rc=%s\n' \ "$(date -u +%FT%TZ)" "$WR_LANE" "$ISSUE" "$LABEL" "$rc" >> "$LOG" printf '%s: coder-llm is queued for a GPU ticket (engine#2496), not dead: :8090 answers when it gets one\n' "$SELF" >&2 rm -rf "$WR_DIR"; exit "$rc" From f21121f686917e7005d6fd3b2060ed65bf54944f Mon Sep 17 00:00:00 2001 From: t Date: Sun, 15 Nov 2026 14:32:29 -0400 Subject: [PATCH 3/4] window-run: an arm for the two states overlapping (held token beats queued) Review ask on this PR: the deferred and queued exemptions were each tested alone, so nothing pinned their precedence -- and with a token in another lane's hand AND a live coder-llm ticket queued, both are true at once (which is exactly the interleaving #2547 measured). The deferred branch is checked before the poll and the token is the more specific fact: the LLM is behind a token someone holds, not waiting for a ticket of its own. The arm asserts restart=deferred exactly once and restart=queued zero times. Mutation-checked: defeating the token check turns it red; restoring gives rc 0. --- scripts/window-run.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/window-run.sh b/scripts/window-run.sh index 75c8cfa20..0cb0eeae6 100755 --- a/scripts/window-run.sh +++ b/scripts/window-run.sh @@ -201,6 +201,14 @@ JLOG PATH=$tmp/tbin2:$tmp/tbin:$PATH GPU_HOLD=$tmp/tbin/gpu-hold GPU_TOKEN_FILE=$tmp/none-token GPU_QUEUE_DIR=$tmp/q WR_HEALTH_URL=http://127.0.0.1:9/x \ WINDOWS_LOG=$tmp/t.log "$0" -m 1 -l x -c true >/dev/null 2>&1 chk "a STALE queue entry (dead pid) does not count as queued" "$(grep -c 'restart=queued' "$tmp/t.log")" 0 + # 3l. BOTH conditions true: another lane holds the token AND a live coder-llm ticket is queued. + # The deferred branch is checked before the poll, so the more specific state must win -- a `queued` here + # would claim the LLM is waiting for a ticket when it is in fact waiting behind a token in someone's hand. + : > "$tmp/t.log"; printf 'gpu-hold cl-other 999999\n' > "$tmp/token"; printf 'label=coder-llm\npid=%s\n' "$$" > "$tmp/q/0179.coder-llm" + PATH=$tmp/tbin2:$tmp/tbin:$PATH GPU_HOLD=$tmp/tbin/gpu-hold GPU_TOKEN_FILE=$tmp/token GPU_QUEUE_DIR=$tmp/q \ + WR_HEALTH_URL=http://127.0.0.1:9/x WINDOWS_LOG=$tmp/t.log "$0" -m 1 -l x -c true >/dev/null 2>&1 + chk "a held token wins over a queued entry (deferred, not queued)" "$(grep -c 'restart=deferred' "$tmp/t.log")" 1 + chk " ... and it is not reported as queued" "$(grep -c 'restart=queued' "$tmp/t.log")" 0 chk "the poll reads the largest child's CPU, not the idle embed child" "$(grep -c 'sort -k2 -n | tai[l] -1' "$me")" 1 # 4. an old enough end opens the gate again (the refusal above is about the gap, not about the file) printf '%s end lane=t issue=0 label=t coder-llm=active :8090=200 rc=0\n' "$(date -u -d '90 minutes ago' +%FT%TZ)" > "$tmp/w.log" From 9603f2934c25e70141ddb2729999ca732efb2b70 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 15 Nov 2026 15:50:52 -0400 Subject: [PATCH 4/4] window-run: the freeze detector could not fire without a llama-server (engine#2566 CI) The poll that decides loading-vs-wedged only counted a repeat when `ps` returned a CPU reading: if [ -n "$_cpu" ] && [ "$_cpu" = "$prev" ]; then froze=... so on a box with NO llama-server -- CI, or a restart whose child never appeared -- an empty reading never counted as no-progress, froze stayed 0, and the loop always ran its whole WR_POLL_MAX x WR_POLL_S bound: 100 x 3 s = 300 s per arm. ctest's 600 s limit is how this surfaced (#2566 build, window_run_self): two such arms and the test dies at 600.09 s. Empty is the strongest possible evidence of no progress. Nothing running for ~30 s after a restart is a wedge, not a load, so empty now counts like any other unchanged reading. - the self-test exports WR_POLL_MAX=3 WR_POLL_S=0.05: these arms assert the verdict, not the wait. Locally :8090 answers 200 so every arm broke instantly; in CI it refuses, which is why the same arms cost 300 s each there and 0 s here -- an ambient difference, not a difference in what was tested. - arm 3l takes its own queue dir ($tmp/q3l): it writes a LIVE coder-llm ticket, which the later arms inherited from the shared $tmp/q. - WR_POLL_TRACE= logs 'poll i= froze= cpu=' per iteration, so an arm can assert the detector fired without timing it; arm 3m (a `ps` stub that reports no llama-server, the CI condition, health refused) requires froze to reach 10 after exactly 10 polls. Mutation-checked: restoring the `-n "$_cpu"` guard gives 'froze reaches 10 (got 0, want 1)' and 'after 10 polls (got 100, want 10)' -- the 100 is the 300 s, measured. --- scripts/window-run.sh | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/scripts/window-run.sh b/scripts/window-run.sh index 0cb0eeae6..2cdbe635c 100755 --- a/scripts/window-run.sh +++ b/scripts/window-run.sh @@ -68,6 +68,11 @@ die() { printf '%s: %s\n' "$SELF" "$*" >&2; exit 2; } self_test() { local fail=0 chk() { local label=$1 got=$2 want=$3; if [ "$got" = "$want" ]; then echo "ok $label"; else echo "FAIL $label (got $got, want $want)"; fail=$((fail+1)); fi; } + # A SELF-TEST MUST NOT POLL FOR REAL: these arms assert the verdict, not the wait. Every arm that reaches + # restore() would otherwise poll :8090 -- 200 on a working box (instant break), refused in CI (the full bound). + # That ambient difference is exactly how engine#2566's build died at ctest's 600 s: same arms, 3 s x 100 each. + export WR_POLL_MAX=3 WR_POLL_S=0.05 + # arm 3l gets its own queue: a LIVE coder-llm ticket left in the shared $tmp/q is inherited by later arms. mkdir -p "${WR_TMPDIR:-$HOME/.cache}"; tmp=$(mktemp -d "${WR_TMPDIR:-$HOME/.cache}/window-run-test.XXXXXX"); trap 'rm -rf "$tmp"' RETURN # 0. THE RESTART PROBE ASKS FOR A COMPLETION. No device is touched here, so this arm reads the artifact the # gate ships. /v1/models is a static registry listing that answers 200 while the weights load (app/serve.cpp @@ -204,11 +209,22 @@ JLOG # 3l. BOTH conditions true: another lane holds the token AND a live coder-llm ticket is queued. # The deferred branch is checked before the poll, so the more specific state must win -- a `queued` here # would claim the LLM is waiting for a ticket when it is in fact waiting behind a token in someone's hand. - : > "$tmp/t.log"; printf 'gpu-hold cl-other 999999\n' > "$tmp/token"; printf 'label=coder-llm\npid=%s\n' "$$" > "$tmp/q/0179.coder-llm" - PATH=$tmp/tbin2:$tmp/tbin:$PATH GPU_HOLD=$tmp/tbin/gpu-hold GPU_TOKEN_FILE=$tmp/token GPU_QUEUE_DIR=$tmp/q \ + mkdir -p "$tmp/q3l" # own queue: a LIVE ticket left in $tmp/q is inherited by later arms + : > "$tmp/t.log"; printf 'gpu-hold cl-other 999999\n' > "$tmp/token"; printf 'label=coder-llm\npid=%s\n' "$$" > "$tmp/q3l/0179.coder-llm" + PATH=$tmp/tbin2:$tmp/tbin:$PATH GPU_HOLD=$tmp/tbin/gpu-hold GPU_TOKEN_FILE=$tmp/token GPU_QUEUE_DIR=$tmp/q3l \ WR_HEALTH_URL=http://127.0.0.1:9/x WINDOWS_LOG=$tmp/t.log "$0" -m 1 -l x -c true >/dev/null 2>&1 chk "a held token wins over a queued entry (deferred, not queued)" "$(grep -c 'restart=deferred' "$tmp/t.log")" 1 chk " ... and it is not reported as queued" "$(grep -c 'restart=queued' "$tmp/t.log")" 0 + # 3m. NO llama-server on the box (the CI condition): the freeze detector must fire instead of burning the whole + # bound. With the old `-n "$_cpu"` guard an empty reading never counted as no-progress, so the loop ran + # WR_POLL_MAX x WR_POLL_S whatever happened -- 300 s per arm; ctest's 600 s limit is how #2566 found it. + mkdir -p "$tmp/nops" "$tmp/q3m"; printf '#!/bin/bash\nexit 1\n' > "$tmp/nops/ps"; chmod +x "$tmp/nops/ps" + : > "$tmp/tr.log" + PATH=$tmp/nops:$tmp/tbin2:$tmp/tbin:$PATH GPU_HOLD=$tmp/tbin/gpu-hold GPU_TOKEN_FILE=$tmp/none-token \ + GPU_QUEUE_DIR=$tmp/q3m WR_HEALTH_URL=http://127.0.0.1:9/x WR_POLL_MAX=100 WR_POLL_S=0.01 \ + WR_POLL_TRACE=$tmp/tr.log WINDOWS_LOG=$tmp/t3m.log "$0" -m 1 -l x -c true >/dev/null 2>&1 + chk "no llama-server: the freeze detector fires (froze reaches 10)" "$(grep -c 'froze=10' "$tmp/tr.log")" 1 + chk " ... after 10 polls, not the whole WR_POLL_MAX" "$(grep -c '^poll ' "$tmp/tr.log")" 10 chk "the poll reads the largest child's CPU, not the idle embed child" "$(grep -c 'sort -k2 -n | tai[l] -1' "$me")" 1 # 4. an old enough end opens the gate again (the refusal above is about the gap, not about the file) printf '%s end lane=t issue=0 label=t coder-llm=active :8090=200 rc=0\n' "$(date -u -d '90 minutes ago' +%FT%TZ)" > "$tmp/w.log" @@ -446,8 +462,13 @@ restore() { "${WR_HEALTH_URL:-http://127.0.0.1:8090/v1/chat/completions}" 2>/dev/null || true) [ "$code" = 200 ] && break _cpu=$(cpu_of) - if [ -n "$_cpu" ] && [ "$_cpu" = "$prev" ]; then froze=$((froze + 1)); else froze=0; fi + # NO PROCESS IS NO PROGRESS. An empty reading used to skip this check (`-n "$_cpu"`), so with no + # llama-server on the box -- CI, or a restart whose child never appeared -- the detector could never fire and + # the loop always burned its whole bound (WR_POLL_MAX 100 x WR_POLL_S 3 s = 300 s per arm; ctest's 600 s limit + # is how engine#2566's build found this). Nothing running for ~30 s after a restart is a wedge, not a load. + if [ "$_cpu" = "$prev" ]; then froze=$((froze + 1)); else froze=0; fi prev=$_cpu + [ -n "${WR_POLL_TRACE:-}" ] && printf 'poll i=%s froze=%s cpu=%s\n' "$i" "$froze" "${_cpu:-none}" >> "$WR_POLL_TRACE" [ "$froze" -ge 10 ] && break # ~30 s without CPU progress: a wedge, not a load sleep "$_poll_s" done