diff --git a/docs/ci-runners.md b/docs/ci-runners.md index c8d993d8e..f3195c82a 100644 --- a/docs/ci-runners.md +++ b/docs/ci-runners.md @@ -302,3 +302,36 @@ and `--list`. It cannot be a ctest itself — it needs the network and a token, which the GitHub-hosted job does not have. A *scheduled* red (a cron workflow that fails once an hour while a runner is down) is the complete answer and is not here: a new workflow changes what the repository's token may do, which is an RFC under CONTRIBUTING.md. + +## Two rows, one question: are the runners healthy? + +Since #1682 and #1546 both landed there are two checks, and neither alone is enough — each is blind to what the +other sees. Both run in `scripts/weekly.sh`'s preflight, so a release refuses to start on a runner it cannot trust. + +| | `scripts/check-ci-runners.sh` | `scripts/ci-runner-health.sh` | +|---|---|---| +| reads | the GitHub API (`…/actions/runners`) | systemd (`systemctl show`) | +| says | is it **registered** and `online` | is the unit **active**, `Result=`, `NRestarts` | +| notices | a registration that was revoked, expired or never made | `Result=oom-kill`, a restart window, a unit that stopped | +| cannot see | *why* it died — no `oom-kill`, no restart count | a unit that is running fine while its registration is gone | +| speed | after the heartbeat times out | the moment the unit dies | +| exit | 0 online · 1 an offender, named · **2 cannot tell** (unreadable or truncated list) | 0 all accepting · 1 one is not · 2 no such units on this host | + +Run either by hand — both are read-only and neither restarts anything: + +```sh +scripts/check-ci-runners.sh --require strixhalo-engine strixhalo-bench --list +scripts/ci-runner-health.sh # every actions.runner.* unit +scripts/ci-runner-health.sh --self-test # the verdict logic, systemctl stubbed +``` + +`WEEKLY_SKIP_RUNNER_CHECK=1` releases during an outage, and it skips **both** rows: they fail for the same dead +runner, so a flag that skipped only one of them could not do the job it documents. It also says so in the log -- +`preflight: runner checks SKIPPED (WEEKLY_SKIP_RUNNER_CHECK=1)` -- because a release log where the checks quietly did +not run reads exactly like one where they passed. Their exit 2 means "nothing to +inspect on this host" — a fresh box or a fork — which is a note, not a failure; a check that cannot see has to say so +rather than report green. + +One case neither row covers: a runner that is `online` in the API *and* active locally, but wedged so it takes no +jobs. That is a hung process, not a registration or unit state; the practical signal for it is a queue that stops +moving while both rows stay green. diff --git a/scripts/weekly.sh b/scripts/weekly.sh index e3f6b076c..ea861f9a8 100755 --- a/scripts/weekly.sh +++ b/scripts/weekly.sh @@ -680,8 +680,14 @@ preflight() { done # engine#1504: a self-hosted runner that is dead makes every check that needs this box read as PENDING, never # red (2026-11-08: both dead 3h45m behind 24 jobs), so the state that damages a release most is the one nothing - # reports. scripts/check-ci-runners.sh asks the API the release lanes already use; it exits 0 when no runners - # are registered, so a fresh box or a fork is unaffected. WEEKLY_SKIP_RUNNER_CHECK=1 releases during an outage. + # reports. Two rows, because each is blind to what the other sees: + # scripts/check-ci-runners.sh - GitHub's view: is it REGISTERED and online. Cannot say why, and it only + # notices after the heartbeat times out. Exit 0 when none are registered. + # scripts/ci-runner-health.sh - systemd's view (#1546): Result=oom-kill, a restart window, a unit that is not + # active. Says WHY, and fires the moment a unit dies. Exit 2 = no such units on + # this host, i.e. a fresh box, which is a note rather than a failure. + # WEEKLY_SKIP_RUNNER_CHECK=1 is the release-during-an-outage escape, and it skips BOTH: a flag that skipped only + # one of them would be useless in exactly the outage it exists for, because both fail for the same dead runner. if [ "${WEEKLY_SKIP_RUNNER_CHECK:-0}" != 1 ]; then if runner_out=$("$(cd "$(dirname "$0")" && pwd)/check-ci-runners.sh" \ --require strixhalo-engine strixhalo-bench 2>&1); then @@ -691,25 +697,24 @@ preflight() { echo "preflight: a self-hosted runner is not online, so the checks after this release would queue for ever (WEEKLY_SKIP_RUNNER_CHECK=1 to release anyway)" bad=1 fi - fi - # The same fact from the other side: scripts/ci-runner-health.sh reads systemd's view (#1546, merged) and says WHY - # a runner is down -- Result=oom-kill, a restart window -- which the API above cannot see, and it fires the moment - # a unit dies instead of after the heartbeat timeout. Neither was ever invoked from anywhere until this row, so - # running both here is what makes the two halves one check at release time. Its exit 2 means "no such units on this - # host", which is a fresh box, not a failure. - if [ -x "$(cd "$(dirname "$0")" && pwd)/ci-runner-health.sh" ]; then - if health_out=$("$(cd "$(dirname "$0")" && pwd)/ci-runner-health.sh" 2>&1); then - printf '%s\n' "$health_out" | grep -c '^PASS' | xargs -r -I{} echo "preflight: runner units: {} PASS in systemd's view" - else - health_rc=$? - if [ "$health_rc" = 2 ]; then - say "preflight: no actions.runner.* units on this host: systemd has nothing to inspect" + if [ -x "$(cd "$(dirname "$0")" && pwd)/ci-runner-health.sh" ]; then + if health_out=$("$(cd "$(dirname "$0")" && pwd)/ci-runner-health.sh" 2>&1); then + printf '%s\n' "$health_out" | grep -c '^PASS' | xargs -r -I{} echo "preflight: runner units: {} PASS in systemd's view" else - printf '%s\n' "$health_out" | sed 's/^/preflight: /' - echo "preflight: a runner unit is not accepting jobs (scripts/ci-runner-health.sh): $health_rc" - bad=1 + health_rc=$? + if [ "$health_rc" = 2 ]; then + say "preflight: no actions.runner.* units on this host: systemd has nothing to inspect" + else + printf '%s\n' "$health_out" | sed 's/^/preflight: /' + echo "preflight: a runner unit is not accepting jobs (scripts/ci-runner-health.sh): $health_rc" + bad=1 + fi fi fi + else + # a skip that says nothing is the failure mode these rows exist for: a release log where the runner checks + # silently did not run reads exactly like one where they passed. + say "preflight: runner checks SKIPPED (WEEKLY_SKIP_RUNNER_CHECK=1) - a dead runner will read as a pending check, never red" fi return $bad }