diff --git a/CMakeLists.txt b/CMakeLists.txt index 610142905..ce9aeeed1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -374,6 +374,10 @@ add_test(NAME pack_version COMMAND ${CMAKE_SOURCE_DIR}/tests/pack_version_test.s # arm asserts the row's message, and a clean tree is the control. No package build, no GPU, runs in CI. add_test(NAME pack_check COMMAND ${CMAKE_SOURCE_DIR}/tests/pack_check_test.sh) add_test(NAME check_ci_runners COMMAND ${CMAKE_SOURCE_DIR}/tests/check_ci_runners_test.sh) +# engine #1504: the release preflight blocks on a runner unit that is down and only warns on a short restart policy. +# The real scripts/ci-runner-health.sh and weekly.sh's preflight_runner_units() against a stub systemctl; no systemd. +add_test(NAME weekly_runner_preflight COMMAND ${CMAKE_SOURCE_DIR}/tests/weekly_runner_preflight_test.sh) +add_test(NAME ci_runner_health_selftest COMMAND bash ${CMAKE_SOURCE_DIR}/scripts/ci-runner-health.sh --self-test) # engine #1750: the carried amdxdna series applies in order to the pinned driver, and the context-restart code it # produces fails a context's jobs (and logs it) when the restart fails, instead of leaving them parked or running # them at close on a half-built firmware context. The driver's own functions in a user-space harness: no NPU, no diff --git a/docs/ci-runners.md b/docs/ci-runners.md index f3195c82a..107760817 100644 --- a/docs/ci-runners.md +++ b/docs/ci-runners.md @@ -306,7 +306,9 @@ here: a new workflow changes what the repository's token may do, which is an RFC ## Two rows, one question: are the runners healthy? Since #1682 and #1546 both landed there are two checks, and neither alone is enough — each is blind to what the -other sees. Both run in `scripts/weekly.sh`'s preflight, so a release refuses to start on a runner it cannot trust. +other sees. Both run in `scripts/weekly.sh`'s preflight. A runner that is **not accepting jobs** stops the release +before it starts. A runner that is up but whose **restart policy is short** is a warning, and the release goes on +(owner decision on engine#1504, 2026-11-11): see [the next section](#a-short-restart-policy-warns-it-does-not-block). | | `scripts/check-ci-runners.sh` | `scripts/ci-runner-health.sh` | |---|---|---| @@ -315,7 +317,7 @@ other sees. Both run in `scripts/weekly.sh`'s preflight, so a release refuses to | notices | a registration that was revoked, expired or never made | `Result=oom-kill`, a restart window, a unit that stopped | | cannot see | *why* it died — no `oom-kill`, no restart count | a unit that is running fine while its registration is gone | | speed | after the heartbeat times out | the moment the unit dies | -| exit | 0 online · 1 an offender, named · **2 cannot tell** (unreadable or truncated list) | 0 all accepting · 1 one is not · 2 no such units on this host | +| exit | 0 online · 1 an offender, named · **2 cannot tell** (unreadable or truncated list) | 0 all accepting, policy right · 1 one is not accepting jobs · 2 no such units on this host · **3 all accepting, a restart policy is short** | Run either by hand — both are read-only and neither restarts anything: @@ -335,3 +337,72 @@ rather than report green. One case neither row covers: a runner that is `online` in the API *and* active locally, but wedged so it takes no jobs. That is a hung process, not a registration or unit state; the practical signal for it is a queue that stops moving while both rows stay green. + +## A short restart policy warns, it does not block + +`scripts/ci-runner-health.sh` prints two lines per unit: its state (`PASS`/`FAIL`) and its restart policy +(`PASS`/`WARN`). The exit code keeps them apart, because they call for different things: + +| exit | meaning | `scripts/weekly.sh` preflight | +|---|---|---| +| 0 | every unit is accepting jobs and would come back after an OOM kill | passes | +| 1 | a unit is not accepting jobs now (dead, or in its restart window) | **blocks**: the checks after the release would queue for ever | +| 2 | no `actions.runner.*` units on this host | a note | +| 3 | every unit is accepting jobs, but a `WARN` line names a policy setting | **warns** and goes on | + +A unit that is down outranks a policy line, so 1 wins over 3. The warning in the release log reads +`preflight: WARNING, not blocking: every runner unit is accepting jobs, but a restart policy would leave one down +after the next OOM kill`, with the `WARN` lines above it naming the unit and the setting. + +Why a warning is enough: the policy is a box configuration change, which the owner applies (below), and a release +that cannot start does not make the unit any safer. Meanwhile the heavy builds run under +`scripts/mem-guard.sh --term`, which ends the build before the kernel's OOM killer reaches the runner. **A build step +that ends with exit 143 was stopped by that guard**: the log has a `GUARD: MemAvailable N GB < floor GB` line, the +runner is still alive, and the job is re-run once memory frees. The guard covers only the steps that use it. A server +or a bench started without it can still take the unit down, which is why the drop-ins below are the real fix. + +`tests/weekly_runner_preflight_test.sh` (ctest `weekly_runner_preflight`) runs the real health script and the real +`preflight_runner_units()` from `scripts/weekly.sh` against a stub `systemctl`: the four exit codes, block on a dead +unit, warn on a short policy. + +### The two drop-ins (owner step) + +On strixhalo, `strixhalo-bench` and `strixhalo-org` still run the units that predate `scripts/install-ci-runner.sh`: +`Restart=no`, `OOMPolicy=stop`. One OOM kill of any process in the unit stops it for good. `strixhalo-engine` got +the same drop-in on 2026-11-10. This needs `sudo`, so it is the owner's to run. It does not re-register a runner, +change its user, or stop a job in flight. + +Both files have the same content, [`scripts/ci-runner-restart.conf`](../scripts/ci-runner-restart.conf): + +```ini +# /etc/systemd/system/actions.runner.1bit-MONSTER-engine.strixhalo-bench.service.d/restart.conf +# /etc/systemd/system/actions.runner.1bit-MONSTER.strixhalo-org.service.d/restart.conf +[Unit] +StartLimitIntervalSec=0 + +[Service] +Restart=always +RestartSec=15 +OOMPolicy=continue +``` + +The three commands, from the root of an engine checkout: + +```sh +sudo install -Dm644 scripts/ci-runner-restart.conf /etc/systemd/system/actions.runner.1bit-MONSTER-engine.strixhalo-bench.service.d/restart.conf +sudo install -Dm644 scripts/ci-runner-restart.conf /etc/systemd/system/actions.runner.1bit-MONSTER.strixhalo-org.service.d/restart.conf +sudo systemctl daemon-reload +``` + +Check it took (read-only, no `sudo`): + +```sh +systemctl show actions.runner.1bit-MONSTER-engine.strixhalo-bench.service actions.runner.1bit-MONSTER.strixhalo-org.service -p Id -p Restart -p OOMPolicy +scripts/ci-runner-health.sh; echo "exit $?" +``` + +Want `Restart=always` and `OOMPolicy=continue` on both. The health script still exits 3 afterwards, with one `WARN` +per unit naming only `MemoryMax=infinity(want a ceiling)`: the drop-in sets no memory ceiling. The ceiling this page +documents (`MemoryHigh=43G`, `MemoryMax=48G`) comes with a full `scripts/install-ci-runner.sh` install, or with +`sudo systemctl set-property MemoryHigh=43G MemoryMax=48G` per unit. To undo a drop-in, delete the +`restart.conf` and run `sudo systemctl daemon-reload`. diff --git a/docs/hw-gate.md b/docs/hw-gate.md index 1d7afdeef..67402e213 100644 --- a/docs/hw-gate.md +++ b/docs/hw-gate.md @@ -72,7 +72,9 @@ hrx-system) through `cmake/hrx.cmake`, then links `onebit` with its HRX wiring. - **Memory.** The box is shared, and on Strix Halo GTT is host RAM. Before it compiles, the job waits for 24 GB of `MemAvailable` (`scripts/mem-wait.sh`, polled once a minute for up to 15 minutes). If memory doesn't free up, the job fails with the reading, and a re-run is the fix. The compile itself runs under `scripts/mem-guard.sh --term 12`, - which stops the whole `thermal-run` scope if `MemAvailable` falls below 12 GB. On 2026-11-08 both self-hosted + which stops the whole `thermal-run` scope if `MemAvailable` falls below 12 GB. **Exit 143 in a build step is + that guard, not a compiler failure**: the step log has a `GUARD: MemAvailable N GB < 12 GB` line just above it, the + runner is still alive, and a re-run once memory frees is the fix. On 2026-11-08 both self-hosted runners were OOM-killed, the second time 76 s after a restart handed this job a compile (engine#1504). - **What it is not.** It is a compile and link check of the public plumbing only. It does not build the private HIP add-on (the gate builds that) and does not dispatch anything. Green means the pinned pair builds into the engine. It diff --git a/scripts/ci-runner-health.sh b/scripts/ci-runner-health.sh index 20f47c874..0cd0fa2cf 100755 --- a/scripts/ci-runner-health.sh +++ b/scripts/ci-runner-health.sh @@ -30,8 +30,12 @@ # scripts/ci-runner-health.sh [...] # named units, e.g. for a specific runner # scripts/ci-runner-health.sh --self-test # the verdict logic, with systemctl stubbed # -# Exit: 0 when every runner is active/running with no failure result; 1 when any is not; 2 when there is nothing to inspect -# (no such units on this host, so a silent pass would be the defect). +# Exit: 0 when every runner is active/running with a restart policy that survives an OOM kill; 1 when a runner is not +# accepting jobs NOW; 2 when there is nothing to inspect (no such units on this host, so a silent pass would be the defect); +# 3 when every runner is accepting jobs but a restart policy would leave one down after the next OOM kill. 1 and 3 are +# separate because they call for different things (engine#1504, owner 2026-11-11): a dead runner blocks a release, since +# its checks would queue for ever, while a policy line is a box configuration change the owner applies +# (docs/ci-runners.md "The two drop-ins"), so scripts/weekly.sh warns on 3 and goes on. The policy lines print as WARN. set -uo pipefail # -> one verdict line. The state checks above answer "is it alive", which is what the @@ -53,6 +57,13 @@ policy_verdict() { # an infinite ceiling means the cgroup never hits its own limit, so the box's OOM killer chooses - and the bench runner # is the unit whose jobs are memory-heavy by definition. [ -n "${_mmax:-}" ] && [ "$_mmax" != infinity ] || _bad="$_bad MemoryMax=${_mmax:-?}(want a ceiling);" + case "$_bad" in + "") ;; + *Restart=*|*OOMPolicy=*) ;; + # Restart and OOMPolicy are right and only the ceiling is missing: the unit does come back, so do not say it will not. + *) printf 'FAIL the runner would come back after an OOM kill, but its cgroup has no ceiling:%s the box-wide OOM killer picks the victim, not the limit of this unit' "$_bad" + return 0 ;; + esac if [ -n "$_bad" ]; then printf 'FAIL the restart policy cannot restart this runner after an OOM kill:%s one OOM kill and the unit stops for good, while jobs read as pending' "$_bad" return 0 @@ -109,7 +120,12 @@ self_test() { chk FAIL "$(policy_verdict always stop 51539607552)" # an OOM kill of a JOB stops the RUNNER chk FAIL "$(policy_verdict always continue infinity)" # no cgroup ceiling: the box's OOM killer chooses chk FAIL "$(policy_verdict '' '' '')" # nothing readable is not a pass - [ "$rc" = 0 ] && echo "ci-runner-health self-test: ok (15 cases)" || echo "ci-runner-health self-test: FAIL" + # a missing ceiling alone must not be reported as "the unit stops for good" (the live strixhalo-engine unit, 2026-11-11) + case "$(policy_verdict always continue infinity)" in + *'would come back'*) printf 'ok a missing ceiling alone says the runner comes back\n' ;; + *) printf 'FAIL a missing ceiling alone should not say the unit stops for good\n'; rc=1 ;; + esac + [ "$rc" = 0 ] && echo "ci-runner-health self-test: ok (16 cases)" || echo "ci-runner-health self-test: FAIL" return $rc } @@ -125,7 +141,7 @@ if [ ${#units[@]} -eq 0 ]; then exit 2 fi -rc=0 +rc=0 policy=0 for u in "${units[@]}"; do line=$("$SC" show "$u" -p ActiveState -p SubState -p Result -p NRestarts -p Restart -p OOMPolicy -p MemoryMax 2>/dev/null) st=$(printf '%s\n' "$line" | sed -n 's/^ActiveState=//p') @@ -144,7 +160,9 @@ for u in "${units[@]}"; do p=$(policy_verdict "$rst" "$oom" "$mmax") case "$p" in PASS*) printf 'PASS %-52s %s\n' "" "${p#PASS }" ;; - *) printf 'FAIL %-52s %s\n' "" "${p#FAIL }"; rc=1 ;; + *) printf 'WARN %-52s %s\n' "" "${p#FAIL }"; policy=1 ;; esac done +# a runner that is down outranks a policy line: 1 if any unit is not accepting jobs, else 3 if only a policy is short. +[ "$rc" = 0 ] && [ "$policy" = 1 ] && rc=3 exit "$rc" diff --git a/scripts/ci-runner-restart.conf b/scripts/ci-runner-restart.conf new file mode 100644 index 000000000..ee193a792 --- /dev/null +++ b/scripts/ci-runner-restart.conf @@ -0,0 +1,29 @@ +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# systemd drop-in for a self-hosted runner unit that predates scripts/install-ci-runner.sh (engine#1504). Installed as +# /etc/systemd/system/.d/restart.conf by the owner; docs/ci-runners.md "The two drop-ins" has the commands. +# It gives the unit the crash-recovery lines of the documented unit and changes nothing else (same user, same paths). +[Unit] +# never give up restarting: a runner that hit the start limit is as dead as one that was never restarted +StartLimitIntervalSec=0 + +[Service] +# always, not on-failure: runsvc.sh exits 0 when it is killed, so on-failure never fires +Restart=always +RestartSec=15 +# the kernel's OOM victim is one process; with the default (stop) that ends the unit. continue lets the job die and +# the runner keep running, so the job fails with a reason instead of the runner vanishing +OOMPolicy=continue diff --git a/scripts/weekly.sh b/scripts/weekly.sh index 468864481..d8ce116fc 100755 --- a/scripts/weekly.sh +++ b/scripts/weekly.sh @@ -635,6 +635,29 @@ stage_release() { say "released $tag" } +# The systemd row of the runner preflight (engine#1504): $1 is scripts/ci-runner-health.sh. Returns 1 only when a runner +# unit is not accepting jobs NOW (its exit 1): the checks after the release would queue for ever. Its exit 3 - every +# runner up, but a restart policy that would leave one down after the next OOM kill - is a WARNING and the release goes +# on (owner 2026-11-11): the policy is a box configuration change the owner applies (docs/ci-runners.md "The two +# drop-ins"), every heavy build already runs under scripts/mem-guard.sh --term, which ends the build (exit 143) before +# the kernel's OOM killer reaches the runner, and a release that cannot start does not make the unit any safer. +# A function of its own so tests/weekly_runner_preflight_test.sh can run it against a stub. +preflight_runner_units() { + local health=$1 health_out health_rc=0 + [ -x "$health" ] || return 0 + health_out=$("$health" 2>&1) || health_rc=$? + case "$health_rc" in + 0) printf '%s\n' "$health_out" | grep -c '^PASS' | xargs -r -I{} echo "preflight: runner units: {} PASS in systemd's view" ;; + 2) say "preflight: no actions.runner.* units on this host: systemd has nothing to inspect" ;; + 3) printf '%s\n' "$health_out" | sed 's/^/preflight: /' + echo "preflight: WARNING, not blocking: every runner unit is accepting jobs, but a restart policy would leave one down after the next OOM kill (scripts/ci-runner-health.sh exit 3; the WARN lines above name the unit and the setting, docs/ci-runners.md \"The two drop-ins\" has the fix)" ;; + *) printf '%s\n' "$health_out" | sed 's/^/preflight: /' + echo "preflight: a runner unit is not accepting jobs, so the checks after this release would queue for ever (scripts/ci-runner-health.sh exit $health_rc; the FAIL lines above say which; WEEKLY_SKIP_RUNNER_CHECK=1 to release anyway)" + return 1 ;; + esac + return 0 +} + # --- preflight ---------------------------------------------------------------------------- # What the stages need from the machine, checked before the first one runs, so that a missing # tool stops the run at the start rather than after an hour of build (2026-10-19 rehearsal). @@ -686,6 +709,7 @@ preflight() { # scripts/ci-runner-health.sh - systemd's view (#1546): Result=oom-kill, a restart window, a unit that is not # active. Says WHY, and fires the moment a unit dies. Exit 2 = no such units on # this host, i.e. a fresh box, which is a note rather than a failure. + # Exit 3 = all up but a restart policy is short: a warning (preflight_runner_units). # WEEKLY_SKIP_RUNNER_CHECK=1 is the release-during-an-outage escape, and it skips BOTH: a flag that skipped only # one of them would be useless in exactly the outage it exists for, because both fail for the same dead runner. if [ "${WEEKLY_SKIP_RUNNER_CHECK:-0}" != 1 ]; then @@ -697,23 +721,7 @@ preflight() { echo "preflight: a self-hosted runner is not online, so the checks after this release would queue for ever (WEEKLY_SKIP_RUNNER_CHECK=1 to release anyway)" bad=1 fi - if [ -x "$(cd "$(dirname "$0")" && pwd)/ci-runner-health.sh" ]; then - if health_out=$("$(cd "$(dirname "$0")" && pwd)/ci-runner-health.sh" 2>&1); then - printf '%s\n' "$health_out" | grep -c '^PASS' | xargs -r -I{} echo "preflight: runner units: {} PASS in systemd's view" - else - health_rc=$? - if [ "$health_rc" = 2 ]; then - say "preflight: no actions.runner.* units on this host: systemd has nothing to inspect" - else - printf '%s\n' "$health_out" | sed 's/^/preflight: /' - # Two kinds of FAIL reach here (engine#1504): a unit that is down NOW, and since #2082 a unit that is up - # but whose restart policy would leave it down after the next OOM kill. The FAIL lines above say which; - # this line must not claim "not accepting jobs" for the second, where every runner is active/running. - echo "preflight: a runner unit is down, or its restart policy would leave it down after the next OOM kill (scripts/ci-runner-health.sh exit $health_rc; the FAIL lines above say which, and scripts/install-ci-runner.sh fixes a policy line)" - bad=1 - fi - fi - fi + preflight_runner_units "$(cd "$(dirname "$0")" && pwd)/ci-runner-health.sh" || bad=1 else # a skip that says nothing is the failure mode these rows exist for: a release log where the runner checks # silently did not run reads exactly like one where they passed. diff --git a/tests/weekly_runner_preflight_test.sh b/tests/weekly_runner_preflight_test.sh new file mode 100755 index 000000000..381f590a5 --- /dev/null +++ b/tests/weekly_runner_preflight_test.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# weekly_runner_preflight_test.sh: the release preflight BLOCKS on a runner unit that is down and only WARNS on a restart +# policy that is short (engine#1504, owner 2026-11-11). Two halves, both the real code: +# 1. scripts/ci-runner-health.sh against a stub systemctl: exit 0 / 1 / 2 / 3, and a down unit outranks a policy line. +# 2. scripts/weekly.sh's preflight_runner_units(), cut out of the script and run against the real health script on the +# same stub: returns 0 with a WARNING line on exit 3, returns 1 on exit 1. +# No systemd, no network, no build. +set -u +root=$(cd "$(dirname "$0")/.." && pwd) +tmp=$(mktemp -d); trap 'rm -rf "$tmp"' EXIT +fails=0 +ok() { printf 'ok %s\n' "$1"; } +bad() { printf 'FAIL %s\n' "$1"; fails=$((fails + 1)); } + +# stub systemctl: STUB_UNITS = lines "unit state sub result restart oompolicy memorymax" +cat > "$tmp/systemctl" <<'STUB' +#!/usr/bin/env bash +case "$1" in + list-units) awk '{print $1, "loaded", $2, $3}' "$STUB_UNITS" ;; + show) awk -v u="$2" '$1 == u { + printf "ActiveState=%s\nSubState=%s\nResult=%s\nNRestarts=0\nRestart=%s\nOOMPolicy=%s\nMemoryMax=%s\n", $2, $3, $4, $5, $6, $7 }' "$STUB_UNITS" ;; +esac +STUB +chmod +x "$tmp/systemctl" +export SYSTEMCTL="$tmp/systemctl" STUB_UNITS="$tmp/units" +health="$root/scripts/ci-runner-health.sh" + +good="actions.runner.a.service active running success always continue 51539607552" +policy="actions.runner.b.service active running success no stop infinity" # the live bench unit, 2026-11-11 +dead="actions.runner.c.service failed failed oom-kill no stop infinity" # the 11-08 shape + +expect_rc() { # + local name=$1 want=$2; shift 2 + printf '%s\n' "$@" > "$STUB_UNITS"; [ $# -eq 0 ] && : > "$STUB_UNITS" + out=$(bash "$health" 2>&1); rc=$? + if [ "$rc" = "$want" ]; then ok "health: $name -> exit $rc"; else bad "health: $name: want exit $want, got $rc: $out"; fi +} +expect_rc "all healthy" 0 "$good" +expect_rc "a unit is down" 1 "$good" "$dead" +expect_rc "no units" 2 +expect_rc "all up, one policy short" 3 "$good" "$policy" +expect_rc "down outranks policy" 1 "$policy" "$dead" +printf '%s\n' "$good" "$policy" > "$STUB_UNITS" +out=$(bash "$health" 2>&1) +if printf '%s\n' "$out" | grep -q '^WARN .*Restart=no(want always)' && ! printf '%s\n' "$out" | grep -q '^FAIL'; then + ok "health: a short policy prints WARN with the setting, and no FAIL line" +else bad "health: a short policy should print WARN and no FAIL: $out"; fi + +# half 2: the function weekly.sh really runs +sed -n '/^preflight_runner_units() {$/,/^}$/p' "$root/scripts/weekly.sh" > "$tmp/fn.sh" +grep -q 'health_rc' "$tmp/fn.sh" || { bad "could not cut preflight_runner_units() out of scripts/weekly.sh"; echo "weekly_runner_preflight: FAIL"; exit 1; } +grep -q 'preflight_runner_units "' "$root/scripts/weekly.sh" || bad "scripts/weekly.sh's preflight no longer calls preflight_runner_units" +say() { echo "$*"; } +# shellcheck disable=SC1091 +. "$tmp/fn.sh" +row() { # + local name=$1 want=$2 text=$3; shift 3 + printf '%s\n' "$@" > "$STUB_UNITS"; [ $# -eq 0 ] && : > "$STUB_UNITS" + out=$(preflight_runner_units "$health" 2>&1); rc=$? + if [ "$rc" = "$want" ] && printf '%s\n' "$out" | grep -q -- "$text"; then ok "weekly: $name -> return $rc" + else bad "weekly: $name: want return $want with '$text', got $rc: $out"; fi +} +row "all healthy passes" 0 "PASS in systemd's view" "$good" +row "a short policy warns and does not block" 0 "WARNING, not blocking" "$good" "$policy" +row "the warning names the setting" 0 "OOMPolicy=stop(want continue)" "$good" "$policy" +row "a dead unit blocks" 1 "not accepting jobs" "$good" "$dead" +row "a dead unit blocks even beside a short policy" 1 "after an OOM kill" "$policy" "$dead" +row "no units is a note" 0 "nothing to inspect" +out=$(preflight_runner_units "$tmp/absent.sh" 2>&1); rc=$? +[ "$rc" = 0 ] && [ -z "$out" ] && ok "weekly: no health script, no row" || bad "weekly: a missing health script should be silent: $rc $out" + +[ "$fails" = 0 ] && { echo "weekly_runner_preflight: ok"; exit 0; } +echo "weekly_runner_preflight: FAIL ($fails)"; exit 1