Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
107 changes: 107 additions & 0 deletions scripts/ci-runner-health.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
#!/bin/bash
# Copyright 2026 bong-water-water-bong
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# ci-runner-health.sh - say whether the self-hosted GitHub runners are RUNNING, and refuse to be silent about a dead one.
#
# Why this exists (engine#1504): both runners died on an OOM kill at 08:37 on 2026-11-08 and stayed dead for ~3h45m with
# `Result=oom-kill` and `NRestarts=0`. Every check that needs the box then read as **pending** - never green, never red -
# which is indistinguishable, at a glance, from "queued behind a busy runner". The issue's own words for the gap are
# "nothing notices", and this is the thing that notices: one line per runner, a printed verdict, and a non-zero exit.
#
# It reads systemd's view, which is the only place `Result=oom-kill` survives (the journal has it too, but the unit property
# is what a restart policy would consult). It never restarts anything: `systemctl start` needs root, and whether to restart a
# runner that OOM-killed twice in two minutes is a decision, not a check (engine#1504 measured that: the fourth restart
# survived only because the GPU ticket was empty).
#
# scripts/ci-runner-health.sh # every actions.runner.* unit
# scripts/ci-runner-health.sh <unit> [...] # named units, e.g. for a specific runner
# scripts/ci-runner-health.sh --self-test # the verdict logic, with systemctl stubbed
#
# Exit: 0 when every runner is active/running with no failure result; 1 when any is not; 2 when there is nothing to inspect
# (no such units on this host, so a silent pass would be the defect).
set -uo pipefail

# <state> <substate> <result> <nrestarts> -> one verdict line
runner_verdict() {
_st=${1:-} _sub=${2:-} _res=${3:-} _nr=${4:-}
if [ -z "$_st" ]; then
printf 'FAIL no systemd state could be read for this runner, so whether it is alive is unknown'
return 0
fi
if [ "$_st" = active ]; then
printf 'PASS active/running, last result %s, %s restart(s) - the runner is accepting jobs' "${_res:-?}" "${_nr:-0}"
return 0
fi
# `activating (auto-restart)` is the transient where a FLAPPING runner looks busy: it is not accepting jobs, but a naive
# check sees a unit that systemd is already dealing with. Say which it is, and say what a repeat means (pi-os3's addition).
if [ "$_st" = activating ] || [ "${_sub:-}" = auto-restart ]; then
printf 'FAIL %s/%s - the runner is in its RESTART WINDOW and not accepting jobs (%s restarts so far); one line is transient, a log full of them is a flapping runner' "$_st" "${_sub:-?}" "${_nr:-0}"
return 0
fi
case "$_res" in
oom-kill) printf 'FAIL %s/%s after an OOM kill (%s restarts) - the kernel killed it and nothing restarted it; jobs read as pending, not as failed' "$_st" "$_sub" "${_nr:-0}" ;;
*) printf 'FAIL %s/%s with last result %s (%s restarts) - the runner is not accepting jobs' "$_st" "${_sub:-?}" "${_res:-?}" "${_nr:-0}" ;;
esac
}

self_test() {
rc=0
chk() { # <want prefix> <got>
case "$2" in "$1"*) printf 'ok %s\n' "$(printf '%s' "$2" | cut -c1-78)" ;;
*) printf 'FAIL want %s, got: %s\n' "$1" "$2"; rc=1 ;; esac
}
chk PASS "$(runner_verdict active running success 0)"
chk FAIL "$(runner_verdict failed failed oom-kill 0)" # the 08:37 shape
chk FAIL "$(runner_verdict failed failed oom-kill 0)"
chk FAIL "$(runner_verdict inactive dead success 0)"
chk FAIL "$(runner_verdict '' '' '' '')" # nothing readable is not a pass
chk FAIL "$(runner_verdict activating auto-restart success 4)" # the window where a flapping runner looks busy
chk FAIL "$(runner_verdict activating auto-restart oom-kill 2)" # same window, last result a kill
if printf '%s' "$(runner_verdict activating auto-restart success 4)" | grep -q 'RESTART WINDOW'; then
printf 'ok the restart window is named, not just failed\n'
else printf 'FAIL the restart window should be named in the message\n'; rc=1; fi
# a restart policy that works shows NRestarts>0 and result success - which is a PASS with the count visible
chk PASS "$(runner_verdict active running success 3)"
[ "$rc" = 0 ] && echo "ci-runner-health self-test: ok (9 cases)" || echo "ci-runner-health self-test: FAIL"
return $rc
}

[ "${1:-}" = --self-test ] && { self_test; exit $?; }

SC=${SYSTEMCTL:-systemctl}
units=("$@")
if [ ${#units[@]} -eq 0 ]; then
mapfile -t units < <("$SC" list-units 'actions.runner.*' --all --no-legend --plain 2>/dev/null | awk '{print $1}')
fi
if [ ${#units[@]} -eq 0 ]; then
echo "FAIL no actions.runner.* units on this host - either the runners were never installed here or systemd has dropped them entirely; a silent pass would be the defect this check exists for"
exit 2
fi

rc=0
for u in "${units[@]}"; do
line=$("$SC" show "$u" -p ActiveState -p SubState -p Result -p NRestarts 2>/dev/null)
st=$(printf '%s\n' "$line" | sed -n 's/^ActiveState=//p')
sub=$(printf '%s\n' "$line" | sed -n 's/^SubState=//p')
res=$(printf '%s\n' "$line" | sed -n 's/^Result=//p')
nr=$(printf '%s\n' "$line" | sed -n 's/^NRestarts=//p')
v=$(runner_verdict "$st" "$sub" "$res" "$nr")
case "$v" in
PASS*) printf 'PASS %-52s %s\n' "$u" "${v#PASS }" ;;
*) printf 'FAIL %-52s %s\n' "$u" "${v#FAIL }"; rc=1 ;;
esac
done
exit "$rc"
Loading