Skip to content
4 changes: 4 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -379,6 +379,10 @@ add_test(NAME pack_version COMMAND ${CMAKE_SOURCE_DIR}/tests/pack_version_test.s
# arm asserts the row's message, and a clean tree is the control. No package build, no GPU, runs in CI.
add_test(NAME pack_check COMMAND ${CMAKE_SOURCE_DIR}/tests/pack_check_test.sh)
add_test(NAME check_ci_runners COMMAND ${CMAKE_SOURCE_DIR}/tests/check_ci_runners_test.sh)
# engine #1504: the hardware gate's build does not start below its memory floor, a memory-guard stop is not
# resumed like a thermal stop, and the stopped compile (or GPU step) is really gone. A fake root with a stub
# cmake, a stub thermal-run and a fake meminfo: no GPU, no compiler, runs in CI.
add_test(NAME hw_gate_mem COMMAND bash ${CMAKE_SOURCE_DIR}/tests/hw_gate_mem_test.sh ${CMAKE_SOURCE_DIR})
# engine #1504: the release preflight blocks on a runner unit that is down and only warns on a short restart policy.
# The real scripts/ci-runner-health.sh and weekly.sh's preflight_runner_units() against a stub systemctl; no systemd.
add_test(NAME weekly_runner_preflight COMMAND ${CMAKE_SOURCE_DIR}/tests/weekly_runner_preflight_test.sh)
Expand Down
10 changes: 9 additions & 1 deletion docs/hw-gate.md
Original file line number Diff line number Diff line change
Expand Up @@ -214,12 +214,20 @@ Each of those steps:
byte-exact, zero resets). The lock is what the **NPU phase** takes, below, because an NPU dispatch must
not overlap another NPU dispatch.
- **Heat.** It waits for Tctl to drop to 75 C, then runs under `~/lb/thermal-run.sh`, which freezes it at 78 C and kills it at 93 C.
- **Memory.** It runs under `scripts/mem-guard.sh` (kills the step below 12 GB MemAvailable). Before a step starts, `tools/mem_budget.py` estimates it. A step that would take GTT to 95 GiB or more (the engine#315 fault zone), or past MemAvailable, is refused. The gate then waits for memory to free up and retries every minute, up to ten times.
- **Memory.** It runs under `scripts/mem-guard.sh --term` (stops the step below 12 GB MemAvailable; `--term` because the guard's default SIGKILL ends `thermal-run` but leaves the step it started running, with its memory). Before a step starts, `tools/mem_budget.py` estimates it. A step that would take GTT to 95 GiB or more (the engine#315 fault zone), or past MemAvailable, is refused. The gate then waits for memory to free up and retries every minute, up to ten times (`HW_GATE_MEM_RETRIES`). It gives the GPU token back for each of those waits and takes it again before the next try, because the jobs holding the memory may be queued behind the token. Under the `gpu-hold` wrapper the ticket belongs to the wrapper and stays held.
- **The PR-Agent model.** coder-llm (Vulkan, :8090) loses its device when long HRX dispatches starve it. The gate runs on `strixhalo-engine`, the runner that also runs the pr-agent jobs, and a runner takes one job at a time, so no review is generating during the gate. Before each step the gate also waits for coder-llm's slots to be idle. Afterwards it reports any new `ring ... timeout` in the kernel log.

The build runs under thermal-run too, and resumes after a thermal kill. Build trees and ccache
persist under `~/.cache/hw-gate`, so a PR that does not move the pins rebuilds in seconds.

The build has its own memory check (engine#1504: on 2026-11-08 both self-hosted runners were OOM-killed 76 s
after a restart handed them a compile). Before it compiles, the gate waits for 24 GB of `MemAvailable`
(`scripts/mem-wait.sh`, polled once a minute for up to 10 minutes; `HW_GATE_BUILD_MEM_FLOOR`,
`HW_GATE_BUILD_MEM_WAIT`). If memory does not free up, the build is not started and the gate exits 2 with the
reading. The compile runs under `scripts/mem-guard.sh --term`, which stops the whole `thermal-run` scope
below 12 GB (`HW_GATE_MEM_FLOOR`). A guard stop is exit 2 too and is not resumed: a re-run once memory frees
is the fix. `tests/hw_gate_mem_test.sh` (ctest `hw_gate_mem`) checks all of it with a fake meminfo.

## Goldens: where they live

- **kat goldens are in git:** `tests/golden/hw-gate/kat-*.json`. They are the text and per-token logprobs of every case, a few KB per model.
Expand Down
40 changes: 34 additions & 6 deletions scripts/hw-gate.sh
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,9 @@ token_held_by_ancestor() {
}
THERMAL=${HW_GATE_THERMAL:-$HOME/lb/thermal-run.sh}
MEM_FLOOR=${HW_GATE_MEM_FLOOR:-12}
# the build (engine#1504): do not start a compile below this much MemAvailable; wait this many minutes for it
BUILD_MEM_FLOOR=${HW_GATE_BUILD_MEM_FLOOR:-24}
BUILD_MEM_WAIT=${HW_GATE_BUILD_MEM_WAIT:-10}
CODER_URL=${HW_GATE_CODER_URL:-http://127.0.0.1:8090}
CODER_WAIT=${HW_GATE_CODER_WAIT:-900}
TOKEN=${HW_GATE_TOKEN:-$HOME/GPU-TOKEN}
Expand Down Expand Up @@ -336,7 +339,9 @@ trap release_token EXIT
trap 'exit 143' TERM INT HUP # so a killed run (timeout, CI cancel) still releases it

# run "$@" on the GPU: hold the GPU token, wait for coder-llm, run it under mem-guard + thermal-run
# (no ~/npu.lock: engine#682, measured in engine#673)
# (no ~/npu.lock: engine#682, measured in engine#673). The guard is --term: its default SIGKILL ends
# thermal-run but not thermal-run's scope, so the step it "killed" kept running and kept its memory
# (engine#1504; tests/hw_gate_mem_test.sh).
gpu() {
if ! take_token; then
result gpu-token FAIL "$TOKEN_WAIT" "$TOKEN held by another job for ${TOKEN_WAIT}s: $(cat "$TOKEN" 2> /dev/null)"
Expand All @@ -350,11 +355,20 @@ gpu() {
export HW_GATE_STAMP=$OUT/.stamp
rm -f "$HW_GATE_STAMP"
while :; do
"$0" --cool --mem "${GPU_MODEL:--}" "$ROOT/scripts/mem-guard.sh" "$MEM_FLOOR" "$THERMAL" "$@"
"$0" --cool --mem "${GPU_MODEL:--}" "$ROOT/scripts/mem-guard.sh" --term "$MEM_FLOOR" "$THERMAL" "$@"
rc=$?
# 76: the memory pre-check said no; nothing is held, so other work can finish first
# 76: the memory pre-check said no, and the step holds nothing. The gate still holds the GPU token, though,
# and the jobs whose memory it is waiting for may be queued behind that token: so the token is given back
# for the wait and taken again before the next try (review of engine#1936). Under the gpu-hold wrapper the
# ticket is the wrapper's and cannot be given back from here; release_token is then a no-op.
[ $rc = 76 ] && [ $tries -lt "${HW_GATE_MEM_RETRIES:-10}" ] || break
tries=$((tries + 1)); sleep 60
tries=$((tries + 1))
release_token
sleep "${HW_GATE_MEM_RETRY_SLEEP:-60}"
if ! take_token; then
result gpu-token FAIL "$TOKEN_WAIT" "$TOKEN held by another job for ${TOKEN_WAIT}s: $(cat "$TOKEN" 2> /dev/null)"
exit 2
fi
done
[ $rc = 76 ] && { note "::error::memory pre-check refused the step ${tries} times"; infra=1; }
t1=$(date +%s)
Expand Down Expand Up @@ -429,7 +443,16 @@ if [ -z "$BIN" ]; then
[ "$(cat "$build/.hip-flags" 2>/dev/null)" = "$hip_flags" ] || rm -f "$build/CMakeCache.txt"
# the same configuration as cmake/hrx.cmake (llama_hrx), plus the two test tools
: > "$OUT/build.log"
built=1
built=1 guarded=0
# Memory (engine#1504). On 2026-11-08 this job was handed to a just-restarted runner on a box with 0 swap
# free and both runners were OOM-killed 76 s later; the compile was the one step here with no memory check.
# So: no compile below BUILD_MEM_FLOOR GB MemAvailable (wait BUILD_MEM_WAIT min, then exit 2 with the
# reading), and the compile runs under mem-guard --term, which ends the whole thermal-run scope below
# MEM_FLOOR GB. A guard stop is not heat: the loop below does not resume it.
if ! "$ROOT/scripts/mem-wait.sh" "$BUILD_MEM_FLOOR" "$BUILD_MEM_WAIT" >> "$OUT/build.log" 2>&1; then
result build FAIL $(( $(date +%s) - t0 )) "not started: $(tail -n 1 "$OUT/build.log")"
exit 2
fi
# thermal-run kills a build at 93 C (the box is shared); ninja resumes where it stopped
for attempt in 1 2 3 4 5 6; do
cool
Expand All @@ -441,7 +464,8 @@ if [ -z "$BIN" ]; then
-DGGML_HRX_HIP_COMPILER="$TOOLCHAIN/bin/amdclang++" "-DGGML_HRX_HIP_FLAGS=$hip_flags" \
-DLLAMA_BUILD_SERVER=ON -DLLAMA_BUILD_TESTS=ON -DLLAMA_CURL=OFF >> "$OUT/build.log" 2>&1 &&
echo "$addon_pin" > "$build/.gpu-kernels-pin" && echo "$hip_flags" > "$build/.hip-flags"; }; } &&
"$THERMAL" cmake --build "$build" -j "$JOBS" --target "${targets[@]}" >> "$OUT/build.log" 2>&1 && { built=0; break; }
"$ROOT/scripts/mem-guard.sh" --term "$MEM_FLOOR" "$THERMAL" cmake --build "$build" -j "$JOBS" --target "${targets[@]}" >> "$OUT/build.log" 2>&1 && { built=0; break; }
grep -q "^GUARD:" <(tail -n 40 "$OUT/build.log") && { guarded=1; break; } # out of memory, not heat
grep -q "error:" <(tail -n 40 "$OUT/build.log") && break # a compile error, not heat
note "build stopped (attempt $attempt, Tctl $(tctl) C); resuming"
done
Expand All @@ -455,6 +479,10 @@ if [ -z "$BIN" ]; then
BUILT=1
result build PASS $(( $(date +%s) - t0 )) "$build"
else
if [ $guarded = 1 ]; then
result build FAIL $(( $(date +%s) - t0 )) "stopped by the memory guard: $(grep "^GUARD:" "$OUT/build.log" | tail -n 1); re-run once memory frees (engine#1504)"
exit 2
fi
result build FAIL $(( $(date +%s) - t0 )) "see $OUT/build.log"
# the compiler's error lines only, not its source excerpts: the job log is public and the
# add-on's sources are not
Expand Down
5 changes: 4 additions & 1 deletion scripts/mem-guard.sh
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,14 @@
# --term sends SIGTERM first and SIGKILL 10 s later. Use it when cmd is ~/lb/thermal-run.sh: a
# SIGKILL'd thermal-run cannot run its trap, so its scope (ninja and every compiler under it) keeps
# running and keeps the memory; on SIGTERM it kills the whole scope (engine#1504).
# MEM_GUARD_MEMINFO names the file read instead of /proc/meminfo (the tests: "plenty, then starved" cannot be
# produced on demand on a shared box).
meminfo=${MEM_GUARD_MEMINFO:-/proc/meminfo}
term=0; [ "$1" = --term ] && { term=1; shift; }
floor=$1; shift
"$@" & pid=$!
while kill -0 $pid 2>/dev/null; do
avail=$(awk '/MemAvailable/ {print int($2/1048576)}' /proc/meminfo)
avail=$(awk '/MemAvailable/ {print int($2/1048576)}' "$meminfo")
if [ "$avail" -lt "$floor" ]; then
echo "GUARD: MemAvailable ${avail} GB < ${floor} GB, killing $pid" >&2
if [ $term = 1 ]; then
Expand Down
6 changes: 4 additions & 2 deletions scripts/mem-wait.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,17 @@
# compile into an exhausted box is what turned one OOM into two on 2026-11-08 (engine#1504; both self-hosted
# runners were OOM-killed 76 s after a restart, with 0 swap free). Each poll prints what the box looks like, so a
# refusal says why. The floor is MemAvailable only: on this APU GTT is host RAM, and MemAvailable already counts it.
# MEM_GUARD_MEMINFO names the file read instead of /proc/meminfo, and MEM_WAIT_POLL the seconds between polls
# (default 60); both are for the tests.
floor=${1:?usage: mem-wait.sh <min_available_gb> <max_wait_min>}; max=${2:?usage: mem-wait.sh <min_available_gb> <max_wait_min>}
meminfo() { awk -v k="$1" '$1 == k":" {print int($2/1048576)}' /proc/meminfo; }
meminfo() { awk -v k="$1" '$1 == k":" {print int($2/1048576)}' "${MEM_GUARD_MEMINFO:-/proc/meminfo}"; }
for i in $(seq 0 "$max"); do
avail=$(meminfo MemAvailable)
line="MemAvailable ${avail} GB (floor ${floor} GB), swap free $(meminfo SwapFree) of $(meminfo SwapTotal) GB"
if [ "$avail" -ge "$floor" ]; then echo "mem-wait: ok: $line"; exit 0; fi
[ "$i" = "$max" ] && break
echo "mem-wait: waiting ($((i + 1)) of $max min): $line"
sleep 60
sleep "${MEM_WAIT_POLL:-60}"
done
echo "mem-wait: REFUSED after ${max} min: $line. The box is short of memory, so this job did not start its build;" \
"re-run it once memory frees (engine#1504)." >&2
Expand Down
Loading
Loading