Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -635,6 +635,11 @@ add_test(NAME hw_gate_mem COMMAND bash ${CMAKE_SOURCE_DIR}/tests/hw_gate_mem_tes
# the first version overwrote the previous run's evidence), and a failed copy names nothing. No GPU, no ticket.
add_test(NAME hw_gate_kept_log COMMAND bash ${CMAKE_SOURCE_DIR}/tests/hw_gate_kept_log_test.sh ${CMAKE_SOURCE_DIR})
add_test(NAME hw_gate_phase_rows COMMAND bash ${CMAKE_SOURCE_DIR}/tests/hw_gate_phase_rows_test.sh ${CMAKE_SOURCE_DIR})
# engine#2523's rule reached main without its runtime arm: the source-reading hw_gate_test.py asserts the SKIP rows'
# text and their `infra=1`, and hw_gate_phase_rows asserts phase parsing, but nothing ran the gate end to end with a
# backend that reports "no test case ran". That is the case the fix was for -- zero cases must read INCOMPLETE and exit
# 2, never PASS. Fakes only (three stub binaries on PATH); no GPU, no device, no ticket.
add_test(NAME hw_gate_zero_cases COMMAND bash ${CMAKE_SOURCE_DIR}/tests/hw_gate_zero_cases_test.sh ${CMAKE_SOURCE_DIR})
# engine #1504: the release preflight blocks on a runner unit that is down and only warns on a short restart policy.
# The gate's "is the coder busy?" probe: `0` idle, `N` busy, `-1` cannot tell. The parser answered `0` on any
# exception while `curl -s` exits 0 on a 404, so a wrong URL, a refused connection and an idle coder were one answer.
Expand Down
80 changes: 80 additions & 0 deletions tests/hw_gate_zero_cases_test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
#!/usr/bin/env bash
# Copyright 2026 bong-water-water-bong
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# A phase that tested NOTHING must not leave the gate green (review of engine#2523; engine#2522's rule).
#
# The pin's `test-backend-ops` refuses to report a pass over an empty selection: it exits 1 and prints
# "error: no test case ran (...) : refusing to report a pass". The gate turns that into `ops SKIP` with the selection
# that produced it -- correct, because it is a filter problem rather than a kernel result -- but a SKIP with no
# `infra=1` leaves the verdict reading PASS, so a run that measured nothing is indistinguishable from a clean one.
#
# No GPU, no model, no compiler: a fake --bin whose test-backend-ops prints that line, an empty model dir, and
# HW_GATE_TOKEN=none so no ticket is taken. The arm is the verdict and the exit code, which is what CI reads.
set -u
root=${1:-$(cd "$(dirname "$0")/.." && pwd)} # the family's convention: ctest passes ${CMAKE_SOURCE_DIR}
fail=0
scratch=$(mktemp -d "${TMPDIR:-$HOME/.cache}/hwgate-zc.XXXXXX") || exit 1
trap 'rm -rf "$scratch"' EXIT
mkdir -p "$scratch/bin"
for b in llama-server test-backend-ops llama-perplexity; do
{
echo '#!/bin/bash'
echo 'if [ "$(basename "$0")" = test-backend-ops ]; then'
echo ' echo "error: no test case ran (check -b/-o/-p and --test-file): refusing to report a pass"'
echo ' exit 1'
echo 'fi'
echo 'exit 0'
} > "$scratch/bin/$b"
chmod +x "$scratch/bin/$b"
done
# thermal-run's contract is "run the rest of the argv"; /bin/true exits 0 and the step then reports nothing at all,
# which is how my first version of this arm turned a passing test into rc 1 with no rows.
printf '#!/bin/sh\nexec "$@"\n' > "$scratch/thermal.sh"
chmod +x "$scratch/thermal.sh"
mkdir -p "$scratch/run"
# Every input this arm depends on is pinned, and each line is a hang that was measured on CI (e2e build 43752965195,
# 120.02 s, rc 124): the step loop retries rc 76 up to HW_GATE_MEM_RETRIES times, and a runner whose MemAvailable sits
# under the 12 GB floor never reaches the zero-case path at all -- the arm then reported rc 124 instead of the verdict
# it was written for. thermal-run is not on a CI runner either, so a real one would have been 127. The queue is not
# what this test is about: HW_GATE_TOKEN=none says so, and the memory floor is 1 GB so it cannot wait.
# The no-model memory pre-check refuses under a hardcoded 16 GiB MemAvailable (hw_gate_manifest.cmd_memcheck), and a CI
# runner is routinely under it: e2e 43756744545 showed `ops FAIL 0s could not run: the memory pre-check refused it
# (rc 76)` even with HW_GATE_MEM_FLOOR=1, because that variable is not what the pre-check reads. Supply the reading the
# way the fleet already supplies it -- MEM_GUARD_MEMINFO, the same variable scripts/mem-guard.sh has honoured all
# along -- instead of pretending the runner has memory.
printf 'MemTotal: 67108864 kB\nMemAvailable: 50331648 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\n' > "$scratch/meminfo"

env HW_GATE_MODELS="$scratch" HW_GATE_TOKEN=none HW_GATE_PRIVATE=0 HW_GATE_COOL_C=1000 \
HW_GATE_MEM_FLOOR=1 HW_GATE_MEM_RETRIES=0 HW_GATE_THERMAL="$scratch/thermal.sh" \
HW_GATE_CODER_URL=http://127.0.0.1:9 HW_GATE_LOCK="$scratch/npu.lock" \
MEM_GUARD_MEMINFO="$scratch/meminfo" \
HW_GATE_STORE="$scratch/run/store" HW_GATE_OUT="$scratch/run/out" GITHUB_STEP_SUMMARY= \
timeout 90 "$root/scripts/hw-gate.sh" --phases ops --bin "$scratch/bin" > "$scratch/run/stdout" 2>&1
rc=$?
out=$scratch/run/stdout

grep -q '| ops | SKIP |' "$out" || { echo "FAIL: a zero-case ops run did not produce an ops SKIP row"; fail=1; }
grep -q 'INCOMPLETE' "$out" || { echo "FAIL: a run that tested nothing did not read as INCOMPLETE"; fail=1; }
grep -q '## HRX hardware gate: PASS' "$out" && { echo "FAIL: a run that tested nothing read as PASS"; fail=1; }
[ "$rc" = 2 ] || { echo "FAIL: rc $rc for a zero-case phase, want 2 (infra, not green)"; fail=1; }
# A fixture that fails without saying why costs a CI cycle each time (measured: 8 minutes to learn "rc 124"). The gate's
# own output is what names the cause, so print it whenever an arm failed.
if [ "$fail" != 0 ]; then
echo "--- the gate's output follows ---"
tail -40 "$out" 2> /dev/null || echo "(no output file at $out)"
fi
[ "$fail" = 0 ] && echo "ok a zero-case ops phase reads INCOMPLETE and exits 2, not PASS"
[ "$fail" = 0 ]
10 changes: 9 additions & 1 deletion tools/mem_budget.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,16 @@ def read_int(path, default=None):


def meminfo():
"""The box's memory, from /proc/meminfo or from MEM_GUARD_MEMINFO when set.

scripts/mem-guard.sh has honoured MEM_GUARD_MEMINFO since it was written (its comment says why: "the tests:
'plenty, then starved' cannot be produced on demand"). This reader did not, so a test could starve one half of
the memory accounting and not the other -- and the hw-gate zero-cases arm could not run on a CI runner at all,
because its no-model pre-check refuses under 16 GiB MemAvailable (measured: e2e 43756744545, `ops FAIL 0s
could not run: the memory pre-check refused it (rc 76)`). Same variable, same meaning, one definition."""
path = os.environ.get("MEM_GUARD_MEMINFO", "/proc/meminfo")
out = {}
with open("/proc/meminfo") as f:
with open(path) as f:
for line in f:
k, v = line.split(":", 1)
out[k] = int(v.split()[0]) * 1024
Expand Down
Loading