From e7984f658de4fcbfd16c4ce7e281bdccb8ffb4d8 Mon Sep 17 00:00:00 2001 From: pi-release Date: Sun, 15 Nov 2026 15:32:00 -0400 Subject: [PATCH 1/3] tests: the zero-cases INCOMPLETE rule gets the end-to-end arm it landed without (engine#2523) #2523 fixed a real hole -- a phase that ran nothing reported PASS and exited 0 -- and it reached main with two source-reading checks and a phase-parsing test, but nothing ran the gate end to end with a backend that says 'no test case ran'. The fix therefore had no arm that could fail on a regression, which is the rule it was itself enforcing. tests/hw_gate_zero_cases_test.sh calls scripts/hw-gate.sh from a scratch dir with three stub binaries on PATH (test-backend-ops reports 'no test case ran' and exits 1) and asserts both halves of the rule: the last line the operator reads says INCOMPLETE, and the exit code is 2 rather than 0. Mutations, both caught by name: - the infra override removed -> 'a run that tested nothing read as PASS' - '[ "$infra" = 1 ] && exit 2' -> exit 0 -> 'rc 0 for a zero-case phase, want 2' Registered in CMakeLists.txt next to hw_gate_phase_rows so ctest runs it (green: 1/1, 0.74 s). No GPU, no device, no ticket: fakes only. --- CMakeLists.txt | 5 +++ tests/hw_gate_zero_cases_test.sh | 55 ++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100755 tests/hw_gate_zero_cases_test.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 061e3f38e..2a99e7833 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -635,6 +635,11 @@ add_test(NAME hw_gate_mem COMMAND bash ${CMAKE_SOURCE_DIR}/tests/hw_gate_mem_tes # the first version overwrote the previous run's evidence), and a failed copy names nothing. No GPU, no ticket. add_test(NAME hw_gate_kept_log COMMAND bash ${CMAKE_SOURCE_DIR}/tests/hw_gate_kept_log_test.sh ${CMAKE_SOURCE_DIR}) add_test(NAME hw_gate_phase_rows COMMAND bash ${CMAKE_SOURCE_DIR}/tests/hw_gate_phase_rows_test.sh ${CMAKE_SOURCE_DIR}) +# engine#2523's rule reached main without its runtime arm: the source-reading hw_gate_test.py asserts the SKIP rows' +# text and their `infra=1`, and hw_gate_phase_rows asserts phase parsing, but nothing ran the gate end to end with a +# backend that reports "no test case ran". That is the case the fix was for -- zero cases must read INCOMPLETE and exit +# 2, never PASS. Fakes only (three stub binaries on PATH); no GPU, no device, no ticket. +add_test(NAME hw_gate_zero_cases COMMAND bash ${CMAKE_SOURCE_DIR}/tests/hw_gate_zero_cases_test.sh ${CMAKE_SOURCE_DIR}) # engine #1504: the release preflight blocks on a runner unit that is down and only warns on a short restart policy. # The gate's "is the coder busy?" probe: `0` idle, `N` busy, `-1` cannot tell. The parser answered `0` on any # exception while `curl -s` exits 0 on a 404, so a wrong URL, a refused connection and an idle coder were one answer. diff --git a/tests/hw_gate_zero_cases_test.sh b/tests/hw_gate_zero_cases_test.sh new file mode 100755 index 000000000..1e64c7af9 --- /dev/null +++ b/tests/hw_gate_zero_cases_test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# A phase that tested NOTHING must not leave the gate green (review of engine#2523; engine#2522's rule). +# +# The pin's `test-backend-ops` refuses to report a pass over an empty selection: it exits 1 and prints +# "error: no test case ran (...) : refusing to report a pass". The gate turns that into `ops SKIP` with the selection +# that produced it -- correct, because it is a filter problem rather than a kernel result -- but a SKIP with no +# `infra=1` leaves the verdict reading PASS, so a run that measured nothing is indistinguishable from a clean one. +# +# No GPU, no model, no compiler: a fake --bin whose test-backend-ops prints that line, an empty model dir, and +# HW_GATE_TOKEN=none so no ticket is taken. The arm is the verdict and the exit code, which is what CI reads. +set -u +root=${1:-$(cd "$(dirname "$0")/.." && pwd)} # the family's convention: ctest passes ${CMAKE_SOURCE_DIR} +fail=0 +scratch=$(mktemp -d "${TMPDIR:-$HOME/.cache}/hwgate-zc.XXXXXX") || exit 1 +trap 'rm -rf "$scratch"' EXIT +mkdir -p "$scratch/bin" +for b in llama-server test-backend-ops llama-perplexity; do + { + echo '#!/bin/bash' + echo 'if [ "$(basename "$0")" = test-backend-ops ]; then' + echo ' echo "error: no test case ran (check -b/-o/-p and --test-file): refusing to report a pass"' + echo ' exit 1' + echo 'fi' + echo 'exit 0' + } > "$scratch/bin/$b" + chmod +x "$scratch/bin/$b" +done +mkdir -p "$scratch/run" +env HW_GATE_MODELS="$scratch" HW_GATE_TOKEN=none HW_GATE_PRIVATE=0 HW_GATE_COOL_C=1000 \ + HW_GATE_STORE="$scratch/run/store" HW_GATE_OUT="$scratch/run/out" GITHUB_STEP_SUMMARY= \ + timeout 120 "$root/scripts/hw-gate.sh" --phases ops --bin "$scratch/bin" > "$scratch/run/stdout" 2>&1 +rc=$? +out=$scratch/run/stdout + +grep -q '| ops | SKIP |' "$out" || { echo "FAIL: a zero-case ops run did not produce an ops SKIP row"; fail=1; } +grep -q 'INCOMPLETE' "$out" || { echo "FAIL: a run that tested nothing did not read as INCOMPLETE"; fail=1; } +grep -q '## HRX hardware gate: PASS' "$out" && { echo "FAIL: a run that tested nothing read as PASS"; fail=1; } +[ "$rc" = 2 ] || { echo "FAIL: rc $rc for a zero-case phase, want 2 (infra, not green)"; fail=1; } +[ "$fail" = 0 ] && echo "ok a zero-case ops phase reads INCOMPLETE and exits 2, not PASS" +[ "$fail" = 0 ] From c539c3b5791578b1459c96af8f09ee0364551016 Mon Sep 17 00:00:00 2001 From: pi-release Date: Sun, 15 Nov 2026 16:12:20 -0400 Subject: [PATCH 2/3] tests: the zero-cases arm is hermetic, and says why when it fails (CI 43752965195) The arm passed here and failed on CI with 'rc 124 for a zero-case phase, want 2' after 120.02 s. It was not the fixture's subject at all: hw-gate's step loop retries rc 76 (memory pre-check refused) up to HW_GATE_MEM_RETRIES times, and a runner whose MemAvailable sits under the 12 GB floor never reached the ops phase. Its HOME also has no ~/lb/thermal-run.sh, so the wrapping command was 127. That is the rule this file's subject taught me, applied to the file itself: a fixture green only where the state cannot exist. It was green here because the token was free and the box had memory. Pinned now: HW_GATE_MEM_FLOOR=1, HW_GATE_MEM_RETRIES=0, a scratch thermal stand-in that really execs its argv (/bin/true exited 0 and swallowed the step, which gave rc 1 and no rows), HW_GATE_CODER_URL at a closed port, HW_GATE_LOCK in the scratch dir, HW_GATE_TOKEN=none. Verified green in a clean HOME with no thermal-run, no store and no token (0.76 s) and greens with another lane's token in place -- the production state that broke it. And the arm now dumps the gate's own output when it fails, because learning 'rc 124' cost one CI cycle: the mutation that removes the infra override prints the row that says '## HRX hardware gate: PASS', which is the defect in one line. --- tests/hw_gate_zero_cases_test.sh | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/hw_gate_zero_cases_test.sh b/tests/hw_gate_zero_cases_test.sh index 1e64c7af9..796f31cc3 100755 --- a/tests/hw_gate_zero_cases_test.sh +++ b/tests/hw_gate_zero_cases_test.sh @@ -40,10 +40,21 @@ for b in llama-server test-backend-ops llama-perplexity; do } > "$scratch/bin/$b" chmod +x "$scratch/bin/$b" done +# thermal-run's contract is "run the rest of the argv"; /bin/true exits 0 and the step then reports nothing at all, +# which is how my first version of this arm turned a passing test into rc 1 with no rows. +printf '#!/bin/sh\nexec "$@"\n' > "$scratch/thermal.sh" +chmod +x "$scratch/thermal.sh" mkdir -p "$scratch/run" +# Every input this arm depends on is pinned, and each line is a hang that was measured on CI (e2e build 43752965195, +# 120.02 s, rc 124): the step loop retries rc 76 up to HW_GATE_MEM_RETRIES times, and a runner whose MemAvailable sits +# under the 12 GB floor never reaches the zero-case path at all -- the arm then reported rc 124 instead of the verdict +# it was written for. thermal-run is not on a CI runner either, so a real one would have been 127. The queue is not +# what this test is about: HW_GATE_TOKEN=none says so, and the memory floor is 1 GB so it cannot wait. env HW_GATE_MODELS="$scratch" HW_GATE_TOKEN=none HW_GATE_PRIVATE=0 HW_GATE_COOL_C=1000 \ + HW_GATE_MEM_FLOOR=1 HW_GATE_MEM_RETRIES=0 HW_GATE_THERMAL="$scratch/thermal.sh" \ + HW_GATE_CODER_URL=http://127.0.0.1:9 HW_GATE_LOCK="$scratch/npu.lock" \ HW_GATE_STORE="$scratch/run/store" HW_GATE_OUT="$scratch/run/out" GITHUB_STEP_SUMMARY= \ - timeout 120 "$root/scripts/hw-gate.sh" --phases ops --bin "$scratch/bin" > "$scratch/run/stdout" 2>&1 + timeout 90 "$root/scripts/hw-gate.sh" --phases ops --bin "$scratch/bin" > "$scratch/run/stdout" 2>&1 rc=$? out=$scratch/run/stdout @@ -51,5 +62,11 @@ grep -q '| ops | SKIP |' "$out" || { echo "FAIL: a zero-case ops run did not pro grep -q 'INCOMPLETE' "$out" || { echo "FAIL: a run that tested nothing did not read as INCOMPLETE"; fail=1; } grep -q '## HRX hardware gate: PASS' "$out" && { echo "FAIL: a run that tested nothing read as PASS"; fail=1; } [ "$rc" = 2 ] || { echo "FAIL: rc $rc for a zero-case phase, want 2 (infra, not green)"; fail=1; } +# A fixture that fails without saying why costs a CI cycle each time (measured: 8 minutes to learn "rc 124"). The gate's +# own output is what names the cause, so print it whenever an arm failed. +if [ "$fail" != 0 ]; then + echo "--- the gate's output follows ---" + tail -40 "$out" 2> /dev/null || echo "(no output file at $out)" +fi [ "$fail" = 0 ] && echo "ok a zero-case ops phase reads INCOMPLETE and exits 2, not PASS" [ "$fail" = 0 ] From 179d17d26104c0b7db3aec8ccbaad67fcf5deae6 Mon Sep 17 00:00:00 2001 From: pi-release Date: Sun, 15 Nov 2026 16:32:23 -0400 Subject: [PATCH 3/3] mem_budget: honour MEM_GUARD_MEMINFO, the variable mem-guard.sh has always honoured CI said, in the arm's own new dump (e2e 43756744545): ops FAIL 0s could not run: the memory pre-check refused it (rc 76) The no-model pre-check refuses under a hardcoded 16 GiB MemAvailable (hw_gate_manifest.cmd_memcheck: 'used + OPS_HEADROOM_GIB >= FAULT_ZONE_GIB or avail < 16'), so no HW_GATE_MEM_FLOOR setting reaches it, and a CI runner is routinely under 16 GiB. The arm could not run there at all. Two memory readers existed and only one could be fed: scripts/mem-guard.sh has honoured MEM_GUARD_MEMINFO since it was written -- its comment says why, 'the tests: plenty, then starved cannot be produced on demand' -- while tools/mem_budget.py read /proc/meminfo directly. Same variable, same meaning, one definition now. Proven both ways: patched + MEM_GUARD_MEMINFO=2.5 GiB -> 'too close to the engine#315 zone', rc 1 unpatched + the same file -> reads /proc, 'MemAvailable 96.7 GiB', rc 0 The arm supplies a 48 GiB reading instead of pretending the runner has memory, so the product path runs with a controlled input rather than being skipped. Green: hw_gate_zero_cases, hw_gate_mem, gpu_hold_mem_abort, and ctest 'hw_gate_mem + hw_gate_zero_cases: 100% tests passed, 0 failed out of 2'. --- tests/hw_gate_zero_cases_test.sh | 8 ++++++++ tools/mem_budget.py | 10 +++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/hw_gate_zero_cases_test.sh b/tests/hw_gate_zero_cases_test.sh index 796f31cc3..257c5b0c9 100755 --- a/tests/hw_gate_zero_cases_test.sh +++ b/tests/hw_gate_zero_cases_test.sh @@ -50,9 +50,17 @@ mkdir -p "$scratch/run" # under the 12 GB floor never reaches the zero-case path at all -- the arm then reported rc 124 instead of the verdict # it was written for. thermal-run is not on a CI runner either, so a real one would have been 127. The queue is not # what this test is about: HW_GATE_TOKEN=none says so, and the memory floor is 1 GB so it cannot wait. +# The no-model memory pre-check refuses under a hardcoded 16 GiB MemAvailable (hw_gate_manifest.cmd_memcheck), and a CI +# runner is routinely under it: e2e 43756744545 showed `ops FAIL 0s could not run: the memory pre-check refused it +# (rc 76)` even with HW_GATE_MEM_FLOOR=1, because that variable is not what the pre-check reads. Supply the reading the +# way the fleet already supplies it -- MEM_GUARD_MEMINFO, the same variable scripts/mem-guard.sh has honoured all +# along -- instead of pretending the runner has memory. +printf 'MemTotal: 67108864 kB\nMemAvailable: 50331648 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\n' > "$scratch/meminfo" + env HW_GATE_MODELS="$scratch" HW_GATE_TOKEN=none HW_GATE_PRIVATE=0 HW_GATE_COOL_C=1000 \ HW_GATE_MEM_FLOOR=1 HW_GATE_MEM_RETRIES=0 HW_GATE_THERMAL="$scratch/thermal.sh" \ HW_GATE_CODER_URL=http://127.0.0.1:9 HW_GATE_LOCK="$scratch/npu.lock" \ + MEM_GUARD_MEMINFO="$scratch/meminfo" \ HW_GATE_STORE="$scratch/run/store" HW_GATE_OUT="$scratch/run/out" GITHUB_STEP_SUMMARY= \ timeout 90 "$root/scripts/hw-gate.sh" --phases ops --bin "$scratch/bin" > "$scratch/run/stdout" 2>&1 rc=$? diff --git a/tools/mem_budget.py b/tools/mem_budget.py index e0c88dcb8..78f6425eb 100755 --- a/tools/mem_budget.py +++ b/tools/mem_budget.py @@ -51,8 +51,16 @@ def read_int(path, default=None): def meminfo(): + """The box's memory, from /proc/meminfo or from MEM_GUARD_MEMINFO when set. + + scripts/mem-guard.sh has honoured MEM_GUARD_MEMINFO since it was written (its comment says why: "the tests: + 'plenty, then starved' cannot be produced on demand"). This reader did not, so a test could starve one half of + the memory accounting and not the other -- and the hw-gate zero-cases arm could not run on a CI runner at all, + because its no-model pre-check refuses under 16 GiB MemAvailable (measured: e2e 43756744545, `ops FAIL 0s + could not run: the memory pre-check refused it (rc 76)`). Same variable, same meaning, one definition.""" + path = os.environ.get("MEM_GUARD_MEMINFO", "/proc/meminfo") out = {} - with open("/proc/meminfo") as f: + with open(path) as f: for line in f: k, v = line.split(":", 1) out[k] = int(v.split()[0]) * 1024