From a6cbf403eb79501a1c077e6ce34ab4ece23b76ec Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Fri, 13 Nov 2026 07:18:06 -0400 Subject: [PATCH 1/2] hw-gate: the "is the coder busy?" probe can now say "I cannot tell" `wait_coder_idle` exists so an HRX step never starts while the PR-Agent model is generating, and its parser answered **0 on any exception** while `curl -s` exits 0 on a 404. So a wrong URL, a refused connection, an HTML error page and a genuinely idle coder were all the same answer: idle. The probe could not fail, and the gate proceeded believing it had checked - the shape that bit me earlier tonight in a readiness probe of my own (`/api/v1/health` is a 404 and `curl -s` calls it success). Now the four answers are distinct, and only one of them means "go": [] -> 0 idle [{"is_processing":true}, ...] -> N busy 404 Not Found -> -1 cannot tell "" / {"error":{...}} -> -1 cannot tell `-1` is reported with a warning and the gate proceeds, unchanged in behaviour and bounded by CODER_WAIT - the policy (proceed vs wait out the window) stays the owner's call, this only stops the answer from being a silent lie. Measured, `-f` is not what protects: removing it leaves every arm passing, because `-1` already covers an unparseable body; `-f` only lets the message say "did not answer" instead of "answered something that is not a slots list". `tests/hw_gate_coder_idle_test.sh` extracts both functions from the real script (so it cannot drift) and fails loudly if the extraction stops matching; the end-to-end arm points `CODER_URL` at a discard port, so a refused connection must return within 30 s having warned. Mutation-checked: reverting `-1` to `0` fails 3 arms. Row `hw_gate_coder_idle` Passed 0.33 s beside `hw_gate_mem` 14.10 s; copyright, ci_targets and tests_referenced rc 0. --- CMakeLists.txt | 3 ++ scripts/hw-gate.sh | 26 ++++++++++-- tests/hw_gate_coder_idle_test.sh | 69 ++++++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 4 deletions(-) create mode 100755 tests/hw_gate_coder_idle_test.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 72b23a0da..34a6c0130 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -554,6 +554,9 @@ add_test(NAME check_ci_runners COMMAND ${CMAKE_SOURCE_DIR}/tests/check_ci_runner # cmake, a stub thermal-run and a fake meminfo: no GPU, no compiler, runs in CI. add_test(NAME hw_gate_mem COMMAND bash ${CMAKE_SOURCE_DIR}/tests/hw_gate_mem_test.sh ${CMAKE_SOURCE_DIR}) # engine #1504: the release preflight blocks on a runner unit that is down and only warns on a short restart policy. +# The gate's "is the coder busy?" probe: `0` idle, `N` busy, `-1` cannot tell. The parser answered `0` on any +# exception while `curl -s` exits 0 on a 404, so a wrong URL, a refused connection and an idle coder were one answer. +add_test(NAME hw_gate_coder_idle COMMAND bash ${CMAKE_SOURCE_DIR}/tests/hw_gate_coder_idle_test.sh ${CMAKE_SOURCE_DIR}) # The real scripts/ci-runner-health.sh and weekly.sh's preflight_runner_units() against a stub systemctl; no systemd. add_test(NAME weekly_runner_preflight COMMAND ${CMAKE_SOURCE_DIR}/tests/weekly_runner_preflight_test.sh) add_test(NAME ci_runner_health_selftest COMMAND bash ${CMAKE_SOURCE_DIR}/scripts/ci-runner-health.sh --self-test) diff --git a/scripts/hw-gate.sh b/scripts/hw-gate.sh index 2e4f35a28..4ad0e7a81 100755 --- a/scripts/hw-gate.sh +++ b/scripts/hw-gate.sh @@ -306,13 +306,31 @@ want_model() { [ -z "$ONLY_MODELS" ] && return 0; case ",$ONLY_MODELS," in *",$1 ring_timeouts() { journalctl -k --since "@$START" --no-pager 2>/dev/null | grep -c "ring .*timeout" || true; } +coder_busy_from_json() { # stdin: the /slots body. prints the busy count, or -1 when it cannot be read + python3 -c 'import json,sys +try: print(sum(1 for s in json.load(sys.stdin) if s.get("is_processing"))) +except Exception: print(-1)' +} + wait_coder_idle() { # the PR-Agent's Vulkan model: never start an HRX step while it is generating local waited=0 while [ "$waited" -lt "$CODER_WAIT" ]; do - busy=$(curl -s --max-time 5 "$CODER_URL/slots" 2>/dev/null | python3 -c \ - 'import json,sys -try: print(sum(1 for s in json.load(sys.stdin) if s.get("is_processing"))) -except Exception: print(0)' 2>/dev/null) + # `curl -s` exits 0 on a 404 and the old parser answered 0 for every unreadable body, so "cannot ask" and + # "idle" were the same answer and this probe could not fail. The guard is coder_busy_from_json's `-1` for an + # unparseable body; `-f` is not what protects, it is what lets the message say "did not answer" rather than + # "answered something that is not a slots list". Measured: with `-f` removed the safety arms still pass, which + # is why the test pins the parse and not the flag. + local body rc + body=$(curl -sf --max-time 5 "$CODER_URL/slots" 2>/dev/null); rc=$? + if [ "$rc" != 0 ]; then + note "::warning::$CODER_URL/slots did not answer (curl exit $rc) - cannot tell whether the coder is busy, proceeding" + return 0 + fi + busy=$(printf '%s' "$body" | coder_busy_from_json) + if [ "${busy:-}" = -1 ]; then + note "::warning::$CODER_URL/slots answered something that is not a slots list - cannot tell whether the coder is busy, proceeding" + return 0 + fi [ "${busy:-0}" = 0 ] && return 0 [ "$waited" = 0 ] && note "waiting for coder-llm ($CODER_URL) to finish a request" sleep 15; waited=$((waited + 15)) diff --git a/tests/hw_gate_coder_idle_test.sh b/tests/hw_gate_coder_idle_test.sh new file mode 100755 index 000000000..2a9524a1c --- /dev/null +++ b/tests/hw_gate_coder_idle_test.sh @@ -0,0 +1,69 @@ +#!/bin/bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# tests/hw_gate_coder_idle_test.sh -- the gate's "is the coder busy?" probe must be able to say "I cannot tell". +# +# What it pins, and why it is a test at all: `wait_coder_idle` exists so an HRX step never starts while the +# PR-Agent's model is generating. Its parser used to answer `0` on ANY exception, and `curl -s` exits 0 on a 404, so +# a wrong URL, a refused connection, a HTML error page and a genuinely idle coder were all the same answer -- the +# probe could not fail, and the gate proceeded believing the coder was idle. The four inputs below separate them: +# `0` idle, `N` busy, **`-1` cannot tell** (reported, not silently read as idle). +# +# The two functions are extracted from the real script rather than copied, so the test cannot drift from the code; +# if the extraction stops matching, this test fails and says so instead of passing on stale text. +# +# Usage: hw_gate_coder_idle_test.sh [repo root] +set -u +root=${1:-.} +script="$root/scripts/hw-gate.sh" +fails=0 +chk() { # want got what + if [ "$1" = "$2" ]; then echo "ok $3"; else echo "FAIL $3: want '$1', got '$2'"; fails=$((fails + 1)); fi +} + +json_fn=$(awk '/^coder_busy_from_json\(\)/,/^}/' "$script") +idle_fn=$(awk '/^wait_coder_idle\(\)/,/^}/' "$script") +if [ -z "$json_fn" ] || [ -z "$idle_fn" ]; then + echo "FAIL cannot extract coder_busy_from_json/wait_coder_idle from $script -- update the extraction" >&2 + exit 1 +fi +eval "$json_fn" +eval "$idle_fn" + +chk 0 "$(printf '[]' | coder_busy_from_json)" "an empty slots list is idle" +chk 1 "$(printf '[{"is_processing":true},{"is_processing":false}]' | coder_busy_from_json)" "one processing slot is busy" +chk -1 "$(printf '404 Not Found' | coder_busy_from_json)" "an HTML error page is NOT idle" +chk -1 "$(printf '' | coder_busy_from_json)" "an empty body is NOT idle" +chk -1 "$(printf '{"error":{"message":"nf"}}' | coder_busy_from_json)" "a JSON error object is NOT idle" + +# End to end through wait_coder_idle: a refused connection must return (bounded, it is not the gate's job to hang on a +# dead coder) but it must SAY it could not tell, and it must get there without sleeping through CODER_WAIT. +notes="" +note() { notes="$notes$*|"; } +CODER_URL="http://127.0.0.1:9" # discard port: nothing listens, so curl -sf exits 7 +CODER_WAIT=900 +start=$(date +%s) +wait_coder_idle +elapsed=$(( $(date +%s) - start )) +chk 0 "$?" "a refused connection still lets the gate proceed" +chk yes "$(case "$notes" in *"did not answer"*) echo yes ;; *) echo no ;; esac)" "it warns that it could not ask the coder" +chk yes "$([ "$elapsed" -lt 30 ] && echo yes || echo no)" "and it does not wait out CODER_WAIT to say so (took ${elapsed}s)" + +if [ "$fails" != 0 ]; then + echo "hw_gate_coder_idle: $fails failure(s)" + exit 1 +fi +echo "hw_gate_coder_idle: ok" From 3339b0fc5c1cefcc30e52f086a9e5abc26c0e5e2 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Fri, 13 Nov 2026 07:21:36 -0400 Subject: [PATCH 2/2] window11: an empty transcript is not an identical one The #799 identical-text gate compares sha256 of each prompt's transcript. When the response cannot be parsed the writer records the empty string, whose sha is the same in every run, so two failed runs compare "3/3 identical" - and with a matching prof row the gate prints PASS. Measured on two logs of three empty rows: texts: 3/3 identical prof rows: 1 identical (phase, role, M) window11: PASS rc 0 <- having compared nothing The script's own comment says two empty strings "would compare identical without having tested anything"; the comment stated the intent and nothing enforced it. `load()` now keeps the `chars` field the line regex already captured, and a row with `chars=0` fails by name. That is exact rather than heuristic: the writer joins content, reasoning, tool_calls and finish_reason with separators, so a real answer is never 0 characters - only an unparseable one is. three empty rows -> FAIL prompt [1, 2, 3] recorded an empty transcript rc 1 normal rows -> PASS rc 0 --control pair -> still derives its tolerance rc 0 The third line matters: `load()` gained a return value, and the script's own `--self-test` caught the two other call sites in `spread()` (ValueError: too many values to unpack) before this was pushed as green. Fixed, and the self-test now passes. --- scripts/npu-window11.sh | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/scripts/npu-window11.sh b/scripts/npu-window11.sh index 72e26d95f..df83b22d3 100755 --- a/scripts/npu-window11.sh +++ b/scripts/npu-window11.sh @@ -105,11 +105,12 @@ if tol is None and control is None: def load(path): - texts, prof, order, taps, head = {}, {}, [], [], {} + texts, prof, order, taps, head, chars = {}, {}, [], [], {}, {} for line in open(path, encoding="utf-8", errors="replace"): m = re.match(r"text (\d+) sha256=(\S+) chars=(\d+) text=(.*)$", line) if m: texts.setdefault(int(m.group(1)), []).append(m.group(2)) + chars[int(m.group(1))] = int(m.group(3)) continue m = re.match(r"prof (dec|pre) (\S+) M=(\d+) ms=([-\d.]+) launches=(\d+)$", line) if m: @@ -125,7 +126,7 @@ def load(path): continue if line.startswith("tap "): taps.append(line[4:].rstrip("\n")) - return texts, prof, order, taps, head + return texts, prof, order, taps, head, chars def stat_row(pa, pb, paired_ok): @@ -143,8 +144,8 @@ def stat_row(pa, pb, paired_ok): def spread(c1, c2): """One binary run twice: its own spread is the only tolerance this box can justify.""" - _, p1, o1, _, _ = load(c1) - _, p2, _, _, _ = load(c2) + _, p1, o1, _, _, _ = load(c1) + _, p2, _, _, _, _ = load(c2) rows = [k for k in o1 if k in p2] if not rows: die("compare: the control logs share no prof rows") @@ -187,8 +188,8 @@ if control is not None: else: tol_from = "--tol" -ta, pa, oa, ka, ha = load(a) -tb, pb, ob, kb, hb = load(b) +ta, pa, oa, ka, ha, ca = load(a) +tb, pb, ob, kb, hb, cb = load(b) rounds = {k: min(len(pa[k]), len(pb[k])) for k in set(pa) & set(pb)} paired_ok = bool(rounds) and min(rounds.values()) >= 2 @@ -199,6 +200,15 @@ fail = 0 if len(ta) != 3 or len(tb) != 3: print(f"FAIL only {len(ta)}/{len(tb)} transcript prompts recorded (3 expected)"); fail = 1 +# A row with chars=0 is a response the writer could not parse: the request failed, or came back as something that is +# not a chat completion. Its sha is the empty string's, so two such runs compare "3/3 identical" and this gate passes +# having tested nothing - measured on two logs of three empty rows: "window11: PASS", rc 0. An answer that is merely +# empty is not this: the writer joins content, reasoning, tool_calls and finish_reason with separators, so a real +# answer is never 0 characters. +empty = sorted(i for i, n in list(ca.items()) + list(cb.items()) if n == 0) +if empty: + print(f"FAIL prompt {sorted(set(empty))} recorded an empty transcript - the response could not be parsed, so " + f"there is nothing to compare"); fail = 1 same = sum(1 for i in ta if i in tb and ta[i] == tb[i]) print(f"texts: {same}/3 identical") for i in sorted(ta):