diff --git a/CMakeLists.txt b/CMakeLists.txt index 8144e013e..bc5cf6b11 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -554,6 +554,9 @@ add_test(NAME check_ci_runners COMMAND ${CMAKE_SOURCE_DIR}/tests/check_ci_runner # cmake, a stub thermal-run and a fake meminfo: no GPU, no compiler, runs in CI. add_test(NAME hw_gate_mem COMMAND bash ${CMAKE_SOURCE_DIR}/tests/hw_gate_mem_test.sh ${CMAKE_SOURCE_DIR}) # engine #1504: the release preflight blocks on a runner unit that is down and only warns on a short restart policy. +# The gate's "is the coder busy?" probe: `0` idle, `N` busy, `-1` cannot tell. The parser answered `0` on any +# exception while `curl -s` exits 0 on a 404, so a wrong URL, a refused connection and an idle coder were one answer. +add_test(NAME hw_gate_coder_idle COMMAND bash ${CMAKE_SOURCE_DIR}/tests/hw_gate_coder_idle_test.sh ${CMAKE_SOURCE_DIR}) # The real scripts/ci-runner-health.sh and weekly.sh's preflight_runner_units() against a stub systemctl; no systemd. add_test(NAME weekly_runner_preflight COMMAND ${CMAKE_SOURCE_DIR}/tests/weekly_runner_preflight_test.sh) add_test(NAME ci_runner_health_selftest COMMAND bash ${CMAKE_SOURCE_DIR}/scripts/ci-runner-health.sh --self-test) diff --git a/scripts/hw-gate.sh b/scripts/hw-gate.sh index 2e4f35a28..4ad0e7a81 100755 --- a/scripts/hw-gate.sh +++ b/scripts/hw-gate.sh @@ -306,13 +306,31 @@ want_model() { [ -z "$ONLY_MODELS" ] && return 0; case ",$ONLY_MODELS," in *",$1 ring_timeouts() { journalctl -k --since "@$START" --no-pager 2>/dev/null | grep -c "ring .*timeout" || true; } +coder_busy_from_json() { # stdin: the /slots body. prints the busy count, or -1 when it cannot be read + python3 -c 'import json,sys +try: print(sum(1 for s in json.load(sys.stdin) if s.get("is_processing"))) +except Exception: print(-1)' +} + wait_coder_idle() { # the PR-Agent's Vulkan model: never start an HRX step while it is generating local waited=0 while [ "$waited" -lt "$CODER_WAIT" ]; do - busy=$(curl -s --max-time 5 "$CODER_URL/slots" 2>/dev/null | python3 -c \ - 'import json,sys -try: print(sum(1 for s in json.load(sys.stdin) if s.get("is_processing"))) -except Exception: print(0)' 2>/dev/null) + # `curl -s` exits 0 on a 404 and the old parser answered 0 for every unreadable body, so "cannot ask" and + # "idle" were the same answer and this probe could not fail. The guard is coder_busy_from_json's `-1` for an + # unparseable body; `-f` is not what protects, it is what lets the message say "did not answer" rather than + # "answered something that is not a slots list". Measured: with `-f` removed the safety arms still pass, which + # is why the test pins the parse and not the flag. + local body rc + body=$(curl -sf --max-time 5 "$CODER_URL/slots" 2>/dev/null); rc=$? + if [ "$rc" != 0 ]; then + note "::warning::$CODER_URL/slots did not answer (curl exit $rc) - cannot tell whether the coder is busy, proceeding" + return 0 + fi + busy=$(printf '%s' "$body" | coder_busy_from_json) + if [ "${busy:-}" = -1 ]; then + note "::warning::$CODER_URL/slots answered something that is not a slots list - cannot tell whether the coder is busy, proceeding" + return 0 + fi [ "${busy:-0}" = 0 ] && return 0 [ "$waited" = 0 ] && note "waiting for coder-llm ($CODER_URL) to finish a request" sleep 15; waited=$((waited + 15)) diff --git a/scripts/npu-window11.sh b/scripts/npu-window11.sh index 72e26d95f..df83b22d3 100755 --- a/scripts/npu-window11.sh +++ b/scripts/npu-window11.sh @@ -105,11 +105,12 @@ if tol is None and control is None: def load(path): - texts, prof, order, taps, head = {}, {}, [], [], {} + texts, prof, order, taps, head, chars = {}, {}, [], [], {}, {} for line in open(path, encoding="utf-8", errors="replace"): m = re.match(r"text (\d+) sha256=(\S+) chars=(\d+) text=(.*)$", line) if m: texts.setdefault(int(m.group(1)), []).append(m.group(2)) + chars[int(m.group(1))] = int(m.group(3)) continue m = re.match(r"prof (dec|pre) (\S+) M=(\d+) ms=([-\d.]+) launches=(\d+)$", line) if m: @@ -125,7 +126,7 @@ def load(path): continue if line.startswith("tap "): taps.append(line[4:].rstrip("\n")) - return texts, prof, order, taps, head + return texts, prof, order, taps, head, chars def stat_row(pa, pb, paired_ok): @@ -143,8 +144,8 @@ def stat_row(pa, pb, paired_ok): def spread(c1, c2): """One binary run twice: its own spread is the only tolerance this box can justify.""" - _, p1, o1, _, _ = load(c1) - _, p2, _, _, _ = load(c2) + _, p1, o1, _, _, _ = load(c1) + _, p2, _, _, _, _ = load(c2) rows = [k for k in o1 if k in p2] if not rows: die("compare: the control logs share no prof rows") @@ -187,8 +188,8 @@ if control is not None: else: tol_from = "--tol" -ta, pa, oa, ka, ha = load(a) -tb, pb, ob, kb, hb = load(b) +ta, pa, oa, ka, ha, ca = load(a) +tb, pb, ob, kb, hb, cb = load(b) rounds = {k: min(len(pa[k]), len(pb[k])) for k in set(pa) & set(pb)} paired_ok = bool(rounds) and min(rounds.values()) >= 2 @@ -199,6 +200,15 @@ fail = 0 if len(ta) != 3 or len(tb) != 3: print(f"FAIL only {len(ta)}/{len(tb)} transcript prompts recorded (3 expected)"); fail = 1 +# A row with chars=0 is a response the writer could not parse: the request failed, or came back as something that is +# not a chat completion. Its sha is the empty string's, so two such runs compare "3/3 identical" and this gate passes +# having tested nothing - measured on two logs of three empty rows: "window11: PASS", rc 0. An answer that is merely +# empty is not this: the writer joins content, reasoning, tool_calls and finish_reason with separators, so a real +# answer is never 0 characters. +empty = sorted(i for i, n in list(ca.items()) + list(cb.items()) if n == 0) +if empty: + print(f"FAIL prompt {sorted(set(empty))} recorded an empty transcript - the response could not be parsed, so " + f"there is nothing to compare"); fail = 1 same = sum(1 for i in ta if i in tb and ta[i] == tb[i]) print(f"texts: {same}/3 identical") for i in sorted(ta): diff --git a/tests/hw_gate_coder_idle_test.sh b/tests/hw_gate_coder_idle_test.sh new file mode 100755 index 000000000..2a9524a1c --- /dev/null +++ b/tests/hw_gate_coder_idle_test.sh @@ -0,0 +1,69 @@ +#!/bin/bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# tests/hw_gate_coder_idle_test.sh -- the gate's "is the coder busy?" probe must be able to say "I cannot tell". +# +# What it pins, and why it is a test at all: `wait_coder_idle` exists so an HRX step never starts while the +# PR-Agent's model is generating. Its parser used to answer `0` on ANY exception, and `curl -s` exits 0 on a 404, so +# a wrong URL, a refused connection, a HTML error page and a genuinely idle coder were all the same answer -- the +# probe could not fail, and the gate proceeded believing the coder was idle. The four inputs below separate them: +# `0` idle, `N` busy, **`-1` cannot tell** (reported, not silently read as idle). +# +# The two functions are extracted from the real script rather than copied, so the test cannot drift from the code; +# if the extraction stops matching, this test fails and says so instead of passing on stale text. +# +# Usage: hw_gate_coder_idle_test.sh [repo root] +set -u +root=${1:-.} +script="$root/scripts/hw-gate.sh" +fails=0 +chk() { # want got what + if [ "$1" = "$2" ]; then echo "ok $3"; else echo "FAIL $3: want '$1', got '$2'"; fails=$((fails + 1)); fi +} + +json_fn=$(awk '/^coder_busy_from_json\(\)/,/^}/' "$script") +idle_fn=$(awk '/^wait_coder_idle\(\)/,/^}/' "$script") +if [ -z "$json_fn" ] || [ -z "$idle_fn" ]; then + echo "FAIL cannot extract coder_busy_from_json/wait_coder_idle from $script -- update the extraction" >&2 + exit 1 +fi +eval "$json_fn" +eval "$idle_fn" + +chk 0 "$(printf '[]' | coder_busy_from_json)" "an empty slots list is idle" +chk 1 "$(printf '[{"is_processing":true},{"is_processing":false}]' | coder_busy_from_json)" "one processing slot is busy" +chk -1 "$(printf '404 Not Found' | coder_busy_from_json)" "an HTML error page is NOT idle" +chk -1 "$(printf '' | coder_busy_from_json)" "an empty body is NOT idle" +chk -1 "$(printf '{"error":{"message":"nf"}}' | coder_busy_from_json)" "a JSON error object is NOT idle" + +# End to end through wait_coder_idle: a refused connection must return (bounded, it is not the gate's job to hang on a +# dead coder) but it must SAY it could not tell, and it must get there without sleeping through CODER_WAIT. +notes="" +note() { notes="$notes$*|"; } +CODER_URL="http://127.0.0.1:9" # discard port: nothing listens, so curl -sf exits 7 +CODER_WAIT=900 +start=$(date +%s) +wait_coder_idle +elapsed=$(( $(date +%s) - start )) +chk 0 "$?" "a refused connection still lets the gate proceed" +chk yes "$(case "$notes" in *"did not answer"*) echo yes ;; *) echo no ;; esac)" "it warns that it could not ask the coder" +chk yes "$([ "$elapsed" -lt 30 ] && echo yes || echo no)" "and it does not wait out CODER_WAIT to say so (took ${elapsed}s)" + +if [ "$fails" != 0 ]; then + echo "hw_gate_coder_idle: $fails failure(s)" + exit 1 +fi +echo "hw_gate_coder_idle: ok"