Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -554,6 +554,9 @@ add_test(NAME check_ci_runners COMMAND ${CMAKE_SOURCE_DIR}/tests/check_ci_runner
# cmake, a stub thermal-run and a fake meminfo: no GPU, no compiler, runs in CI.
add_test(NAME hw_gate_mem COMMAND bash ${CMAKE_SOURCE_DIR}/tests/hw_gate_mem_test.sh ${CMAKE_SOURCE_DIR})
# engine #1504: the release preflight blocks on a runner unit that is down and only warns on a short restart policy.
# The gate's "is the coder busy?" probe: `0` idle, `N` busy, `-1` cannot tell. The parser answered `0` on any
# exception while `curl -s` exits 0 on a 404, so a wrong URL, a refused connection and an idle coder were one answer.
add_test(NAME hw_gate_coder_idle COMMAND bash ${CMAKE_SOURCE_DIR}/tests/hw_gate_coder_idle_test.sh ${CMAKE_SOURCE_DIR})
# The real scripts/ci-runner-health.sh and weekly.sh's preflight_runner_units() against a stub systemctl; no systemd.
add_test(NAME weekly_runner_preflight COMMAND ${CMAKE_SOURCE_DIR}/tests/weekly_runner_preflight_test.sh)
add_test(NAME ci_runner_health_selftest COMMAND bash ${CMAKE_SOURCE_DIR}/scripts/ci-runner-health.sh --self-test)
Expand Down
26 changes: 22 additions & 4 deletions scripts/hw-gate.sh
Original file line number Diff line number Diff line change
Expand Up @@ -306,13 +306,31 @@ want_model() { [ -z "$ONLY_MODELS" ] && return 0; case ",$ONLY_MODELS," in *",$1

ring_timeouts() { journalctl -k --since "@$START" --no-pager 2>/dev/null | grep -c "ring .*timeout" || true; }

coder_busy_from_json() { # stdin: the /slots body. prints the busy count, or -1 when it cannot be read
python3 -c 'import json,sys
try: print(sum(1 for s in json.load(sys.stdin) if s.get("is_processing")))
except Exception: print(-1)'
}

wait_coder_idle() { # the PR-Agent's Vulkan model: never start an HRX step while it is generating
local waited=0
while [ "$waited" -lt "$CODER_WAIT" ]; do
busy=$(curl -s --max-time 5 "$CODER_URL/slots" 2>/dev/null | python3 -c \
'import json,sys
try: print(sum(1 for s in json.load(sys.stdin) if s.get("is_processing")))
except Exception: print(0)' 2>/dev/null)
# `curl -s` exits 0 on a 404 and the old parser answered 0 for every unreadable body, so "cannot ask" and
# "idle" were the same answer and this probe could not fail. The guard is coder_busy_from_json's `-1` for an
# unparseable body; `-f` is not what protects, it is what lets the message say "did not answer" rather than
# "answered something that is not a slots list". Measured: with `-f` removed the safety arms still pass, which
# is why the test pins the parse and not the flag.
local body rc
body=$(curl -sf --max-time 5 "$CODER_URL/slots" 2>/dev/null); rc=$?
if [ "$rc" != 0 ]; then
note "::warning::$CODER_URL/slots did not answer (curl exit $rc) - cannot tell whether the coder is busy, proceeding"
return 0
fi
busy=$(printf '%s' "$body" | coder_busy_from_json)
if [ "${busy:-}" = -1 ]; then
note "::warning::$CODER_URL/slots answered something that is not a slots list - cannot tell whether the coder is busy, proceeding"
return 0
fi
[ "${busy:-0}" = 0 ] && return 0
[ "$waited" = 0 ] && note "waiting for coder-llm ($CODER_URL) to finish a request"
sleep 15; waited=$((waited + 15))
Expand Down
22 changes: 16 additions & 6 deletions scripts/npu-window11.sh
Original file line number Diff line number Diff line change
Expand Up @@ -105,11 +105,12 @@ if tol is None and control is None:


def load(path):
texts, prof, order, taps, head = {}, {}, [], [], {}
texts, prof, order, taps, head, chars = {}, {}, [], [], {}, {}
for line in open(path, encoding="utf-8", errors="replace"):
m = re.match(r"text (\d+) sha256=(\S+) chars=(\d+) text=(.*)$", line)
if m:
texts.setdefault(int(m.group(1)), []).append(m.group(2))
chars[int(m.group(1))] = int(m.group(3))
continue
m = re.match(r"prof (dec|pre) (\S+) M=(\d+) ms=([-\d.]+) launches=(\d+)$", line)
if m:
Expand All @@ -125,7 +126,7 @@ def load(path):
continue
if line.startswith("tap "):
taps.append(line[4:].rstrip("\n"))
return texts, prof, order, taps, head
return texts, prof, order, taps, head, chars


def stat_row(pa, pb, paired_ok):
Expand All @@ -143,8 +144,8 @@ def stat_row(pa, pb, paired_ok):

def spread(c1, c2):
"""One binary run twice: its own spread is the only tolerance this box can justify."""
_, p1, o1, _, _ = load(c1)
_, p2, _, _, _ = load(c2)
_, p1, o1, _, _, _ = load(c1)
_, p2, _, _, _, _ = load(c2)
rows = [k for k in o1 if k in p2]
if not rows:
die("compare: the control logs share no prof rows")
Expand Down Expand Up @@ -187,8 +188,8 @@ if control is not None:
else:
tol_from = "--tol"

ta, pa, oa, ka, ha = load(a)
tb, pb, ob, kb, hb = load(b)
ta, pa, oa, ka, ha, ca = load(a)
tb, pb, ob, kb, hb, cb = load(b)

rounds = {k: min(len(pa[k]), len(pb[k])) for k in set(pa) & set(pb)}
paired_ok = bool(rounds) and min(rounds.values()) >= 2
Expand All @@ -199,6 +200,15 @@ fail = 0

if len(ta) != 3 or len(tb) != 3:
print(f"FAIL only {len(ta)}/{len(tb)} transcript prompts recorded (3 expected)"); fail = 1
# A row with chars=0 is a response the writer could not parse: the request failed, or came back as something that is
# not a chat completion. Its sha is the empty string's, so two such runs compare "3/3 identical" and this gate passes
# having tested nothing - measured on two logs of three empty rows: "window11: PASS", rc 0. An answer that is merely
# empty is not this: the writer joins content, reasoning, tool_calls and finish_reason with separators, so a real
# answer is never 0 characters.
empty = sorted(i for i, n in list(ca.items()) + list(cb.items()) if n == 0)
if empty:
print(f"FAIL prompt {sorted(set(empty))} recorded an empty transcript - the response could not be parsed, so "
f"there is nothing to compare"); fail = 1
same = sum(1 for i in ta if i in tb and ta[i] == tb[i])
print(f"texts: {same}/3 identical")
for i in sorted(ta):
Expand Down
69 changes: 69 additions & 0 deletions tests/hw_gate_coder_idle_test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
#!/bin/bash
# Copyright 2026 bong-water-water-bong
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# tests/hw_gate_coder_idle_test.sh -- the gate's "is the coder busy?" probe must be able to say "I cannot tell".
#
# What it pins, and why it is a test at all: `wait_coder_idle` exists so an HRX step never starts while the
# PR-Agent's model is generating. Its parser used to answer `0` on ANY exception, and `curl -s` exits 0 on a 404, so
# a wrong URL, a refused connection, a HTML error page and a genuinely idle coder were all the same answer -- the
# probe could not fail, and the gate proceeded believing the coder was idle. The four inputs below separate them:
# `0` idle, `N` busy, **`-1` cannot tell** (reported, not silently read as idle).
#
# The two functions are extracted from the real script rather than copied, so the test cannot drift from the code;
# if the extraction stops matching, this test fails and says so instead of passing on stale text.
#
# Usage: hw_gate_coder_idle_test.sh [repo root]
set -u
root=${1:-.}
script="$root/scripts/hw-gate.sh"
fails=0
chk() { # want got what
if [ "$1" = "$2" ]; then echo "ok $3"; else echo "FAIL $3: want '$1', got '$2'"; fails=$((fails + 1)); fi
}

json_fn=$(awk '/^coder_busy_from_json\(\)/,/^}/' "$script")
idle_fn=$(awk '/^wait_coder_idle\(\)/,/^}/' "$script")
if [ -z "$json_fn" ] || [ -z "$idle_fn" ]; then
echo "FAIL cannot extract coder_busy_from_json/wait_coder_idle from $script -- update the extraction" >&2
exit 1
fi
eval "$json_fn"
eval "$idle_fn"

chk 0 "$(printf '[]' | coder_busy_from_json)" "an empty slots list is idle"
chk 1 "$(printf '[{"is_processing":true},{"is_processing":false}]' | coder_busy_from_json)" "one processing slot is busy"
chk -1 "$(printf '<html>404 Not Found</html>' | coder_busy_from_json)" "an HTML error page is NOT idle"
chk -1 "$(printf '' | coder_busy_from_json)" "an empty body is NOT idle"
chk -1 "$(printf '{"error":{"message":"nf"}}' | coder_busy_from_json)" "a JSON error object is NOT idle"

# End to end through wait_coder_idle: a refused connection must return (bounded, it is not the gate's job to hang on a
# dead coder) but it must SAY it could not tell, and it must get there without sleeping through CODER_WAIT.
notes=""
note() { notes="$notes$*|"; }
CODER_URL="http://127.0.0.1:9" # discard port: nothing listens, so curl -sf exits 7
CODER_WAIT=900
start=$(date +%s)
wait_coder_idle
elapsed=$(( $(date +%s) - start ))
chk 0 "$?" "a refused connection still lets the gate proceed"
chk yes "$(case "$notes" in *"did not answer"*) echo yes ;; *) echo no ;; esac)" "it warns that it could not ask the coder"
chk yes "$([ "$elapsed" -lt 30 ] && echo yes || echo no)" "and it does not wait out CODER_WAIT to say so (took ${elapsed}s)"

if [ "$fails" != 0 ]; then
echo "hw_gate_coder_idle: $fails failure(s)"
exit 1
fi
echo "hw_gate_coder_idle: ok"
Loading