From 9dd6077bff5415bcfe557bb78ac8b56922f640f9 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 06:58:32 -0400 Subject: [PATCH 1/4] hw-gate: `--phases placement` ran nothing and reported PASS; a run that could not test is not a pass MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I dispatched the gate on main to measure engine#2450's placement row after both halves of it merged: gh workflow run hw-gate.yml -f ref=main -f phases=build,placement -> run 43700775742 The run printed `PHASES: build,placement`, a summary table containing `build` and nothing else, and ## HRX hardware gate: PASS Three faults, all of the same kind: a gate reporting success over a question it never asked. 1. **`placement` was not in the per-model guard.** The loop that computes `doplace=1` was guarded by `has kat || has needle || has kld`, so `placement` alone (or with only `build`) entered nothing and produced no row. The gate has only ever had a placement row because its *default* phases include `kat` -- which is also why #2450's failures appeared in every labelled `hw-gate` run and in no other: the row was a side effect of a phase nobody asked for. 2. **A requested phase that produced no row is now a FAIL naming it** -- `requested but produced no row: --phases placement ran nothing for it`. The alias is deliberate: per-model phases report a missing model as `model `, and that row satisfies the request; only a phase with no row at all is a slip. 3. **A run that could not test an input is not a PASS.** `infra=1` marks an absent model/text or a refused memory pre-check, and it was only consulted *inside* `if [ "$nfail" != 0 ]`. Once an absent input became a SKIP row (engine#2522, merged while this was in review) a run with nothing to test had no FAIL to enter that branch and exited 0. Measured on the merged tree: `--phases placement` with an empty models dir gave `| model … | SKIP |` and `## HRX hardware gate: PASS`, rc 0. The verdict is now `INCOMPLETE (an input was absent: nothing was tested)` and the exit is 2, independent of the failure count. This PR's own guard fix is what makes that reachable: before it, `placement` produced no row at all; after it, it produces an honest SKIP that a reader would otherwise read as green. Device-free reproduction (the gate's own `HW_GATE_TOKEN=none` hook, real `--bin`, empty models dir, no GPU, no queue): --phases build -> | build | PASS | ## PASS rc 0 --phases placement -> | model qwen3-0.6b | SKIP | … ## INCOMPLETE (an input was absent) … rc 2 guard reverted -> | placement | FAIL | requested but produced no row … ## FAIL (1) rc 1 Rebased onto main after engine#2522; the conflict was one line -- this keeps both sides: `… || has kld || has placement` with the landed `SKIP 0 "text missing (input absent)"` row. Not claimed: that placement passes. It still needs a run with a real model on the box. But a run that asks for placement and cannot do it now says INCOMPLETE -- never PASS. --- scripts/hw-gate.sh | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/scripts/hw-gate.sh b/scripts/hw-gate.sh index 8a75101ce..a6abe5e69 100755 --- a/scripts/hw-gate.sh +++ b/scripts/hw-gate.sh @@ -643,8 +643,11 @@ if has ops; then fi fi -# ---- kat + needle + KLD: one step per model ----------------------------------------------- -if has kat || has needle || has kld; then +# ---- kat + needle + KLD + placement: one step per model ------------------------------------ +# `placement` must be in this guard. It was not, so `--phases build,placement` (and `placement` alone) entered nothing, +# produced no row, and the gate reported PASS over a run that never asked the question (2026-11-15, dispatch 43700775742: +# "PHASES: build,placement" then a summary table with `build` and nothing else, `## HRX hardware gate: PASS`). +if has kat || has needle || has kld || has placement; then [ -f "$TEXT" ] || { has kld && { result kld SKIP 0 "text missing (input absent): $TEXT"; infra=1; }; } while IFS='|' read -r name file repeat donee srvargs; do want_model "$name" || continue @@ -796,10 +799,31 @@ fi # ---- summary --------------------------------------------------------------------------------- total=$(( $(date +%s) - START )) +# A REQUESTED PHASE THAT PRODUCED NO ROW IS A FAILURE, not a pass. `--phases placement` entered no model loop until the +# guard above was fixed, and the gate reported `## HRX hardware gate: PASS` over a run that never asked the question +# (dispatch 43700775742). Whatever the next such slip is, it must not be able to read as green: every phase named on the +# command line has to appear as a row (PASS, FAIL, SKIP, or a `model …` row), or the gate says so by name. +for _ph in $(printf '%s' "$PHASES" | tr ',' ' '); do + case "$_ph" in ''|*' '*) continue ;; esac + # The per-model phases report their missing/throttled models as a `model ` row, so that row satisfies the + # request for any of them; only a phase that produced NO row at all is a slip. + case "$_ph" in + placement|kat|kld|needle) _phre="^($_ph|model )" ;; + *) _phre="^$_ph" ;; + esac + grep -qE "$_phre" "$RESULTS" 2>/dev/null || + printf '%s\tFAIL\t0\trequested but produced no result row: --phases %s ran nothing for it\n' "$_ph" "$_ph" >> "$RESULTS" +done nfail=$(awk -F'\t' '$2 == "FAIL"' "$RESULTS" | wc -l) rings=$(ring_timeouts) { - echo "## HRX hardware gate: $([ "$nfail" = 0 ] && echo PASS || echo "FAIL ($nfail)")" + # A run that could not test an input is NOT a pass. `infra=1` marks that (an absent model, absent text, a refused + # memory pre-check); before this, the flag was only consulted inside `if [ "$nfail" != 0 ]`, so once an absent input + # became a SKIP row (engine#2522) a run with nothing to test reported `## HRX hardware gate: PASS` and exited 0. + _verdict=PASS + [ "$nfail" != 0 ] && _verdict="FAIL ($nfail)" + [ "$infra" = 1 ] && [ "$_verdict" = PASS ] && _verdict="INCOMPLETE (an input was absent: nothing was tested)" + echo "## HRX hardware gate: $_verdict" echo echo "Commit \`$(git -C "$ROOT" rev-parse --short HEAD 2>/dev/null)\`, llama.cpp \`$(git -C "$ROOT" ls-tree HEAD third_party/llama.cpp | awk '{print substr($3,1,12)}')\`, hrx-system \`$(git -C "$ROOT" ls-tree HEAD third_party/hrx-system | awk '{print substr($3,1,12)}')\`, binaries \`$BIN\`, ${total}s, Tctl at end $(tctl) C, amdgpu ring timeouts during the run: $rings." echo @@ -813,8 +837,6 @@ rings=$(ring_timeouts) } > "$OUT/summary.md" [ -n "${GITHUB_STEP_SUMMARY:-}" ] && cat "$OUT/summary.md" >> "$GITHUB_STEP_SUMMARY" cat "$OUT/summary.md" -if [ "$nfail" != 0 ]; then - [ "$infra" = 1 ] && exit 2 - exit 1 -fi +[ "$infra" = 1 ] && exit 2 # not green: an input was absent, so the gate did not test what it was asked to +[ "$nfail" != 0 ] && exit 1 exit 0 From c08d0b06301525a62b079087ee45e4938956bdce Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 08:49:15 -0400 Subject: [PATCH 2/4] hw-gate: the ops zero-cases SKIP carries infra=1 The zero-cases guard is a *filter* problem, not a kernel result: nothing was tested. Without the flag a run whose -o pattern matched nothing still produced '## HRX hardware gate: PASS' and exit 0 -- the same defect engine#2522 fixed for the absent-input SKIPs, missed on this row. Reported by the queue pass on engine#2523. --- scripts/hw-gate.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/scripts/hw-gate.sh b/scripts/hw-gate.sh index a6abe5e69..d13afa36e 100755 --- a/scripts/hw-gate.sh +++ b/scripts/hw-gate.sh @@ -589,7 +589,15 @@ if has ops; then # pass". That is a *filter* problem, not a kernel result - and it must never read as a pass either. # Report it as SKIP with the selection that produced it, so neither an empty run nor a broken filter # can be mistaken for a measured one. + # infra=1: nothing was tested, so this is "could not run", not a measured SKIP. Without the flag a filter + # that matches no case leaves the gate reporting PASS (engine#2522 set it on the absent-input SKIPs for the + # same reason; the zero-cases SKIP was the one that was missed). + infra=1 result ops SKIP $(( $(date +%s) - t0 )) "no case matched -b HRX0 -o \"$OPS\": the zero-cases guard refused to report a pass (fork engine#657) - fix the filter or the build" + # engine#2522's rule, applied to this row: a SKIP means the run did not test what it claims to test, so the run + # as a whole is infra-incomplete and must not read as PASS. The verdict block now consults infra, so setting it + # here is what turns "the ops phase had nothing to run" into a non-green gate instead of a quiet row. + infra=1 else unknown=$(printf '%s\n' "$fails" | grep -v -F "$FLAKY_CASES" | grep -c . || true) st=FAIL detail="${passed:-rc=$rc}: $(printf '%s' "$fails" | head -n 3 | tr '\n' ' ')" @@ -807,9 +815,13 @@ for _ph in $(printf '%s' "$PHASES" | tr ',' ' '); do case "$_ph" in ''|*' '*) continue ;; esac # The per-model phases report their missing/throttled models as a `model ` row, so that row satisfies the # request for any of them; only a phase that produced NO row at all is a slip. + # `placement` has its OWN row kind, so a `model …` row must not satisfy a request for it: with the prefix rule + # below covering all four phases, `--phases placement` was satisfied by any model row and a run with no placement + # row at all still read as green -- the same defect one level in (review at 9dd6077). The per-model phases DO + # report their missing/throttled models as `model ` rows, so only those three accept that alternative. case "$_ph" in - placement|kat|kld|needle) _phre="^($_ph|model )" ;; - *) _phre="^$_ph" ;; + kat|kld|needle) _phre="^($_ph|model )" ;; + *) _phre="^$_ph" ;; esac grep -qE "$_phre" "$RESULTS" 2>/dev/null || printf '%s\tFAIL\t0\trequested but produced no result row: --phases %s ran nothing for it\n' "$_ph" "$_ph" >> "$RESULTS" From 21d99f7bed7f817f9c3e9fe81dd2736dc9e32c70 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 08:49:52 -0400 Subject: [PATCH 3/4] A phase is the row's first field, not a prefix of it (the last open finding at this head) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Your `c08d0b063` already carries ORCH's `infra=1` on the ops zero-case SKIP and already took `placement` out of the `model ` alternation, so this is only the third finding — the one still live at this head: **`npu` is a prefix of `npuserve`**, and `grep -qE "^$_ph"` therefore read an `npuserve` row as proof that a requested `npu` phase had run. Rows are `namestatus…`, so the match is now on the field: ```sh kat|kld|needle) _phre="^(${_ph}|model)[[:space:]]" ;; *) _phre="^${_ph}[[:space:]]" ;; ``` `tests/hw_gate_phase_rows_test.sh` (registered as `hw_gate_phase_rows`) pins it by **extracting the loop from this script and running it**, so it pins the gate rather than a copy that would keep passing while the real check drifted: the npu/npuserve slip and the earlier model/placement one both fail before the change, and the three legitimate satisfactions (`placement `, `model ` for kat, a plain row for its own phase) still pass after it. --- CMakeLists.txt | 1 + scripts/hw-gate.sh | 7 ++- tests/hw_gate_phase_rows_test.sh | 74 ++++++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+), 2 deletions(-) create mode 100755 tests/hw_gate_phase_rows_test.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 7b917f82e..45b5a2f93 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -629,6 +629,7 @@ add_test(NAME hw_gate_mem COMMAND bash ${CMAKE_SOURCE_DIR}/tests/hw_gate_mem_tes # engine#2500/#2513: a failing row's kept server log gets a name unique per failure (CI's $OUT basename is constant, so # the first version overwrote the previous run's evidence), and a failed copy names nothing. No GPU, no ticket. add_test(NAME hw_gate_kept_log COMMAND bash ${CMAKE_SOURCE_DIR}/tests/hw_gate_kept_log_test.sh ${CMAKE_SOURCE_DIR}) +add_test(NAME hw_gate_phase_rows COMMAND bash ${CMAKE_SOURCE_DIR}/tests/hw_gate_phase_rows_test.sh ${CMAKE_SOURCE_DIR}) # engine #1504: the release preflight blocks on a runner unit that is down and only warns on a short restart policy. # The gate's "is the coder busy?" probe: `0` idle, `N` busy, `-1` cannot tell. The parser answered `0` on any # exception while `curl -s` exits 0 on a 404, so a wrong URL, a refused connection and an idle coder were one answer. diff --git a/scripts/hw-gate.sh b/scripts/hw-gate.sh index d13afa36e..e77416760 100755 --- a/scripts/hw-gate.sh +++ b/scripts/hw-gate.sh @@ -819,9 +819,12 @@ for _ph in $(printf '%s' "$PHASES" | tr ',' ' '); do # below covering all four phases, `--phases placement` was satisfied by any model row and a run with no placement # row at all still read as green -- the same defect one level in (review at 9dd6077). The per-model phases DO # report their missing/throttled models as `model ` rows, so only those three accept that alternative. + # A phase is the ROW'S FIRST FIELD, not a prefix of it: `npu` is a prefix of `npuserve`, so `^npu` read an + # `npuserve` row as proof that `npu` had run (review of engine#2523, measured on a results file holding only + # `build` and `npuserve`). Rows are `namestatus...`, so `[[:space:]]` is what makes it a field match. case "$_ph" in - kat|kld|needle) _phre="^($_ph|model )" ;; - *) _phre="^$_ph" ;; + kat|kld|needle) _phre="^(${_ph}|model)[[:space:]]" ;; + *) _phre="^${_ph}[[:space:]]" ;; esac grep -qE "$_phre" "$RESULTS" 2>/dev/null || printf '%s\tFAIL\t0\trequested but produced no result row: --phases %s ran nothing for it\n' "$_ph" "$_ph" >> "$RESULTS" diff --git a/tests/hw_gate_phase_rows_test.sh b/tests/hw_gate_phase_rows_test.sh new file mode 100755 index 000000000..a0bfe3232 --- /dev/null +++ b/tests/hw_gate_phase_rows_test.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# hw-gate keeps a failing row's server log (engine#2500, #2513) under $HW_GATE_STORE/failed and names it in the +# failure line. The name must be unique per failure: CI sets HW_GATE_OUT=$RUNNER_TEMP/hw-gate, so a name built from +# basename "$OUT" was the constant "hw-gate-.log" and every failing run on every PR overwrote the last one's +# evidence. These arms run the CI shape (same model, same run, same second), and check that a failed copy prints no +# path. A recording fake gpu-hold is first on PATH, so the internal mode is also proven to take no GPU ticket. +# No GPU, no model, runs in CI. +# +# The gate's phase check must match the phase FIELD, not a prefix of it, and only the phases that actually write +# `model ` rows may be satisfied by one (review of engine#2523): +# +# * `npu` is a prefix of `npuserve`, so `grep -qE "^npu"` read an `npuserve` row as proof that `npu` had run; +# * `placement` writes `placement `, not `model `, so grouping it with kat|kld|needle let any model row +# satisfy a requested placement phase -- the same silent pass, one name over. +# +# The loop is EXTRACTED FROM THE SCRIPT and run, so this pins the shipped code rather than a copy of it. A copy would +# pass while the real check drifted. +set -u +root=$(cd "$(dirname "$0")/.." && pwd) +gate=$root/scripts/hw-gate.sh +fail=0 + +# the check, verbatim, as it ships +snippet=$(mktemp) +sed -n '/^for _ph in \$(printf .%s. "\$PHASES"/,/^done$/p' "$gate" > "$snippet" +[ -s "$snippet" ] || { echo "FAIL: the phase-check loop is no longer where this test finds it"; exit 1; } + +check() { # $1 results file, $2 PHASES -> prints the FAIL rows the check appended + ( PHASES=$2; RESULTS=$1; . "$snippet" ) 2>&1 + awk -F'\t' '$2 == "FAIL" {print $1}' "$1" +} + +run() { # name, results content, phases, expected-substring-in-output + local t; t=$(mktemp) + printf '%b' "$2" > "$t" + local out; out=$(check "$t" "$3") + rm -f "$t" + if [ -n "$4" ]; then + case "$out" in *"$4"*) ;; *) echo "FAIL $1: expected a FAIL row for '$4', got '${out:-}'"; fail=1; return ;; esac + else + [ -z "$out" ] || { echo "FAIL $1: expected no failure, got '$out'"; fail=1; return; } + fi + echo "ok $1" +} + +# the two slips the review measured, both must now fail +run "a npuserve row does not satisfy a requested npu phase" \ + 'build\tPASS\t1\tok\nnpuserve\tPASS\t1\tok\n' "build,npu,npuserve" "npu" +run "a model row does not satisfy a requested placement phase" \ + 'build\tPASS\t1\tok\nmodel gemma\tPASS\t1\tok\n' "build,placement" "placement" + +# and the three legitimate satisfactions must keep working +run "a placement row satisfies placement" 'build\tPASS\t1\tok\nplacement gemma\tPASS\t1\tok\n' "build,placement" "" +run "a model row still satisfies kat" 'build\tPASS\t1\tok\nmodel gemma\tPASS\t1\tok\n' "build,kat" "" +run "a plain row satisfies its own phase" 'build\tPASS\t1\tok\nops\tPASS\t1\tok\n' "build,ops" "" + +rm -f "$snippet" +[ "$fail" = 0 ] && echo "hw_gate_phase_rows: ok (a phase is a field, not a prefix; only kat/kld/needle accept a model row)" +exit "$fail" From 6a5161079e4ca95955f6d995cc6f937cad7de0b3 Mon Sep 17 00:00:00 2001 From: pi-lane Date: Sun, 15 Nov 2026 09:23:08 -0400 Subject: [PATCH 4/4] tests: hybrid_route stops the serve's process GROUP, so the backend cannot outlive it Fixes #2537. `stop()` signalled `1bit serve`'s pid and reaped it; `tests/fake_backend.py` is a CHILD, so it was reparented rather than signalled and kept its listener. The next section's backend wanted the same derived port (`backend 0`) and failed to bind: `OSError: [Errno 98]` in the child, reported by serve as "backend 0 did not become ready" -- which reads like a hang. Seen on engine#2523's build, job 132793096698. - `start` runs serve under `setsid`, so the command's pid IS its process group (setsid execs in place here, so `$!` is the command, not a wrapper). - `stop_tree`/`cleanup` signal the group (`kill -- -$pid`, falling back to the pid alone), which is what reaches the backend. - an arm tests the property, not the mechanism: after `stop`, the serve's process group must be gone. --- tests/hybrid_route.sh | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/hybrid_route.sh b/tests/hybrid_route.sh index 1034e67ae..d23ee1798 100755 --- a/tests/hybrid_route.sh +++ b/tests/hybrid_route.sh @@ -33,7 +33,11 @@ touch "$scratch/tiny.gguf" fail=0 check() { if eval "$2"; then echo "ok $1"; else echo "FAIL $1"; fail=1; fi; } pid= -cleanup() { [ -n "$pid" ] && { kill "$pid" 2>/dev/null; wait "$pid" 2>/dev/null; }; rm -rf "$scratch"; } +# stop_tree: the serve runs in its own process group (see start), so TERM the GROUP. Killing only the pid +# reaped `1bit serve` and left `fake_backend.py` orphaned with its listener up; the next section then failed +# to bind the same derived port and serve reported it as "backend 0 did not become ready" (engine#2537). +stop_tree() { [ -n "${1:-}" ] || return 0; kill -- -"$1" 2>/dev/null || kill "$1" 2>/dev/null; wait "$1" 2>/dev/null; } +cleanup() { stop_tree "$pid"; rm -rf "$scratch"; } trap cleanup EXIT start() { # start [env...]: serve on the port it reports, with the fake backend and env applied @@ -41,9 +45,9 @@ start() { # start [env...]: serve on the port it reports, with the fake local port code deadline # --port 0: the kernel picks and serve reports it, so nothing here picks a port and releases it before serve binds - # the release-then-bind race this test used to retry around is gone rather than mitigated (engine #1444/#1485). - env FAKE_SLOT_MARKER="$scratch/slot.json" "$@" "$bin" serve -m "$scratch/tiny.gguf" --device hybrid --port 0 --alias hybrid-model \ + setsid env FAKE_SLOT_MARKER="$scratch/slot.json" "$@" "$bin" serve -m "$scratch/tiny.gguf" --device hybrid --port 0 --alias hybrid-model \ --llama-server "$here/fake_backend.py" --no-mem-check >"$log" 2>&1 & - pid=$! + pid=$! pgid=$pid # setsid execs in place, so the command's pid IS its process group (engine#2537) port=$(serve_port_wait "$log" "$pid" 900) || { echo "FAIL serve never reported a port; log=$log"; tail -5 "$log" 2>/dev/null; return 1; } api="http://127.0.0.1:$port" # A deadline in seconds, not a try count: the old 100-try loop took 6.7 s on a quiet box and 15.2 s on the @@ -69,7 +73,7 @@ start() { # start [env...]: serve on the port it reports, with the fake done | head -1) [ "$code" = 200 ] } -stop() { kill "$pid" 2>/dev/null; wait "$pid" 2>/dev/null; pid=; } +stop() { stop_tree "$pid"; pid=; } # 1. the handoff: provider named, the backend sees id_slot 0 and cache_prompt true check "serve starts with the fake provider" 'start "$scratch/a.log" ONEBIT_HYBRID_PREFILL="$here/fake_prefill_provider.py" FAKE_PREFILL_COPY=1' @@ -93,6 +97,12 @@ check "streamed reply relayed ($chunks chunks)" '[ "$chunks" = 7 ]' check "no handoff file left in the slot directory" '[ -n "$slotdir" ] && [ -z "$(ls "$slotdir" 2>/dev/null)" ]' stop check "the slot directory is removed at exit" '[ -n "$slotdir" ] && [ ! -d "$slotdir" ]' +# engine#2537: the backend is a CHILD of serve, so a teardown that signals only serve's pid leaves it holding +# the port and the next section's bind fails. Nothing of this run's scratch tree may still be listening. +# setsid makes serve's pid its own process group, so the GROUP is the thing to test: with a teardown that +# signals only serve's pid the orphaned backend stays in the group and this fails. +for _ in $(seq 50); do kill -0 -- -"$pgid" 2>/dev/null || break; sleep 0.1; done +check "the serve's process group is gone, so no backend survives its stop" '! kill -0 -- -"$pgid" 2>/dev/null' # 2. no provider: the device serves, the backend prefills (said once) check "serve starts without a provider" 'start "$scratch/b.log"'