From 832fa4c00c132d399284d59de00ae5987758faff Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Fri, 13 Nov 2026 07:34:07 -0400 Subject: [PATCH 1/3] hw-gate memcheck: when the estimator cannot be read, say so (the refusal policy stays) `cmd_memcheck`'s contract is explicit - *"the estimator failing is not a refusal: scripts/mem-guard.sh still guards the step"* - and that policy is right, so this changes nothing about it. What it changes is that both unreadable paths now print a line instead of `return 0` in silence: memory pre-check: could not read tools/mem_budget.py box output - not refusing (scripts/mem-guard.sh still guards this step) Measured with a stubbed `mem_budget.py` that prints non-JSON: **rc is still 0** and the log now carries the reason. Silence was the defect: `memory pre-check: ok` versus nothing at all are different statements, and a log that cannot tell "I measured 0 GiB in use" from "I could not measure" is the shape this session kept finding in gates. Third instance of one class, all in gates whose failure path reads as fine: hw-gate coder probe answered `0` = idle for a refused connection (#2360) window11 answered the empty sha = "identical" for a failed request (#2360) hw-gate memcheck answered nothing = passes (policy kept, now stated) this commit `hw_gate_judges` 6.59 s, `hw_gate_mem` 13.48 s, `hw_gate_npuserve` 15.61 s all still pass; copyright and ci_targets rc 0. --- tools/hw_gate_manifest.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/hw_gate_manifest.py b/tools/hw_gate_manifest.py index 1085a5188..024e1f235 100644 --- a/tools/hw_gate_manifest.py +++ b/tools/hw_gate_manifest.py @@ -189,6 +189,11 @@ def cmd_memcheck(model, llama_dir): try: led = json.loads(p.stdout) except ValueError: + # Not a refusal, by this function's contract: mem-guard.sh still guards the step. But say so - a + # pre-check that fails silently reads as "ok" in the log, and "I could not measure" is not "there is + # 0 GiB in use" (tools/ci's own doc_section_loss/lane_syntax rule, one level up). + print("memory pre-check: could not read tools/mem_budget.py box output - not refusing " + "(scripts/mem-guard.sh still guards this step)", file=sys.stderr) return 0 used = led.get("gtt_used", 0) / g avail = led["mem_available"] / g @@ -203,6 +208,9 @@ def cmd_memcheck(model, llama_dir): try: r = json.loads(p.stdout) except ValueError: + print(f"memory pre-check: could not read tools/mem_budget.py model output for " + f"{os.path.basename(model)} - not refusing (scripts/mem-guard.sh still guards this step)", + file=sys.stderr) return 0 msg = (f"memory pre-check: {os.path.basename(model)} needs {r['need'] / g:.1f} GiB, GTT after load " f"{r['gtt_used_after'] / g:.1f} GiB, MemAvailable {r['mem_available'] / g:.1f} GiB") From b406eeceaaa648863e15e352f1201a5f8e7c41b8 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Fri, 13 Nov 2026 18:09:31 -0400 Subject: [PATCH 2/3] hw-gate manifest: an unreadable pin is null and named, not the string "unknown" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The third silent path the review found, in the same file as the memcheck fix: `pins()` recorded `"unknown"` into the manifest's `recorded_at` when `git ls-tree` failed, and **nothing consumes that field** (`grep -rn recorded_at` finds the writer and one doc sentence), so the artifact kept looking like a record of what ran while holding a placeholder. `recorded_at` is a name that promises commit ids, so the two failure modes now differ visibly: ``` $ python3 -c "import hw_gate_manifest as h; print(h.pins(\"/tmp/not-a-repo\"))" hw_gate_manifest: could not read the pin of third_party/llama.cpp - recording null (CalledProcessError) hw_gate_manifest: could not read the pin of third_party/hrx-system - recording null (CalledProcessError) {"llama.cpp": None, "hrx-system": None} $ … h.pins() {"llama.cpp": "e6c80014eb1c00bd4c1900d18379752227303fbb", "hrx-system": "7f4236924aadc069fb7425681525dddbd1535567"} ``` Both are asserted in the run above: failure -> `None`, success -> 40 hex, and the exception type is named so a reader can tell "not a git repo" from "no such submodule row". `docs/hw-gate.md` is the one place that reads this field out loud, and it now says what a `null` means. copyright --check rc 0. --- docs/hw-gate.md | 3 ++- tools/hw_gate_manifest.py | 13 +++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/docs/hw-gate.md b/docs/hw-gate.md index a8869c179..389a6b4e1 100644 --- a/docs/hw-gate.md +++ b/docs/hw-gate.md @@ -383,7 +383,8 @@ What the goldens record at the release pin: - ZAYA1-8B and Qwen3.8-27B answer every case. The MoE and IQ rows (`qwen3-coder-30b-a3b`, `qwen3-4b-iq4xs`) were recorded on 2026-10-20 at -llama.cpp 00f2a68d / hrx-system 82804197 (`manifest.json` `recorded_at`). Both answer every kat case. +llama.cpp 00f2a68d / hrx-system 82804197 (`manifest.json` `recorded_at`; a submodule whose pin could not be read is +recorded as `null` and named on stderr, never as the string "unknown"). Both answer every kat case. Their HRX base logits against a CPU run of the same file (`--device none`) give: | model | mean KLD | KLD p99 | same top token | diff --git a/tools/hw_gate_manifest.py b/tools/hw_gate_manifest.py index 024e1f235..b3372c9b9 100644 --- a/tools/hw_gate_manifest.py +++ b/tools/hw_gate_manifest.py @@ -62,14 +62,23 @@ def model_fingerprint(path): def pins(root): + """The two submodule pins, or None for a submodule whose pin could not be read. + + This used to write the string "unknown" into the manifest's `recorded_at`, and NOTHING consumes that field + (`grep -rn recorded_at` finds only this writer), so a `git ls-tree` failure produced an artifact that still + looked like a record of what ran. `recorded_at` is a name that promises commit ids: an unreadable pin is + recorded as **null** and said out loud, rather than as a value a reader would have to know is a placeholder. + """ out = {} for sub in ("llama.cpp", "hrx-system"): try: line = subprocess.run(["git", "-C", root, "ls-tree", "HEAD", "third_party/" + sub], capture_output=True, text=True, check=True).stdout.split() out[sub] = line[2] - except (subprocess.CalledProcessError, IndexError): - out[sub] = "unknown" + except (subprocess.CalledProcessError, IndexError) as exc: + print(f"hw_gate_manifest: could not read the pin of third_party/{sub} - recording null " + f"({type(exc).__name__})", file=sys.stderr) + out[sub] = None return out From 6e43db2614a04fcf67ac96a9ceb481e2827e2fb6 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Fri, 13 Nov 2026 22:14:55 -0400 Subject: [PATCH 3/3] tests: hw_gate_judges asserts the recorded_at contract for an unreadable pin (#2362) Review at b406eec: nothing consumes recorded_at, so only a test notices if "unknown" comes back or the stderr line goes. The new arm reads pins() from two repos where the ref cannot resolve: one with no commit (CalledProcessError) and one with a commit but no submodule rows (IndexError). For both submodules it asserts the value is null AND that stderr names third_party/ and the exception type. Mutants (not committed): "unknown" in place of None fails 4 checks; dropping the stderr line fails the other 4. The clean tree passes. Co-Authored-By: Claude Opus 5.5 --- tests/hw_gate_test.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/hw_gate_test.py b/tests/hw_gate_test.py index c1bddfc07..173deef37 100644 --- a/tests/hw_gate_test.py +++ b/tests/hw_gate_test.py @@ -190,6 +190,36 @@ def main(): f.write(b"x") if subprocess.run([sys.executable, MAN, "verify", man2, "m", base, model, text], capture_output=True).returncode != 1: bad.append("verify must refuse a changed base") + + # recorded_at: a pin that cannot be read is null AND named on stderr with the exception type (#2362). + # Nothing consumes recorded_at, so this arm is the only thing that would notice "unknown" coming back, + # or a None with the message dropped. Two unresolvable refs, each in a repo of its own (a bare temp dir + # could sit inside some other checkout and read that checkout's pins): + # no commit at all -> `git ls-tree HEAD` fails -> CalledProcessError + # a commit without the submodule rows -> no line -> IndexError + git_env = dict(os.environ, GIT_AUTHOR_NAME="t", GIT_AUTHOR_EMAIL="t@t", GIT_COMMITTER_NAME="t", + GIT_COMMITTER_EMAIL="t@t") + empty, norows = os.path.join(d, "pin-empty"), os.path.join(d, "pin-norows") + for repo in (empty, norows): + subprocess.run(["git", "init", "-q", repo], check=True, env=git_env) + subprocess.run(["git", "-C", norows, "commit", "-q", "--allow-empty", "-m", "x"], check=True, env=git_env) + read_pins = ("import json, sys; sys.path.insert(0, sys.argv[1]); import hw_gate_manifest as h; " + "print(json.dumps(h.pins(sys.argv[2])))") + for repo, exc in ((empty, "CalledProcessError"), (norows, "IndexError")): + p = subprocess.run([sys.executable, "-c", read_pins, os.path.join(ROOT, "tools"), repo], + capture_output=True, text=True) + name = os.path.basename(repo) + try: + got = json.loads(p.stdout) + except ValueError: + bad.append(f"pins {name}: rc {p.returncode}, no JSON on stdout: {p.stdout!r} {p.stderr!r}") + continue + for sub in ("llama.cpp", "hrx-system"): + if sub not in got or got[sub] is not None: + bad.append(f"pins {name}: {sub} must be null when unreadable, got {got.get(sub, '')!r}") + said = [l for l in p.stderr.splitlines() if "third_party/" + sub in l] + if not said or exc not in said[0]: + bad.append(f"pins {name}: stderr must name third_party/{sub} and {exc}; said {p.stderr.strip()!r}") srv.shutdown() for b in bad: print("FAIL:", b)