diff --git a/docs/hw-gate.md b/docs/hw-gate.md index a8869c179..389a6b4e1 100644 --- a/docs/hw-gate.md +++ b/docs/hw-gate.md @@ -383,7 +383,8 @@ What the goldens record at the release pin: - ZAYA1-8B and Qwen3.8-27B answer every case. The MoE and IQ rows (`qwen3-coder-30b-a3b`, `qwen3-4b-iq4xs`) were recorded on 2026-10-20 at -llama.cpp 00f2a68d / hrx-system 82804197 (`manifest.json` `recorded_at`). Both answer every kat case. +llama.cpp 00f2a68d / hrx-system 82804197 (`manifest.json` `recorded_at`; a submodule whose pin could not be read is +recorded as `null` and named on stderr, never as the string "unknown"). Both answer every kat case. Their HRX base logits against a CPU run of the same file (`--device none`) give: | model | mean KLD | KLD p99 | same top token | diff --git a/tests/hw_gate_test.py b/tests/hw_gate_test.py index c1bddfc07..173deef37 100644 --- a/tests/hw_gate_test.py +++ b/tests/hw_gate_test.py @@ -190,6 +190,36 @@ def main(): f.write(b"x") if subprocess.run([sys.executable, MAN, "verify", man2, "m", base, model, text], capture_output=True).returncode != 1: bad.append("verify must refuse a changed base") + + # recorded_at: a pin that cannot be read is null AND named on stderr with the exception type (#2362). + # Nothing consumes recorded_at, so this arm is the only thing that would notice "unknown" coming back, + # or a None with the message dropped. Two unresolvable refs, each in a repo of its own (a bare temp dir + # could sit inside some other checkout and read that checkout's pins): + # no commit at all -> `git ls-tree HEAD` fails -> CalledProcessError + # a commit without the submodule rows -> no line -> IndexError + git_env = dict(os.environ, GIT_AUTHOR_NAME="t", GIT_AUTHOR_EMAIL="t@t", GIT_COMMITTER_NAME="t", + GIT_COMMITTER_EMAIL="t@t") + empty, norows = os.path.join(d, "pin-empty"), os.path.join(d, "pin-norows") + for repo in (empty, norows): + subprocess.run(["git", "init", "-q", repo], check=True, env=git_env) + subprocess.run(["git", "-C", norows, "commit", "-q", "--allow-empty", "-m", "x"], check=True, env=git_env) + read_pins = ("import json, sys; sys.path.insert(0, sys.argv[1]); import hw_gate_manifest as h; " + "print(json.dumps(h.pins(sys.argv[2])))") + for repo, exc in ((empty, "CalledProcessError"), (norows, "IndexError")): + p = subprocess.run([sys.executable, "-c", read_pins, os.path.join(ROOT, "tools"), repo], + capture_output=True, text=True) + name = os.path.basename(repo) + try: + got = json.loads(p.stdout) + except ValueError: + bad.append(f"pins {name}: rc {p.returncode}, no JSON on stdout: {p.stdout!r} {p.stderr!r}") + continue + for sub in ("llama.cpp", "hrx-system"): + if sub not in got or got[sub] is not None: + bad.append(f"pins {name}: {sub} must be null when unreadable, got {got.get(sub, '')!r}") + said = [l for l in p.stderr.splitlines() if "third_party/" + sub in l] + if not said or exc not in said[0]: + bad.append(f"pins {name}: stderr must name third_party/{sub} and {exc}; said {p.stderr.strip()!r}") srv.shutdown() for b in bad: print("FAIL:", b) diff --git a/tools/hw_gate_manifest.py b/tools/hw_gate_manifest.py index 1085a5188..b3372c9b9 100644 --- a/tools/hw_gate_manifest.py +++ b/tools/hw_gate_manifest.py @@ -62,14 +62,23 @@ def model_fingerprint(path): def pins(root): + """The two submodule pins, or None for a submodule whose pin could not be read. + + This used to write the string "unknown" into the manifest's `recorded_at`, and NOTHING consumes that field + (`grep -rn recorded_at` finds only this writer), so a `git ls-tree` failure produced an artifact that still + looked like a record of what ran. `recorded_at` is a name that promises commit ids: an unreadable pin is + recorded as **null** and said out loud, rather than as a value a reader would have to know is a placeholder. + """ out = {} for sub in ("llama.cpp", "hrx-system"): try: line = subprocess.run(["git", "-C", root, "ls-tree", "HEAD", "third_party/" + sub], capture_output=True, text=True, check=True).stdout.split() out[sub] = line[2] - except (subprocess.CalledProcessError, IndexError): - out[sub] = "unknown" + except (subprocess.CalledProcessError, IndexError) as exc: + print(f"hw_gate_manifest: could not read the pin of third_party/{sub} - recording null " + f"({type(exc).__name__})", file=sys.stderr) + out[sub] = None return out @@ -189,6 +198,11 @@ def cmd_memcheck(model, llama_dir): try: led = json.loads(p.stdout) except ValueError: + # Not a refusal, by this function's contract: mem-guard.sh still guards the step. But say so - a + # pre-check that fails silently reads as "ok" in the log, and "I could not measure" is not "there is + # 0 GiB in use" (tools/ci's own doc_section_loss/lane_syntax rule, one level up). + print("memory pre-check: could not read tools/mem_budget.py box output - not refusing " + "(scripts/mem-guard.sh still guards this step)", file=sys.stderr) return 0 used = led.get("gtt_used", 0) / g avail = led["mem_available"] / g @@ -203,6 +217,9 @@ def cmd_memcheck(model, llama_dir): try: r = json.loads(p.stdout) except ValueError: + print(f"memory pre-check: could not read tools/mem_budget.py model output for " + f"{os.path.basename(model)} - not refusing (scripts/mem-guard.sh still guards this step)", + file=sys.stderr) return 0 msg = (f"memory pre-check: {os.path.basename(model)} needs {r['need'] / g:.1f} GiB, GTT after load " f"{r['gtt_used_after'] / g:.1f} GiB, MemAvailable {r['mem_available'] / g:.1f} GiB")