Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion docs/hw-gate.md
Original file line number Diff line number Diff line change
Expand Up @@ -383,7 +383,8 @@ What the goldens record at the release pin:
- ZAYA1-8B and Qwen3.8-27B answer every case.

The MoE and IQ rows (`qwen3-coder-30b-a3b`, `qwen3-4b-iq4xs`) were recorded on 2026-10-20 at
llama.cpp 00f2a68d / hrx-system 82804197 (`manifest.json` `recorded_at`). Both answer every kat case.
llama.cpp 00f2a68d / hrx-system 82804197 (`manifest.json` `recorded_at`; a submodule whose pin could not be read is
recorded as `null` and named on stderr, never as the string "unknown"). Both answer every kat case.
Their HRX base logits against a CPU run of the same file (`--device none`) give:

| model | mean KLD | KLD p99 | same top token |
Expand Down
30 changes: 30 additions & 0 deletions tests/hw_gate_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,36 @@ def main():
f.write(b"x")
if subprocess.run([sys.executable, MAN, "verify", man2, "m", base, model, text], capture_output=True).returncode != 1:
bad.append("verify must refuse a changed base")

# recorded_at: a pin that cannot be read is null AND named on stderr with the exception type (#2362).
# Nothing consumes recorded_at, so this arm is the only thing that would notice "unknown" coming back,
# or a None with the message dropped. Two unresolvable refs, each in a repo of its own (a bare temp dir
# could sit inside some other checkout and read that checkout's pins):
# no commit at all -> `git ls-tree HEAD` fails -> CalledProcessError
# a commit without the submodule rows -> no line -> IndexError
git_env = dict(os.environ, GIT_AUTHOR_NAME="t", GIT_AUTHOR_EMAIL="t@t", GIT_COMMITTER_NAME="t",
GIT_COMMITTER_EMAIL="t@t")
empty, norows = os.path.join(d, "pin-empty"), os.path.join(d, "pin-norows")
for repo in (empty, norows):
subprocess.run(["git", "init", "-q", repo], check=True, env=git_env)
subprocess.run(["git", "-C", norows, "commit", "-q", "--allow-empty", "-m", "x"], check=True, env=git_env)
read_pins = ("import json, sys; sys.path.insert(0, sys.argv[1]); import hw_gate_manifest as h; "
"print(json.dumps(h.pins(sys.argv[2])))")
for repo, exc in ((empty, "CalledProcessError"), (norows, "IndexError")):
p = subprocess.run([sys.executable, "-c", read_pins, os.path.join(ROOT, "tools"), repo],
capture_output=True, text=True)
name = os.path.basename(repo)
try:
got = json.loads(p.stdout)
except ValueError:
bad.append(f"pins {name}: rc {p.returncode}, no JSON on stdout: {p.stdout!r} {p.stderr!r}")
continue
for sub in ("llama.cpp", "hrx-system"):
if sub not in got or got[sub] is not None:
bad.append(f"pins {name}: {sub} must be null when unreadable, got {got.get(sub, '<missing>')!r}")
said = [l for l in p.stderr.splitlines() if "third_party/" + sub in l]
if not said or exc not in said[0]:
bad.append(f"pins {name}: stderr must name third_party/{sub} and {exc}; said {p.stderr.strip()!r}")
srv.shutdown()
for b in bad:
print("FAIL:", b)
Expand Down
21 changes: 19 additions & 2 deletions tools/hw_gate_manifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,14 +62,23 @@ def model_fingerprint(path):


def pins(root):
"""The two submodule pins, or None for a submodule whose pin could not be read.

This used to write the string "unknown" into the manifest's `recorded_at`, and NOTHING consumes that field
(`grep -rn recorded_at` finds only this writer), so a `git ls-tree` failure produced an artifact that still
looked like a record of what ran. `recorded_at` is a name that promises commit ids: an unreadable pin is
recorded as **null** and said out loud, rather than as a value a reader would have to know is a placeholder.
"""
out = {}
for sub in ("llama.cpp", "hrx-system"):
try:
line = subprocess.run(["git", "-C", root, "ls-tree", "HEAD", "third_party/" + sub],
capture_output=True, text=True, check=True).stdout.split()
out[sub] = line[2]
except (subprocess.CalledProcessError, IndexError):
out[sub] = "unknown"
except (subprocess.CalledProcessError, IndexError) as exc:
print(f"hw_gate_manifest: could not read the pin of third_party/{sub} - recording null "
f"({type(exc).__name__})", file=sys.stderr)
out[sub] = None
return out


Expand Down Expand Up @@ -189,6 +198,11 @@ def cmd_memcheck(model, llama_dir):
try:
led = json.loads(p.stdout)
except ValueError:
# Not a refusal, by this function's contract: mem-guard.sh still guards the step. But say so - a
# pre-check that fails silently reads as "ok" in the log, and "I could not measure" is not "there is
# 0 GiB in use" (tools/ci's own doc_section_loss/lane_syntax rule, one level up).
print("memory pre-check: could not read tools/mem_budget.py box output - not refusing "
"(scripts/mem-guard.sh still guards this step)", file=sys.stderr)
return 0
used = led.get("gtt_used", 0) / g
avail = led["mem_available"] / g
Expand All @@ -203,6 +217,9 @@ def cmd_memcheck(model, llama_dir):
try:
r = json.loads(p.stdout)
except ValueError:
print(f"memory pre-check: could not read tools/mem_budget.py model output for "
f"{os.path.basename(model)} - not refusing (scripts/mem-guard.sh still guards this step)",
file=sys.stderr)
return 0
msg = (f"memory pre-check: {os.path.basename(model)} needs {r['need'] / g:.1f} GiB, GTT after load "
f"{r['gtt_used_after'] / g:.1f} GiB, MemAvailable {r['mem_available'] / g:.1f} GiB")
Expand Down
Loading