From f8a17a8aeb6a5cf7d3e379f2b6eaed5281106e21 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 10 Nov 2026 02:40:42 -0400 Subject: [PATCH 1/4] tools: fconst19_prescreen.py -- decide whether a twin-diff row is possible without downloading either pack Every row in kFconst19Rules came from tools/fconst19_measure.py, and its premise is that a 1.9 pack's block is its 1.8 twin's block plus the per-layer header. On amd/Phi-4-mini-instruct that premise fails: the 1.8 Token .fconst carries ZERO occurrences of the per-head marker "00 00 7f ff" against the 1.9's 768 (32 x 24), and past the header 751,858 of the first 1,000,000 B differ. That was learned after downloading 9.5 GB of packs. The first header sits megabytes into the file (8.7 MB on that pack), not at byte 0, so the same question is answerable with an HTTP range request for the first 12-48 MB of each pack's Token .fconst -- which is what this tool does: ./tools/fconst19_prescreen.py amd/Phi-4-mini-instruct_rai_1.9.0_npu amd/Phi-4-mini-instruct_rai_1.8.0_npu_4K 1.9 ... header markers in first 12 MB: 24 1.8 ... header markers in first 12 MB: 0 -> the twin carries NO per-head header in this window: the twin-diff route needs the geometry-agnostic path, or another 1.8 reference. Do not spend the download. It grows the window when the first header is further in (Qwen3-8B puts it past 12 MB, inside 24 MB), never claims a rule, and says "fetch more before judging" rather than guessing. Three of the six remaining 1.9 shapes have no published 1.8 twin at all and one has a twin with no Token .fconst, so for those it prints nothing to diff and exits -- which is also the answer. Why it belongs in the repo rather than in one lane's scratch: any lane that wants a rule row needs this decision first, and "the twin exists" does not imply "the twin diffs". It is the cheap gate in front of the 9 GB one. --- tools/fconst19_prescreen.py | 129 ++++++++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100755 tools/fconst19_prescreen.py diff --git a/tools/fconst19_prescreen.py b/tools/fconst19_prescreen.py new file mode 100755 index 000000000..73b993f26 --- /dev/null +++ b/tools/fconst19_prescreen.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# SPDX-License-Identifier: Apache-2.0 +"""Pre-screen a 1.9 pack + its 1.8 twin for the twin-diff route WITHOUT downloading either pack. + +Why: `tools/fconst19_measure.py` aligns the two packs' first block, and that needs the 1.8 twin's Token `.fconst` to be the same +layout minus the 1.9 headers. On amd/Phi-4-mini-instruct that premise failed (1.8 has 0 occurrences of the per-head header +`00 00 7f ff`, the 1.9 has 768, and past the header 751,858 of the first 1,000,000 B differ) -- discovered only after a 9.5 GB +download. The first block of the header is 8.7 MB into the file, so a range request for the first ~12 MB answers the same +question for a few MB of traffic. + +What it reports, per pack: the 1.9 side's header count in the fetched window (should be `layers` if the window covers one block), +whether the 1.8 side has any header at all, and the first differing offset between the two windows. It does NOT measure a rule -- +it says whether measuring one is worth the download. + +Usage: ./fconst19-prescreen.py amd/Model_rai_1.9.0_npu amd/Model_rai_1.8.0_npu_4K +""" +import json +import os +import re +import sys +import urllib.request + +HDR = b"\x00\x00\x7f\xff" +WINDOW = 12 * 1024 * 1024 + + +def token(): + if os.environ.get("HF_TOKEN"): + return os.environ["HF_TOKEN"] + path = os.path.expanduser("~/.cache/huggingface/token") + return open(path).read().strip() if os.path.exists(path) else "" + + +def api(repo, path=""): + req = urllib.request.Request(f"https://huggingface.co/api/models/{repo}{path}", + headers={"Authorization": f"Bearer {token()}"}) + return json.load(urllib.request.urlopen(req, timeout=90)) + + +def token_fconst(repo): + files = [i["path"] for i in api(repo, "/tree/main") if i["type"] == "file"] + cand = [f for f in files if f.startswith("dd_metastate_Llm_Token_") and f.endswith(".fconst") and "_Prefill_" not in f] + exact = [f for f in cand if f.startswith("dd_metastate_Llm_Token_Token_")] + return (exact or cand or [None])[0] + + +def head_bytes(repo, name, n=WINDOW): + """First `n` bytes of a file, by range request.""" + url = f"https://huggingface.co/{repo}/resolve/main/{name}" + req = urllib.request.Request(url, headers={"Authorization": f"Bearer {token()}", "Range": f"bytes=0-{n-1}"}) + with urllib.request.urlopen(req, timeout=120) as r: + return r.read() + + +def geometry(repo): + for fn in ("config.json", "model_config.json"): + try: + c = api(repo, f"/resolve/main/{fn}") + except Exception: + continue + c = c.get("text_config") or c + if all(k in c for k in ("hidden_size", "num_attention_heads", "num_hidden_layers")): + return int(c["hidden_size"]), int(c["num_attention_heads"]), int(c["num_hidden_layers"]) + return None, None, None + + +def main(): + if len(sys.argv) < 3: + print(__doc__.strip().splitlines()[-1], file=sys.stderr) + return 2 + r19, r18 = sys.argv[1], sys.argv[2] + h, nh, nl = geometry(r19) + try: + f19, f18 = token_fconst(r19), token_fconst(r18) + except Exception as e: + print(f"{r19}: cannot list files ({e}) -- check the twin's exact repo name") + return 1 + if not f19 or not f18: + print(f"{r19}: missing Token .fconst ({f19}, {f18})") + return 1 + # The first header can sit past the first window (a bigger hidden size means a bigger prefix), so grow the window until a + # header appears on the 1.9 side or the cap is reached. Still tens of MB, not gigabytes. + win = WINDOW + while True: + try: + b = head_bytes(r19, f19, win) + a = head_bytes(r18, f18, win) + except Exception as e: + print(f" fetch failed at {win//1048576} MB: {e}") + return 1 + n19, n18 = b.count(HDR), a.count(HDR) + if n19 or win >= 48 * 1024 * 1024: + break + win *= 2 + print(f"{r19.split('/')[-1][:44]:46s} h{h} nh{nh} L{nl}") + print(f" 1.9 {f19[:52]:54s} header markers in first {win//1048576} MB: {n19}") + print(f" 1.8 {f18[:52]:54s} header markers in first {win//1048576} MB: {n18}") + if n18 == 0 and n19 == 0: + print(" -> neither window contains a header; fetch more before judging") + return 0 + if n18 == 0: + print(" -> the twin carries NO per-head header in this window: the twin-diff route needs the geometry-agnostic path " + "(engine#1740 W1a) or another 1.8 reference. Do not spend the download.") + return 0 + # first difference between the two windows, which is what the tool reports as the prefix + same = 0 + while same < min(len(a), len(b)) and a[same] == b[same]: + same += 1 + print(f" -> both carry headers; windows agree for {same:,} B, then differ. A twin-diff measurement is plausible: run " + f"tools/fconst19_measure.py on the full packs.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 571c0e327d51cb5748a6fd7d3de0df22ec5eb430 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 10 Nov 2026 07:01:26 -0400 Subject: [PATCH 2/4] tools: fconst19_prescreen decides by CONTENT alignment, not by counting the twin's headers (the CHANGES on #1828) The review was right and the bug was the kind only running the tool on a known-good pair finds. A 1.8 Token .fconst never carries the per-head header -- the 1.9 block is the 1.8 block PLUS it, which is why tools/fconst19_measure.py computes stride8 = stride9 - hdr_len. So "the twin has no markers" is the normal case, and the first version of this tool treated it as fatal: it rejected every pair, including amd/Qwen2.5_3B_Instruct's, which had already been measured into the table as {2048, 16, 36, 4160, 9, 1, 295936} (#1774). What actually separates a usable twin is whether the bytes line up once the 1.9 header is removed. The tool now: * locates the first header run in the 1.9 window (growing the window past 12 MB when the header is further in); * finds the header LENGTH by asking which head count makes the two files agree past the header -- not by reading a config, which returned None in the first version and left a 24-head fallback that mis-aligned every pair by 32 B; * reports the agreement and decides at 99 %. Three pairs, two of them with a known answer: Qwen2.5_3B_Instruct heads=16 100.00 % -> plausible (already measured into the table, #1774) Qwen2.5-7B-Instruct heads=28 100.00 % -> plausible (already measured into the table, #1758) Phi-4-mini-instruct heads=24 23.42 % -> not plausible (the finding this tool was built for, now on evidence that matches what the full-pack measurement showed) The head counts recovered by the alignment (16, 28, 24) are the packs' real ones, which is the check that the search is not fitting noise. The Phi-4-mini conclusion in #1740 stands; the reasoning behind it is now right, and the tool no longer rejects the two pairs whose rows are already merged. --- tools/fconst19_prescreen.py | 64 ++++++++++++++++++++++++------------- 1 file changed, 42 insertions(+), 22 deletions(-) diff --git a/tools/fconst19_prescreen.py b/tools/fconst19_prescreen.py index 73b993f26..4975954f4 100755 --- a/tools/fconst19_prescreen.py +++ b/tools/fconst19_prescreen.py @@ -79,11 +79,19 @@ def geometry(repo): def main(): + """Decide whether a twin-diff is plausible, by CONTENT past the 1.9 header -- not by counting the twin's headers. + + Two corrections from review are baked in here: + 1. a 1.8 Token .fconst never carries the per-head header (the 1.9 block is the 1.8 block PLUS it), so "the twin has no + markers" is the normal case, not a bad twin. The first version treated it as fatal and rejected every pair, + including amd/Qwen2.5_3B_Instruct's, which is already measured into the table as {2048, 16, 36, ...} (#1774); + 2. the head count is not read from a config (that returned None and made the fallback mis-align everything) -- it is + found by asking which header length makes the two files agree past the header. + """ if len(sys.argv) < 3: print(__doc__.strip().splitlines()[-1], file=sys.stderr) return 2 r19, r18 = sys.argv[1], sys.argv[2] - h, nh, nl = geometry(r19) try: f19, f18 = token_fconst(r19), token_fconst(r18) except Exception as e: @@ -92,36 +100,48 @@ def main(): if not f19 or not f18: print(f"{r19}: missing Token .fconst ({f19}, {f18})") return 1 - # The first header can sit past the first window (a bigger hidden size means a bigger prefix), so grow the window until a - # header appears on the 1.9 side or the cap is reached. Still tens of MB, not gigabytes. - win = WINDOW - while True: + win, at, b, a = WINDOW, -1, b"", b"" + while win <= 48 * 1024 * 1024: try: b = head_bytes(r19, f19, win) a = head_bytes(r18, f18, win) except Exception as e: - print(f" fetch failed at {win//1048576} MB: {e}") + print(f" fetch failed at {win // 1048576} MB: {e}") return 1 - n19, n18 = b.count(HDR), a.count(HDR) - if n19 or win >= 48 * 1024 * 1024: + at = b.find(HDR * 2) if at < 0 else at # any run of at least two markers is a header + if at >= 0: break win *= 2 - print(f"{r19.split('/')[-1][:44]:46s} h{h} nh{nh} L{nl}") - print(f" 1.9 {f19[:52]:54s} header markers in first {win//1048576} MB: {n19}") - print(f" 1.8 {f18[:52]:54s} header markers in first {win//1048576} MB: {n18}") - if n18 == 0 and n19 == 0: - print(" -> neither window contains a header; fetch more before judging") + if at < 0: + print(f" the 1.9 window ({win // 1048576} MB) contains no header markers: fetch more before judging") return 0 - if n18 == 0: - print(" -> the twin carries NO per-head header in this window: the twin-diff route needs the geometry-agnostic path " - "(engine#1740 W1a) or another 1.8 reference. Do not spend the download.") + n = min(1_000_000, len(a) - at, len(b) - at - 512) + if n <= 0: + print(" the windows are too small to compare past the first header") return 0 - # first difference between the two windows, which is what the tool reports as the prefix - same = 0 - while same < min(len(a), len(b)) and a[same] == b[same]: - same += 1 - print(f" -> both carry headers; windows agree for {same:,} B, then differ. A twin-diff measurement is plausible: run " - f"tools/fconst19_measure.py on the full packs.") + best = None + for heads in range(1, 129): + hdr_len = heads * 4 + if at + hdr_len + n > len(b): + continue + step = max(1, n // 20000) + idx = range(0, n, step) + same = sum(1 for i in idx if a[at + i] == b[at + hdr_len + i]) + agree = same / len(range(0, n, step)) + if best is None or agree > best[0]: + best = (agree, heads, hdr_len) + agree, heads, hdr_len = best + print(f"{r19.split('/')[-1][:44]:46s}") + print(f" 1.9 {f19[:52]:54s} first header at {at:,} in the first {win // 1048576} MB") + print(f" 1.8 {f18[:52]:54s} best alignment at heads={heads} ({hdr_len} B header): {100.0 * agree:.2f} % of the " + f"{n:,} sampled bytes past the header identical") + if agree > 0.99: + print(" -> the twin lines up past the header: a twin-diff measurement is plausible. Run " + "tools/fconst19_measure.py on the full packs.") + else: + print(" -> the twin does NOT line up past the header: its payload is a different division or rotation, so the " + "twin-diff route cannot produce this row. It needs the geometry-agnostic path (engine#1740 W1a) or another " + "1.8 reference. Do not spend the download.") return 0 From 07aee53684ca8ae98e1afb252eec873d89bbaac2 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 10 Nov 2026 08:10:34 -0400 Subject: [PATCH 3/4] tools: validate the Range response, and give the prescreen a --self-test (the CHANGES on #1828, second pass) Two review points, both about the tool's own safety rather than its verdict. 1. head_bytes() trusted `r.read()`. A CDN that ignores `Range`, a proxy that strips it, or a redirect that drops the header all answer 200 with the ENTIRE file -- 1.76 GB for a 7B pack -- and the caller would treat it as a 12 MB window, i.e. this tool would make the download it exists to prevent. It now requires status 206, reads at most `n` bytes, and raises with the status in the message when the server ignores the range. 2. The tool had no way to be checked, which for a 149-line decision helper is the same complaint it makes about other people's claims. `--self-test` runs the alignment on a synthetic pair whose answer is known -- no network, no packs: aligned pair: heads=16 agree=1.00 PASS rotated twin: agree=0.00 PASS (must not line up) 3-marker run: heads=3 PASS (not the declared 16) The decision is now a pure function (`align_past_header`) so the self-test exercises the same code path as the real run. The self-test earned its place immediately: the FIRST version used a periodic payload ((i*7+3) % 251), which aligns with itself after a one-byte shift, so the rotated arm passed at agree=1.00 -- the negative control was broken and nothing else would have said so. It is pseudo-random now (sha256 stream) and the rotated arm scores 0.00. Re-verified against the pair whose answer is known: Qwen2.5_3B_Instruct still comes out 100.00 % at heads=16, and a nonexistent repo still fails with a message that says which side was missing. --- tools/fconst19_prescreen.py | 86 ++++++++++++++++++++++++++++++------- 1 file changed, 71 insertions(+), 15 deletions(-) diff --git a/tools/fconst19_prescreen.py b/tools/fconst19_prescreen.py index 4975954f4..c46f9a026 100755 --- a/tools/fconst19_prescreen.py +++ b/tools/fconst19_prescreen.py @@ -28,6 +28,7 @@ Usage: ./fconst19-prescreen.py amd/Model_rai_1.9.0_npu amd/Model_rai_1.8.0_npu_4K """ +import hashlib import json import os import re @@ -59,11 +60,24 @@ def token_fconst(repo): def head_bytes(repo, name, n=WINDOW): - """First `n` bytes of a file, by range request.""" + """First `n` bytes of a file, by range request -- and REFUSE anything that is not a range response. + + `r.read()` returns whatever the server sends. A CDN that ignores `Range`, a proxy or edge node that strips it, or a redirect + that drops the header all answer 200 with the ENTIRE file -- 1.76 GB for a 7B pack -- and the caller would treat it as a 12 MB + window. Two independent checks, because the point of this tool is to avoid exactly that download: the status must be 206, and + the body must be no larger than the window asked for. + """ url = f"https://huggingface.co/{repo}/resolve/main/{name}" - req = urllib.request.Request(url, headers={"Authorization": f"Bearer {token()}", "Range": f"bytes=0-{n-1}"}) + req = urllib.request.Request(url, headers={"Authorization": f"Bearer {token()}", "Range": f"bytes=0-{n - 1}"}) with urllib.request.urlopen(req, timeout=120) as r: - return r.read() + status = getattr(r, "status", None) or r.getcode() + body = r.read(n) # never read past the window, even if the server sends more + if status != 206: + raise RuntimeError(f"{repo}/{name}: the server answered {status}, not 206 -- it ignored the Range request, so this " + f"would have been a full-file download. Refusing (this tool exists to avoid that).") + if len(body) > n: + raise RuntimeError(f"{repo}/{name}: {len(body)} B returned for a {n} B range request") + return body def geometry(repo): @@ -119,18 +133,7 @@ def main(): if n <= 0: print(" the windows are too small to compare past the first header") return 0 - best = None - for heads in range(1, 129): - hdr_len = heads * 4 - if at + hdr_len + n > len(b): - continue - step = max(1, n // 20000) - idx = range(0, n, step) - same = sum(1 for i in idx if a[at + i] == b[at + hdr_len + i]) - agree = same / len(range(0, n, step)) - if best is None or agree > best[0]: - best = (agree, heads, hdr_len) - agree, heads, hdr_len = best + agree, heads, hdr_len = align_past_header(a, b, at, n) print(f"{r19.split('/')[-1][:44]:46s}") print(f" 1.9 {f19[:52]:54s} first header at {at:,} in the first {win // 1048576} MB") print(f" 1.8 {f18[:52]:54s} best alignment at heads={heads} ({hdr_len} B header): {100.0 * agree:.2f} % of the " @@ -145,5 +148,58 @@ def main(): return 0 +def align_past_header(a, b, at, n): + """Best (agreement, heads, header_bytes) over plausible head counts, comparing the bytes that follow the 1.9 header. + + The head count is not read from a config: the first version did, got None, and left a 24-head fallback that mis-aligned a + known-good pair by 32 B. Searching is self-validating -- the right count makes the files agree, the wrong ones do not, and the + count it recovers is the pack's real head count, which is the check that it is not fitting noise. + """ + best = None + for heads in range(1, 129): + hdr_len = heads * 4 + if at + hdr_len + n > len(b): + continue + step = max(1, n // 20000) + same = sum(1 for i in range(0, n, step) if a[at + i] == b[at + hdr_len + i]) + agree = same / len(range(0, n, step)) + if best is None or agree > best[0]: + best = (agree, heads, hdr_len) + return best + + +def self_test(): + """Check the alignment on a synthetic pair whose answer is known -- no network, no packs. + + A 1.8 block with no header, a 1.9 block that is the same bytes with a header inserted at offset `hdr_off`: the aligner must + recover the head count and 100 % agreement. Then the same pair with the 1.8 payload rotated by one byte must score ~0, which is + what "the twin does not line up" means. + """ + heads, block = 16, 400_000 + hdr_off = 100_000 + # A pseudo-random payload, not a periodic one: an arithmetic pattern like (i*7+3) % 251 aligns with itself after a shift, so a + # rotated twin would score like an aligned one and the negative arm would pass the bug it is meant to catch (it did, first run). + payload = b"".join(hashlib.sha256(str(i).encode()).digest() for i in range(block // 32 + 1))[:block] + a18 = payload[:hdr_off] + payload[hdr_off:] # no header at all + a19 = payload[:hdr_off] + b"\x00\x00\x7f\xff" * heads + payload[hdr_off:] + at = a19.find(b"\x00\x00\x7f\xff") + agree, got_heads, hdr_len = align_past_header(a18, a19, at, 200_000) + ok = (at == hdr_off and got_heads == heads and hdr_len == heads * 4 and agree > 0.99) + print(f" self-test aligned pair: at={at} heads={got_heads} agree={agree:.2f} -> {'PASS' if ok else 'FAIL'}") + rotated = payload[:hdr_off] + payload[hdr_off + 1:] + payload[hdr_off:hdr_off + 1] + agree2, _, _ = align_past_header(rotated[:len(a18)], a19, at, 200_000) + ok2 = agree2 < 0.5 + print(f" self-test rotated twin: agree={agree2:.2f} -> {'PASS' if ok2 else 'FAIL'} (must not line up)") + # a header run that is not a whole number of heads must not be mistaken for the 1.9 form + short = payload[:hdr_off] + b"\x00\x00\x7f\xff" * 3 + payload[hdr_off:] + at_short = short.find(b"\x00\x00\x7f\xff") + agree3, heads3, _ = align_past_header(a18, short, at_short, 100_000) + ok3 = heads3 != heads + print(f" self-test 3-marker run: heads={heads3} -> {'PASS' if ok3 else 'FAIL'} (not the declared {heads})") + return 0 if (ok and ok2 and ok3) else 1 + + if __name__ == "__main__": + if "--self-test" in sys.argv: + sys.exit(self_test()) sys.exit(main()) From e2bbe8e9d374f3424004c8264186df54be5fed12 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 10 Nov 2026 11:01:29 -0400 Subject: [PATCH 4/4] tests: register fconst19_prescreen's --self-test, which engine#1901's rule now requires The review was right and the failure is mine in the sense that matters: the file declares a --self-test and nothing ran it, so tests_referenced went red on a rule that landed on main after this branch was cut. Registration, in main's own idiom: add_python_test(fconst19_prescreen_selftest ${CMAKE_SOURCE_DIR}/tools/fconst19_prescreen.py --self-test) (Using the helper main introduced rather than a raw add_test -- the merge brought the helper in, and using it keeps the Python3_EXECUTABLE plumbing in one place.) Verified with the rule's own checker after merging main: $ python3 tools/ci/tests_referenced_check.py tests_referenced: all 74 test script(s) are referenced (1 deliberately not a row: anchor_rerun.sh); 22 tool self-test(s) are registered and the self-test itself, which needs no network and no packs: aligned pair: at=100000 heads=16 agree=1.00 PASS rotated twin: agree=0.00 PASS (must not line up) 3-marker run: heads=3 PASS (not the declared 16) The merge also had a CMakeLists conflict with main's own device_holders registration; resolved by keeping main's call and adding mine beside it, both through add_python_test. --- CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index e20a86864..6b69d754c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -323,6 +323,10 @@ add_test(NAME release_check_hrx_init_reason COMMAND bash ${CMAKE_SOURCE_DIR}/tes # then two lanes each starting an HRX server outside the token within one half hour). The ancestry test is the # check; the two permanent services are exempt by name. --self-test drives the decision, no device needed. add_python_test(device_holders ${CMAKE_SOURCE_DIR}/tools/device_holders.py --self-test) +# engine#1901's rule: a tool that declares --self-test must be registered where something runs it. engine#1828's +# fconst19_prescreen has one (the alignment decision on a synthetic pair whose answer is known -- the control that caught a +# broken negative arm in its first version), and it needs no network and no packs, so it belongs in the default suite. +add_python_test(fconst19_prescreen_selftest ${CMAKE_SOURCE_DIR}/tools/fconst19_prescreen.py --self-test) # engine #1668: the bench preflight must pin the load mode, because `-lm none` is +17.5 % prefill at -1.6 % decode for +604 MiB of # GTT. The script existed with a test that nothing ran; a test nobody runs is a comment, so the loader/flag assertions are registered # here (the test fakes the box, so it needs no device and no ticket).