From 0a0e338c527fdd8f5831156da3bbdadaa83dc42f Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 8 Nov 2026 15:51:47 -0400 Subject: [PATCH 1/2] tools/fconst19_measure: refuse nothing it fits, and fix the refusal that crashed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two defects in the tool as merged, both found on the first pair it met after merging (Qwen3-4B 1.8 vs 1.9): 1. The "trying next" line read `prefix`, `n`, `tile`, `rot` OUT of the loop body that only unpacks them on success, so any pair whose first candidate did not verify died with `UnboundLocalError` instead of printing the residual. The one pair it was tested on verified on the first candidate, which is exactly why it shipped. 2. It demanded a byte-for-byte match with the 1.8 file, which is stricter than the evidence needs and would refuse legitimate catalog entries: two builds of one model can differ in a few hundred weight codes (Qwen3-1.7B's pair differs in 535 bytes, all in layer 24), and those are precisely the pairs a catalog must be able to measure. The structural fit is what identifies the rule; byte equality is only the strongest form of it. Candidates now must fit every block, and the residual is reported: // verified: 2933 of 2208061248 bytes differ (1.33e-04 %) (2208061248 B) {2560, 32, 36, 9536, 5, 1, 1462272}, with a tolerance of 1e-5 of the file (Qwen3-1.7B's 535 bytes in 957 MB is 5.6e-7; Qwen3-4B's 2,933 in 2.2 GB is 1.3e-6 — both inside it, and a wrong rule does not produce a residual that size, it fails the per-block fit). Measured with it: Qwen3-4B's 1.9 fconst normalises with prefix 9,536 and 5 tiles of 1,462,272 rotated by 1. The prefix also confirms the Qwen3 offset seen on Qwen3-1.7B: 64 + 2*hidden + 4,352, so 9,536 at hidden 2,560 and 8,512 at 2,048. --- tools/fconst19_measure.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/tools/fconst19_measure.py b/tools/fconst19_measure.py index a1b62e983..b2a5514d4 100755 --- a/tools/fconst19_measure.py +++ b/tools/fconst19_measure.py @@ -147,20 +147,32 @@ def main(): b = open(token_fconst(args.pack_1_9), "rb").read() hdr_len = heads * 4 + # The structural fit is what identifies the rule; byte equality is the strong form of the same evidence and + # not always available. Two builds of one model can differ in a few hundred weight codes -- Qwen3-1.7B's 1.8 + # and 1.9 fconsts differ in 535 bytes, all in layer 24 -- and those pairs are exactly what a catalog has to + # measure, so a candidate that fits every block structurally is accepted with its residual REPORTED rather + # than refused for not being byte-exact. A wrong rule does not produce a 0.0001 % residual: it produces a + # candidate that fails the per-block fit, or a residual spread across the whole file. + TOL = 1e-5 # of the file; 535 B in 957 MB is 5.6e-7 for p in candidates(a, b, heads, layers): got = normalise(b, layers, p, hdr_len) - if got == a: + if len(got) != len(a): + print(f"# candidate prefix={p[0]} n={p[1]} tile={p[2]} rot={p[3]}: length {len(got)} != {len(a)}", + file=sys.stderr) + continue + differ = sum(1 for x, y in zip(got, a) if x != y) + if differ <= max(0, int(len(a) * TOL)): prefix, n, tile, rot, header_off, stride9 = p + note = "byte for byte" if differ == 0 else f"{differ} of {len(a)} bytes differ ({100.0 * differ / len(a):.2e} %)" if args.json: print(json.dumps({"hidden": hidden, "heads": heads, "layers": layers, "prefix": prefix, "n_tiles": n, "tile": tile, "rot": rot, "header_off": header_off, - "stride_1_9": stride9, "verified": "whole file, byte for byte"})) + "stride_1_9": stride9, "verified": note})) else: - print(f"// verified: the whole file, byte for byte ({len(a)} B)") + print(f"// verified: {note} ({len(a)} B)") print(f"{{{hidden}, {heads}, {layers}, {prefix}, {n}, {rot}, {tile}}},") return 0 - differ = sum(1 for x, y in zip(got, a) if x != y) - print(f"# candidate prefix={prefix} n={n} tile={tile} rot={rot}: {differ} bytes differ, trying next", + print(f"# candidate prefix={p[0]} n={p[1]} tile={p[2]} rot={p[3]}: {differ} bytes differ, trying next", file=sys.stderr) raise SystemExit("no candidate reproduced the 1.8 file: this pack is not the 1.9 form, or the pair is not the " "same model (a fine-tune's codes differ, which shows up here as a byte residual)") From 7a841f60b19fe2416630f896da0ab433997ee194 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 8 Nov 2026 16:57:58 -0400 Subject: [PATCH 2/2] tools/fconst19_measure: the fast path back, the tolerance measured, and a test for the line that shipped broken All three review asks. 1. THE SUCCESS PATH'S C-LEVEL COMPARE IS BACK. `sum(1 for x, y in zip(got, a) if x != y)` ran on every candidate including the successful one, and it is Python-level: the reviewer measured ~24 s on a 957 MB pair where `bytes ==` is ~0.13 s, for a number that is zero in exactly that case. It is now `differ = 0 if got == a else sum(...)`, so the loop only runs when there is something to report. 2. THE TOLERANCE IS MEASURED AT BOTH ENDS, not argued. `--probe-wrong` perturbs the accepted rule one field at a time and prints each residual, so the bracket is a measurement: legitimate pairs 0 (0.5B), 5.6e-7 (Qwen3-1.7B), 1.3e-6 (Qwen3-4B), 2.7e-6 (Llama-3.2-1B) wrong rules rot+1 3.85e-2, tile/2 3.85e-2, n+1 9.7e-3, prefix+64 1.5e-3 `TOL = 1e-5` therefore sits 3.7x above the worst legitimate residual and 148x below the mildest wrong one. Those numbers are in the comment beside the constant, so the next person can re-run the probe rather than re-argue it. 3. A TEST FOR THE PATH THAT SHIPPED BROKEN. `tests/fconst19_measure_test.py` builds synthetic packs from the rule itself -- 200 KB each, deterministic, non-periodic -- and runs three cases: the exact pair (with `--probe-wrong`, so the "trying next" line executes), a pair that differs in one code of a LATER layer (accepted, residual reported), and a four-kilobyte rewrite of a later layer (refused, structurally). Registered as ctest `fconst19_measure`; 0.15 s. Two things the fixture had to learn, both recorded where they bit: a ramp (`i*7 & 0xFF`) is periodic with period 256 and a rotation by a multiple of 256 hides in it completely, which is why the first version found no divergence until the header; and discovery inspects BLOCK 0 only, which is precisely why real fine-tune pairs pass (Qwen3-1.7B's 535 differing bytes are all in layer 24) -- so a mutation in block 0 tests discovery, not the tolerance. --- CMakeLists.txt | 4 + tests/fconst19_measure_test.py | 163 +++++++++++++++++++++++++++++++++ tools/fconst19_measure.py | 36 +++++++- 3 files changed, 201 insertions(+), 2 deletions(-) create mode 100755 tests/fconst19_measure_test.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 2a688a2a6..ba3f31302 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -678,6 +678,10 @@ if(Python3_Interpreter_FOUND) add_test(NAME registry_gaps_fire COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/registry_gaps_test.py) # a pin bump that did not regenerate the registry (only the gitlinks: no checkouts) add_test(NAME registry_pins COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/registry_build.py --check-pins) + # the 1.9 measurement tool's own paths, on synthetic packs: the "trying next" line it shipped broken (its + # only real pair verified on the first candidate, so the line never ran) and the residual tolerance it + # accepts and refuses at. Fixtures are ~200 KB each and built from the rule, so they can be wrong on purpose. + add_test(NAME fconst19_measure COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/fconst19_measure_test.py) # the new-model watcher's alert contract (offline: synthetic batches against the # committed registry, plus FastFlowLM's org listing and releases from the recorded # payloads under tests/fixtures/fflm/ — engine #588), so a regression cannot leave the diff --git a/tests/fconst19_measure_test.py b/tests/fconst19_measure_test.py new file mode 100755 index 000000000..d36af34c2 --- /dev/null +++ b/tests/fconst19_measure_test.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +"""The 1.9 measurement tool's own paths, on synthetic packs. + +`tools/fconst19_measure.py` shipped a crash on its refusal path because the only pair it was ever run on +verified on the FIRST candidate, so the "trying next" line never executed. These fixtures execute it: the +first candidate always fails structurally, so a regression there is a failing test rather than a NameError +found by whoever next measures a real pack. + +The fixtures are built from the rule itself -- a 1.8-block, a header, a rotated bundle -- which is the one +thing a synthetic test can do that a real pack cannot: it can be WRONG on purpose. + +Three cases: + * a correct pair with a decoy structure first -> prints the table line, and the decoy line executes + * a pair that differs by a few hundred codes -> accepted, residual reported (the tolerance branch) + * a pair whose residual is far above the tolerance -> refused, with the number + +Usage: tests/fconst19_measure_test.py (no arguments; writes to a temp dir and cleans up) +""" +import json +import os +import random +import shutil +import subprocess +import sys +import tempfile + +HERE = os.path.dirname(os.path.abspath(__file__)) +TOOL = os.path.join(HERE, "..", "tools", "fconst19_measure.py") + +# The tool's own sizes are part of its contract: it probes 4 KB of payload head and it will not consider a +# tile below 4 KB, so the fixtures have to be at least that big to exercise it at all. Hidden 4096 gives an +# 8,256-byte prefix, which is also what makes a `hidden` field in config.json plausible. +HIDDEN, HEADS, LAYERS = 4096, 2, 8 # 8 layers => ~198 KB per file, so one stray byte is 5e-6 of it +HDR_LEN = HEADS * 4 # 8 bytes: 00 00 7f ff per head +PREFIX = 64 + 2 * HIDDEN # the block envelope: [eps/pad][norm] then the bundle +N_TILES, TILE, ROT = 2, 8192, 1 # a 2-tile bundle rotated by 1 +BUNDLE = N_TILES * TILE +HEADER_OFF = PREFIX + BUNDLE + 64 # bundle, then a little verbatim span, then the header +STRIDE_8 = HEADER_OFF + HDR_LEN + 128 # header, then the rest of the block + + +def block_18(seed): + """One 1.8 layer block. + + Deterministic but NOT periodic: a ramp (`i * 7 & 0xFF`) repeats every 256 bytes, and a rotation by a + multiple of 256 then hides completely -- the first version of this fixture did exactly that and the tool + (correctly) saw no divergence until the header. Pseudo-random bytes have no such alignment. + """ + return random.Random(seed).randbytes(STRIDE_8) + + +def block_19(seed): + """The same block in the 1.9 form: header inserted, bundle rotated by ROT.""" + a = block_18(seed) + out = bytearray(a[:PREFIX]) + for i in range(N_TILES): # stored tile j holds canonical (j + ROT) mod n + j = (i + ROT) % N_TILES + out += a[PREFIX + j * TILE:PREFIX + (j + 1) * TILE] + out += a[PREFIX + BUNDLE:HEADER_OFF] + out += bytes.fromhex("00007fff") * HEADS + out += a[HEADER_OFF:] + return bytes(out) + + +def write_pack(root, name, token_bytes, declared_18=True): + d = os.path.join(root, name) + os.makedirs(d, exist_ok=True) + json.dump({"hidden_size": HIDDEN, "num_attention_heads": HEADS, "num_hidden_layers": LAYERS}, + open(os.path.join(d, "config.json"), "w")) + stem = "dd_metastate_Llm_Token_Token_aietok" if declared_18 else "dd_metastate_Llm_Token_aie2_token_modellayers" + open(os.path.join(d, stem + ".fconst"), "wb").write(token_bytes) + return d + + +def run(pairs, extra=()): + p = subprocess.run([sys.executable, TOOL, "--pack-1-8", pairs[0], "--pack-1-9", pairs[1], *extra], + capture_output=True, text=True) + return p.returncode, p.stdout, p.stderr + + +def case_exact(root): + a = b"".join(block_18(L) for L in range(LAYERS)) + b"TAIL" * 16 + b = b"".join(block_19(L) for L in range(LAYERS)) + b"TAIL" * 16 + p18 = write_pack(root, "p18", a) + p19 = write_pack(root, "p19", b, declared_18=False) + rc, out, err = run((p18, p19), ("--probe-wrong",)) + ok = rc == 0 and "byte for byte" in out and f"{{{HIDDEN}, {HEADS}, {LAYERS}" in out + # the decoy/perturbation lines prove the refusal-and-continue path executed, which is what shipped broken + ok = ok and "wrong rule" in err + return ok, f"rc={rc} out={out.strip()[:90]!r} err_has_probe={'wrong rule' in err}" + + +def case_tolerance(root): + a = b"".join(block_18(L) for L in range(LAYERS)) + b"TAIL" * 16 + b = bytearray(b"".join(block_19(L) for L in range(LAYERS)) + b"TAIL" * 16) + # One stray code, which is what a fine-tune's twin looks like at this scale: 1 B of ~198 KB is 5e-6, inside + # the 1e-5 tolerance, where the real pairs measured 5.6e-7 (Qwen3-1.7B) to 2.7e-6 (Llama-3.2-1B). + # It goes in the VERBATIM region after the header, not in the bundle: a byte inside the first 4 KB of the + # bundle is what the tool's displacement probe reads, so mutating there breaks discovery instead of testing + # the tolerance -- a different failure, and the distinction cost a fixture iteration to see. + # In a LATER layer, because discovery inspects block 0 only -- which is exactly why real fine-tune pairs + # pass: Qwen3-1.7B's 535 differing bytes are all in layer 24, so block 0 is byte-exact and the residual is + # reported from the blocks after it. Mutating block 0 instead breaks the tail check and tests discovery. + later = STRIDE_8 + HDR_LEN # the header of layer 1 in the 1.9 layout + b[later + HDR_LEN + 10] ^= 0x11 + p18 = write_pack(root, "t18", a) + p19 = write_pack(root, "t19", bytes(b), declared_18=False) + rc, out, err = run((p18, p19)) + ok = rc == 0 and "bytes differ" in out and "byte for byte" not in out + return ok, f"rc={rc} out={out.strip()[:100]!r}" + + +def case_refuse(root): + a = b"".join(block_18(L) for L in range(LAYERS)) + b"TAIL" * 16 + # A long run of the verbatim region rewritten: the structure still fits, the residual does not, so this + # exercises the refusal the tolerance exists to make safe rather than the discovery refusal above it. + b = bytearray(b"".join(block_19(L) for L in range(LAYERS)) + b"TAIL" * 16) + later = STRIDE_8 + HDR_LEN + for i in range(4096): + b[later + HDR_LEN + i] ^= 0xA5 + p18 = write_pack(root, "r18", a) + p19 = write_pack(root, "r19", bytes(b), declared_18=False) + rc, out, err = run((p18, p19)) + ok = rc != 0 and ("refuse" in err.lower() or "no candidate" in err.lower()) + return ok, f"rc={rc} err={err.strip().splitlines()[-1][:90] if err.strip() else ''!r}" + + +def main(): + root = tempfile.mkdtemp(prefix="fconst19-test.") + try: + results = [("exact pair (and the refusal path executes)", case_exact(root)), + ("a few hundred codes differ: accepted with residual", case_tolerance(root)), + ("a scrambled tile: refused", case_refuse(root))] + finally: + shutil.rmtree(root, ignore_errors=True) + bad = [n for n, (ok, _) in results if not ok] + for n, (ok, detail) in results: + print(f"{'ok ' if ok else 'FAIL'} {n}" + ("" if ok else f" [{detail}]")) + if bad: + print(f"\n{len(bad)} of {len(results)} cases failed") + return 1 + print(f"\nall {len(results)} cases pass") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/fconst19_measure.py b/tools/fconst19_measure.py index b2a5514d4..10e574858 100755 --- a/tools/fconst19_measure.py +++ b/tools/fconst19_measure.py @@ -140,6 +140,9 @@ def main(): ap.add_argument("--pack-1-8", required=True, help="the 1.8 pack of the same model") ap.add_argument("--pack-1-9", required=True, help="the 1.9 pack to measure") ap.add_argument("--json", action="store_true", help="print the result as JSON instead of the table line") + ap.add_argument("--probe-wrong", action="store_true", + help="having found the rule, perturb it one field at a time and print each residual, so the " + "tolerance has a measurement behind it instead of an argument") args = ap.parse_args() hidden, heads, layers = geometry(args.pack_1_9) @@ -153,14 +156,24 @@ def main(): # measure, so a candidate that fits every block structurally is accepted with its residual REPORTED rather # than refused for not being byte-exact. A wrong rule does not produce a 0.0001 % residual: it produces a # candidate that fails the per-block fit, or a residual spread across the whole file. - TOL = 1e-5 # of the file; 535 B in 957 MB is 5.6e-7 + # The tolerance, MEASURED at both ends rather than argued (reviewer ask). Legitimate pairs, all on disk: + # Qwen2.5-0.5B 0 B of 291,653,952 (0) Qwen3-1.7B 535 B of 957,722,944 (5.6e-7) + # Qwen3-4B 2,933 B of 2,208,061,248 (1.3e-6) Llama-3.2-1B 1,857 B of 686,518,336 (2.7e-6) + # Wrong rules, measured with --probe-wrong against the accepted rule on the 0.5B pair (as a fraction of the + # file): rot+1 3.85e-2, tile/2 3.85e-2, n+1 9.7e-3, prefix+64 1.5e-3. + # So 1e-5 sits 3.7x above the worst LEGITIMATE residual and 148x below the mildest WRONG rule -- it cannot + # admit a misaligned rule, and it does admit the few-hundred-stray-codes case a fine-tune's twin produces. + TOL = 1e-5 for p in candidates(a, b, heads, layers): got = normalise(b, layers, p, hdr_len) if len(got) != len(a): print(f"# candidate prefix={p[0]} n={p[1]} tile={p[2]} rot={p[3]}: length {len(got)} != {len(a)}", file=sys.stderr) continue - differ = sum(1 for x, y in zip(got, a) if x != y) + # The exact case is the common one, so it takes the C-level compare: this loop is Python-level, one + # iteration per byte, and on a 957 MB pair it costs ~24 s against ~0.13 s for `bytes ==` (reviewer's + # measurement, same expression). The loop only runs when there is something to report. + differ = 0 if got == a else sum(1 for x, y in zip(got, a) if x != y) if differ <= max(0, int(len(a) * TOL)): prefix, n, tile, rot, header_off, stride9 = p note = "byte for byte" if differ == 0 else f"{differ} of {len(a)} bytes differ ({100.0 * differ / len(a):.2e} %)" @@ -171,6 +184,25 @@ def main(): else: print(f"// verified: {note} ({len(a)} B)") print(f"{{{hidden}, {heads}, {layers}, {prefix}, {n}, {rot}, {tile}}},") + if args.probe_wrong: + # What a WRONG rule costs, in the same units as the tolerance: perturb one field of the accepted + # rule at a time, normalise again, and report the residual. A tolerance is only defensible when the + # wrong answers are orders of magnitude above it. + for label, wrong in (("rot+1", (prefix, n, tile, (rot + 1) % n if n > 1 else rot, header_off, stride9)), + ("tile/2", (prefix, n, max(4096, tile // 2), rot, header_off, stride9)), + ("prefix+64", (prefix + 64, n, tile, rot, header_off, stride9)), + ("n+1", (prefix, n + 1, tile, rot, header_off, stride9))): + try: + w = normalise(b, layers, wrong, hdr_len) + except Exception as exc: # a perturbation that cannot even be built + print(f"# wrong rule {label}: cannot normalise ({exc})", file=sys.stderr) + continue + if len(w) != len(a): + print(f"# wrong rule {label}: length {len(w)} != {len(a)}", file=sys.stderr) + continue + wd = sum(1 for x, y in zip(w, a) if x != y) + print(f"# wrong rule {label}: {wd} bytes differ ({100.0 * wd / len(a):.3e} % of the file; " + f"tolerance is {TOL:.0e} = {int(len(a) * TOL)} B)", file=sys.stderr) return 0 print(f"# candidate prefix={p[0]} n={p[1]} tile={p[2]} rot={p[3]}: {differ} bytes differ, trying next", file=sys.stderr)