diff --git a/CMakeLists.txt b/CMakeLists.txt index 9975caf17..5d2d3ce74 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -571,6 +571,10 @@ if(Python3_Interpreter_FOUND) # pooling those logs by eye produced two wrong numbers in this lane. Fixture tickets only -- no model, no # device; the recorded 8x64 window is the tool's known-answer check and is run by hand with --selftest. add_test(NAME mtp_depth_pool COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/mtp_depth_pool_test.py) + # The repack's tile-order claim (engine#1110): `npu_plan.py` writes container.tile_order=engine-1 for every + # plan, so the load-time guard (#1130) is green on a container whose TILES are another order. This pins the + # mapping from q4nx_diff's verdict to (stamp value, fatal), stdlib only: no model, no numpy, no device. + add_test(NAME repack_tile_order COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/repack_tile_order_test.py) # the t2 chunk kind (docs/npu.md "The t2 chunk kind"): synthetic only, because the real-model # leg needs 14 GB of Bonsai-2 images. Run tests/q4nx_t2_test.py without --no-model for the gate. add_test(NAME q4nx_t2 COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/q4nx_t2_test.py --no-model) diff --git a/docs/npu.md b/docs/npu.md index afaec386c..3bb2ce1e0 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -543,7 +543,7 @@ own, with a negative control. A cached directory is checked again on every hit, with the same metadata-only scan, and it is rebuilt if the scan fails or if its `repack-stamp.txt` (converter path and commit, architecture, -repack script) names a different converter than this build would use (`repack_gguf.py +the measured `tile_order`, repack script) names a different converter than this build would use (`repack_gguf.py --check-stamp `: exit 0 current, 1 stale, 2 cannot tell). If the checker or the converter cannot be found, serve warns once per process and reuses the directory, so a missing script cannot stop serving. A Q4NX directory served as it is (`1bit serve -m `), such as a @@ -553,6 +553,27 @@ quantized tensor's chunk kind, and refuses an unknown chunk size or a tensor sho needs. It cannot tell the two Q8 layouts apart ("Q4NX chunks", above), because the container does not record which one it holds. +**The tile order is measured, not declared.** Two converters can write `q4_1` tiles in the two +different orders this engine and FastFlowLM use, and a container in the wrong order repacks cleanly +and decodes as noise — HTTP 200, nothing in the log (engine#1110; measured with FLM_Q4NX_Converter +`d1d5232`, whose containers come back `matches NO layout`). The load-time field +`container.tile_order` (engine#1130) cannot catch it, because `tools/npu_plan.py` writes `engine-1` +for every plan: it guards against a future writer, not against the converter that produced the +tiles. So the repack decodes a sample of the container against the source GGUF with +`tools/q4nx_diff.py`'s own verdict — one pass per layout, once per directory — and: + +* **refuses** the directory (`rc=1`) when no layout matches, or when the layout that matches is not +the `engine-1` order `npu/forward/src/q4nx.cpp` decodes; +* records `tile_order=engine-1` and `tile_check=` in `repack-stamp.txt` when it +does match, so the claim in `plan.json` is measured rather than assumed; +* records `tile_order=unverified` when nothing was comparable (an architecture the checker cannot +dequantize) instead of asserting a layout nobody checked. + +The same measurement runs on its own over an existing directory: +`python3 scripts/repack_gguf.py --tile-check ` (exit 0 ours, 1 another order or +none, 2 nothing compared). A cache hit re-checks metadata only, so the cost is paid once per +directory, on top of the converter's own pass. + ### From a Ryzen AI OGA NPU pack `1bit npu-pack import -o ` reads a pack AMD publishes for Ryzen AI diff --git a/scripts/repack_gguf.py b/scripts/repack_gguf.py index 4b236cdc3..e0e9c362b 100644 --- a/scripts/repack_gguf.py +++ b/scripts/repack_gguf.py @@ -20,6 +20,8 @@ # tokenizer.json; config.json is derived here from the GGUF metadata. # # Usage: python3 repack_gguf.py [converter_dir] +# python3 repack_gguf.py --check-stamp (converter still current?) +# python3 repack_gguf.py --tile-check (is this dir's tile order ours?) import json import os import subprocess @@ -43,6 +45,66 @@ ATTENTION_BIAS_ARCHES = {"qwen2"} +def tile_order_policy(rc, line): + """(what the stamp may claim, is the repack fatal) from tools/q4nx_diff.py's verdict. + + Why this exists here and not in the plan generator: `tools/npu_plan.py` writes + `container.tile_order = "engine-1"` for every plan it generates (engine#1130 carries that field + and refuses an id it does not implement at load time), so the load-time guard cannot catch a + container whose tiles are a DIFFERENT order -- the declaration is unconditional. A converter + that writes another order repacks cleanly and decodes as noise at HTTP 200 with nothing in the + log (measured: FLM_Q4NX_Converter `d1d5232`, engine#1110). The repack is the only step holding + both the source GGUF and the tiles, so the claim is either measured true here or the repack + fails; a directory nobody could check says `unverified` instead of claiming `engine-1`. + """ + if rc == 0 and line.startswith("matches layout ours"): + return "engine-1", False # the order npu/forward/src/q4nx.cpp decodes + if rc == 2: + return "unverified", False # nothing comparable: say so, do not claim a layout + return "", True # another layout, or none: these tiles are not these weights + + +def tile_check_exit(rc, line): + """Exit code for `--tile-check`: 0 ours, 1 another order or none, 2 nothing compared. + + The checker's own rc is not this: `verdict()` returns 0 for ANY matching layout, including the + one this engine does not decode, so the CLI goes through the policy above instead of passing it + through (engine#1110 review, blocker 1). + """ + order, fatal = tile_order_policy(rc, line) + if fatal: + return 1 + return 2 if order == "unverified" else 0 + + +def tile_order_check(gguf, out_dir): + """(rc, line) from the same verdict `tools/q4nx_diff.py decode --layout both` prints. + + Imported rather than reimplemented so the repack and the tool an operator runs cannot disagree; + the verdict's tolerance and ratio are that file's, and its logic is what tests/q4nx_diff_grid_test.py + and `q4nx_diff.py --self-test` already cover. + """ + tools = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, "tools") + sys.path.insert(0, os.path.abspath(tools)) + try: + import q4nx_diff as _D + except Exception as e: # no numpy, no gguf-py, a pruned package + return 2, f"tile check unavailable ({e})" + + class _Args: + pass + + a = _Args() + a.q4nx, a.gguf, a.tensor = out_dir, gguf, None + results = {} + for layout in sorted(_D.DECODERS): + try: + results[layout] = _D._decode_pass(a, layout, quiet=True) + except Exception as e: + return 2, f"tile check unavailable ({e})" + return _D.verdict(results) + + def _field(reader, name): for f in reader.fields.values(): if f.name == name: @@ -212,6 +274,8 @@ def _ssm(key, dflt=0): def main(): if len(sys.argv) < 3: print("usage: repack_gguf.py [converter_dir]") + print(" repack_gguf.py --check-stamp ") + print(" repack_gguf.py --tile-check ") return 2 gguf, out_dir = sys.argv[1], sys.argv[2] # The Q4NX converter is not vendored in this repo. Resolve it in order: @@ -305,6 +369,17 @@ def converter_commit(): print("stamp: current") return 0 + # `--tile-check `: the same measurement the repack runs, on a directory that + # already exists (a published container, a dir from another converter). Same exit codes as the + # repack: 0 the engine-1 order, 1 another order or none, 2 nothing comparable. + if sys.argv[1] == "--tile-check": + if len(sys.argv) < 4: + print("usage: repack_gguf.py --tile-check ") + return 2 + rc, line = tile_order_check(sys.argv[3], sys.argv[2]) + print(line) + return tile_check_exit(rc, line) + os.makedirs(out_dir, exist_ok=True) convert_py = os.path.join(converter_dir, "convert.py") if not os.path.exists(convert_py): @@ -398,6 +473,23 @@ def converter_commit(): cfg = write_config(gguf, out_dir, arch, vocab_size) + # The tile order is measured before the directory is cached, for the same reason the scale scan + # below is: a wrong order is silent all the way to a 200 (engine#1110). Cost is one dequantize of + # every comparable tensor per layout -- the same order as the repack itself, paid once per dir + # because a cache hit re-checks metadata only (docs/npu.md, "Declared scales are checked"). + _tile_rc, _tile_line = tile_order_check(gguf, out_dir) + tile_order, _tile_fatal = tile_order_policy(_tile_rc, _tile_line) + if _tile_fatal: + print(f"repack: {_tile_line}") + print("repack: the converter wrote q4_1 tiles in an order this engine does not decode -- refusing " + "the directory. Point --converter / ONEBIT_Q4NX_CONVERTER at a checkout that writes the " + "engine-1 order (engine#1110; measured good: FLM_Q4NX_Converter 81be119e).") + return 1 + if tile_order == "unverified": + print(f"repack: tile order not verified: {_tile_line}; the stamp records that instead of claiming engine-1") + else: + print(f"repack: tile order engine-1, measured: {_tile_line}") + # Record the provenance the cache-hit gate needs (see --check-stamp above). with open(os.path.join(out_dir, "repack-stamp.txt"), "w") as f: f.write(f"converter_path={converter_dir}\n") @@ -405,6 +497,8 @@ def converter_commit(): f.write(f"arch={arch}\n") f.write(f"model_type={cfg.get('model_type', '')}\n") f.write("q4nx_default_tensor_type=Q4_1\n") + f.write(f"tile_order={tile_order}\n") + f.write(f"tile_check={_tile_line}\n") f.write(f"repack_script={os.path.abspath(__file__)}\n") # Enforce the scale-key guard before the dir is cached (and before anyone diff --git a/tests/repack_tile_order_test.py b/tests/repack_tile_order_test.py new file mode 100644 index 000000000..4a63a656d --- /dev/null +++ b/tests/repack_tile_order_test.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""scripts/repack_gguf.py: what the repack may claim about a container's tile order (engine#1110). + +The load-time guard (#1130) refuses a `container.tile_order` this decoder does not implement, but +`tools/npu_plan.py` writes `engine-1` unconditionally, so the guard is green on a container whose +TILES are another order -- which repacks cleanly and decodes as noise at HTTP 200 (measured with +FLM_Q4NX_Converter `d1d5232`). The repack now measures the order and refuses the directory; this +test pins the mapping from the checker's verdict to (stamp value, fatal) with the four shapes that +occur, so the policy cannot drift from the tool an operator runs by hand. + +usage: tests/repack_tile_order_test.py (stdlib only: no numpy, no gguf-py, no model, no device) +""" +import importlib.util +import os +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +spec = importlib.util.spec_from_file_location( + "repack_gguf", os.path.join(HERE, os.pardir, "scripts", "repack_gguf.py")) +R = importlib.util.module_from_spec(spec) +spec.loader.exec_module(R) + +FAIL = [] + + +def check(ok, what, why=""): + if ok: + print("ok ", what) + return + FAIL.append(what) + print("FAIL", what, (" -- " + why) if why else "") + + +# (rc, line) -> (stamp value, fatal), with the lines tools/q4nx_diff.py verdict() really returns. +CASES = [ + # a correct requantized container: the honest Q4_K_M -> q4_1 error, the other order far above it + (0, "matches layout ours (worst max|err| 5.600e-02, next layout flm worst max|err| 2.6e+00)", + "engine-1", False, "the measured order is the one npu_plan.py declares"), + # a round trip, exact on every tile + (0, "matches layout ours (168 tensor(s) exactly)", "engine-1", False, "an exact round trip is ours"), + # the d1d5232 converter: no order matches, because the tiles are not these weights + (1, "matches NO layout: closest is flm with worst max|err| 2.474e+00 (> 1.000e-01, or not 3x " + "better than ours) - the container's tiles are not these weights", "", True, + "a container that is not these weights must not become a cached directory"), + # a container written in the other order: it MATCHES, but not the order npu/q4nx.cpp decodes + (0, "matches layout flm (worst max|err| 5.6e-02, next layout ours worst max|err| 2.6e+00)", + "", True, "a matching FLM container is still one this engine cannot read"), + # an architecture the checker cannot dequantize: nothing compared, so nothing may be claimed + (2, "nothing was compared", "unverified", False, "no evidence is not evidence of the right order"), +] + +for rc, line, want_order, want_fatal, why in CASES: + got_order, got_fatal = R.tile_order_policy(rc, line) + check(got_order == want_order and got_fatal == want_fatal, + f"{line[:44]!r} -> ({want_order!r}, fatal={want_fatal})", why) + +# `unverified` must never be spelled `engine-1`: the whole point is that the stamp stops asserting. +check(R.tile_order_policy(2, "nothing was compared")[0] != "engine-1", + "an unverified directory does not claim engine-1") + +# `--tile-check`'s exit codes are the policy's, not the checker's rc: verdict() returns 0 for ANY +# matching layout, including the one this engine cannot decode (engine#1110 review, blocker 1). +EXIT_CASES = [ + ("matches layout ours (worst max|err| 5.600e-02, next layout flm worst max|err| 2.6e+00)", 0, 0, + "the engine-1 order exits 0"), + ("matches layout flm (worst max|err| 5.600e-02, next layout ours worst max|err| 2.6e+00)", 0, 1, + "a matching layout this decoder does not implement exits 1, not 0"), + # the exact line pi-prs measured: a 0-tensor pair used to come back as this 'match' + ("matches layout flm (worst max|err| 0.000e+00, next layout ours worst max|err| 0.000e+00)", 0, 1, + "the 0-vs-0 tie is not a pass"), + ("matches NO layout: closest is flm ...", 1, 1, "no match exits 1"), + ("nothing was compared", 2, 2, "nothing compared exits 2"), +] +for line, rc, want_exit, why in EXIT_CASES: + got = R.tile_check_exit(rc, line) + check(got == want_exit, f"--tile-check exit {want_exit} for {line[:40]!r}", why) + +print(f"repack_tile_order_test: {len(FAIL)} failure(s)") +sys.exit(1 if FAIL else 0) diff --git a/tools/q4nx_diff.py b/tools/q4nx_diff.py index 105913da8..a8143d1d0 100755 --- a/tools/q4nx_diff.py +++ b/tools/q4nx_diff.py @@ -244,6 +244,14 @@ def verdict(results): container. So name the layout that matched, and when none did, name the closest and say so instead of implying the container is broken. """ + # A layout that compared NO tensor is not evidence of anything: `_decode_pass` returns + # (0, 0, 0.0) when every tensor was skipped, and worst=0.0 with rival=0.0 satisfies + # "rival >= worst * RATIO", so such a layout used to be reported as a MATCH on whichever one + # sorted first -- measured: a 0.6B container against a 4B GGUF came back + # `matches layout flm (worst max|err| 0.000e+00, ...)` with 0 tensors compared (engine#1110 + # review). Drop the layouts with no evidence before choosing, and if none is left say so: + # every caller already has a branch for rc=2. + results = {k: v for k, v in results.items() if v[0] + v[1] > 0} if not results: return 2, "nothing was compared" order = sorted(results, key=lambda k: results[k][2]) @@ -350,7 +358,7 @@ def decode_cmd(args): def self_test(): - """The verdict logic, with the four cases this lane measured on real containers.""" + """The verdict logic, with the cases measured on real containers plus the 0-tensor tie.""" bad = [] cases = [ # what a correct container looks like: one layout exact (a round trip), the other noise @@ -363,6 +371,10 @@ def self_test(): # ambiguous: close to the tolerance and not clearly better than its rival ({"ours": (0, 12, 0.09), "flm": (0, 12, 0.12)}, 1, "NO", "not clearly better than the rival"), ({}, 2, "nothing", "nothing compared"), + # both layouts skipped every tensor: a 0-vs-0 tie is not a match (engine#1110 review) + ({"ours": (0, 0, 0.0), "flm": (0, 0, 0.0)}, 2, "nothing", "two layouts that compared no tensor"), + # one layout compared tensors and the other did not: the one with evidence decides + ({"ours": (0, 0, 0.0), "flm": (0, 12, 1.6)}, 1, "NO", "a layout with no evidence cannot win"), ] for results, want_rc, want_word, what in cases: rc, line = verdict(results) @@ -374,7 +386,7 @@ def self_test(): print(" " + b) return 1 print("self-test passed: the verdict names the matching layout, accepts a lossy-but-correct pack, " - "and refuses tiles from other weights") + "refuses tiles from other weights, and treats a layout that compared nothing as no evidence") return 0