From 0f0033a8f7ac7e8a4b2d10139cff315ed6af9bc5 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 11:41:41 -0400 Subject: [PATCH] tools: dflash_draft_info, the gate before a DFlash draft loader: the package is in forms we already read (engine#1780) A DFlash draft package (FastFlowLM 1.0.12's two-package tags) is config.json (model_type dflash) plus one Q4NX container. The thread on #1780 stopped at two byte-identical readings of the projections (q4_1 4096x2560 against a half-float 1280x2560) and at a claim that the BF16 labels are really F16. Both are settled from the bytes: - the I8 [tiles, 5120] projections are the ordinary q4_1 tile grid: all 35 have bytes == (N/32)(K/256) x 5120 for the config's N x K, 10,240,000 groups have a finite positive scale, and minimum/scale is -7.59 .. -7.33 per tensor (a main-model container our forward serves reads -7.41 the same way). The half-float reading has 25.7 % of values above 100 in magnitude. So head_dim is 128, as the config says. - the BF16 labels are bf16: the 'tight as half-float' pattern is also present in a main model that runs correctly with the bf16 reading, because half-float folds bf16's exponent range into one octave. tools/dflash_draft_info.py checks a package against its own config (tile-grid sizes by role, tile-header sanity, the container's scalars, fc and norm sizes, no unexplained tensor) and fails closed; 19 of 19 on the Bolt draft in 3.4 s. --selftest builds a synthetic package with eight negative controls (ctest tool_selftest_dflash_draft_info). Stdlib only. docs/amd-drafters.md records the answers and what is actually missing: the draft's forward, not a format. Co-Authored-By: Claude Fable 5.2 --- CMakeLists.txt | 3 + docs/amd-drafters.md | 28 +++ tools/dflash_draft_info.py | 359 +++++++++++++++++++++++++++++++++++++ 3 files changed, 390 insertions(+) create mode 100755 tools/dflash_draft_info.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 79a6b1c44..f8873232f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -317,6 +317,9 @@ add_python_test(tool_selftest_logprob_agree ${CMAKE_SOURCE_DIR}/tools/logprob_ag add_python_test(tool_selftest_mtp_head_ref ${CMAKE_SOURCE_DIR}/tools/mtp_head_ref.py --selftest) add_python_test(tool_selftest_spec_decode_acceptance ${CMAKE_SOURCE_DIR}/tools/spec_decode_acceptance.py --selftest) add_python_test(tool_selftest_whisper_encoder_baseline ${CMAKE_SOURCE_DIR}/tools/whisper_encoder_baseline.py --self-test) +# engine#1780: the DFlash draft package's gate before a loader -- tile-grid sizes from the config, tile-header sanity and +# the container's scalars against config.json, on a synthetic package with eight negative controls. Stdlib only. +add_python_test(tool_selftest_dflash_draft_info ${CMAKE_SOURCE_DIR}/tools/dflash_draft_info.py --selftest) # engine#1777: the release row's own arm, which merged unregistered -- this sweep is what found it (engine#1820). add_test(NAME release_check_hrx_init_reason COMMAND bash ${CMAKE_SOURCE_DIR}/tests/release_check_hrx_init_reason_test.sh ${CMAKE_SOURCE_DIR}/os/mini/release-check.sh) # engine#1615: every process holding the GPU/NPU device must belong to a live gpu-hold/npu-hold ticket. Three diff --git a/docs/amd-drafters.md b/docs/amd-drafters.md index ee89be046..699edf009 100644 --- a/docs/amd-drafters.md +++ b/docs/amd-drafters.md @@ -88,6 +88,34 @@ So the Gumiho row is *"fails as published, because our verifier is the wrong sha quality (its published numbers are the best in its class: 2.97× mean over Eagle-2's 2.74× on vicuna-7B). Running it would mean implementing a tree verifier first, which is its own project. +## DFlash draft packages on the NPU route: **the container is in forms we already read; the forward is what is missing** (engine#1780) + +FastFlowLM 1.0.12 ships speculative decode as two-package tags: a small DFlash draft beside the main model's package. +A draft package is `config.json` (`model_type: dflash`) plus one Q4NX container. `tools/dflash_draft_info.py ` +checks such a package against its own config and prints what it declares; on +`Nemotron-3.5-Bolt-E4B-DFlash-NPU2` (0.27 GB, sha256 matching the repo's `MANIFEST.sha256`) it passes 19 of 19 +checks in 3.4 s. What that settles, each with the evidence the tool prints: + +| question | answer | evidence | +|---|---|---| +| are the `I8 [tiles, 5120]` projections a new packing? | **No.** They are the ordinary Q4NX q4_1 tile grid (32 x 256 weights per 5120-byte tile), the form every main-model container uses. | all 35 projections have `bytes == (N/32)(K/256) x 5120` for the N x K the config gives the role; 10,240,000 groups with every scale finite and positive; minimum/scale median -7.59 .. -7.33 per tensor (a known-good main model reads -7.41 the same way) | +| is `q_proj` 4096 x 2560 or a half-float 1280 x 2560? | **4096 x 2560**, so `head_dim` is 128 as the config says. | both readings are 6,553,600 bytes, but the half-float one has 25.7 % of its values above 100 in magnitude and 1.8 % non-finite, and 1280 x 2560 as a tile grid would be 400 tiles, not 1280 | +| are the `BF16` labels wrong (really F16)? | **No, they are bf16.** | a norm gain read as half-float looks tight (1.6-2.4) and as bf16 looks wide (0.22-6.3), which reads like a mislabel. The same pattern appears in a main-model container our forward serves correctly with the bf16 reading (1.49-1.88 against 0.12-1.05): half-float folds bf16's exponent range into one octave, so "tight" proves nothing | +| does the container agree with `config.json`? | **Yes.** | `dflash.block_size`, `target_layer_ids`, `sliding_window`, `rope_theta`, `rms_eps`, `embedding_scale` all equal | +| is anything in it unexplained? | **No.** | 64 tensors: 35 projections, 22 norms, `model.fc.weight` (2560 x 12800, the fusion of five target layers' features) and six scalars | + +So importing a DFlash draft is a config-schema translation, not a format project. What our runtime does not have is the +draft's **forward**: fuse the target's hidden states at `target_layer_ids` through `fc`, run five non-causal layers +over a block of `block_size` positions filled with `mask_token_id`, and score the block with the target's own +embedding table and head. `forward-serve` stops on such a package at the first key it expects +(`key 'model' not found`), which is the schema, before any of that. + +The tool is the gate before a loader, and it fails closed: a projection that is a tile short, a `head_dim` that +disagrees with the stored tiles, a NaN scale, a plain half-float matrix of the same size, a scalar that differs from +the config, or a tensor it cannot explain (the second draft family, `DFlash2DraftModel`, has selector and +convolution tensors this tool does not know) each fail by name. `--selftest` runs those controls on a synthetic +package (ctest `tool_selftest_dflash_draft_info`). + ## PARD-2's repos are weights-only — borrow the target's tokenizer `amd/PARD2-*` ship **`README.md`, `config.json`, `model.safetensors`, `warp_model.bin`** and nothing else: **no tokenizer diff --git a/tools/dflash_draft_info.py b/tools/dflash_draft_info.py new file mode 100755 index 000000000..09090ab2b --- /dev/null +++ b/tools/dflash_draft_info.py @@ -0,0 +1,359 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""dflash_draft_info.py -- what a DFlash draft package declares, checked against its own config (engine#1780). + +A DFlash draft is a two-file model source: `config.json` (model_type "dflash": block size, mask token, the target +layers it taps, the draft's own geometry) and a Q4NX container with the draft's weights. Before a forward for it is +written, three questions have to be answered from the bytes, not from the names: + + 1. is every projection in a form our reader already decodes? The container labels them `I8 [tiles, 5120]`, which + reads as a row x column int8 matrix and is not one: it is the ordinary Q4NX q4_1 tile grid (32 rows x 256 + columns per 5120-byte tile: 512 B of bf16 scales, 512 B of bf16 minimums, 4096 B of 4-bit codes), so the tile + count must equal (N/32) * (K/256) for the N x K the config gives that role. + 2. do the container's own scalars agree with config.json? (block size, taps, rope theta, eps, window, scale.) + 3. are the tile headers sane? Every scale finite and positive, every minimum finite. A wrong layout fails this at + once; the right one also shows min/scale near -7.5, the q4_1 signature of a symmetric 16-level group. + + tools/dflash_draft_info.py # PASS/FAIL lines, exit 1 on any FAIL + tools/dflash_draft_info.py --json # the census as JSON + tools/dflash_draft_info.py --selftest # a synthetic package and its negative controls; no model + +Standard library only, and it reads tile headers rather than whole tensors, so the 0.27 GB package takes seconds. +It decodes nothing into a model and runs nothing: it is the gate before a loader, not a loader. +""" +import argparse +import json +import math +import os +import struct +import sys +import tempfile + +TILE_ROWS, TILE_COLS, TILE_BYTES, HDR_BYTES = 32, 256, 5120, 1024 +ROLES = ("self_attn.q_proj", "self_attn.k_proj", "self_attn.v_proj", "self_attn.o_proj", + "mlp.gate_proj", "mlp.up_proj", "mlp.down_proj") +REQUIRED = ("hidden_size", "intermediate_size", "num_hidden_layers", "num_attention_heads", "num_key_value_heads", + "head_dim", "block_size", "target_layer_ids", "draft_weight") + + +def bf16(u16): + """A bf16 value: the top 16 bits of an IEEE float32.""" + return struct.unpack(" size - 8: + raise ValueError(f"header length {n} does not fit a {size}-byte file") + return json.loads(f.read(n)), 8 + n, size + + +def tile_header_stats(f, base, lo, ntiles): + """Scale/minimum sanity over every tile header of one tensor. Reads 1024 of each 5120 bytes.""" + bad_scale = bad_min = 0 + ratios = [] + for t in range(ntiles): + f.seek(base + lo + t * TILE_BYTES) + h = f.read(HDR_BYTES) + if len(h) != HDR_BYTES: + return {"short_read_at_tile": t} + vals = struct.unpack("<512H", h) + for i in range(256): + s, m = bf16(vals[i]), bf16(vals[256 + i]) + if not math.isfinite(s) or s <= 0.0: + bad_scale += 1 + elif not math.isfinite(m): + bad_min += 1 + elif t % 16 == 0 and i % 16 == 0: # a sample is enough for the median + ratios.append(m / s) + ratios.sort() + return {"groups": ntiles * 256, "bad_scale": bad_scale, "bad_min": bad_min, + "min_over_scale_median": round(ratios[len(ratios) // 2], 3) if ratios else None} + + +def inspect(pkg): + """Returns (checks, census). `checks` is a list of (ok, text); the census is JSON-serialisable.""" + checks, census = [], {"package": pkg} + + def check(ok, text): + checks.append((bool(ok), text)) + return bool(ok) + + cfg_path = os.path.join(pkg, "config.json") + if not check(os.path.isfile(cfg_path), f"config.json is present in {pkg}"): + return checks, census + c = json.load(open(cfg_path)) + if not check(c.get("model_type") == "dflash", f"config model_type is dflash (got {c.get('model_type')!r})"): + return checks, census + missing = [k for k in REQUIRED if k not in c] + if not check(not missing, f"config carries the geometry and the interface keys (missing: {missing or 'none'})"): + return checks, census + census["config"] = {k: c[k] for k in REQUIRED + ("mask_token_id", "sliding_window", "causal") if k in c} + wpath = os.path.join(pkg, c["draft_weight"]) + if not check(os.path.isfile(wpath), f"the draft weight {c['draft_weight']} is present"): + return checks, census + try: + hdr, base, size = read_header(wpath) + except (ValueError, json.JSONDecodeError) as e: + check(False, f"the weight file parses as a Q4NX table: {e}") + return checks, census + names = [k for k in hdr if k != "__metadata__"] + check(True, f"the weight file parses as a Q4NX table: {len(names)} tensors, {base}-byte header, {size} bytes") + end = max((hdr[k]["data_offsets"][1] for k in names), default=0) + check(base + end <= size, f"every tensor lies inside the file (payload ends at {base + end} of {size})") + + def span(name): + lo, hi = hdr[name]["data_offsets"] + return lo, hi - lo + + # 1. the projections: an ordinary q4_1 tile grid, sized by the config + L = c["num_hidden_layers"] + proj, wrong, absent = [], [], [] + with open(wpath, "rb") as f: + for layer in range(L): + for role in ROLES: + name = f"model.layers.{layer}.{role}.weight" + if name not in hdr: + absent.append(name) + continue + n, k = role_shape(role, c) + want = tiles_for(n, k) + lo, nbytes = span(name) + row = {"name": name, "rows": n, "cols": k, "tiles_expected": want, + "bytes": nbytes, "declared": [hdr[name]["dtype"], hdr[name]["shape"]]} + if want is None or nbytes != want * TILE_BYTES: + wrong.append(f"{name}: {nbytes} B, expected {want} tiles x {TILE_BYTES}") + else: + row.update(tile_header_stats(f, base, lo, want)) + proj.append(row) + check(not absent, f"all {L} x {len(ROLES)} projections are in the container (absent: {absent[:3] or 'none'})") + check(not wrong and proj, + f"each projection is a whole q4_1 tile grid of the config's N x K: bytes == (N/32)(K/256) x 5120 " + f"({len(proj) - len(wrong)} of {len(proj)}{'; ' + wrong[0] if wrong else ''})") + bad = [r["name"] for r in proj if r.get("bad_scale") or r.get("bad_min") or "short_read_at_tile" in r] + groups = sum(r.get("groups", 0) for r in proj) + check(not bad and groups, f"every tile header is sane: scale finite and positive, minimum finite, over {groups} " + f"32-weight groups (first bad: {bad[0] if bad else 'none'})") + ratios = sorted(r["min_over_scale_median"] for r in proj if r.get("min_over_scale_median") is not None) + if ratios: + check(-8.5 <= ratios[0] and ratios[-1] <= -6.5, + f"minimum/scale sits at the q4_1 signature of a symmetric 16-level group: median per tensor " + f"{ratios[0]:.3f} .. {ratios[-1]:.3f} (expected near -7.5)") + census["projections"] = proj + + # 2. the container's own scalars against config.json + def scalars(name, fmt): + if name not in hdr: + return None + lo, nbytes = span(name) + with open(wpath, "rb") as f: + f.seek(base + lo) + return list(struct.unpack("<" + fmt * (nbytes // 4), f.read(nbytes))) + + pairs = (("dflash.block_size", "i", "block_size", None), ("dflash.target_layer_ids", "i", "target_layer_ids", None), + ("dflash.sliding_window", "i", "sliding_window", None), ("dflash.rope_theta", "f", "rope_theta", 1e-6), + ("dflash.rms_eps", "f", "rms_norm_eps", 1e-6), ("dflash.embedding_scale", "f", "embedding_scale", 1e-6)) + stated = {} + for tname, fmt, key, rel in pairs: + got = scalars(tname, fmt) + if got is None or key not in c: + continue + want = c[key] if isinstance(c[key], list) else [c[key]] + same = len(got) == len(want) and all( + (g == w) if rel is None else abs(g - w) <= rel * max(1.0, abs(w)) for g, w in zip(got, want)) + stated[tname] = got + check(same, f"{tname} in the container equals config.json's {key} ({got} vs {want})") + census["container_scalars"] = stated + + # 3. the feature projection and the norms: 2-byte tensors with the sizes the config implies + h, taps = c["hidden_size"], len(c["target_layer_ids"]) + if "model.fc.weight" in hdr: + _, nbytes = span("model.fc.weight") + check(nbytes == h * taps * h * 2, + f"model.fc.weight is hidden x (taps x hidden) at 2 bytes: {nbytes} B == {h} x {taps * h} x 2 " + f"(declared {hdr['model.fc.weight']['dtype']} {hdr['model.fc.weight']['shape']})") + else: + check(False, "model.fc.weight (the tap-fusing projection) is in the container") + norms = [k for k in names if k.endswith("norm.weight")] + off = [k for k in norms if span(k)[1] not in (2 * h, 2 * c["head_dim"])] + check(norms and not off, f"{len(norms)} norm weights are 2 bytes x hidden or x head_dim (off-size: {off[:2] or 'none'})") + known = {f"model.layers.{layer}.{r}.weight" for layer in range(L) for r in ROLES} | set(norms) | \ + {"model.fc.weight"} | {p[0] for p in pairs} + extra = sorted(set(names) - known) + check(not extra, f"no tensor is left unexplained (unexplained: {extra[:4] or 'none'})") + census["tensors"] = len(names) + census["unexplained"] = extra + return checks, census + + +# ---- self-test: a synthetic package small enough to write in milliseconds ---------------------------------------- + +def _write_package(d, mutate=None): + c = {"model_type": "dflash", "architectures": ["DFlashDraftModel"], "draft_weight": "dflash_weight.q4nx", + "block_size": 8, "mask_token_id": 4, "target_layer_ids": [1, 3], "num_hidden_layers": 1, + "hidden_size": 256, "intermediate_size": 512, "num_attention_heads": 2, "num_key_value_heads": 1, + "head_dim": 128, "rms_norm_eps": 1e-06, "rope_theta": 1000000.0, "sliding_window": 1024, + "embedding_scale": 16.0, "causal": False} + tensors = {} + + def tile(): + s = struct.pack("> 16) * 256 + m = struct.pack("> 16) * 256 + return s + m + bytes(4096) + + for role in ROLES: + n, k = role_shape(role, c) + nt = tiles_for(n, k) + tensors[f"model.layers.0.{role}.weight"] = ("I8", [nt, TILE_BYTES], tile() * nt) + for nm, w in (("model.norm.weight", 256), ("model.hidden_norm.weight", 256), + ("model.layers.0.input_layernorm.weight", 256), ("model.layers.0.post_attention_layernorm.weight", 256), + ("model.layers.0.self_attn.q_norm.weight", 128), ("model.layers.0.self_attn.k_norm.weight", 128)): + tensors[nm] = ("BF16", [w], struct.pack("= 14, f"a well-formed synthetic package passes every check ({len(checks)} checks, failing: {bad})") + expect(len(census.get("projections", [])) == 7 and all(r["min_over_scale_median"] == -7.5 for r in census["projections"]), + "its census lists the 7 projections with minimum/scale -7.5") + + def short_q(c, t): + dt, sh, data = t["model.layers.0.self_attn.q_proj.weight"] + t["model.layers.0.self_attn.q_proj.weight"] = (dt, sh, data[:-TILE_BYTES]) + bad, _, _ = run(short_q) + expect(any("whole q4_1 tile grid" in b for b in bad), "a projection one tile short of its config shape FAILS by name") + + def head_dim_40(c, t): + c["head_dim"] = 40 + bad, _, _ = run(head_dim_40) + expect(any("whole q4_1 tile grid" in b for b in bad), + "a config whose head_dim disagrees with the stored tiles FAILS (the geometry is checked, not assumed)") + + def nan_scale(c, t): + dt, sh, data = t["model.layers.0.mlp.up_proj.weight"] + t["model.layers.0.mlp.up_proj.weight"] = (dt, sh, struct.pack(" [--json] | --selftest", file=sys.stderr) + return 2 + checks, census = inspect(a.package) + failed = [t for ok, t in checks if not ok] + if a.json: + census["checks"] = [{"ok": ok, "what": t} for ok, t in checks] + print(json.dumps(census, indent=1)) + else: + for ok, t in checks: + print(f"{'PASS' if ok else 'FAIL'} {t}") + print(f"{len(checks) - len(failed)} of {len(checks)} checks passed") + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:]))