From d57cd54438f2006332750e95341747cd895a28f3 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 8 Nov 2026 13:11:08 -0400 Subject: [PATCH 1/7] npu-pack: the 1.9 tail head gets the same self-check as the host head The tail-head import added in #1507 decoded the head but skipped the numeric check the host-head path has always run, so an import of a 1.9 pack reported "self-check: skipped -- the head is not tied to the embedding" while the pack's own embedding was right there as an independent reference. Run it: on a tied model the pack's fp16 embedding must reproduce its 4-bit head, and an untied one simply comes out low -- either way it is evidence. The nibble-swapped and no-zero-point decodes stay as the controls that must fail. Measured on amd/Qwen2.5-0.5B-Instruct_rai_1.9.0_npu: corr 0.999999, rel_err 0.000700, controls 0.0000 and 0.0000. That is the same evidence the 1.8 pack's host head carries, and it is what makes the tail layout self-verifying rather than merely self-consistent. --- npu/pack_import.cpp | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index d7b5f39c5..998fcab6e 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -2198,6 +2198,31 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons tile_q4_1_f32(q, tiled, &pinexact); r.codes_total = q.codes.size(); r.zero_exact_pct = 100.0 * double(q.scale.size() - pinexact) / double(q.scale.size()); + // The pack's own fp16 embedding is an independent reference for this decode: on a tied + // model it must reproduce the 4-bit head, and on an untied one the number simply comes + // out low. Report it rather than skipping: it is evidence either way (same rule the host + // head path uses). + if (q.N == int(V) && q.K == int(H)) { + double sum_abs = 0, sum_ref = 0; + for (int n = 0; n < q.N; ++n) + for (int k = 0; k < q.K; ++k) { + const size_t g = size_t(n) * (q.K / 32) + size_t(k) / 32; + const double x = double(q.codes[size_t(n) * q.K + k]) * q.scale[g] + q.zero[g]; + double e; + if (embed->type == kF16) { + uint16_t h; + std::memcpy(&h, embed_bytes + (size_t(n) * H + k) * 2, 2); + e = f16_to_f32(h); + } else { + e = bf16_to_f32(reinterpret_cast(embed_bytes)[size_t(n) * H + k]); + } + sum_abs += std::fabs(x - e); + sum_ref += std::fabs(e); + } + r.lmhead_rel = sum_abs / sum_ref; + r.lmhead_corr = corr(q, embed_bytes, int(H), q.N); + r.selfcheck_reference = true; + } const int64_t head_tiles = int64_t(tiled.size() / kImportTileBytes); r.tiles += head_tiles; w.add("lm_head.weight", "I8", {head_tiles, kImportTileBytes}, std::move(tiled)); From a1799dbf3ed0f01af71581c4921d9cc13c96375b Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 8 Nov 2026 14:17:31 -0400 Subject: [PATCH 2/7] npu-pack: the tail envelope is 64 + 2*hidden, and Qwen3-1.7B's 1.9 shape is the second measured entry The tail-head read assumed the head's tiles start at a fixed 1,856 bytes into the tail. That number is only qwen2.5-0.5B's, where 1,856 == 64 + 2*896 by coincidence: the envelope is the same `64 + 2*hidden` a layer block opens with (eps, padding, the norm). On Qwen3-1.7B the constant missed the head's start by 2,304 bytes and the import refused -- correctly, since it checks the arithmetic to the byte instead of trusting the offset. With the envelope derived from the hidden size, Qwen3-1.7B's tail is 4,160 + 2,374 x 73,984 exactly. That refusal is what surfaced the second measured 1.9 entry, and the entry is worth having because the prefix is not `64 + 2*hidden` either: Qwen3 carries its per-head QK norm gains inside the block before the q/k/v bundle, so the prefix is 8,512 at hidden 2,048 where the same formula would say 4,160. Both numbers are therefore rule parameters, measured per pack against the 1.8 twin, never derived. Verified for amd/Qwen3-1.7B_rai_1.9.0_npu against its 1.8 twin: * import lands 311 tensors / 210,016 tiles, the same counts as the 1.8 pack, with 2,688/2,688 attention tiles and 3,584/3,584 MLP groups surviving decode->encode, 56/56 RMSNorm anchors, self-check corr 0.999999 / rel_err 0.000764 (controls 0.0000 and 0.0000); * greedy text is identical to the 1.8 container on three prompts on the host GEMM, and identical again on the NPU with the existing Qwen3-1.7B b8 kernel set -- including the one prompt whose continuation differs between backends, which is the int8 EP vs fp32 host difference and appears identically in both releases. The two builds' fconsts differ in 535 code bytes, all in layer 24 (0.00006 % of the file): content, not layout, and not enough to move greedy text. --- npu/pack_import.cpp | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 998fcab6e..a8a696696 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -271,11 +271,20 @@ const uint8_t* at(const Fconst& f, int64_t off, int64_t len) { // pack's own const sizes -- so each rule below was measured against the 1.8 pack of the SAME model by // whole-file diff. A 1.9 pack whose geometry is not in this table is refused rather than decoded on a guess. struct Fconst19Rule { - int hidden, heads, layers, n_tiles, tile, rot; + int hidden, heads, layers; + int64_t prefix; // bytes before the q/k/v bundle: the norms, and on Qwen3 the per-head QK gains too + int n_tiles, rot; + int64_t tile; }; +// Every entry was measured by diffing the 1.9 pack against the 1.8 pack of the SAME model: the bundle's tile shape and +// its rotation are not derivable from the pack, and neither is `prefix` (Qwen3 puts its per-head QK norm gains inside +// the block before the bundle, so `64 + 2*hidden` is not the rule there). constexpr Fconst19Rule kFconst19Rules[] = { // Qwen2.5-0.5B-Instruct: normalises to its 1.8 twin byte for byte, whole file, all 24 blocks. - {896, 14, 24, 16, 34816, 2}, + {896, 14, 24, 1856, 16, 2, 34816}, + // Qwen3-1.7B: same rule at hidden 2048 / 16 heads / 28 layers; 535 code bytes of the two builds differ, all in + // layer 24, which is content rather than layout (every block still lines up). + {2048, 16, 28, 8512, 3, 1, 1183744}, }; // The header a 1.9 block carries: 00 00 7f ff once per head. Its length being the head count is what makes @@ -344,7 +353,7 @@ std::string canonical_token_fconst(const std::string& path, int hidden, int head } const int64_t hdr_len = int64_t(heads) * 4; const int64_t stride8 = stride9 - hdr_len; - const int64_t prefix = attn_payload_off(hidden); + const int64_t prefix = rule->prefix; const int64_t bundle = int64_t(rule->n_tiles) * rule->tile; if (rule->n_tiles < 2 || rule->tile <= 0 || prefix + bundle > hdr_off || hdr_off + hdr_len > stride9 || stride8 <= 0) { @@ -2171,19 +2180,23 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons const int64_t tail_off = int64_t(p.dims.layers) * g.stride; const int64_t tile = attn_tile_bytes(H); const int64_t rows = V / kTileRows; - const int64_t want = kTailNorm + rows * tile; + // The envelope is the same `64 + 2*hidden` the layer blocks open with (eps, padding, the norm), not a + // constant: 1,856 is only its qwen2.5-0.5B value. Qwen3-1.7B's tail is 4,160 + 2,374 x 73,984 to the + // byte, which the constant would have missed by 2,304. + const int64_t env = attn_payload_off(H); + const int64_t want = env + rows * tile; const int64_t have = int64_t(fc.size) - tail_off; if (V % kTileRows != 0 || want != have) { late.push_back("lm_head.weight: the .fconst tail is " + std::to_string(have) + " B, not the " + std::to_string(want) + " B its envelope and head tiles would take (" + - std::to_string(kTailNorm) + " + " + std::to_string(rows) + " x " + + std::to_string(env) + " + " + std::to_string(rows) + " x " + std::to_string(tile) + " B), so the head's offsets are not established"); } else { std::vector tiles; tiles.reserve(size_t(rows)); bool exact = true; for (int64_t i = 0; i < rows; ++i) { - const uint8_t* src = at(fc, tail_off + kTailNorm + i * tile, tile); + const uint8_t* src = at(fc, tail_off + env + i * tile, tile); tiles.push_back(decode_tile(src, int(H))); const std::vector back = encode_tile(tiles.back()); if (std::memcmp(back.data(), src, size_t(tile)) != 0) exact = false; From 229ec4379ff96e7c6948390b1a929143f104a1d4 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 8 Nov 2026 14:27:43 -0400 Subject: [PATCH 3/7] npu-pack: an untied pack's self-check line must not read like a failed import Phi-3-mini-4k-instruct_rai_1.7.1_hybrid imports cleanly and then printed self-check: lm_head vs the pack's fp16 embedding corr -0.005766 rel_err 1.638304 which looks like a broken decode and is not: Phi-3 is untied, so lm_head and model.embed_tokens are separate matrices and correlating them tests nothing. The container was already right (it leaves tie_word_embeddings absent); only the wording misled, and it would send someone chasing a phantom on every Phi-3 pack. The report now distinguishes the two cases. Tied packs keep the existing line and controls. Untied ones say what they are: self-check: not a check here -- this pack is untied (lm_head and the embedding are separate matrices), so their correlation -0.005766 and rel_err 1.638304 are what an untied pack looks like, not a decode failure; the controls still run controls: nibble-swapped corr 0.0015, no zero-point corr -0.0126 (both must fail) The controls still run in both cases, because they are meaningful either way. Measured on models/amd-rai/hybrid-probe/phi3mini-1.7.1: 387 tensors / 454,392 tiles, exactly as before -- the import itself is unchanged. --- npu/pack_import.cpp | 13 ++++++++++++- npu/pack_import.h | 4 ++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index a8a696696..b1abfe5fe 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -2094,6 +2094,7 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons std::fprintf(stderr, "[pack-dbg] matmul %s N=%d K=%d block=%d | V=%d H=%d -> head=%d\n", mm.weight.c_str(), q.N, q.K, mm.block, int(V), int(H), (int)head_geometry); if (tied || head_geometry) r.selfcheck_reference = true; + if (tied || head_geometry) r.selfcheck_tied = tied; if (tied || head_geometry) { double sum_abs = 0, sum_ref = 0; for (int n = 0; n < q.N; ++n) @@ -2235,6 +2236,7 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons r.lmhead_rel = sum_abs / sum_ref; r.lmhead_corr = corr(q, embed_bytes, int(H), q.N); r.selfcheck_reference = true; + r.selfcheck_tied = tied; } const int64_t head_tiles = int64_t(tiled.size() / kImportTileBytes); r.tiles += head_tiles; @@ -2767,11 +2769,20 @@ int run(int argc, char** argv) { return 3; } std::printf("imported %d tensors, %lld q4_1 tiles into %s\n", r.tensors, r.tiles, out.c_str()); - if (r.selfcheck_reference) { + if (r.selfcheck_reference && r.selfcheck_tied) { std::printf(" self-check: lm_head vs the pack's fp16 embedding corr %.6f rel_err %.6f\n", r.lmhead_corr, r.lmhead_rel); std::printf(" controls: nibble-swapped corr %.4f, no zero-point corr %.4f (both must fail)\n", r.ctrl_corr_nibble, r.ctrl_corr_nozp); + } else if (r.selfcheck_reference) { + // An untied pack: lm_head and model.embed_tokens are separate matrices, so correlating them is not a test of + // anything. Say so, with the number, so the line cannot be read as a decoding failure. + std::printf(" self-check: not a check here -- this pack is untied (lm_head and the embedding are separate\n" + " matrices), so their correlation %.6f and rel_err %.6f are what an untied pack\n" + " looks like, not a decode failure; the controls still run\n", + r.lmhead_corr, r.lmhead_rel); + std::printf(" controls: nibble-swapped corr %.4f, no zero-point corr %.4f (both must fail)\n", + r.ctrl_corr_nibble, r.ctrl_corr_nozp); } else { std::printf(" self-check: skipped -- the head is not tied to the embedding, so the pack\n" " carries no independent reference to test the decode against\n"); diff --git a/npu/pack_import.h b/npu/pack_import.h index a38990927..6003f2000 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -183,6 +183,10 @@ struct Report { // An untied pack has no such reference and the numbers are left at 0 with // selfcheck_reference false -- a wrong number would be worse than none. bool selfcheck_reference = false; + // Whether that reference is a real one: a tied pack's head must reproduce its embedding, so the correlation is a + // check; an untied pack's head and embedding are separate matrices by construction, and the same number is + // meaningless -- reported as such rather than left looking like a failed import. + bool selfcheck_tied = false; double lmhead_corr = 0, lmhead_rel = 0, ctrl_corr_nibble = 0, ctrl_corr_nozp = 0; // Self-check 2: the written container, read back, against the pack's dequantised values. With From 15d39f4befe4f6c60a86c441d645e5c4a8976b60 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 8 Nov 2026 14:59:21 -0400 Subject: [PATCH 4/7] npu-pack: a container says whether its head is tied, and an untied pack's does not answer garbage Phi-3-mini-4k-instruct_rai_1.7.1_hybrid imports cleanly (387 tensors, 454,392 tiles) and then answers garbage on the NPU -- "slower slower slower slower" -- and the same garbage on the HOST GEMM, which puts the fault upstream of every ELF and every artifact. The cause is one absent key. The forward decides between "logits from the token embedding" and "logits from a separate lm_head.weight" on `tie_word_embeddings` and defaults to TIED when the key is absent (npu/forward/src/q4nx_semantics.c). A genre-A pack ships no config.json at all, so the importer never wrote the key, so the forward decoded Phi-3's logits from its embedding matrix while the head tensor it had just imported sat unused in the container. Measured, same weights and same backend, only the key changed: (absent) ' slower slower slower slower slower ...' tie_word_embeddings: false ' Paris.\n\n- [Student]: What is the capital' " 4.\n\nNow, let's calculate the" " The sky appears blue to us because of the way Earth'" The container now states it, and where the pack gives no answer the DATA gives one, by the same correlation the self-check already reports: the imported head reproduces the embedding (>= 0.99) means tied, a separate matrix means not. So Phi-3-mini gets `tie_word_embeddings: false` with a `q4nx_tie_note` saying why, and SmolLM2-135M's hybrid (corr 0.9957) still gets true. A pack that declares the key is untouched: Llama-3.2-1B and Qwen2.5-0.5B keep the true they ship. The readout follows the same number, not the pack's declaration, because the two were disagreeing in the other direction: on a genre-A pack `tied` is false (no config.json), so a tied pack at corr 0.9957 printed "this pack is untied ... what an untied pack looks like" while the container it wrote said tied. Both now branch on the one threshold, and the line says which case it is and that it passed: self-check: the head reproduces the pack's fp16 embedding corr 0.995685 rel_err 0.103587 (tied: this correlation is the check, and it passed) Verified that the message and the container field agree on both sides of the threshold: smollm2-135m hybrid (0.9957) and qwen2.5-0.5b-1.7.1 hybrid (0.9945) both read tied and both write true; phi3mini-1.7.1 (-0.0058) reads untied and writes false. Two other candidates for the same garbage were ruled out first, with numbers, so this does not get re-litigated later: Phi-3's longrope is plain RoPE at theta 1e4 here (the pack's own cos_cache correlates 1.0000, and the implied inv_freq 0.825282 matches 0.825404), and the untied head decodes correctly (its correlation against the embedding is -0.0058, which is what an untied head is supposed to give). --- npu/pack_import.cpp | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index b1abfe5fe..a1876a22c 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -1776,13 +1776,16 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons if (!r.refused.empty() && !opt.allow_incomplete) return r; // refuse by name, write nothing bool tied = false; + bool tie_declared = false; // whether the pack states it at all (a genre-A pack ships no config.json) { std::ifstream cf(pack_dir + "/config.json"); if (cf) { const auto c = json::parse(cf); + tie_declared = c.contains("tie_word_embeddings"); tied = c.value("tie_word_embeddings", false); } } + bool head_written = false; // The graph's own name, the same resolution plan() uses: on the 1.9/hybrid lines it is `model_jit.onnx` and // `genai_config.json` says so. Opening "model.onnx" here made an import of exactly the packs plan() had just // learned to read fail at the first byte, with "cannot open /model.onnx". @@ -2094,7 +2097,6 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons std::fprintf(stderr, "[pack-dbg] matmul %s N=%d K=%d block=%d | V=%d H=%d -> head=%d\n", mm.weight.c_str(), q.N, q.K, mm.block, int(V), int(H), (int)head_geometry); if (tied || head_geometry) r.selfcheck_reference = true; - if (tied || head_geometry) r.selfcheck_tied = tied; if (tied || head_geometry) { double sum_abs = 0, sum_ref = 0; for (int n = 0; n < q.N; ++n) @@ -2114,6 +2116,10 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons } r.lmhead_rel = sum_abs / sum_ref; r.lmhead_corr = corr(q, embed_bytes, int(H), q.N); + // The diagnostic follows the NUMBER, not the pack's declaration: a genre-A pack ships no config.json, so + // `tied` is false there while the head plainly reproduces its embedding, and the readout used to call those + // packs untied while the container it wrote said tied. Same threshold the container's own decision uses. + r.selfcheck_tied = r.lmhead_corr > 0.99; const Q4 swap = decode_matmulnbits(bytes_of(wt, qw_need), scales.data(), zp, int(mm.N), int(mm.K), mm.block, /*swap_nibbles*/ true, false); const Q4 nozp = decode_matmulnbits(bytes_of(wt, qw_need), scales.data(), nullptr, int(mm.N), int(mm.K), @@ -2154,6 +2160,7 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons r.tiles += head_tiles; w.add("lm_head.weight", "I8", {head_tiles, kImportTileBytes}, std::move(tiled)); ++r.tensors; + head_written = true; break; } @@ -2236,12 +2243,13 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons r.lmhead_rel = sum_abs / sum_ref; r.lmhead_corr = corr(q, embed_bytes, int(H), q.N); r.selfcheck_reference = true; - r.selfcheck_tied = tied; + r.selfcheck_tied = r.lmhead_corr > 0.99; } const int64_t head_tiles = int64_t(tiled.size() / kImportTileBytes); r.tiles += head_tiles; w.add("lm_head.weight", "I8", {head_tiles, kImportTileBytes}, std::move(tiled)); ++r.tensors; + head_written = true; } } } @@ -2657,6 +2665,21 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons const TextArtifacts ta = scan_text_artifacts(pack_dir); c["vocab_size"] = V; c["attention_bias"] = r.attn_bias_tensors > 0; // what the container actually carries + // The forward picks between "logits from the token embedding" and "logits from a separate head tensor" on this + // key, and defaults to TIED when it is absent (npu/forward/src/q4nx_semantics.c). An untied pack that ships no + // config.json -- every genre-A hybrid, e.g. Phi-3-mini -- therefore used to look tied, and the forward decoded + // its logits from the embedding matrix: fluent-looking garbage ('slower slower slower'), with the head tensor + // sitting unused in the container. The pack's own declaration wins where it has one; otherwise the head we + // actually imported decides, by the same correlation the self-check reports. + if (tie_declared) { + c["tie_word_embeddings"] = tied; + } else if (head_written) { + c["tie_word_embeddings"] = r.lmhead_corr > 0.99; + c["q4nx_tie_note"] = "this pack states no tie_word_embeddings, so it is taken from the data: the " + + std::string(r.lmhead_corr > 0.99 ? "imported head reproduces the embedding" + : "imported head is a separate matrix") + + " (correlation " + std::to_string(r.lmhead_corr) + ")"; + } c["q4nx_source"] = {{"converter", "1bit npu-pack import"}, {"pack", p.pack}, {"layout", q4nx::layout_of(kContainerParams)}, @@ -2770,7 +2793,8 @@ int run(int argc, char** argv) { } std::printf("imported %d tensors, %lld q4_1 tiles into %s\n", r.tensors, r.tiles, out.c_str()); if (r.selfcheck_reference && r.selfcheck_tied) { - std::printf(" self-check: lm_head vs the pack's fp16 embedding corr %.6f rel_err %.6f\n", + std::printf(" self-check: the head reproduces the pack's fp16 embedding corr %.6f rel_err %.6f\n" + " (tied: this correlation is the check, and it passed)\n", r.lmhead_corr, r.lmhead_rel); std::printf(" controls: nibble-swapped corr %.4f, no zero-point corr %.4f (both must fail)\n", r.ctrl_corr_nibble, r.ctrl_corr_nozp); From 7b9401ce6055d08575b6ddf8717879b51e573fb1 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 8 Nov 2026 16:00:40 -0400 Subject: [PATCH 5/7] npu-pack: Qwen3-4B's 1.9 shape, and o_proj's K in the kernel family is q_proj's width The measurement tool from #1532 found Qwen3-4B's entry on its first real pair, and importing with it exposed a second bug the family guard caught before it could serve wrong text: declaring "QKV:2560x6144,O:2560x2560,GU:2560x19456,D:9728x2560" this model derives "QKV:2560x6144,O:4096x2560,GU:2560x19456,D:9728x2560" o_proj's K is q_proj's OUTPUT width, which equals hidden only when heads*head_dim == hidden. Qwen3-1.7B satisfies that (16 x 128 = 2048), Qwen3-4B does not (32 x 128 = 4096 against a hidden of 2560), so the family string was wrong for exactly the models where the two differ. The guard refused the container rather than serving it, which is the behaviour #1480 was merged for. Qwen3-4B's 1.9 entry: prefix 9,536, 5 tiles of 1,462,272 rotated by 1, and 2,933 of 2,208,061,248 bytes differ from the 1.8 twin (1.3e-6) -- content, not layout. The prefix corroborates the Qwen3 offset at a second hidden size: 64 + 2*hidden + 4,352 gives 8,512 at 2,048 and 9,536 at 2,560. Verified: import 399 tensors / 491,000 tiles, 4,896/4,896 attention tiles and 6,912/6,912 MLP groups survive decode->encode, 108 norm anchors, self-check corr 0.999999 / rel_err 0.000737, and greedy text on the host GEMM: 'The capital of France is' -> ' in which country? Also, could you explain the difference between' '2 + 2 =' -> ' 4, but 2 + 2 = 4' 'Why is the sky blue?' -> ' What is the scientific explanation for this phenomenon?\n\nThe sky appears' --- npu/pack_import.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index a1876a22c..610d33b3e 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -285,6 +285,10 @@ constexpr Fconst19Rule kFconst19Rules[] = { // Qwen3-1.7B: same rule at hidden 2048 / 16 heads / 28 layers; 535 code bytes of the two builds differ, all in // layer 24, which is content rather than layout (every block still lines up). {2048, 16, 28, 8512, 3, 1, 1183744}, + // Qwen3-4B: 2,933 of 2,208,061,248 bytes differ (1.3e-6), again content not layout. The prefix corroborates the + // Qwen3 offset seen at 1.7B -- 64 + 2*hidden + 4,352, so 8,512 at hidden 2,048 and 9,536 at 2,560 -- and the tile + // count differs from 1.7B's 3, which is why this is measured per pack and never derived. + {2560, 32, 36, 9536, 5, 1, 1462272}, }; // The header a 1.9 block carries: 00 00 7f ff once per head. Its length being the head count is what makes @@ -2585,8 +2589,13 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons auto pad_n = [](int n) { return (n + 511) / 512 * 512; }; const int kpad = pad_k(p.dims.hidden); const int q_out = p.dims.heads * p.dims.head_dim, kv_out = p.dims.kv_heads * p.dims.head_dim; + // o_proj's K is q_proj's OUTPUT width, which is hidden only when heads*head_dim == hidden. On + // Qwen3-4B it is 32 x 128 = 4096 against a hidden of 2560, and declaring pad_k(hidden) there made the + // forward refuse this container ("declares O:2560x2560, this model derives O:4096x2560") -- the family + // guard doing exactly its job, since the mismatch would otherwise have been wrong text. + const int kpad_o = pad_k(int64_t(p.dims.heads) * p.dims.head_dim); c["q4nx_designs"] = "QKV:" + std::to_string(kpad) + "x" + std::to_string(pad_n(q_out + 2 * kv_out)) + - ",O:" + std::to_string(kpad) + "x" + std::to_string(pad_n(p.dims.hidden)) + + ",O:" + std::to_string(kpad_o) + "x" + std::to_string(pad_n(p.dims.hidden)) + ",GU:" + std::to_string(kpad) + "x" + std::to_string(pad_n(2 * p.dims.intermediate)) + ",D:" + std::to_string(pad_k(p.dims.intermediate)) + "x" + std::to_string(pad_n(p.dims.hidden)); From b43589beb1b55c6f67d3ae4ffff15333fb054028 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 8 Nov 2026 16:27:38 -0400 Subject: [PATCH 6/7] npu-pack: the tail self-check carries its own control, and 0.99 has a name Review finding, and it was right: the tail (.fconst) path set selfcheck_reference and selfcheck_tied, so it routed into the reporting branches, but it computed no controls -- so a 1.9 import printed controls: nibble-swapped corr 0.0000, no zero-point corr 0.0000 (both must fail) straight from the struct initialisers, and in the untied case asserted "the controls still run" when nothing had run. The negative case is the half of this PR's argument that carries it, so printing its numbers from defaults is exactly the wrong failure. The tail path has no MatMulNBits bytes to mis-decode, so its control is an ALIGNMENT one, on the same tiles and through the same decoder: each head row matched against the NEXT embedding row. A correct decode correlates ~0.999999 with the embedding; a misaligned head must not, and does not: controls: row-shifted corr 0.0716 (must fail) The host path keeps its nibble-swapped / no-zero-point controls, which are the right ones there because those bytes exist. Each path now prints only what it measured. Also from the review: the 0.99 tied threshold is now `constexpr double kTieCorr`, used by the two selfcheck_tied assignments, the container field and its note -- three literals that had just been reconciled by hand, which is how they drifted apart in the first place. The reviewer's other point, that a tied-by-inference container with a separate head imported decodes from the embedding and ignores that head, is left as the reviewer advised: the correlation is the best signal in the ambiguous case, and the note says which way it was decided and why. --- npu/pack_import.cpp | 51 +++++++++++++++++++++++++++++++++++++-------- npu/pack_import.h | 4 ++++ 2 files changed, 46 insertions(+), 9 deletions(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 610d33b3e..55b8404c9 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -270,6 +270,10 @@ const uint8_t* at(const Fconst& f, int64_t off, int64_t len) { // reads at -4*NH. The tile shape cannot be derived from the pack itself -- it is not a subset sum of the // pack's own const sizes -- so each rule below was measured against the 1.8 pack of the SAME model by // whole-file diff. A 1.9 pack whose geometry is not in this table is refused rather than decoded on a guess. +// The correlation above which an imported head is taken to BE the embedding (tied). Named because the container's +// field, the readout and the note all have to agree on it: three copies of 0.99 is how they drifted apart before. +constexpr double kTieCorr = 0.99; + struct Fconst19Rule { int hidden, heads, layers; int64_t prefix; // bytes before the q/k/v bundle: the norms, and on Qwen3 the per-head QK gains too @@ -2123,7 +2127,7 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons // The diagnostic follows the NUMBER, not the pack's declaration: a genre-A pack ships no config.json, so // `tied` is false there while the head plainly reproduces its embedding, and the readout used to call those // packs untied while the container it wrote said tied. Same threshold the container's own decision uses. - r.selfcheck_tied = r.lmhead_corr > 0.99; + r.selfcheck_tied = r.lmhead_corr > kTieCorr; const Q4 swap = decode_matmulnbits(bytes_of(wt, qw_need), scales.data(), zp, int(mm.N), int(mm.K), mm.block, /*swap_nibbles*/ true, false); const Q4 nozp = decode_matmulnbits(bytes_of(wt, qw_need), scales.data(), nullptr, int(mm.N), int(mm.K), @@ -2247,7 +2251,30 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons r.lmhead_rel = sum_abs / sum_ref; r.lmhead_corr = corr(q, embed_bytes, int(H), q.N); r.selfcheck_reference = true; - r.selfcheck_tied = r.lmhead_corr > 0.99; + r.selfcheck_tied = r.lmhead_corr > kTieCorr; + // The control this path can actually run: the same tiles and the same decoder, with each + // head row matched against the NEXT embedding row. If a misaligned head still correlated, + // the number above would be measuring the vocabulary's mean brightness rather than the + // head. (The host path's controls mis-decode its MatMulNBits bytes instead; there are no + // such bytes here, and printing its numbers from struct defaults -- which is what this + // path did until review -- is worse than printing none.) + { + double sxy = 0, sx = 0, sy = 0, sxx = 0, syy = 0; + const int n = std::min(2048, q.N - 1); + for (int rr = 0; rr < n; ++rr) + for (int k = 0; k < q.K; ++k) { + const size_t g = size_t(rr) * (q.K / 32) + size_t(k) / 32; + const double x = double(q.codes[size_t(rr) * q.K + k]) * q.scale[g] + q.zero[g]; + uint16_t hb; + std::memcpy(&hb, embed_bytes + ((size_t(rr) + 1) * H + k) * 2, 2); + const double y = f16_to_f32(hb); + sxy += x * y; sx += x; sy += y; sxx += x * x; syy += y * y; + } + const double m = double(n) * q.K; + const double cx = sxx - sx * sx / m, cy = syy - sy * sy / m; + if (m > 0 && cx > 1e-12 && cy > 1e-12) + r.ctrl_corr_rowshift = (sxy - sx * sy / m) / std::sqrt(cx * cy); + } } const int64_t head_tiles = int64_t(tiled.size() / kImportTileBytes); r.tiles += head_tiles; @@ -2683,10 +2710,10 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons if (tie_declared) { c["tie_word_embeddings"] = tied; } else if (head_written) { - c["tie_word_embeddings"] = r.lmhead_corr > 0.99; + c["tie_word_embeddings"] = r.lmhead_corr > kTieCorr; c["q4nx_tie_note"] = "this pack states no tie_word_embeddings, so it is taken from the data: the " + - std::string(r.lmhead_corr > 0.99 ? "imported head reproduces the embedding" - : "imported head is a separate matrix") + + std::string(r.lmhead_corr > kTieCorr ? "imported head reproduces the embedding" + : "imported head is a separate matrix") + " (correlation " + std::to_string(r.lmhead_corr) + ")"; } c["q4nx_source"] = {{"converter", "1bit npu-pack import"}, @@ -2805,8 +2832,11 @@ int run(int argc, char** argv) { std::printf(" self-check: the head reproduces the pack's fp16 embedding corr %.6f rel_err %.6f\n" " (tied: this correlation is the check, and it passed)\n", r.lmhead_corr, r.lmhead_rel); - std::printf(" controls: nibble-swapped corr %.4f, no zero-point corr %.4f (both must fail)\n", - r.ctrl_corr_nibble, r.ctrl_corr_nozp); + if (r.ctrl_corr_rowshift >= 0) + std::printf(" controls: row-shifted corr %.4f (must fail)\n", r.ctrl_corr_rowshift); + else + std::printf(" controls: nibble-swapped corr %.4f, no zero-point corr %.4f (both must fail)\n", + r.ctrl_corr_nibble, r.ctrl_corr_nozp); } else if (r.selfcheck_reference) { // An untied pack: lm_head and model.embed_tokens are separate matrices, so correlating them is not a test of // anything. Say so, with the number, so the line cannot be read as a decoding failure. @@ -2814,8 +2844,11 @@ int run(int argc, char** argv) { " matrices), so their correlation %.6f and rel_err %.6f are what an untied pack\n" " looks like, not a decode failure; the controls still run\n", r.lmhead_corr, r.lmhead_rel); - std::printf(" controls: nibble-swapped corr %.4f, no zero-point corr %.4f (both must fail)\n", - r.ctrl_corr_nibble, r.ctrl_corr_nozp); + if (r.ctrl_corr_rowshift >= 0) + std::printf(" controls: row-shifted corr %.4f (must fail)\n", r.ctrl_corr_rowshift); + else + std::printf(" controls: nibble-swapped corr %.4f, no zero-point corr %.4f (both must fail)\n", + r.ctrl_corr_nibble, r.ctrl_corr_nozp); } else { std::printf(" self-check: skipped -- the head is not tied to the embedding, so the pack\n" " carries no independent reference to test the decode against\n"); diff --git a/npu/pack_import.h b/npu/pack_import.h index 6003f2000..87306bdf4 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -188,6 +188,10 @@ struct Report { // meaningless -- reported as such rather than left looking like a failed import. bool selfcheck_tied = false; double lmhead_corr = 0, lmhead_rel = 0, ctrl_corr_nibble = 0, ctrl_corr_nozp = 0; + // The tail (.fconst) path has no MatMulNBits bytes to mis-decode, so its control is an ALIGNMENT one: the same + // tiles, the same decoder, rows matched against the neighbouring embedding row. A correct decode correlates ~1 + // with the embedding; if a row-misaligned head still did, the check would be measuring nothing. -1 = not run. + double ctrl_corr_rowshift = -1; // Self-check 2: the written container, read back, against the pack's dequantised values. With // the fp32 q4_1f form (engine#1185) q4nx_rel and q4nx_max_abs are 0 and zero_exact_pct is 100: From 0019f0a4988420bf17aa6ba0a61424d967cd4c46 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 8 Nov 2026 16:34:20 -0400 Subject: [PATCH 7/7] npu-pack: Llama-3.2-1B's 1.9 shape, and the row that refutes my own explanation of the prefix The measurement tool found this entry on a non-Qwen family for the first time: {2048, 32, 16, 8512, 5, 1, 591872}, with 1,857 of 686,518,336 bytes differing from the 1.8 twin (2.7e-6 -- content, not layout). It also corrects a comment I wrote two entries ago. I had explained the extra 4,352 bytes in the prefix as Qwen3's per-head QK norm gains sitting inside the block before the bundle. Llama-3.2-1B has no QK norms at all and carries the same prefix at the same hidden size (8,512 at hidden 2,048), so the extra is a property of the geometry and not of the architecture. The table keeps doing its job -- it is measured -- but the explanation next to it was a plausible story that would have become the reason some future row was added wrongly. That is the second time this week a tidy explanation survived exactly one model. What differs from Qwen3-1.7B at the same hidden size is the bundle: 5 tiles of 591,872 against 3 of 1,183,744, so the two models cut the same payload into different pieces and the tile count alone identifies neither. Verified: import 147 tensors / 150,848 tiles (the same counts as its 1.8 twin), 1,280/1,280 attention tiles survive decode->encode, self-check corr 0.999999, and greedy text on the host GEMM: 'The capital of France is' -> ' Paris.\nThe capital of France is Paris. \nThe capital' '2 + 2 =' -> ' ?\nTo add 2 + 2, you simply count' 'Why is the sky blue?' -> ' The sky appears blue because of a phenomenon called Rayleigh scattering' --- npu/pack_import.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 55b8404c9..e3884f92b 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -281,8 +281,10 @@ struct Fconst19Rule { int64_t tile; }; // Every entry was measured by diffing the 1.9 pack against the 1.8 pack of the SAME model: the bundle's tile shape and -// its rotation are not derivable from the pack, and neither is `prefix` (Qwen3 puts its per-head QK norm gains inside -// the block before the bundle, so `64 + 2*hidden` is not the rule there). +// its rotation are not derivable from the pack, and neither is `prefix`. It was tempting to read the extra bytes as +// Qwen3's per-head QK norm gains, but Llama-3.2-1B -- which has no QK norms at all -- carries the same +4,352 at hidden +// 2,048, so the extra is a property of the geometry, not of the architecture. Measured anyway, because a plausible +// explanation that survives one model is how this table would get a wrong row. constexpr Fconst19Rule kFconst19Rules[] = { // Qwen2.5-0.5B-Instruct: normalises to its 1.8 twin byte for byte, whole file, all 24 blocks. {896, 14, 24, 1856, 16, 2, 34816}, @@ -293,6 +295,10 @@ constexpr Fconst19Rule kFconst19Rules[] = { // Qwen3 offset seen at 1.7B -- 64 + 2*hidden + 4,352, so 8,512 at hidden 2,048 and 9,536 at 2,560 -- and the tile // count differs from 1.7B's 3, which is why this is measured per pack and never derived. {2560, 32, 36, 9536, 5, 1, 1462272}, + // Llama-3.2-1B-Instruct: hidden 2,048 with 32/16 heads and no QK norms -- the first non-Qwen family here. Same + // prefix as Qwen3-1.7B at this hidden size (8,512), a different bundle (5 tiles of 591,872, not 3 of 1,183,744), + // and 1,857 of 686,518,336 bytes differ (2.7e-6). This row is the one that disproved the QK-norm explanation above. + {2048, 32, 16, 8512, 5, 1, 591872}, }; // The header a 1.9 block carries: 00 00 7f ff once per head. Its length being the head count is what makes