From aa2d9686fff58258c6f327eed7c96921e27b01ef Mon Sep 17 00:00:00 2001 From: t Date: Tue, 10 Nov 2026 09:35:12 -0400 Subject: [PATCH] npu-pack: claim the per-head QK gains out of the attention gap, so Hunyuan-7B imports with its q/k norms (engine#1889) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #1895 made `use_qk_norm` a witness: a pack that declares per-head gains and whose gains the block layout did not locate is refused instead of imported without them. Hunyuan-7B-Instruct is exactly that pack **and the gains are there** — the importer's own note has said where since it was measured: ``` note: attention section is 96 tiles at K=4096 plus 640 B of per-head QK norms before o_proj ``` It computed that gap and then stepped over it without claiming it, so the refusal fired on a pack that carries everything. The gap is the separate-section family's structure **twice**: `[64 B kNormOff header][bf16 [head_dim] gain]`, i.e. `2 * (kNormOff + head_dim*2)` = **640 B at head_dim 128** — which is exactly this pack's `rem`. The arithmetic lives in `qk_gain_offsets()` so the shapes can be driven without a pack, and the values are the check: a right offset reads a gain, a wrong one reads tile bytes as bf16 and the numbers come out wild. ## Measured `Hunyuan-7B-Instruct_rai_1.9.0_npu`, `--weight-layout b8 --allow-incomplete`, this head: ``` attention: 5120/5120 .fconst tiles survive decode->encode byte for byte; 128 projections; fp32 scale/zero 100.00% MLP: 9216/9216 .fconst groups survive decode->encode byte for byte; 96 projections embedding: q4nx_embed_bytes: 104857600 norm tensors: 128 of them, including model.layers.0.self_attn.q_norm.weight [128] bf16 min 0.000 max 1.703 mean 1.371 model.layers.0.self_attn.k_norm.weight [128] bf16 min -0.000 max 1.703 mean 1.370 model.layers.31.self_attn.q_norm.weight [128] bf16 min -0.000 max 1.516 mean 1.285 ``` Those means are what a norm gain looks like; a one-block-off offset would put 4-bit tile bytes where a bf16 gain belongs and the numbers would be garbage rather than 1.0-2.0. **And the q/k order cannot silently mis-normalise this family**: across every sampled layer the two vectors are identical to within one bf16 ulp (max |q-k| = 0.0078 at ~1.3, 0.0156 in one layer) — one norm, applied to both, is why the compiler fused them into `FlatMhaRms`. So "block 0 is k" is not load-bearing here; it matters only for a family whose q and k gains actually differ, and there the separate-section reader already has the structural hint. ## Arms `test_qk_gain_offsets()`: the right shape gives 1064 and 1384 (320 B apart), and four neighbours must NOT be claimed — a gap one byte short, one byte long, two bare gains with no headers, and the right byte count at the wrong `head_dim`. Mutation: making the helper never claim fails the three positive arms. Full suite: `all npu-pack checks passed`; the reference 1.9 pack leg is unchanged. Refs #1889, #1895, #1900. `Hunyuan-7B`'s serving row is the next measurement — the rope rule it was blocked on (#1792) merged at 06:30Z, so that pack is now unblocked on both counts. --- npu/pack_import.cpp | 24 ++++++++++++++++++++++++ npu/pack_import.h | 4 ++++ tests/npu_pack_import_test.cpp | 19 +++++++++++++++++++ 3 files changed, 47 insertions(+) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 248c2f6ce..66ffe955d 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -2493,6 +2493,19 @@ std::string Plan::manifest() const { return j.dump(2); } +// engine#1889: where the per-head QK gains sit when the attention section carries them as the 640 B between the last +// qkv tile and o_proj. Split out and exposed because this is the arithmetic a wrong answer silently mis-rotates every +// Hunyuan pack with: a right offset reads a gain (~1.0-2.0), a wrong one reads tile bytes as bf16 and the gains come out +// wild, and the test drives both shapes without a pack. +QkGainOffsets qk_gain_offsets(int64_t gap_start, int64_t head_dim, int64_t gap_bytes) { + const int64_t per_head = head_dim * 2; + if (gap_bytes != 2 * (kNormOff + per_head)) return QkGainOffsets{}; + QkGainOffsets o; + o.k_norm = gap_start + kNormOff; // block 0 is k, as the separate-section family + o.q_norm = gap_start + (kNormOff + per_head) + kNormOff; // block 1 is q + return o; +} + std::string qk_norm_refusal(bool declared, bool found) { if (declared && !found) return "this pack declares use_qk_norm and its per-head q/k gains are not where the block layout says " @@ -2813,6 +2826,17 @@ Plan plan(const std::string& pack_dir) { p.attn_tiles = n_kv * 2 + n_q; p.o_off = attn_off + p.attn_tiles * tile_bytes + rem; p.o_k = int(H); + // engine#1889: that gap IS the per-head QK norms -- the note below has said so since it was + // measured, and its shape is the separate-section family's, twice: [64 B header][bf16 + // [head_dim] gain], i.e. 2 * (kNormOff + head_dim*2) = 640 B at head_dim 128, which is + // exactly the `rem` Hunyuan-7B reports. Naming it is what turns #1895's refusal ("declares + // use_qk_norm, gains not located") into an import; the values are the check (a wrong offset + // reads tile bytes as bf16 and the gains come out wild, a right one gives the 1.0-2.0 a norm + // gain has). + const QkGainOffsets qkg = + qk_gain_offsets(attn_off + p.attn_tiles * tile_bytes, int64_t(d.head_dim), rem); + p.knorm_off = qkg.k_norm; + p.qnorm_off = qkg.q_norm; static bool said2 = false; if (!said2) { said2 = true; diff --git a/npu/pack_import.h b/npu/pack_import.h index 01541c3d7..bb50c25ab 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -86,6 +86,10 @@ struct Dims { // REFUSAL (importing without the gains is the silent-garbage path), found-and-not-declared is allowed only with the // measurement behind it (Qwen3 packs ship the sections without the key), and neither is the ordinary case. Exposed // so the three can be pinned without a 4 GB pack. +struct QkGainOffsets { + int64_t k_norm = 0, q_norm = 0; // 0,0 = the gap is not the gains' shape +}; +QkGainOffsets qk_gain_offsets(int64_t gap_start, int64_t head_dim, int64_t gap_bytes); std::string qk_norm_refusal(bool declared, bool found); // A hybrid pack's embedding table lives in its own graph (`embedding.onnx` + `.data`, gathered with diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index a86bf9b9d..57c95e4f9 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -3468,8 +3468,27 @@ void test_qk_norm_refusal() { std::printf(" qk norm: 7 arms (declared+not-found refuses; found-only and neither are allowed)\n"); } +// engine#1889: the arithmetic that claims the per-head QK gains out of the 640 B Hunyuan-7B carries between its last qkv +// tile and o_proj. A wrong offset there reads tile bytes as bf16 and mis-rotates every pack of the family, so both the +// right shape (2 * (64 + head_dim*2) = 640 at head_dim 128) and the neighbours that must NOT be taken for it are driven. +void test_qk_gain_offsets() { + const pi::QkGainOffsets o = pi::qk_gain_offsets(1000, 128, 640); + check(o.k_norm == 1064, "qk gains: block 0 starts one kNormOff (64 B) after the gap"); + check(o.q_norm == 1384, "qk gains: block 1 starts after block 0 (64 B header + a 256 B bf16 [128] gain)"); + check(o.q_norm - o.k_norm == 320, "qk gains: the two blocks are 320 B apart, the same stride as the separate family"); + const pi::QkGainOffsets n1 = pi::qk_gain_offsets(1000, 128, 639); + const pi::QkGainOffsets n2 = pi::qk_gain_offsets(1000, 128, 641); + const pi::QkGainOffsets n3 = pi::qk_gain_offsets(1000, 128, 512); // just the two gains, no headers + const pi::QkGainOffsets n4 = pi::qk_gain_offsets(1000, 256, 640); // the right byte count at the wrong head_dim + check(n1.k_norm == 0 && n1.q_norm == 0, "qk gains: a gap one byte short is not the gains' shape"); + check(n2.k_norm == 0 && n2.q_norm == 0, "qk gains: a gap one byte long is not either"); + check(n3.k_norm == 0 && n3.q_norm == 0, "qk gains: two bare gains with no headers are not this shape"); + check(n4.k_norm == 0 && n4.q_norm == 0, "qk gains: the byte count alone is not enough, head_dim has to agree"); +} + int main(int argc, char** argv) { test_qk_norm_refusal(); + test_qk_gain_offsets(); test_tail_head_placement(); const std::string golden_dir = argc > 1 ? argv[1] : ""; const std::string pack = argc > 2 ? argv[2] : "";