diff --git a/docs/npu.md b/docs/npu.md index 99af6263b..94ab18a0d 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1136,6 +1136,17 @@ their hidden sizes differ). `plan()` accepts a block of 2 or 4 headers, checks e that arithmetic to the byte, and refuses any other count by name. FACTS §7.24 has the tables and the checkpoint evidence. +**A 2-header block can carry the gains too, in the other order** (engine#1889, engine#1934). The +Hunyuan packs declare `use_qk_norm: true` and keep one header per block: the gains are the 640 B +between the last q/k/v tile and o_proj, which the pack's metadata names as one tensor +(`model.layers.L.attn.q_norm.layernorm.weight.qk_rms_packed_26_L`, bf16 `[320]`). Its layout is +`[64 B zeros][q_norm, bf16 [128]][64 B zeros][k_norm, bf16 [128]]` — **q first**, the reverse of +the Qwen3 sections above. The order is not readable from the bytes (the two gains differ by about +one bf16 ulp in a few elements, so both look like a gain), so it was taken from the pack's own +graph: `model.onnx` carries `attn.q_norm.layernorm.weight` and `attn.k_norm.layernorm.weight` as +FP16 `[128]` initializers, and on Hunyuan-7B-Instruct their bf16 rounding is the first and the +second block bit for bit in 32 of 32 layers. `qk_gain_offsets()` holds that arithmetic. + **Every tile read is its own check.** Before anything is written, each tile is decoded and re-encoded; the encoder has to reproduce the 34,816 bytes it came from, bias slots included. That one comparison holds every field, index and nibble order at once, and it is why a tile whose diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 75e4b7b0b..a9d1b93c5 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -2747,14 +2747,20 @@ std::string Plan::manifest() const { // engine#1889: where the per-head QK gains sit when the attention section carries them as the 640 B between the last // qkv tile and o_proj. Split out and exposed because this is the arithmetic a wrong answer silently mis-rotates every -// Hunyuan pack with: a right offset reads a gain (~1.0-2.0), a wrong one reads tile bytes as bf16 and the gains come out -// wild, and the test drives both shapes without a pack. +// Hunyuan pack with: a wrong offset reads tile bytes as bf16 and the gains come out wild, and a wrong ORDER reads two +// plausible gains under each other's name (engine#1934). The test drives the shapes and the order without a pack. QkGainOffsets qk_gain_offsets(int64_t gap_start, int64_t head_dim, int64_t gap_bytes) { const int64_t per_head = head_dim * 2; if (gap_bytes != 2 * (kNormOff + per_head)) return QkGainOffsets{}; + // engine#1934: block 0 is Q and block 1 is K here -- the reverse of the separate-section family ([k][q], engine#979). + // Measured against the pack's own graph, not inferred from the bytes: Hunyuan-7B's model.onnx carries + // `model.layers.L.attn.{q,k}_norm.layernorm.weight` as FP16 [128] initializers, and rounded to bf16 q_norm is block 0 + // and k_norm is block 1 bit for bit in 32 of 32 layers, each behind a 64 B all-zero header. The two gains differ by + // about one bf16 ulp in ~17 of 128 elements and are identical in no layer, so the wrong order is invisible to a value + // check ("looks like a gain") and to the served text; only the graph tells them apart. QkGainOffsets o; - o.k_norm = gap_start + kNormOff; // block 0 is k, as the separate-section family - o.q_norm = gap_start + (kNormOff + per_head) + kNormOff; // block 1 is q + o.q_norm = gap_start + kNormOff; // block 0: [64 B zeros][q_norm bf16 [head_dim]] + o.k_norm = gap_start + (kNormOff + per_head) + kNormOff; // block 1: [64 B zeros][k_norm bf16 [head_dim]] return o; } @@ -3082,9 +3088,10 @@ Plan plan(const std::string& pack_dir) { // measured, and its shape is the separate-section family's, twice: [64 B header][bf16 // [head_dim] gain], i.e. 2 * (kNormOff + head_dim*2) = 640 B at head_dim 128, which is // exactly the `rem` Hunyuan-7B reports. Naming it is what turns #1895's refusal ("declares - // use_qk_norm, gains not located") into an import; the values are the check (a wrong offset - // reads tile bytes as bf16 and the gains come out wild, a right one gives the 1.0-2.0 a norm - // gain has). + // use_qk_norm, gains not located") into an import. The pack names the whole gap as one + // tensor (`...q_norm.layernorm.weight.qk_rms_packed_26_L`, bf16 [320]); which block is q and + // which is k comes from the pack's graph (engine#1934, see qk_gain_offsets()), because a + // value check cannot tell two gains apart. const QkGainOffsets qkg = qk_gain_offsets(attn_off + p.attn_tiles * tile_bytes, int64_t(d.head_dim), rem); p.knorm_off = qkg.k_norm; diff --git a/npu/pack_import.h b/npu/pack_import.h index 91a24a4b7..dcd0aa86a 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -88,6 +88,8 @@ struct Dims { // REFUSAL (importing without the gains is the silent-garbage path), found-and-not-declared is allowed only with the // measurement behind it (Qwen3 packs ship the sections without the key), and neither is the ordinary case. Exposed // so the three can be pinned without a 4 GB pack. +// The gap is `[64 B zeros][q_norm bf16 [head_dim]][64 B zeros][k_norm bf16 [head_dim]]`: q first (engine#1934, measured +// against the q_norm/k_norm initializers in Hunyuan-7B's own model.onnx). struct QkGainOffsets { int64_t k_norm = 0, q_norm = 0; // 0,0 = the gap is not the gains' shape }; diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index acd95805f..d699b37cb 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -4189,9 +4189,20 @@ void test_qk_norm_refusal() { // right shape (2 * (64 + head_dim*2) = 640 at head_dim 128) and the neighbours that must NOT be taken for it are driven. void test_qk_gain_offsets() { const pi::QkGainOffsets o = pi::qk_gain_offsets(1000, 128, 640); - check(o.k_norm == 1064, "qk gains: block 0 starts one kNormOff (64 B) after the gap"); - check(o.q_norm == 1384, "qk gains: block 1 starts after block 0 (64 B header + a 256 B bf16 [128] gain)"); - check(o.q_norm - o.k_norm == 320, "qk gains: the two blocks are 320 B apart, the same stride as the separate family"); + // engine#1934: block 0 is Q. Hunyuan-7B's own model.onnx carries both gains as FP16 initializers, and q_norm is the + // first block and k_norm the second, bit for bit, in 32 of 32 layers (layer 0: q_norm bytes at 13,795,456 = gap + 64, + // k_norm bytes 320 B later). The order this arm replaced had them the other way round, and nothing else could see + // it: the two gains differ by about one bf16 ulp, so both orders "look like a gain". + check(o.q_norm == 1064, "qk gains: block 0, one kNormOff (64 B) after the gap, is q_norm (engine#1934)"); + check(o.k_norm == 1384, "qk gains: block 1, after block 0 (64 B header + a 256 B bf16 [128] gain), is k_norm"); + check(o.k_norm - o.q_norm == 320, "qk gains: the two blocks are 320 B apart, q first"); + // the two windows are disjoint and inside the gap: neither a copy of the other nor a read past o_proj's start + check(o.q_norm + 256 <= o.k_norm - 64, "qk gains: q's window ends before k's 64 B header starts"); + check(o.q_norm >= 1000 + 64 && o.k_norm + 256 == 1000 + 640, "qk gains: both windows lie inside the 640 B gap"); + // Layer 0's real numbers, so the arm is tied to the measurement rather than to a round example. + const pi::QkGainOffsets hy = pi::qk_gain_offsets(13795392, 128, 640); + check(hy.q_norm == 13795456 && hy.k_norm == 13795776, + "qk gains: Hunyuan-7B layer 0 -- q_norm at 13,795,456 and k_norm at 13,795,776, where its graph's bytes are"); const pi::QkGainOffsets n1 = pi::qk_gain_offsets(1000, 128, 639); const pi::QkGainOffsets n2 = pi::qk_gain_offsets(1000, 128, 641); const pi::QkGainOffsets n3 = pi::qk_gain_offsets(1000, 128, 512); // just the two gains, no headers