From 6e184061acd8f2cc1c1989bfc363562235faff9d Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 22:00:26 -0400 Subject: [PATCH] npu-pack: read the gemma3 1.8 six-header layer block (engine#2604) `npu-pack info` on amd/translategemma-4b-it_rai_1.8.0_npu_4K reported 340 gaps, every one "the layer block has 6 epsilon headers, and this importer reads blocks of 2 ... or 4". The block is Qwen3's four headers plus Gemma-3's pre- and post-feedforward norms. Measured on the pack's own Token .fconst (2,189,399,392 B), by a scan for the envelope header (eps followed by 62 zero bytes -- searching the eps pair alone gives ~47,800 false positives): 205 headers = 34 x 6 + the tail, and per block h0->h1 q/k/v tiles | h1->h2 first per-head gain | h2->h3 second per-head gain [+32 B on the 29 SLIDING-window layers] then o_proj | h3->h4 post_attention | h4->h5 pre/post-feedforward, with the MLP between h4 and h5. Both spans reproduce the file exactly: 64 + 512 + 578*5120 = 2,959,936 on the 5 full-attention layers and + 32 = 2,959,968 on the other 29; the 32 B is 16 bf16 = 2 x num_attention_heads and the reader SKIPS it rather than interpreting it. The five short layers are the full_attention ones (config layer_types puts them at 5, 11, 17, 23, 29, matching the scan). Two changes: * the 6-header attention section; `extra in {0,32}` is derived from the section span itself, never from a version or a config; * mlp_start_in_block(): on this form the MLP starts after the SECOND-TO-LAST header and the block ends one envelope+weight after the MLP does. Reading the last header put the MLP one envelope past its start ("ends at 51,741,088 B in a 51,746,272-byte layer block"), and passing the full stride then made the section 5,184 B too generous. Verified: `npu-pack info` on the real pack -> 0 gaps (was 340). Arm: test_mlp_start_in_block pins the rule on the pack's own measured offsets plus the unchanged 4- and 2-header cases; reverting the helper to last-header makes 4 assertions fail. Not in this PR: the import still refuses, one layer later, on the HEAD -- "best-fit scale 50.325832, rms ratio 0.019792", i.e. the tail decodes ~50x too small (430,026,304 B over 262,208 x 2,560 = 0.6406 B/value, a q4_1 32-group tile at 0.625, so the scale/zero reading is the suspect). No device row: a clean import is not proof (engine#2594). --- npu/pack_import.cpp | 63 +++++++++++++++++++++++++++++++--- npu/pack_import.h | 18 ++++++++++ tests/npu_pack_import_test.cpp | 29 ++++++++++++++++ 3 files changed, 105 insertions(+), 5 deletions(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 9f548eb52..75297af82 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -13,6 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include "pack_import.h" #include "onnx_graph.h" @@ -4525,6 +4526,7 @@ Plan plan(const std::string& pack_dir) { const int64_t attn_off = attn_payload_off(H); const int64_t per_head = int64_t(d.head_dim) * 2; // one QK norm gain, bf16 const int64_t tile_o = attn_tile_bytes(q_out); + std::optional mlp_off_6h; // engine#2604: set only by the 6-header form const int64_t n_kv = kv_out / 64, n_q = q_out / 64; auto refuse = [&](const std::string& s) { p.attn_note = s; @@ -4655,14 +4657,65 @@ Plan plan(const std::string& pack_dir) { p.o_off = g.hdr[1] + kNormOff + per_head; p.o_k = q_out; } + } else if (g.hdr.size() == 5) { + // engine#2604: the gemma3 1.8 Token block -- Qwen3's four headers plus Gemma-3's pre- and + // post-feedforward norms, i.e. six per layer. Measured on amd/translategemma-4b-it_rai_1.8.0_npu_4K + // (hidden 2560, 34 layers, 8 heads, head_dim 256): headers are `eps` + 62 zero bytes, 205 of them + // (34 x 6 + the tail), and the block spans are + // h0->h1 q/k/v tiles | h1->h2 first per-head gain | h2->h3 second per-head gain [+32 B on the 29 + // SLIDING-window layers] then o_proj | h3->h4 post_attention | h4->h5 pre/post-feedforward | MLP. + // Both spans reproduce exactly: 64 + 512 + 578*5120 = 2,959,936 on the 5 full-attention layers, and + // + 32 = 2,959,968 on the other 29. The 32 B is 16 bf16 = 2 x num_attention_heads: a per-head scalar + // pair the reader has to SKIP, not interpret. Its presence is cross-checked against the span itself, + // so nothing here is inferred from a version or a config. + const int64_t kv_tiles = n_kv + n_q + n_kv; + const int64_t s0 = g.hdr[0] - attn_off; + const int64_t s1 = g.hdr[1] - g.hdr[0]; + const int64_t s2 = g.hdr[2] - g.hdr[1]; + const int64_t after_gains = s2 - (kNormOff + per_head); // o_proj tiles, plus that 32 B + const int64_t extra = after_gains >= 0 ? after_gains % tile_o : -1; + const int64_t n_otiles = after_gains >= 0 ? after_gains / tile_o : 0; + if (s0 != kv_tiles * tile_bytes) + refuse("the attention payload before the first per-head gain is " + std::to_string(s0) + + " B; k/q/v at 64 rows each are " + std::to_string(kv_tiles * tile_bytes) + " B"); + else if (s1 != kNormOff + per_head) + refuse("the section between the block's first two gains is " + std::to_string(s1) + " B; a " + + std::to_string(kNormOff) + "-byte header plus a bf16 [head_dim " + + std::to_string(d.head_dim) + "] gain is " + std::to_string(kNormOff + per_head)); + else if (n_otiles != H / kTileRows || (extra != 0 && extra != 32)) + refuse("the o_proj section is " + std::to_string(after_gains) + " B after the gains; hidden " + + std::to_string(H) + " rows at K = q_proj's " + std::to_string(q_out) + " are " + + std::to_string(H / kTileRows) + " tiles of " + std::to_string(tile_o) + + " B, optionally preceded by 32 B of per-head scalars on a sliding-window layer"); + else { + p.attn_tiles = kv_tiles; + p.knorm_off = g.hdr[0] + kNormOff; + p.qnorm_off = g.hdr[1] + kNormOff; + p.o_off = g.hdr[1] + kNormOff + per_head + extra; // skip the per-head scalar pair + p.o_k = q_out; + // engine#2604: the MLP sits BETWEEN the pre- and post-feedforward norms, so `hdr.back()` + // (the post-feedforward header) is PAST it -- measured on translategemma-4b: h3 8,814,816, + // h4 8,820,000, MLP 42,921,088, h5 51,741,088. The MLP therefore starts after h4. + mlp_off_6h = g.hdr[3] + kNormOff + H * 2; + } } else { refuse("the layer block has " + std::to_string(g.hdr.size() + 1) + - " epsilon headers, and this importer reads blocks of 2 (q/k/v/o in one payload) " - "or 4 (q/k/v, two per-head QK norms, o_proj) -- recorded 2026-11-04 on Qwen3"); + " epsilon headers, and this importer reads blocks of 2 (q/k/v/o in one payload), " + "4 (q/k/v, two per-head QK norms, o_proj) or 6 (the same plus pre/post-feedforward " + "norms, engine#2604) -- recorded 2026-11-04 on Qwen3"); } - // the MLP section starts right after the block's post-attention norm - const int64_t mlp_off = g.post_attn_hdr() + kNormOff + H * 2; - const MlpGeom mg = mlp_geom(H, d.intermediate, mlp_off, g.stride); + // the MLP section starts right after the block's post-attention norm -- except on the 6-header + // form, where the pre/post-feedforward norms straddle it and the branch above moves mlp_off. + int64_t mlp_end_trim = 0; + const int64_t mlp_off = mlp_off_6h ? *mlp_off_6h + : mlp_start_in_block(g.hdr, kNormOff, H, &mlp_end_trim); + // engine#2604: on the 6-header form the MLP is followed by the post-feedforward norm, which is an + // envelope + bf16[H] (5,184 B at hidden 2560) INSIDE the block -- so the section ends that much + // before the block does. Passing the full stride made the decoder expect 5,184 B more than the + // file has (measured: "ends at 51,741,088 in a 51,746,272-byte block", the two differing by + // exactly h5's own envelope+weight). + const int64_t mlp_end = g.stride - mlp_end_trim; + const MlpGeom mg = mlp_geom(H, d.intermediate, mlp_off, mlp_end); p.mlp_note = mg.why; if (mg.ok()) { p.mlp_gu_units = mg.gu_units; diff --git a/npu/pack_import.h b/npu/pack_import.h index ae82a0314..043bc9af1 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -960,6 +960,24 @@ struct FconstGeom { int64_t post_attn_hdr() const { return hdr.empty() ? h1 : hdr.back(); } std::string why; // non-empty when the geometry could not be established }; +// engine#2604: where the MLP section starts inside a layer block, and how many bytes of the block's tail are NOT +// part of it. Two forms, and the difference is a whole norm: +// * 2- and 4-header blocks (Qwen2.5/Llama/Qwen3): the MLP follows the block's LAST header (post_attention), and +// nothing trails it -- `end_trim` is 0. +// * the 6-header gemma3 form: the pre-feedforward norm opens the MLP and the POST-feedforward norm closes it, so +// the MLP starts after the SECOND-TO-LAST header and the block ends `kNormOff + 2H` bytes after the MLP does. +// Measured on translategemma-4b (hidden 2560): h3 8,814,816, h4 8,820,000, MLP 42,921,088, h5 51,741,088. +// Reading the last header on the 6-header form put the MLP one envelope past its start, which is the refusal +// "the MLP section ends at 51,741,088 B in a 51,746,272-byte layer block". +inline int64_t mlp_start_in_block(const std::vector& hdr, int64_t kNormOff, int64_t H, int64_t* end_trim) { + const int64_t last = hdr.empty() ? 0 : hdr.back(); + if (hdr.size() == 5) { // gemma3: five in-block headers, the last one AFTER the MLP + *end_trim = kNormOff + H * 2; + return hdr[3] + kNormOff + H * 2; + } + *end_trim = 0; + return last + kNormOff + H * 2; +} unsigned short bf16_bits_of(float v); // engine#1650: config.json may declare an OPTIONAL field as JSON null (AMD's 1.9 diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 2846056cf..cbd7e89e2 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -3904,6 +3904,34 @@ void test_import_fp32_params(const std::string& root) { // payload came out as "whole tiles plus a stray run" and the attention section was refused. The two // numbers below are the packs' OWN measured second-header offsets, so this is a known-answer test: with // the derived offset each section is exactly the tiles q/k/v/o need, and with the old constant it is not. +// engine#2604: the MLP offset rule, with the real pack's own header offsets. The 6-header gemma3 block puts the +// pre-feedforward norm BEFORE the MLP and the post-feedforward norm AFTER it, so the block's last header is past +// the section at BOTH ends -- reading it as the MLP start is the refusal "the MLP section ends at 51,741,088 B in a +// 51,746,272-byte layer block", and reading the full stride as the MLP's length is 5,184 B too generous. +void test_mlp_start_in_block() { + const int64_t kNormOff = 64, H = 2560; + // translategemma-4b layer 0, measured: q/k/v | h1 5,854,272 | h2 5,854,848 | h3 8,814,816 | h4 8,820,000 | + // MLP 42,921,088 | h5 51,741,088 | next block + const std::vector six{5854272, 5854848, 8814816, 8820000, 51741088}; + int64_t trim = -1; + const int64_t start = pi::mlp_start_in_block(six, kNormOff, H, &trim); + check(start == 8820000 + 64 + H * 2, "mlp start: the 6-header form opens the MLP after h4, not h5 (" + + std::to_string(start) + " vs " + std::to_string(8820000 + 64 + H * 2) + ")"); + check(start != six.back() + 64 + H * 2, "mlp start: h5 is past the MLP, so it is not the start"); + check(trim == 64 + H * 2, "mlp start: the block ends one envelope+weight after the MLP (" + std::to_string(trim) + ")"); + check(51746272 - start - trim == 42915904, "mlp start: the section is 42,915,904 B, as the file has"); + // a 4-header block is unchanged by the rule: last header is the post-attention norm, nothing trails it + const std::vector four{1000, 2000, 3000}; + int64_t trim4 = -1; + const int64_t s4 = pi::mlp_start_in_block(four, kNormOff, H, &trim4); + check(s4 == 3000 + 64 + H * 2 && trim4 == 0, "mlp start: a 4-header block keeps the last header and trims nothing"); + // a 2-header block likewise + const std::vector two{5000}; + int64_t trim2 = -1; + check(pi::mlp_start_in_block(two, kNormOff, H, &trim2) == 5000 + 64 + H * 2 && trim2 == 0, + "mlp start: a 2-header block keeps its only in-block header"); +} + void test_attn_payload_off() { check(pi::attn_payload_off(896) == 1856, "attn off: hidden 896 is the 1,856 the importer shipped with (64 + 2 x 896)"); @@ -7055,6 +7083,7 @@ int main(int argc, char** argv) { test_mlp_gu(golden_dir); test_fconst_eps(golden_dir.empty() ? "." : golden_dir); test_fconst_variable_stride(golden_dir.empty() ? "." : golden_dir); + test_mlp_start_in_block(); test_attn_payload_off(); test_text_config_geometry(golden_dir.empty() ? "." : golden_dir); test_use_qk_norm_declaration(golden_dir.empty() ? "." : golden_dir);