diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 7243c0898..1384e72d9 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -1366,6 +1366,24 @@ int intermediate_size_from_graph(const std::string& model_path, int64_t hidden) // with "the epsilon headers do not repeat with a single period" while its own header reported the stride exactly. // So ask the header first on the SOURCE file (the canonical copy has the 1.9 header removed by construction), and keep the eps // vote as the fallback for packs with no 1.9 header -- the 1.8 / R1-Distill line, where it works today. +FconstGeom fconst_layout_choose(const FconstGeom& voted, int64_t stride8, int64_t residue_off, int residue_count) { + FconstGeom g = voted; + g.stride = stride8; // the 1.9 header's stride wins; a disagreement was refused by the caller + g.why.clear(); + // A layout the VOTE found is the only witness to how many in-block headers a pack has: `period` is 2 on the + // Qwen2/Llama packs (q/k/v/o share one payload) and 4 on the Qwen3 ones (three in-block headers, the middle two being + // the per-head QK norms, which `post_attn_hdr()` reads as `hdr.back()`). Rewriting that here is not "populating the + // same fields" -- it discards the measurement that separates the two layouts, and on Qwen3 it pointed + // `post_attn_hdr()` at the k_norm header (engine#1831). Everything below fills in what the vote could NOT find. + if (!g.hdr.empty()) return g; + if (residue_off > 0 && residue_count >= 3) { // repeats across blocks: a real section boundary, not a coincidence + g.h1 = residue_off; + g.hdr.assign(1, residue_off); // 2-header block: q/k/v/o in one payload (Qwen2/Llama) + g.period = 2; + } + return g; +} + FconstGeom locate_fconst_with_header(const std::string& tok, const std::string& pack_dir, const Dims& dims) { FconstGeom g = locate_fconst(tok, dims.eps_bits); const std::string src = find_token_fconst(pack_dir); @@ -1385,38 +1403,23 @@ FconstGeom locate_fconst_with_header(const std::string& tok, const std::string& g.stride = 0; return g; } - g.stride = stride8; - g.why.clear(); // The second header offset (`h1`) is what tells the payload reader whether a block holds 2 eps headers or 4, so the eps // vote failing must not leave it at 0: with the stride known, the anchors' offsets *within* a block repeat, and the mode - // of `pos mod stride` (excluding 0, the block start) is the second header. This is the same pool of anchors the vote - // could not win on, used for a question that does not need a dominance test -- 28 blocks each contribute their residue. + // of `pos mod stride` (excluding 0, the block start) is the second header. The scan window is the first GiB (about 8 + // blocks of this geometry), so the right residue repeats ~8 times against 1-2 for a payload coincidence. const std::vector pos = eps_header_positions(tok, size_t(1) << 30, dims.eps_bits); + int best = 0; + int64_t best_off = 0; if (!pos.empty()) { std::map off; for (const int64_t q : pos) { const int64_t r = q % stride8; if (r > 0 && r < stride8) ++off[r]; } - int best = 0; - int64_t best_off = 0; for (const auto& [o, n] : off) if (n > best) { best = n; best_off = o; } - if (best_off > 0) { - // Populate the same fields `locate_fconst` would have: `hdr` is the in-block header list and the reader - // classifies a pack as a 2-header (one inner anchor) or 4-header (three) block from its SIZE. Setting only - // h1 left `hdr` empty, so a pack whose anchors could not win the vote was classified as "1 epsilon header" - // and every attention projection refused while the MLP imported byte-exact -- measured on Qwen2-7B 1.9. - // The scan window is the first GiB (about 8 blocks of this geometry), so the right residue repeats ~8 times - // against 1-2 for a payload coincidence. Requiring `layers/2` was too strict and silently dropped the real one. - if (best >= 3) { // the residue repeats across blocks: a real section boundary - g.h1 = best_off; - g.hdr.assign(1, best_off); // 2-header block: q/k/v/o in one payload (Qwen2/Llama) - g.period = 2; - } - } } - return g; + return fconst_layout_choose(g, stride8, best_off, best); } std::string find_token_fconst_exported(const std::string& pack_dir) { return find_token_fconst(pack_dir); } diff --git a/npu/pack_import.h b/npu/pack_import.h index 838513fbb..5d635b91b 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -440,6 +440,12 @@ int parse_onnx_matmul_census(const std::string& path, int* matmuls, int* without // engine#1638: the geometry as read, so a test can check WHICH source a number came from. nlohmann::json load_geometry_exported(const std::string& pack_dir, std::string* source); FconstGeom locate_fconst(const std::string& path, unsigned short eps_bits = 0x3586); +// The in-block layout to hand the reader: the epsilon vote's own when it found one, otherwise the 2-header shape a stride +// and a repeated residue imply. Split out so the rule "never rewrite a layout the vote found" is a testable function +// rather than a position inside a sixty-line one -- rewriting it made a Qwen3 pack's k_norm header read as +// post-attention (engine#1831). `residue_off`/`residue_count` are the mode of `eps position mod stride` and how often it +// repeated; the caller is expected to have already refused when the two witnesses disagree on the stride itself. +FconstGeom fconst_layout_choose(const FconstGeom& voted, int64_t stride8, int64_t residue_off, int residue_count); // One measured Ryzen AI 1.9 Token .fconst block shape (engine#1503). Every field is measured against the 1.8 pack of // the same model by tools/fconst19_measure.py; none of it is derivable from the pack, which is why a pack whose shape diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 8afb5cd79..8bc7a48c1 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -3208,8 +3208,44 @@ void test_tail_head_placement() { } +// engine#1831: the layout a 1.9 pack's header list describes is the only witness to how many in-block headers it has, +// and `locate_fconst_with_header()` used to rewrite it unconditionally to the 2-header shape -- which on a Qwen3 pack +// (three in-block headers, `period == 4`) pointed `post_attn_hdr()` at the k_norm section. Four arms: the vote's layout +// survives, a missing one is filled, a weak residue fills nothing, and a 2-header vote is not overwritten either. +static void test_fconst_layout_choose() { + pi::FconstGeom four; // what locate_fconst() returns for a Qwen3 pack + four.stride = 1 << 20; + four.period = 4; + four.h1 = 262144; + four.hdr = {262144, 524288, 786432}; + four.blocks = 8; + + const pi::FconstGeom keep = pi::fconst_layout_choose(four, four.stride, 131072, 8); + check(keep.period == 4 && keep.hdr.size() == 3 && keep.h1 == 262144 && keep.post_attn_hdr() == 786432, + "layout: a voted 4-header layout survives a strong residue (period " + std::to_string(keep.period) + + ", post-attn " + std::to_string(keep.post_attn_hdr()) + ")"); + + pi::FconstGeom none; // the vote found no stride: nothing to preserve + const pi::FconstGeom filled = pi::fconst_layout_choose(none, 1 << 20, 131072, 8); + check(filled.period == 2 && filled.hdr.size() == 1 && filled.h1 == 131072 && filled.post_attn_hdr() == 131072, + "layout: an empty vote is filled from the residue (period " + std::to_string(filled.period) + ")"); + + const pi::FconstGeom weak = pi::fconst_layout_choose(none, 1 << 20, 131072, 2); + check(weak.period == 0 && weak.hdr.empty() && weak.h1 == 0, + "layout: a residue that repeats twice is not a section boundary (" + std::to_string(weak.period) + ")"); + + pi::FconstGeom two = four; // a Qwen2/Llama pack + two.period = 2; + two.h1 = 131072; + two.hdr = {131072}; + const pi::FconstGeom kept2 = pi::fconst_layout_choose(two, two.stride, 65536, 8); + check(kept2.period == 2 && kept2.hdr.size() == 1 && kept2.h1 == 131072, + "layout: a voted 2-header layout is not overwritten either (h1 " + std::to_string(kept2.h1) + ")"); +} + int main(int argc, char** argv) { test_tail_head_placement(); + test_fconst_layout_choose(); const std::string golden_dir = argc > 1 ? argv[1] : ""; const std::string pack = argc > 2 ? argv[2] : ""; // The pack leg is the only one that can regress a real pack, and ctest runs this binary with the