diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index ea8a61d5a..ec2772511 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -1407,34 +1407,51 @@ FconstGeom locate_fconst_with_header(const std::string& tok, const std::string& // vote failing must not leave it at 0: with the stride known, the anchors' offsets *within* a block repeat, and the mode // of `pos mod stride` (excluding 0, the block start) is the second header. This is the same pool of anchors the vote // could not win on, used for a question that does not need a dominance test -- 28 blocks each contribute their residue. + // engine#1831: only when the vote did NOT record the block's headers. When it did, its list is the measurement + // (`plan()` reads 2-header vs 4-header from `hdr.size()`, and `post_attn_hdr()` is `hdr.back()`), and rewriting it + // to one entry turned every Qwen3 4-header 1.9 block into a 2-header one whose "post-attention" header was k_norm's. + if (!g.hdr.empty()) return g; const std::vector pos = eps_header_positions(tok, size_t(1) << 30, dims.eps_bits); - if (!pos.empty()) { - std::map off; - for (const int64_t q : pos) { - const int64_t r = q % stride8; - if (r > 0 && r < stride8) ++off[r]; - } - int best = 0; - int64_t best_off = 0; - for (const auto& [o, n] : off) - if (n > best) { best = n; best_off = o; } - if (best_off > 0) { - // Populate the same fields `locate_fconst` would have: `hdr` is the in-block header list and the reader - // classifies a pack as a 2-header (one inner anchor) or 4-header (three) block from its SIZE. Setting only - // h1 left `hdr` empty, so a pack whose anchors could not win the vote was classified as "1 epsilon header" - // and every attention projection refused while the MLP imported byte-exact -- measured on Qwen2-7B 1.9. - // The scan window is the first GiB (about 8 blocks of this geometry), so the right residue repeats ~8 times - // against 1-2 for a payload coincidence. Requiring `layers/2` was too strict and silently dropped the real one. - if (best >= 3) { // the residue repeats across blocks: a real section boundary - g.h1 = best_off; - g.hdr.assign(1, best_off); // 2-header block: q/k/v/o in one payload (Qwen2/Llama) - g.period = 2; - } - } + // The scan window is the first GiB (about 8 blocks at h3584), so a real section boundary repeats ~8 times against + // 1-2 for a payload coincidence; 3 separates them (requiring `layers/2` was too strict and dropped the real one). + const std::vector inner = fconst_block_headers(pos, stride8, 3); + if (!fconst_set_block_headers(g, inner)) { + g.why = "the 1.9 header gives stride " + std::to_string(stride8) + ", but the epsilon anchors repeat at " + + std::to_string(inner.size()) + " in-block offset(s), and a layer block has 1 (2-header) or 3 (4-header); " + "refusing rather than guessing the layout"; + g.stride = 0; } return g; } +// engine#1831: every in-block header offset the anchors support, sorted. A header at offset `r` of a block of `stride` +// bytes lands at `r` modulo `stride` in every block, so the offsets that recur `min_repeat` or more times are the block's +// section boundaries; offset 0 is the block start, not an inner header. ALL of them are kept rather than the most common +// one: a Qwen3 block has three equally frequent inner headers, and keeping only the first maximum picked k_norm's. +std::vector fconst_block_headers(const std::vector& pos, int64_t stride, int min_repeat) { + std::vector out; + if (stride <= 0) return out; + std::map count; + for (const int64_t q : pos) { + const int64_t r = q % stride; + if (r > 0) ++count[r]; + } + for (const auto& [r, n] : count) + if (n >= min_repeat) out.push_back(r); // std::map iterates in order, so `out` is sorted + return out; +} + +// engine#1831: the fields `locate_fconst()` sets from the same list -- `hdr` (whose SIZE is the block layout), `h1` (its +// first entry) and `period` (headers per block, counting the block's own). One inner header is a 2-header block +// (Qwen2.5/Llama), three is a 4-header block (Qwen3); anything else is not a layout this reader knows, so nothing is set. +bool fconst_set_block_headers(FconstGeom& g, const std::vector& inner) { + if (inner.size() != 1 && inner.size() != 3) return false; + g.hdr = inner; + g.h1 = inner.front(); + g.period = int(inner.size()) + 1; + return true; +} + std::string find_token_fconst_exported(const std::string& pack_dir) { return find_token_fconst(pack_dir); } // engine#1638: the geometry READ (not just its dims), so a test can assert where each number came from -- the diff --git a/npu/pack_import.h b/npu/pack_import.h index 8ffe7a16f..f039305e7 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -444,6 +444,13 @@ int parse_onnx_matmul_census(const std::string& path, int* matmuls, int* without // engine#1638: the geometry as read, so a test can check WHICH source a number came from. nlohmann::json load_geometry_exported(const std::string& pack_dir, std::string* source); FconstGeom locate_fconst(const std::string& path, unsigned short eps_bits = 0x3586); +// engine#1831: the in-block header offsets the epsilon anchors support at a known stride, sorted -- every offset that +// recurs `min_repeat` or more times (offset 0, the block start, excluded). Kept as a list, not a mode: a Qwen3 block has +// three equally frequent inner headers. +std::vector fconst_block_headers(const std::vector& pos, int64_t stride, int min_repeat); +// engine#1831: set `hdr`/`h1`/`period` from that list the way locate_fconst() does -- one inner header is a 2-header block, +// three is a 4-header block. Returns false, and changes nothing, for any other count. +bool fconst_set_block_headers(FconstGeom& g, const std::vector& inner); // One measured Ryzen AI 1.9 Token .fconst block shape (engine#1503). Every field is measured against the 1.8 pack of // the same model by tools/fconst19_measure.py; none of it is derivable from the pack, which is why a pack whose shape diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 6e0ba47d3..de57884a2 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -2644,6 +2644,56 @@ void test_fconst19_rule_table() { std::printf("fconst19 rule: measured row, unmeasured refusal, and the ambiguity guard all behave\n"); } +// engine#1831: the header-derived layout keeps EVERY in-block header the anchors support. #1819's version kept only the most +// common residue and always wrote a 2-header block, so a Qwen3 4-header block (three equally frequent inner headers) was read +// as 2-header with k_norm's header standing in for the post-attention one -- the engine#1003 misread, reintroduced. +void test_fconst_block_headers() { + const int64_t stride = 1000; + auto blocks = [&](std::initializer_list inner, int nblocks, std::initializer_list noise) { + std::vector pos; + for (int b = 0; b < nblocks; ++b) { + pos.push_back(int64_t(b) * stride); // the block's own (first) header + for (const int64_t r : inner) pos.push_back(int64_t(b) * stride + r); + } + for (const int64_t q : noise) pos.push_back(q); // eps-SHAPED payload bytes: once or twice + std::sort(pos.begin(), pos.end()); + return pos; + }; + // 2-header (Qwen2.5/Llama): one inner header, 8 blocks, two coincidences elsewhere in the payload + { + const auto inner = pi::fconst_block_headers(blocks({400}, 8, {1234, 5678}), stride, 3); + check(inner.size() == 1 && inner[0] == 400, "block headers: a 2-header block yields its one inner header"); + pi::FconstGeom g; + g.stride = stride; + check(pi::fconst_set_block_headers(g, inner) && g.period == 2 && g.h1 == 400 && g.hdr.size() == 1 && + g.post_attn_hdr() == 400, + "block headers: and is set as a 2-header block whose post-attention header is h1"); + } + // 4-header (Qwen3): three inner headers repeating equally -- ALL must be kept, in order, so post_attn is the LAST one + { + const auto inner = pi::fconst_block_headers(blocks({300, 320, 340}, 8, {777}), stride, 3); + check(inner.size() == 3 && inner[0] == 300 && inner[1] == 320 && inner[2] == 340, + "block headers: a 4-header block yields all three inner headers, sorted (not just the most common one)"); + pi::FconstGeom g; + g.stride = stride; + check(pi::fconst_set_block_headers(g, inner) && g.period == 4 && g.h1 == 300 && g.hdr.size() == 3 && + g.post_attn_hdr() == 340, + "block headers: and is set as a 4-header block whose post-attention header is the last one, not k_norm's"); + } + // coincidences alone are not a header, and a count that is neither 1 nor 3 sets nothing + { + check(pi::fconst_block_headers(blocks({}, 8, {111, 222, 222}), stride, 3).empty(), + "block headers: payload coincidences that recur fewer than min_repeat times are not headers"); + pi::FconstGeom g; + g.stride = stride; + g.h1 = 7; + check(!pi::fconst_set_block_headers(g, {100, 200}) && g.hdr.empty() && g.h1 == 7 && g.period == 0, + "block headers: two inner headers is not a known layout, and nothing is changed"); + check(pi::fconst_block_headers({0, 1000, 2000}, 0, 1).empty(), "block headers: a zero stride yields nothing"); + } + std::printf(" fconst block headers: 2-header and 4-header layouts kept whole; unknown counts refused\n"); +} + // engine#1801's reviewer asked for the test that would catch the case the fix itself found once: the canonical .fconst copy is a // cache whose size is the pack's own .fconst, and on this box /tmp is tmpfs, so where it lands is a RAM decision. The bug was a // cache directory that was never created -- every pack then fell through to /tmp and nothing noticed. @@ -3436,6 +3486,7 @@ int main(int argc, char** argv) { test_embedding_component(golden_dir.empty() ? "." : golden_dir); test_fconst19_rule_table(); test_fconst_scratch_order(); + test_fconst_block_headers(); test_selfcheck_reads_bf16(); if (const char* rc = std::getenv("NPU_EMBED_COMPONENT")) dump_real_component(rc); test_model_config_geometry(golden_dir.empty() ? "." : golden_dir);