Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 40 additions & 23 deletions npu/pack_import.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1407,34 +1407,51 @@ FconstGeom locate_fconst_with_header(const std::string& tok, const std::string&
// vote failing must not leave it at 0: with the stride known, the anchors' offsets *within* a block repeat, and the mode
// of `pos mod stride` (excluding 0, the block start) is the second header. This is the same pool of anchors the vote
// could not win on, used for a question that does not need a dominance test -- 28 blocks each contribute their residue.
// engine#1831: only when the vote did NOT record the block's headers. When it did, its list is the measurement
// (`plan()` reads 2-header vs 4-header from `hdr.size()`, and `post_attn_hdr()` is `hdr.back()`), and rewriting it
// to one entry turned every Qwen3 4-header 1.9 block into a 2-header one whose "post-attention" header was k_norm's.
if (!g.hdr.empty()) return g;
const std::vector<int64_t> pos = eps_header_positions(tok, size_t(1) << 30, dims.eps_bits);
if (!pos.empty()) {
std::map<int64_t, int> off;
for (const int64_t q : pos) {
const int64_t r = q % stride8;
if (r > 0 && r < stride8) ++off[r];
}
int best = 0;
int64_t best_off = 0;
for (const auto& [o, n] : off)
if (n > best) { best = n; best_off = o; }
if (best_off > 0) {
// Populate the same fields `locate_fconst` would have: `hdr` is the in-block header list and the reader
// classifies a pack as a 2-header (one inner anchor) or 4-header (three) block from its SIZE. Setting only
// h1 left `hdr` empty, so a pack whose anchors could not win the vote was classified as "1 epsilon header"
// and every attention projection refused while the MLP imported byte-exact -- measured on Qwen2-7B 1.9.
// The scan window is the first GiB (about 8 blocks of this geometry), so the right residue repeats ~8 times
// against 1-2 for a payload coincidence. Requiring `layers/2` was too strict and silently dropped the real one.
if (best >= 3) { // the residue repeats across blocks: a real section boundary
g.h1 = best_off;
g.hdr.assign(1, best_off); // 2-header block: q/k/v/o in one payload (Qwen2/Llama)
g.period = 2;
}
}
// The scan window is the first GiB (about 8 blocks at h3584), so a real section boundary repeats ~8 times against
// 1-2 for a payload coincidence; 3 separates them (requiring `layers/2` was too strict and dropped the real one).
const std::vector<int64_t> inner = fconst_block_headers(pos, stride8, 3);
if (!fconst_set_block_headers(g, inner)) {
g.why = "the 1.9 header gives stride " + std::to_string(stride8) + ", but the epsilon anchors repeat at " +
std::to_string(inner.size()) + " in-block offset(s), and a layer block has 1 (2-header) or 3 (4-header); "
"refusing rather than guessing the layout";
g.stride = 0;
}
return g;
}

// engine#1831: every in-block header offset the anchors support, sorted. A header at offset `r` of a block of `stride`
// bytes lands at `r` modulo `stride` in every block, so the offsets that recur `min_repeat` or more times are the block's
// section boundaries; offset 0 is the block start, not an inner header. ALL of them are kept rather than the most common
// one: a Qwen3 block has three equally frequent inner headers, and keeping only the first maximum picked k_norm's.
std::vector<int64_t> fconst_block_headers(const std::vector<int64_t>& pos, int64_t stride, int min_repeat) {
std::vector<int64_t> out;
if (stride <= 0) return out;
std::map<int64_t, int> count;
for (const int64_t q : pos) {
const int64_t r = q % stride;
if (r > 0) ++count[r];
}
for (const auto& [r, n] : count)
if (n >= min_repeat) out.push_back(r); // std::map iterates in order, so `out` is sorted
return out;
}

// engine#1831: the fields `locate_fconst()` sets from the same list -- `hdr` (whose SIZE is the block layout), `h1` (its
// first entry) and `period` (headers per block, counting the block's own). One inner header is a 2-header block
// (Qwen2.5/Llama), three is a 4-header block (Qwen3); anything else is not a layout this reader knows, so nothing is set.
bool fconst_set_block_headers(FconstGeom& g, const std::vector<int64_t>& inner) {
if (inner.size() != 1 && inner.size() != 3) return false;
g.hdr = inner;
g.h1 = inner.front();
g.period = int(inner.size()) + 1;
return true;
}

std::string find_token_fconst_exported(const std::string& pack_dir) { return find_token_fconst(pack_dir); }

// engine#1638: the geometry READ (not just its dims), so a test can assert where each number came from -- the
Expand Down
7 changes: 7 additions & 0 deletions npu/pack_import.h
Original file line number Diff line number Diff line change
Expand Up @@ -444,6 +444,13 @@ int parse_onnx_matmul_census(const std::string& path, int* matmuls, int* without
// engine#1638: the geometry as read, so a test can check WHICH source a number came from.
nlohmann::json load_geometry_exported(const std::string& pack_dir, std::string* source);
FconstGeom locate_fconst(const std::string& path, unsigned short eps_bits = 0x3586);
// engine#1831: the in-block header offsets the epsilon anchors support at a known stride, sorted -- every offset that
// recurs `min_repeat` or more times (offset 0, the block start, excluded). Kept as a list, not a mode: a Qwen3 block has
// three equally frequent inner headers.
std::vector<int64_t> fconst_block_headers(const std::vector<int64_t>& pos, int64_t stride, int min_repeat);
// engine#1831: set `hdr`/`h1`/`period` from that list the way locate_fconst() does -- one inner header is a 2-header block,
// three is a 4-header block. Returns false, and changes nothing, for any other count.
bool fconst_set_block_headers(FconstGeom& g, const std::vector<int64_t>& inner);

// One measured Ryzen AI 1.9 Token .fconst block shape (engine#1503). Every field is measured against the 1.8 pack of
// the same model by tools/fconst19_measure.py; none of it is derivable from the pack, which is why a pack whose shape
Expand Down
51 changes: 51 additions & 0 deletions tests/npu_pack_import_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2644,6 +2644,56 @@ void test_fconst19_rule_table() {
std::printf("fconst19 rule: measured row, unmeasured refusal, and the ambiguity guard all behave\n");
}

// engine#1831: the header-derived layout keeps EVERY in-block header the anchors support. #1819's version kept only the most
// common residue and always wrote a 2-header block, so a Qwen3 4-header block (three equally frequent inner headers) was read
// as 2-header with k_norm's header standing in for the post-attention one -- the engine#1003 misread, reintroduced.
void test_fconst_block_headers() {
const int64_t stride = 1000;
auto blocks = [&](std::initializer_list<int64_t> inner, int nblocks, std::initializer_list<int64_t> noise) {
std::vector<int64_t> pos;
for (int b = 0; b < nblocks; ++b) {
pos.push_back(int64_t(b) * stride); // the block's own (first) header
for (const int64_t r : inner) pos.push_back(int64_t(b) * stride + r);
}
for (const int64_t q : noise) pos.push_back(q); // eps-SHAPED payload bytes: once or twice
std::sort(pos.begin(), pos.end());
return pos;
};
// 2-header (Qwen2.5/Llama): one inner header, 8 blocks, two coincidences elsewhere in the payload
{
const auto inner = pi::fconst_block_headers(blocks({400}, 8, {1234, 5678}), stride, 3);
check(inner.size() == 1 && inner[0] == 400, "block headers: a 2-header block yields its one inner header");
pi::FconstGeom g;
g.stride = stride;
check(pi::fconst_set_block_headers(g, inner) && g.period == 2 && g.h1 == 400 && g.hdr.size() == 1 &&
g.post_attn_hdr() == 400,
"block headers: and is set as a 2-header block whose post-attention header is h1");
}
// 4-header (Qwen3): three inner headers repeating equally -- ALL must be kept, in order, so post_attn is the LAST one
{
const auto inner = pi::fconst_block_headers(blocks({300, 320, 340}, 8, {777}), stride, 3);
check(inner.size() == 3 && inner[0] == 300 && inner[1] == 320 && inner[2] == 340,
"block headers: a 4-header block yields all three inner headers, sorted (not just the most common one)");
pi::FconstGeom g;
g.stride = stride;
check(pi::fconst_set_block_headers(g, inner) && g.period == 4 && g.h1 == 300 && g.hdr.size() == 3 &&
g.post_attn_hdr() == 340,
"block headers: and is set as a 4-header block whose post-attention header is the last one, not k_norm's");
}
// coincidences alone are not a header, and a count that is neither 1 nor 3 sets nothing
{
check(pi::fconst_block_headers(blocks({}, 8, {111, 222, 222}), stride, 3).empty(),
"block headers: payload coincidences that recur fewer than min_repeat times are not headers");
pi::FconstGeom g;
g.stride = stride;
g.h1 = 7;
check(!pi::fconst_set_block_headers(g, {100, 200}) && g.hdr.empty() && g.h1 == 7 && g.period == 0,
"block headers: two inner headers is not a known layout, and nothing is changed");
check(pi::fconst_block_headers({0, 1000, 2000}, 0, 1).empty(), "block headers: a zero stride yields nothing");
}
std::printf(" fconst block headers: 2-header and 4-header layouts kept whole; unknown counts refused\n");
}

// engine#1801's reviewer asked for the test that would catch the case the fix itself found once: the canonical .fconst copy is a
// cache whose size is the pack's own .fconst, and on this box /tmp is tmpfs, so where it lands is a RAM decision. The bug was a
// cache directory that was never created -- every pack then fell through to /tmp and nothing noticed.
Expand Down Expand Up @@ -3436,6 +3486,7 @@ int main(int argc, char** argv) {
test_embedding_component(golden_dir.empty() ? "." : golden_dir);
test_fconst19_rule_table();
test_fconst_scratch_order();
test_fconst_block_headers();
test_selfcheck_reads_bf16();
if (const char* rc = std::getenv("NPU_EMBED_COMPONENT")) dump_real_component(rc);
test_model_config_geometry(golden_dir.empty() ? "." : golden_dir);
Expand Down
Loading