From 969af37f922fd335c2a9c007b9db15c5f024a182 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Mon, 16 Nov 2026 04:22:35 -0400 Subject: [PATCH 1/2] npu-pack: locate a 1.7 headerless canonical copy from the rewrite's arithmetic, not the epsilon vote (engine#2650) The vote's 12-byte anchor (epsilon + 10 zeros) occurs inside 4-bit tiles. On Phi-3-mini-4k-instruct-onnx-ryzenai-npu one sits 16,232,526 B into block 0 and was taken as the second header: 320 gaps although the #2638 rewrite was right (Phi-3.5 / Phi-3-128k, same layout, 0 gaps). Phi-4-mini and Qwen2-7B likewise. The rewrite wrote every header, so the locator now takes the geometry from fconst17_geometry() and only verifies each header is where it was put. Co-Authored-By: Claude Opus 5.5 --- docs/npu.md | 8 +++++ npu/pack_import.cpp | 39 +++++++++++++++++++++ npu/pack_import.h | 3 ++ tests/npu_pack_import_test.cpp | 62 ++++++++++++++++++++++++++++++++++ 4 files changed, 112 insertions(+) diff --git a/docs/npu.md b/docs/npu.md index bfc37430e..e20548f62 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1851,6 +1851,14 @@ With `config.json` empty, the model's numbers come from the pack itself: On the NPU, the result is "The capital of France is Paris.", "12 * 7 = 84" and "Neil Armstrong ... July 20, 1969" (all stop, 3.9-5.0 tok/s). +The rewritten copy is then located from the same arithmetic, not by the epsilon vote (engine#2650). The vote's anchor +is 12 bytes, epsilon then 10 zeros, and that pattern can occur inside 4-bit tiles. On +`Phi-3-mini-4k-instruct-onnx-ryzenai-npu` one occurs 16,232,526 B into block 0, and the vote took it as the second +header. The pack was refused on 320 tensors, although its rewrite was right: Phi-3.5-mini and Phi-3-mini-128k have the +identical layout and read with 0 gaps. Phi-4-mini (320) and Qwen2-7B (280) were refused the same way. The rewrite +wrote every header, so the locator only checks that each one is where it was put, and refuses the copy by block when +one is not. + The **gate/up unit** is the head's unit with 64 row *slots* per strip instead of 32 rows: 4,864 zero bytes, then `2 x (K/32)` strips of 1,216 B. Strip `s` holds K block `s/2`; slots 0..31 are `gate_proj` rows `32 x (s%2) .. +31` of the unit and slots 32..63 are the same rows of `up_proj`. diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 27ce52a7b..b14bfc1a0 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -1939,6 +1939,42 @@ int intermediate_size_from_graph(const std::string& model_path, int64_t hidden) // So ask the header first on the SOURCE file (the canonical copy has the 1.9 header removed by construction), and keep the eps // vote as the fallback for packs with no 1.9 header -- the 1.8 / R1-Distill line, where it works today. FconstGeom locate_fconst_with_header(const std::string& tok, const std::string& pack_dir, const Dims& dims) { + // engine#2650: a 1.7 headerless pack's canonical copy was WRITTEN by canonical_token_fconst17(), so its geometry is + // known exactly; the epsilon vote is not asked. The vote takes block 0's first two anchors as the header pair, and its + // 12-byte anchor (epsilon + 10 zeros) occurs inside 4-bit tiles: on Phi-3-mini-4k-instruct-onnx-ryzenai-npu one did, + // at an odd offset 16,232,526 B into block 0, and the pack was refused on 320 tensors ("attention payload is + // 16,226,318 B, not a whole number of 108,800-byte tiles") although the rewrite was right -- the identical layout of + // Phi-3.5 and Phi-3-128k read with 0 gaps. Every header the rewrite wrote is still checked to be where it put it. + { + const std::string src = find_token_fconst(pack_dir); + std::error_code ec; + const uint64_t src_size = src.empty() ? 0 : uint64_t(fs::file_size(src, ec)); + Fconst17Geom g17; + if (!src.empty() && !ec && tok != src && fconst17_geometry(src_size, dims, &g17).empty()) { + const int64_t H = dims.hidden, env = kNormOff + 2 * H; + FconstGeom g; + g.h1 = env + g17.payload; + g.stride = env + g17.payload + env + g17.gu + 64 + g17.down; + g.hdr = {g.h1}; + g.period = 2; + g.blocks = dims.layers; + Fconst fc = open_fconst(tok); + const bool size_ok = uint64_t(fc.size) == uint64_t(dims.layers) * uint64_t(g.stride) + uint64_t(env) + + uint64_t(g17.head); + int bad = -1; + for (int L = 0; size_ok && L <= dims.layers && bad < 0; ++L) { + const int64_t b = int64_t(L) * g.stride; + if (!eps_at(fc, b, dims.eps_bits) || (L < dims.layers && !eps_at(fc, b + g.h1, dims.eps_bits))) bad = L; + } + unmap_file(fc.map, fc.size); + if (!size_ok || bad >= 0) { + g.why = "the 1.7 headerless pack's canonical copy is not where its rewrite put it (" + + (size_ok ? "block " + std::to_string(bad) + "'s headers" : std::string("the size")) + ")"; + g.stride = 0; + } + return g; + } + } FconstGeom g = locate_fconst(tok, dims.eps_bits); // engine#2604: the 1.8 gemma3 6-header form -- per-layer starts from every header, not one stride. Only on a file with // no 1.9 header (below, a 6-header 1.9 block is refused by name). @@ -7796,6 +7832,9 @@ std::string canonical_token_fconst17_exported(const std::string& path, const std const std::string& work) { return canonical_token_fconst17(path, pack_dir, d, work); } +FconstGeom locate_fconst_with_header_exported(const std::string& tok, const std::string& pack_dir, const Dims& dims) { + return locate_fconst_with_header(tok, pack_dir, dims); +} float graph_norm_eps_exported(const std::string& onnx_path, int* nodes, bool* conflict) { const OnnxModel m = parse_onnx(onnx_path); if (nodes) *nodes = m.norm_eps_nodes; diff --git a/npu/pack_import.h b/npu/pack_import.h index 60a0d6b76..14c1f3bec 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -996,6 +996,9 @@ std::string fconst17_geometry_exported(uint64_t size, const Dims& d); // The 1.8 form of such a file written to `work` (returns `work`), or `path` unchanged when it is not in that form. std::string canonical_token_fconst17_exported(const std::string& path, const std::string& pack_dir, const Dims& d, const std::string& work); +// engine#2650: where the importer puts the layer blocks of `tok` (the Token .fconst it reads -- for a 1.7 headerless pack, +// the canonical copy), given the pack directory it came from. +FconstGeom locate_fconst_with_header_exported(const std::string& tok, const std::string& pack_dir, const Dims& dims); // The epsilon the graph's RMSNorm nodes state (0 when none), how many state it, and whether two disagree. float graph_norm_eps_exported(const std::string& onnx_path, int* nodes, bool* conflict); // The rms_norm_eps to serve with: config.json's when it states one, else the graph's when its nodes agree; when they diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index f640b13cf..114a086d3 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -504,6 +504,67 @@ void test_fconst17_rewrite() { fs::remove_all(dir); } +// engine#2650: the canonical copy of a 1.7 headerless pack is located from the rewrite's arithmetic, not by the epsilon +// vote. A 12-byte anchor (epsilon + 10 zeros) planted inside block 0's q tiles fools the vote -- the control -- and the +// importer's locator still gives the exact stride and second header. +void test_fconst17_known_geometry() { + pi::Dims d; + d.hidden = 128, d.intermediate = 256, d.layers = 3, d.heads = 2, d.kv_heads = 1, d.head_dim = 64, d.vocab = 64; + d.model_type = "llama"; + d.rms_norm_eps = 1e-5f; + d.eps_bits = pi::bf16_bits_of(1e-5f); + const int64_t H = 128, T = pi::attn_tile_bytes(H), nq = 2, nkv = 1, no = 2; + const int64_t gu = (256 / 64) * pi::group_bytes(H, 128), down = (H / 64) * pi::group_bytes(256, 64), head = T; + uint32_t seed = 777; + const auto rnd = [&](int64_t n) { + std::string b(size_t(n), '\0'); + for (auto& ch : b) { seed = seed * 1103515245u + 12345u; ch = char((seed >> 16) | 1); } // never 0: no anchors + return b; + }; + std::string f17; + for (int L = 0; L < 3; ++L) f17 += rnd(nq * T) + rnd(nkv * T) + rnd(nkv * T) + rnd(no * T) + rnd(2 * H) + rnd(gu) + rnd(down) + rnd(2 * H); + f17 += rnd(head); + // the coincidence: epsilon then 10 zeros, 1000 B into q's first tile of block 0 + f17[1000] = char(d.eps_bits & 0xFF), f17[1001] = char(d.eps_bits >> 8); + for (int i = 2; i < 12; ++i) f17[size_t(1000 + i)] = '\0'; + check(pi::fconst17_geometry_exported(f17.size(), d).empty(), "known geometry: the synthetic 3-layer stream fits"); + const fs::path dir = fs::temp_directory_path() / ("fconst17kg." + std::to_string(::getpid())); + fs::remove_all(dir); + fs::create_directories(dir); + const std::string tok = (dir / "dd_metastate_Llm_Token_MatMulNBits_2_0.fconst").string(); + put_file(tok, f17); + std::vector g0(static_cast(H), uint16_t(15 << 10)); // fp16 1.0 + put_file((dir / "genai_config.json").string(), R"({"model":{"decoder":{"filename":"fusion.onnx"}}})"); + put_file((dir / "fusion.onnx.data").string(), std::string(reinterpret_cast(g0.data()), g0.size() * 2)); + put_file((dir / "fusion.onnx").string(), + pb::model({pb::tensor("model.layers.0.input_layernorm.weight", {H}, "fusion.onnx.data", 0, uint64_t(2 * H), 10)}, {})); + const std::string work = (dir / "canonical.fconst").string(); + std::string said; + try { (void)pi::canonical_token_fconst17_exported(tok, dir.string(), d, work); } catch (const std::exception& e) { said = e.what(); } + const int64_t env = 64 + 2 * H, payload = (nq + 2 * nkv) * T + no * T; + const int64_t h1 = env + payload, stride = env + payload + env + gu + 64 + down; + const pi::FconstGeom vote = pi::locate_fconst(work, d.eps_bits); + check(said.empty() && vote.h1 != h1, "known geometry: the planted anchor fools the epsilon vote (h1 " + + std::to_string(vote.h1) + " vs " + std::to_string(h1) + ") -- the control"); + const pi::FconstGeom g = pi::locate_fconst_with_header_exported(work, dir.string(), d); + check(g.stride == stride && g.h1 == h1 && g.hdr.size() == 1 && g.why.empty(), + "known geometry: the canonical copy is located from the rewrite's arithmetic (stride " + + std::to_string(g.stride) + ", h1 " + std::to_string(g.h1) + ", " + g.why + ")"); + // a copy whose headers are not where the rewrite put them is refused, not located some other way + { + std::string bad; + std::ifstream f(work, std::ios::binary); + bad.assign(std::istreambuf_iterator(f), std::istreambuf_iterator()); + bad[size_t(stride + h1)] = 0; // block 1's second header + const std::string work2 = (dir / "canonical-bad.fconst").string(); + put_file(work2, bad); + const pi::FconstGeom gb = pi::locate_fconst_with_header_exported(work2, dir.string(), d); + check(gb.stride == 0 && gb.why.find("block 1") != std::string::npos, + "known geometry: a header missing from the copy is refused by block (" + gb.why + ")"); + } + fs::remove_all(dir); +} + // The conversion half of the writer (engine#1503): the pack's `(code - zp)*scale` and the container's // `code*d + m` are different conventions, and the mapping has to reproduce the pack's own numbers. The obvious // alternative -- dequantise and re-quantise with the container's quantiser -- was measured on a real phi-4 tile and @@ -7256,6 +7317,7 @@ int main(int argc, char** argv) { test_rope_cache_best_theta(); test_rope_cache_llama3(); test_fconst17_rewrite(); + test_fconst17_known_geometry(); test_external_to_container(); test_external_roles(); test_external_container_names(); From 9acc544c3694ab05eb43bf57ec49a6a85b1e5af3 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Mon, 16 Nov 2026 04:30:52 -0400 Subject: [PATCH 2/2] npu-pack: the 1.7 headerless form may zero-pad its head (CodeLlama-7b-Instruct, engine#2650) CodeLlama-7b-Instruct-hf-onnx-ryzenai-npu is 515 x 143,616-B tiles larger than the arithmetic (vocab 32016 = 501 head tiles); range reads show those 515 tiles are all zero, and every layer's gains sit where the arithmetic puts them. The size gate now takes whole head tiles of padding (padded head <= 65,536 rows); the rewrite drops them only after checking every byte is zero. Co-Authored-By: Claude Opus 5.5 --- docs/npu.md | 6 ++++++ npu/pack_import.cpp | 28 +++++++++++++++++++++++++++- tests/npu_pack_import_test.cpp | 26 ++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 1 deletion(-) diff --git a/docs/npu.md b/docs/npu.md index e20548f62..76287636b 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1859,6 +1859,12 @@ identical layout and read with 0 gaps. Phi-4-mini (320) and Qwen2-7B (280) were wrote every header, so the locator only checks that each one is where it was put, and refuses the copy by block when one is not. +The export can also zero-pad the head (engine#2650). `CodeLlama-7b-Instruct-hf-onnx-ryzenai-npu` (vocab 32,016, so +501 head tiles, the last holding 16 rows) carries 515 more tiles of zeros after the head. That is 1,016 tiles, 65,024 +rows, while its 32 layer blocks sit exactly where the arithmetic puts them. The size gate therefore also takes whole +head tiles after the head, up to 65,536 padded head rows. The rewrite drops them, and only once every padding byte is +checked to be zero; with one non-zero byte the file is not rewritten. + The **gate/up unit** is the head's unit with 64 row *slots* per strip instead of 32 rows: 4,864 zero bytes, then `2 x (K/32)` strips of 1,216 B. Strip `s` holds K block `s/2`; slots 0..31 are `gate_proj` rows `32 x (s%2) .. +31` of the unit and slots 32..63 are the same rows of `up_proj`. diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index b14bfc1a0..806a5f458 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -928,8 +928,10 @@ std::string graph_filename(const std::string& pack_dir); // mapped place (a one-unit misplacement scores 8.5 %); the rest is the two exports' re-quantization. struct Fconst17Geom { int64_t tile_h = 0, tile_o = 0, n_q = 0, n_kv = 0, n_o = 0, payload = 0, gu = 0, down = 0, stride = 0, head = 0; + int64_t pad = 0; // engine#2650: whole head tiles after the head (CodeLlama-7b-Instruct: 515); must be all zero }; -// The arithmetic, and whether `size` is exactly layers x stride + head. Empty when it is; otherwise why not. +// The arithmetic, and whether `size` is layers x stride + head, plus whole head tiles of padding after the head (the +// caller checks those are zero). Empty when it is; otherwise why not. std::string fconst17_geometry(uint64_t size, const Dims& d, Fconst17Geom* g) { const int64_t H = d.hidden, IM = d.intermediate, Nq = int64_t(d.heads) * d.head_dim, Nkv = int64_t(d.kv_heads) * d.head_dim, V = d.vocab; @@ -946,6 +948,16 @@ std::string fconst17_geometry(uint64_t size, const Dims& d, Fconst17Geom* g) { g->stride = g->payload + 2 * H + g->gu + g->down + 2 * H; g->head = ((V + 63) / 64) * g->tile_h; const uint64_t want = uint64_t(d.layers) * uint64_t(g->stride) + uint64_t(g->head); + // engine#2650: CodeLlama-7b-Instruct-hf-onnx-ryzenai-npu (vocab 32016, 501 head tiles) carries 515 more tiles of + // zeros after the head; the export pads the head's rows. Its 32 layer blocks are where this arithmetic puts them. + // The padded head stays within 65,536 rows (there, 1,016 tiles = 65,024 rows), so an unrelated larger file is not + // taken for this form by its size. + if (size > want && (size - want) % uint64_t(g->tile_h) == 0 && + uint64_t(g->head) + (size - want) <= uint64_t(65536 / 64) * uint64_t(g->tile_h)) { + g->pad = int64_t(size - want); + return ""; + } + g->pad = 0; if (size != want) return "the file is " + std::to_string(size) + " B, and the 1.7 headerless form of this geometry is " + std::to_string(d.layers) + " x " + std::to_string(g->stride) + " + a " + std::to_string(g->head) + @@ -968,6 +980,20 @@ std::string canonical_token_fconst17(const std::string& path, const std::string& unmap_file(fc.map, fc.size); return path; } + if (g.pad > 0) { + // the padding is dropped from the rewrite, so it must carry nothing: a non-zero byte there means this is not + // the padded form, and the file is left to the 1.8 reader (which refuses it with its own reasons) + const uint8_t* p = fc.map + size_t(d.layers) * size_t(g.stride) + size_t(g.head); + bool zero = true; + for (int64_t i = 0; i < g.pad && zero; ++i) zero = p[i] == 0; + if (!zero) { + std::fprintf(stderr, "1bit npu-pack: note: %s fits the 1.7 headerless arithmetic with %lld B after the " + "head, but those bytes are not zero, so it is not rewritten\n", + path.c_str(), (long long)g.pad); + unmap_file(fc.map, fc.size); + return path; + } + } const int64_t H = d.hidden; // layer 0's input_layernorm, from the graph (either branch: the two copies are one tensor) std::vector in0; diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 114a086d3..85399ecb4 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -457,6 +457,32 @@ void test_fconst17_rewrite() { put_file(tokx, f17 + std::string(64, '\0')); check(pi::canonical_token_fconst17_exported(tokx, dir.string(), d, work + ".3") == tokx, "1.7 fconst: a file that does not fit the arithmetic is not rewritten"); + // engine#2650: a head padded with whole zero tiles (CodeLlama-7b-Instruct: 515 after 501) rewrites to the same 1.8 + // stream; the same padding with one non-zero byte, or padding past 65,536 head rows, is not taken for this form + { + check(pi::fconst17_geometry_exported(f17.size() + 3 * T, d).empty(), "1.7 fconst: whole zero tiles after the head fit"); + const std::string tokp = (dir / "padded.fconst").string(); + put_file(tokp, f17 + std::string(size_t(3 * T), '\0')); + std::string outp, saidp; + try { + const std::string gotp = pi::canonical_token_fconst17_exported(tokp, dir.string(), d, work + ".p"); + check(gotp == work + ".p", "1.7 fconst: a zero-padded head is rewritten"); + } catch (const std::exception& e) { saidp = e.what(); } + { + std::ifstream f(work + ".p", std::ios::binary); + outp.assign(std::istreambuf_iterator(f), std::istreambuf_iterator()); + } + check(saidp.empty() && outp == f18, "1.7 fconst: the zero-padded head rewrites to the 1.8 stream byte for byte (" + + std::to_string(outp.size()) + " B, " + saidp + ")"); + std::string nz = f17 + std::string(size_t(3 * T), '\0'); + nz[nz.size() - 7] = 1; + const std::string tokn = (dir / "padded-nz.fconst").string(); + put_file(tokn, nz); + check(pi::canonical_token_fconst17_exported(tokn, dir.string(), d, work + ".n") == tokn, + "1.7 fconst: padding with a non-zero byte is not rewritten"); + check(!pi::fconst17_geometry_exported(f17.size() + 1024 * T, d).empty(), + "1.7 fconst: padding past 65,536 head rows does not fit"); + } // the graph's RMSNorm epsilon (a node with a float attribute), read back std::string node, attr; pb::str(node, 4, "SimplifiedLayerNormalization");