diff --git a/docs/npu.md b/docs/npu.md index bfc37430e..2350c2215 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1851,6 +1851,21 @@ With `config.json` empty, the model's numbers come from the pack itself: On the NPU, the result is "The capital of France is Paris.", "12 * 7 = 84" and "Neil Armstrong ... July 20, 1969" (all stop, 3.9-5.0 tok/s). +The rewritten copy is then located from the same arithmetic, not by the epsilon vote (engine#2650). The vote's anchor +is 12 bytes, epsilon then 10 zeros, and that pattern can occur inside 4-bit tiles. On +`Phi-3-mini-4k-instruct-onnx-ryzenai-npu` one occurs 16,232,526 B into block 0, and the vote took it as the second +header. The pack was refused on 320 tensors, although its rewrite was right: Phi-3.5-mini and Phi-3-mini-128k have the +identical layout and read with 0 gaps. Phi-4-mini (320) and Qwen2-7B (280) were refused the same way. The rewrite +wrote every header, so the locator only checks that each one is where it was put, and refuses the copy by block when +one is not. When the same process wrote the copy, the stride the writer measured from its own offsets (#2653) is a +second witness: the arithmetic and the measurement must agree, or the pack is refused naming both. + +The export can also zero-pad the head (engine#2650). `CodeLlama-7b-Instruct-hf-onnx-ryzenai-npu` (vocab 32,016, so +501 head tiles, the last holding 16 rows) carries 515 more tiles of zeros after the head. That is 1,016 tiles, 65,024 +rows, while its 32 layer blocks sit exactly where the arithmetic puts them. The size gate therefore also takes whole +head tiles after the head, up to 65,536 padded head rows. The rewrite drops them, and only once every padding byte is +checked to be zero; with one non-zero byte the file is not rewritten. + The **gate/up unit** is the head's unit with 64 row *slots* per strip instead of 32 rows: 4,864 zero bytes, then `2 x (K/32)` strips of 1,216 B. Strip `s` holds K block `s/2`; slots 0..31 are `gate_proj` rows `32 x (s%2) .. +31` of the unit and slots 32..63 are the same rows of `up_proj`. diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index fccdc184b..fd0911a90 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -928,8 +928,10 @@ std::string graph_filename(const std::string& pack_dir); // mapped place (a one-unit misplacement scores 8.5 %); the rest is the two exports' re-quantization. struct Fconst17Geom { int64_t tile_h = 0, tile_o = 0, n_q = 0, n_kv = 0, n_o = 0, payload = 0, gu = 0, down = 0, stride = 0, head = 0; + int64_t pad = 0; // engine#2650: whole head tiles after the head (CodeLlama-7b-Instruct: 515); must be all zero }; -// The arithmetic, and whether `size` is exactly layers x stride + head. Empty when it is; otherwise why not. +// The arithmetic, and whether `size` is layers x stride + head, plus whole head tiles of padding after the head (the +// caller checks those are zero). Empty when it is; otherwise why not. std::string fconst17_geometry(uint64_t size, const Dims& d, Fconst17Geom* g) { const int64_t H = d.hidden, IM = d.intermediate, Nq = int64_t(d.heads) * d.head_dim, Nkv = int64_t(d.kv_heads) * d.head_dim, V = d.vocab; @@ -946,6 +948,16 @@ std::string fconst17_geometry(uint64_t size, const Dims& d, Fconst17Geom* g) { g->stride = g->payload + 2 * H + g->gu + g->down + 2 * H; g->head = ((V + 63) / 64) * g->tile_h; const uint64_t want = uint64_t(d.layers) * uint64_t(g->stride) + uint64_t(g->head); + // engine#2650: CodeLlama-7b-Instruct-hf-onnx-ryzenai-npu (vocab 32016, 501 head tiles) carries 515 more tiles of + // zeros after the head; the export pads the head's rows. Its 32 layer blocks are where this arithmetic puts them. + // The padded head stays within 65,536 rows (there, 1,016 tiles = 65,024 rows), so an unrelated larger file is not + // taken for this form by its size. + if (size > want && (size - want) % uint64_t(g->tile_h) == 0 && + uint64_t(g->head) + (size - want) <= uint64_t(65536 / 64) * uint64_t(g->tile_h)) { + g->pad = int64_t(size - want); + return ""; + } + g->pad = 0; if (size != want) return "the file is " + std::to_string(size) + " B, and the 1.7 headerless form of this geometry is " + std::to_string(d.layers) + " x " + std::to_string(g->stride) + " + a " + std::to_string(g->head) + @@ -981,6 +993,20 @@ std::string canonical_token_fconst17(const std::string& path, const std::string& unmap_file(fc.map, fc.size); return path; } + if (g.pad > 0) { + // the padding is dropped from the rewrite, so it must carry nothing: a non-zero byte there means this is not + // the padded form, and the file is left to the 1.8 reader (which refuses it with its own reasons) + const uint8_t* p = fc.map + size_t(d.layers) * size_t(g.stride) + size_t(g.head); + bool zero = true; + for (int64_t i = 0; i < g.pad && zero; ++i) zero = p[i] == 0; + if (!zero) { + std::fprintf(stderr, "1bit npu-pack: note: %s fits the 1.7 headerless arithmetic with %lld B after the " + "head, but those bytes are not zero, so it is not rewritten\n", + path.c_str(), (long long)g.pad); + unmap_file(fc.map, fc.size); + return path; + } + } const int64_t H = d.hidden; // layer 0's input_layernorm, from the graph (either branch: the two copies are one tensor) std::vector in0; @@ -1974,6 +2000,52 @@ int64_t fconst17_reconcile(int64_t vote, int64_t recorded, std::string* why) { } FconstGeom locate_fconst_with_header(const std::string& tok, const std::string& pack_dir, const Dims& dims) { + // engine#2650: a 1.7 headerless pack's canonical copy was WRITTEN by canonical_token_fconst17(), so its geometry is + // known exactly; the epsilon vote is not asked. The vote takes block 0's first two anchors as the header pair, and its + // 12-byte anchor (epsilon + 10 zeros) occurs inside 4-bit tiles: on Phi-3-mini-4k-instruct-onnx-ryzenai-npu one did, + // at an odd offset 16,232,526 B into block 0, and the pack was refused on 320 tensors ("attention payload is + // 16,226,318 B, not a whole number of 108,800-byte tiles") although the rewrite was right -- the identical layout of + // Phi-3.5 and Phi-3-128k read with 0 gaps. Every header the rewrite wrote is still checked to be where it put it. + { + const std::string src = find_token_fconst(pack_dir); + std::error_code ec; + const uint64_t src_size = src.empty() ? 0 : uint64_t(fs::file_size(src, ec)); + Fconst17Geom g17; + if (!src.empty() && !ec && tok != src && fconst17_geometry(src_size, dims, &g17).empty()) { + const int64_t H = dims.hidden, env = kNormOff + 2 * H; + FconstGeom g; + g.h1 = env + g17.payload; + g.stride = env + g17.payload + env + g17.gu + 64 + g17.down; + g.hdr = {g.h1}; + g.period = 2; + g.blocks = dims.layers; + Fconst fc = open_fconst(tok); + const bool size_ok = uint64_t(fc.size) == uint64_t(dims.layers) * uint64_t(g.stride) + uint64_t(env) + + uint64_t(g17.head); + int bad = -1; + for (int L = 0; size_ok && L <= dims.layers && bad < 0; ++L) { + const int64_t b = int64_t(L) * g.stride; + if (!eps_at(fc, b, dims.eps_bits) || (L < dims.layers && !eps_at(fc, b + g.h1, dims.eps_bits))) bad = L; + } + unmap_file(fc.map, fc.size); + if (!size_ok || bad >= 0) { + g.why = "the 1.7 headerless pack's canonical copy is not where its rewrite put it (" + + (size_ok ? "block " + std::to_string(bad) + "'s headers" : std::string("the size")) + ")"; + g.stride = 0; + return g; + } + // #2653's witness: the stride the writer measured from its own offsets. When this process wrote the copy, the + // arithmetic and the measurement must agree, or the pack is refused naming both (fconst17_reconcile). + if (const auto it = g_geom17_by_pack.find(pack_dir); it != g_geom17_by_pack.end() && it->second.stride > 0) { + std::string why; + if (fconst17_reconcile(g.stride, it->second.stride, &why) == 0) { + g.why = why; + g.stride = 0; + } + } + return g; + } + } FconstGeom g = locate_fconst(tok, dims.eps_bits); // engine#2650: if this process rewrote the pack from its 1.7 headerless form, the stride of the copy is known from the // writer's own offsets, and the epsilon vote is the weaker witness (a 12-byte anchor occurs inside 4-bit tiles: measured @@ -7863,6 +7935,9 @@ std::string canonical_token_fconst17_exported(const std::string& path, const std const std::string& work) { return canonical_token_fconst17(path, pack_dir, d, work); } +FconstGeom locate_fconst_with_header_exported(const std::string& tok, const std::string& pack_dir, const Dims& dims) { + return locate_fconst_with_header(tok, pack_dir, dims); +} float graph_norm_eps_exported(const std::string& onnx_path, int* nodes, bool* conflict) { const OnnxModel m = parse_onnx(onnx_path); if (nodes) *nodes = m.norm_eps_nodes; diff --git a/npu/pack_import.h b/npu/pack_import.h index 1fb43988c..9f4c13c2c 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -996,6 +996,9 @@ std::string fconst17_geometry_exported(uint64_t size, const Dims& d); // The 1.8 form of such a file written to `work` (returns `work`), or `path` unchanged when it is not in that form. std::string canonical_token_fconst17_exported(const std::string& path, const std::string& pack_dir, const Dims& d, const std::string& work); +// engine#2650: where the importer puts the layer blocks of `tok` (the Token .fconst it reads -- for a 1.7 headerless pack, +// the canonical copy), given the pack directory it came from. +FconstGeom locate_fconst_with_header_exported(const std::string& tok, const std::string& pack_dir, const Dims& dims); // engine#2650: the block stride and head size that the 1.7 rewrite above actually WROTE for `pack_dir` (measured from the // writer's own offsets), or 0 when this process did not rewrite that pack. The reader prefers these over the epsilon vote, // which loses when a 12-byte anchor occurs inside a 4-bit tile. diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index cdc41868b..8bfb3211a 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -487,6 +487,32 @@ void test_fconst17_rewrite() { put_file(tokx, f17 + std::string(64, '\0')); check(pi::canonical_token_fconst17_exported(tokx, dir.string(), d, work + ".3") == tokx, "1.7 fconst: a file that does not fit the arithmetic is not rewritten"); + // engine#2650: a head padded with whole zero tiles (CodeLlama-7b-Instruct: 515 after 501) rewrites to the same 1.8 + // stream; the same padding with one non-zero byte, or padding past 65,536 head rows, is not taken for this form + { + check(pi::fconst17_geometry_exported(f17.size() + 3 * T, d).empty(), "1.7 fconst: whole zero tiles after the head fit"); + const std::string tokp = (dir / "padded.fconst").string(); + put_file(tokp, f17 + std::string(size_t(3 * T), '\0')); + std::string outp, saidp; + try { + const std::string gotp = pi::canonical_token_fconst17_exported(tokp, dir.string(), d, work + ".p"); + check(gotp == work + ".p", "1.7 fconst: a zero-padded head is rewritten"); + } catch (const std::exception& e) { saidp = e.what(); } + { + std::ifstream f(work + ".p", std::ios::binary); + outp.assign(std::istreambuf_iterator(f), std::istreambuf_iterator()); + } + check(saidp.empty() && outp == f18, "1.7 fconst: the zero-padded head rewrites to the 1.8 stream byte for byte (" + + std::to_string(outp.size()) + " B, " + saidp + ")"); + std::string nz = f17 + std::string(size_t(3 * T), '\0'); + nz[nz.size() - 7] = 1; + const std::string tokn = (dir / "padded-nz.fconst").string(); + put_file(tokn, nz); + check(pi::canonical_token_fconst17_exported(tokn, dir.string(), d, work + ".n") == tokn, + "1.7 fconst: padding with a non-zero byte is not rewritten"); + check(!pi::fconst17_geometry_exported(f17.size() + 1024 * T, d).empty(), + "1.7 fconst: padding past 65,536 head rows does not fit"); + } // the graph's RMSNorm epsilon (a node with a float attribute), read back std::string node, attr; pb::str(node, 4, "SimplifiedLayerNormalization"); @@ -534,6 +560,67 @@ void test_fconst17_rewrite() { fs::remove_all(dir); } +// engine#2650: the canonical copy of a 1.7 headerless pack is located from the rewrite's arithmetic, not by the epsilon +// vote. A 12-byte anchor (epsilon + 10 zeros) planted inside block 0's q tiles fools the vote -- the control -- and the +// importer's locator still gives the exact stride and second header. +void test_fconst17_known_geometry() { + pi::Dims d; + d.hidden = 128, d.intermediate = 256, d.layers = 3, d.heads = 2, d.kv_heads = 1, d.head_dim = 64, d.vocab = 64; + d.model_type = "llama"; + d.rms_norm_eps = 1e-5f; + d.eps_bits = pi::bf16_bits_of(1e-5f); + const int64_t H = 128, T = pi::attn_tile_bytes(H), nq = 2, nkv = 1, no = 2; + const int64_t gu = (256 / 64) * pi::group_bytes(H, 128), down = (H / 64) * pi::group_bytes(256, 64), head = T; + uint32_t seed = 777; + const auto rnd = [&](int64_t n) { + std::string b(size_t(n), '\0'); + for (auto& ch : b) { seed = seed * 1103515245u + 12345u; ch = char((seed >> 16) | 1); } // never 0: no anchors + return b; + }; + std::string f17; + for (int L = 0; L < 3; ++L) f17 += rnd(nq * T) + rnd(nkv * T) + rnd(nkv * T) + rnd(no * T) + rnd(2 * H) + rnd(gu) + rnd(down) + rnd(2 * H); + f17 += rnd(head); + // the coincidence: epsilon then 10 zeros, 1000 B into q's first tile of block 0 + f17[1000] = char(d.eps_bits & 0xFF), f17[1001] = char(d.eps_bits >> 8); + for (int i = 2; i < 12; ++i) f17[size_t(1000 + i)] = '\0'; + check(pi::fconst17_geometry_exported(f17.size(), d).empty(), "known geometry: the synthetic 3-layer stream fits"); + const fs::path dir = fs::temp_directory_path() / ("fconst17kg." + std::to_string(::getpid())); + fs::remove_all(dir); + fs::create_directories(dir); + const std::string tok = (dir / "dd_metastate_Llm_Token_MatMulNBits_2_0.fconst").string(); + put_file(tok, f17); + std::vector g0(static_cast(H), uint16_t(15 << 10)); // fp16 1.0 + put_file((dir / "genai_config.json").string(), R"({"model":{"decoder":{"filename":"fusion.onnx"}}})"); + put_file((dir / "fusion.onnx.data").string(), std::string(reinterpret_cast(g0.data()), g0.size() * 2)); + put_file((dir / "fusion.onnx").string(), + pb::model({pb::tensor("model.layers.0.input_layernorm.weight", {H}, "fusion.onnx.data", 0, uint64_t(2 * H), 10)}, {})); + const std::string work = (dir / "canonical.fconst").string(); + std::string said; + try { (void)pi::canonical_token_fconst17_exported(tok, dir.string(), d, work); } catch (const std::exception& e) { said = e.what(); } + const int64_t env = 64 + 2 * H, payload = (nq + 2 * nkv) * T + no * T; + const int64_t h1 = env + payload, stride = env + payload + env + gu + 64 + down; + const pi::FconstGeom vote = pi::locate_fconst(work, d.eps_bits); + check(said.empty() && vote.h1 != h1, "known geometry: the planted anchor fools the epsilon vote (h1 " + + std::to_string(vote.h1) + " vs " + std::to_string(h1) + ") -- the control"); + const pi::FconstGeom g = pi::locate_fconst_with_header_exported(work, dir.string(), d); + check(g.stride == stride && g.h1 == h1 && g.hdr.size() == 1 && g.why.empty(), + "known geometry: the canonical copy is located from the rewrite's arithmetic (stride " + + std::to_string(g.stride) + ", h1 " + std::to_string(g.h1) + ", " + g.why + ")"); + // a copy whose headers are not where the rewrite put them is refused, not located some other way + { + std::string bad; + std::ifstream f(work, std::ios::binary); + bad.assign(std::istreambuf_iterator(f), std::istreambuf_iterator()); + bad[size_t(stride + h1)] = 0; // block 1's second header + const std::string work2 = (dir / "canonical-bad.fconst").string(); + put_file(work2, bad); + const pi::FconstGeom gb = pi::locate_fconst_with_header_exported(work2, dir.string(), d); + check(gb.stride == 0 && gb.why.find("block 1") != std::string::npos, + "known geometry: a header missing from the copy is refused by block (" + gb.why + ")"); + } + fs::remove_all(dir); +} + // The conversion half of the writer (engine#1503): the pack's `(code - zp)*scale` and the container's // `code*d + m` are different conventions, and the mapping has to reproduce the pack's own numbers. The obvious // alternative -- dequantise and re-quantise with the container's quantiser -- was measured on a real phi-4 tile and @@ -7286,6 +7373,7 @@ int main(int argc, char** argv) { test_rope_cache_best_theta(); test_rope_cache_llama3(); test_fconst17_rewrite(); + test_fconst17_known_geometry(); test_external_to_container(); test_external_roles(); test_external_container_names();