diff --git a/docs/npu.md b/docs/npu.md index 2350c2215..2422851e8 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1866,6 +1866,18 @@ rows, while its 32 layer blocks sit exactly where the arithmetic puts them. The head tiles after the head, up to 65,536 padded head rows. The rewrite drops them, and only once every padding byte is checked to be zero; with one non-zero byte the file is not rewritten. +A tied model's file may carry no head at all (engine#2658). `Phi-4-mini-instruct-onnx-ryzenai-npu` (vocab 200,064, +tied) is exactly 32 layer strides, 1,758,724,096 B. Its head is the graph's `lm_head` MatMulNBits in +`fusion.onnx.data`, the same arrangement as its 1.8 `npu_4K` twin, whose Token file has no head either and reads with +0 gaps. So a file of exactly layers x stride is taken as this form, and the rewrite ends at the final norm. The +rewrite is the twin's size to the byte (1,758,736,448 B) and 76.7 % byte-equal to it at the mapped place, against +2.6 % with the twin shifted one attention tile; the rest is the two exports' re-quantization. The reader then takes +the head from the graph, or refuses the pack by name when the graph has none. "Tied" is not inferred from the +missing tiles. The head is the graph's MatMulNBits whose geometry is the head's (N = vocab, K = hidden, the #1671 rule); +Phi-4-mini's is `lm_head.weight` [200064, 3072] in `fusion.onnx.data`. With no such node the reader falls to the +`.fconst` tail, and a tail that holds only the final norm's envelope is refused ("shorter than ... its envelope and +head tiles would take"). So an untied pack with no head anywhere still refuses. + The **gate/up unit** is the head's unit with 64 row *slots* per strip instead of 32 rows: 4,864 zero bytes, then `2 x (K/32)` strips of 1,216 B. Strip `s` holds K block `s/2`; slots 0..31 are `gate_proj` rows `32 x (s%2) .. +31` of the unit and slots 32..63 are the same rows of `up_proj`. diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index fd0911a90..c5bc95023 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -929,6 +929,7 @@ std::string graph_filename(const std::string& pack_dir); struct Fconst17Geom { int64_t tile_h = 0, tile_o = 0, n_q = 0, n_kv = 0, n_o = 0, payload = 0, gu = 0, down = 0, stride = 0, head = 0; int64_t pad = 0; // engine#2650: whole head tiles after the head (CodeLlama-7b-Instruct: 515); must be all zero + bool no_head = false; // engine#2658: a tied model's file ends with the last layer; the head is the graph's }; // The arithmetic, and whether `size` is layers x stride + head, plus whole head tiles of padding after the head (the // caller checks those are zero). Empty when it is; otherwise why not. @@ -958,6 +959,15 @@ std::string fconst17_geometry(uint64_t size, const Dims& d, Fconst17Geom* g) { return ""; } g->pad = 0; + // engine#2658: Phi-4-mini-instruct-onnx-ryzenai-npu (tied, vocab 200,064) is exactly 32 layer strides, 1,758,724,096 + // B: no head tiles at all. Its head is the graph's lm_head MatMulNBits, as in its 1.8 npu_4K twin (whose Token file + // has no head either and reads with 0 gaps). The rewrite then ends at the final norm, and the reader takes the head + // from the graph -- or refuses the pack by name when the graph has none. + if (size == uint64_t(d.layers) * uint64_t(g->stride)) { + g->head = 0; + g->no_head = true; + return ""; + } if (size != want) return "the file is " + std::to_string(size) + " B, and the 1.7 headerless form of this geometry is " + std::to_string(d.layers) + " x " + std::to_string(g->stride) + " + a " + std::to_string(g->head) + diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 8bfb3211a..c615f4a63 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -513,6 +513,36 @@ void test_fconst17_rewrite() { check(!pi::fconst17_geometry_exported(f17.size() + 1024 * T, d).empty(), "1.7 fconst: padding past 65,536 head rows does not fit"); } + // engine#2658: a tied model's file ends with the last layer (Phi-4-mini: exactly 32 strides). The rewrite is the 1.8 + // stream without the head -- what that pack's 1.8 twin carries -- and a file one tile short of that is not taken. + { + const std::string f17n = f17.substr(0, f17.size() - size_t(head)); + const std::string f18n = f18.substr(0, f18.size() - size_t(head)); + check(pi::fconst17_geometry_exported(f17n.size(), d).empty(), "1.7 fconst: a file with no head fits"); + check(!pi::fconst17_geometry_exported(f17n.size() - size_t(T), d).empty(), "1.7 fconst: one tile short of the layers does not"); + const std::string tokh = (dir / "nohead.fconst").string(); + put_file(tokh, f17n); + std::string outh, saidh; + try { + check(pi::canonical_token_fconst17_exported(tokh, dir.string(), d, work + ".h") == work + ".h", + "1.7 fconst: a file with no head is rewritten"); + } catch (const std::exception& e) { saidh = e.what(); } + { + std::ifstream f(work + ".h", std::ios::binary); + outh.assign(std::istreambuf_iterator(f), std::istreambuf_iterator()); + } + check(saidh.empty() && outh == f18n, "1.7 fconst: the no-head file rewrites to the 1.8 stream without its head (" + + std::to_string(outh.size()) + " vs " + std::to_string(f18n.size()) + " B, " + saidh + ")"); + // #2666 review, the negative case: the no-head form is NOT a catch-all for a missing head. With no graph matmul of the + // head's geometry (N = V, K = H), the reader falls to the tail; the rewritten copy's tail is the envelope alone, + // and plan_tail_head() refuses it by name -- so an untied pack without a head anywhere still refuses. + const int64_t env18 = 64 + 2 * H, layers_end = int64_t(f18n.size()) - env18; + const pi::TailHeadPlacement th = pi::plan_tail_head(int64_t(f18n.size()), layers_end, env18, 1, T, 0); + check(!th.why.empty() && th.why.find("shorter") != std::string::npos, + "1.7 fconst: with no head in the graph, the no-head copy's tail is refused by name (" + th.why + ")"); + const pi::TailHeadPlacement th2 = pi::plan_tail_head(int64_t(f18.size()), layers_end, env18, 1, T, 0); + check(th2.why.empty(), "1.7 fconst: control -- the copy WITH its head places the head in the tail (" + th2.why + ")"); + } // the graph's RMSNorm epsilon (a node with a float attribute), read back std::string node, attr; pb::str(node, 4, "SimplifiedLayerNormalization");