diff --git a/docs/npu.md b/docs/npu.md index cd92a1c68..0fa479493 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1777,6 +1777,8 @@ then zeros. On SmolLM2-135M the block is 2,360,292 B (q at 1,216, k at 231,616, 385,252, the second envelope at 615,652, gate/up at 616,868, down at 1,784,292), the head is at 70,809,976 and the file is exactly `30 x 2,360,292 + 1,216 + 768 x 25,600 = 90,470,776` B. +**The 1.8 file is the same block in another order (engine#2594).** `SmolLM2-135M-Instruct_rai_1.8.0_npu_4K`'s Token `.fconst` is 90,469,696 B, 1,080 B less than the 1.9 file, which is 30 layers x 4 x 9 heads. It writes `k_proj` before `q_proj` and has no `4 x heads` field: k at 1,216, q at 78,016, v at 308,416, o at 385,216, stride 2,360,256, head at 70,808,896. Rebuilt into the 1.9 order, it equals the 1.9 file byte for byte, except 112 code bytes in layer 3's `o_proj`. `plan()` takes this form only when the file fits it and does not fit the one above. The envelopes move 4 x heads bytes a layer, so on a multi-layer file only one of the two can pass, and `npu-pack info` names the form in `block32_layers.form`. The tied-head self-check cannot see a q/k swap. With the sink field dropped but q still first, the import passed its own checks (head vs embedding corr 0.999999, 60/60 norm anchors), and the device scored 2/64 on the teacher-forced probes. With k first, the device scores 53/64, the same as the 1.9 pack (53/64; RTN int4 of the HF model scores 47/64). It answers "Neil Armstrong … July 20, 1969". It declines the France question, and so do the host f32 and host int8 GEMMs on the same container, so that comes from the pack's 4-bit weights; the HF fp32 model answers Paris. + The **gate/up unit** is the head's unit with 64 row *slots* per strip instead of 32 rows: 4,864 zero bytes, then `2 x (K/32)` strips of 1,216 B. Strip `s` holds K block `s/2`; slots 0..31 are `gate_proj` rows `32 x (s%2) .. +31` of the unit and slots 32..63 are the same rows of `up_proj`. diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 739b70f2e..9f548eb52 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -2967,7 +2967,7 @@ Block32GateUp read_block32_gate_up(const uint8_t* p, int64_t avail, int64_t rows return h; } -Block32Layout block32_layout(const Dims& d) { +Block32Layout block32_layout(const Dims& d, bool form18) { Block32Layout l; const int64_t H = d.hidden, IM = d.intermediate, Nq = int64_t(d.heads) * d.head_dim, Nkv = int64_t(d.kv_heads) * d.head_dim, V = d.vocab; @@ -2986,11 +2986,17 @@ Block32Layout block32_layout(const Dims& d) { return l; } l.env = kNormOff + 2 * H; - l.q_off = l.env; - l.k_off = l.q_off + (Nq / kB32UnitRows) * block32_unit_bytes(H); - l.v_off = l.k_off + (Nkv / kB32UnitRows) * block32_unit_bytes(H); + if (form18) { // engine#2594: the 1.8 writer puts k_proj first and has no sink field + l.k_off = l.env; + l.q_off = l.k_off + (Nkv / kB32UnitRows) * block32_unit_bytes(H); + l.v_off = l.q_off + (Nq / kB32UnitRows) * block32_unit_bytes(H); + } else { + l.q_off = l.env; + l.k_off = l.q_off + (Nq / kB32UnitRows) * block32_unit_bytes(H); + l.v_off = l.k_off + (Nkv / kB32UnitRows) * block32_unit_bytes(H); + } l.sink_off = l.v_off + (Nkv / kB32UnitRows) * block32_unit_bytes(H); - l.o_off = l.sink_off + 4 * int64_t(d.heads); + l.o_off = l.sink_off + (form18 ? 0 : 4 * int64_t(d.heads)); l.post_off = l.o_off + (H / kB32UnitRows) * block32_unit_bytes(Nq); l.gu_off = l.post_off + l.env; l.hdr_off = l.gu_off + (IM / kB32UnitRows) * block32_mlp_unit_bytes(H); @@ -3303,8 +3309,9 @@ std::string Plan::manifest() const { // engine#1503: an all-block-32 pack is read with its own layout, so the block-128 measurements above are 0 for it; // this says which layout was used and what its arithmetic gives. Absent for every other pack. if (block32_layers) { - const Block32Layout lay = block32_layout(dims); - j["block32_layers"] = {{"stride", lay.stride}, + const Block32Layout lay = block32_layout(dims, block32_form18); + j["block32_layers"] = {{"form", block32_form18 ? "1.8 (k, q, v, o; no sink field)" : "1.9 (q, k, v, sink, o)"}, + {"stride", lay.stride}, {"post_attention_envelope", lay.post_off}, {"head_offset", lay.head_off}, {"head_units", lay.head_units}, @@ -4396,6 +4403,16 @@ Plan plan(const std::string& pack_dir) { } else { Fconst fc = open_fconst(tok_path); p.block32_note = block32_file_refusal(fc.map, int64_t(fc.size), p.dims, lay); + // engine#2594: the 1.8 file is the same block with k_proj before q_proj and no sink field. Taken only when + // the file fits that form and not the 1.9 one; the envelopes move by 4*heads bytes a layer, so both cannot + // pass on a multi-layer file. + if (!p.block32_note.empty()) { + const Block32Layout f18 = block32_layout(p.dims, true); + if (block32_file_refusal(fc.map, int64_t(fc.size), p.dims, f18).empty()) { + p.block32_note.clear(); + p.block32_form18 = true; + } + } unmap_file(fc.map, fc.size); } } @@ -6033,7 +6050,7 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons // below writes, in the same form. This layout has no bias field, so no bias tensor is written. std::vector late; if (p.block32_note.empty()) { - const Block32Layout lay = block32_layout(p.dims); + const Block32Layout lay = block32_layout(p.dims, p.block32_form18); Fconst fc = open_fconst(find_token_fconst(pack_dir)); // the file is re-checked on this mapping: plan() looked at its own if (const std::string why = block32_file_refusal(fc.map, int64_t(fc.size), p.dims, lay); !why.empty()) diff --git a/npu/pack_import.h b/npu/pack_import.h index 0e98b5680..ae82a0314 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -224,6 +224,9 @@ struct Plan { // raw .fconst; the block-128 walk and the canonical copy are not used. `block32_note` is why that file does not // fit the layout (empty when it does), and is the refusal every .fconst tensor of the plan then carries. bool block32_layers = false; + // engine#2594: the 1.8 Token .fconst is the 1.9 block with k_proj before q_proj and no 4*heads sink field. plan() + // sets this when only that form fits the file (its size and every envelope); every later reader follows it. + bool block32_form18 = false; std::string block32_note; std::string pack; // *_rai_1.8.0_npu_4K, from the directory name Dims dims; @@ -816,7 +819,9 @@ Block32GateUp read_block32_gate_up(const uint8_t* p, int64_t avail, int64_t rows // q_off (Nq/64) x unit(H) q_proj [Nq][H] // k_off (Nkv/64) x unit(H) k_proj [Nkv][H] // v_off (Nkv/64) x unit(H) v_proj [Nkv][H] -// sink_off 4*heads bytes not a weight; skipped (int32 0xFF7F0000 x heads on the measured pack) +// sink_off 4*heads bytes not a weight; skipped (int32 0xFF7F0000 x heads on the measured pack); +// The 1.8 file (SmolLM2-135M-Instruct 1.8 npu_4K, 90,469,696 B) is this block with k_proj BEFORE q_proj and no sink +// field: rebuilt into the 1.9 order it equals the 1.9 file byte for byte except 112 code bytes of layer 3's o_proj. // o_off (H/64) x unit(Nq) o_proj [H][Nq] // post_off 64 + 2*H envelope: eps, zeros, post_attention_layernorm bf16[H] // gu_off (IM/64) x mlp_unit(H) gate_proj + up_proj, [IM][H] each @@ -833,7 +838,8 @@ struct Block32Layout { int64_t tail_off = 0, head_off = 0, head_units = 0; int64_t min_size = 0; // head_off + head_units x unit(H); the file is this plus zero padding }; -Block32Layout block32_layout(const Dims& d); +// `form18`: the 1.8 Token .fconst's form of the same block -- k_proj before q_proj, no sink field (engine#2594). +Block32Layout block32_layout(const Dims& d, bool form18 = false); // Empty when a mapped file fits `lay`: it is at least `min_size` long, and every envelope (two per layer and the // tail's) opens with the pack's epsilon followed by zeros up to the norm. Otherwise the reason, with the offset. The // zero padding after the head is the head reader's check. diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index fafc445b8..a2b8869bc 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -2672,6 +2672,15 @@ void test_block32_layers(const std::string& root) { std::to_string(l.stride) + ")"); check(l.head_off == 70809976 && l.head_units == 768 && l.min_size == 90470776, "block-32 layout: its head at 70,809,976 and the file's 90,470,776 B (" + std::to_string(l.min_size) + ")"); + // engine#2594: the 1.8 Token .fconst of the same model: k_proj first, then q_proj, v_proj, and no sink field + // (4 * 9 heads = 36 B less per layer). These offsets rebuild the 1.9 file from the 1.8 one byte for byte + // except 112 code bytes of layer 3's o_proj. + const pi::Block32Layout n = pi::block32_layout(d, true); + check(n.why.empty() && n.k_off == 1216 && n.q_off == 78016 && n.v_off == 308416 && n.o_off == 385216 && + n.post_off == 615616 && n.stride == 2360256 && n.min_size == 90469696, + "block-32 layout, 1.8 form: SmolLM2-135M, k at 1,216, q at 78,016, o at 385,216, stride 2,360,256 and " + "90,469,696 B (" + std::to_string(n.q_off) + " / " + std::to_string(n.stride) + " / " + + std::to_string(n.min_size) + ")"); d.intermediate = 1500; check(pi::block32_layout(d).why.find("intermediate 1500") != std::string::npos, "block-32 layout: a width that is not whole 64-row units is refused by name"); @@ -2703,14 +2712,18 @@ void test_block32_layers(const std::string& root) { e[0] = char(0x86), e[1] = char(0x35); // bf16 1e-06 return e + norm_bytes(which); }; - const auto fconst = [&] { + const auto fconst = [&](bool form18 = false) { std::string fc; for (int l = 0; l < LAYERS; ++l) { const auto& m = [&](int i) -> const std::vector& { return mats[size_t(l * NPROJ + i)]; }; fc += envelope(10 * l + 1); - fc += encode_block32_units(m(Q), NQ, H) + encode_block32_units(m(K_), NKV, H) + - encode_block32_units(m(V_), NKV, H); - fc += std::string(4 * 2, '\x7f'); // 4 x heads bytes that are not a weight (their content is not read) + if (form18) // engine#2594: the 1.8 writer puts k_proj first and writes no sink field + fc += encode_block32_units(m(K_), NKV, H) + encode_block32_units(m(Q), NQ, H) + + encode_block32_units(m(V_), NKV, H); + else + fc += encode_block32_units(m(Q), NQ, H) + encode_block32_units(m(K_), NKV, H) + + encode_block32_units(m(V_), NKV, H) + + std::string(4 * 2, '\x7f'); // 4 x heads bytes that are not a weight (not read) fc += encode_block32_units(m(O), H, NQ); fc += envelope(10 * l + 2); fc += encode_block32_gate_up(m(G), m(U), IM, H); @@ -2804,6 +2817,7 @@ void test_block32_layers(const std::string& root) { const pi::Plan p = pi::plan(dir.string()); check(p.block32_layers && p.block32_note.empty() && p.refused().empty(), "all-block-32: the plan takes the block-32 layout and refuses nothing (" + p.block32_note + ")"); + check(!p.block32_form18, "all-block-32: a 1.9-form file keeps the 1.9 layout"); r = pi::import_pack(dir.string(), out, pi::Options{}); } catch (const std::exception& e) { said = e.what(); @@ -2857,6 +2871,49 @@ void test_block32_layers(const std::string& root) { fs::remove_all(out); fs::remove_all(dir); } + { // 5n: the 1.8 writer's file (engine#2594): k_proj before q_proj and no sink field. The plan takes the 1.8 + // form, and every projection comes back as its own matrix -- q and k are not swapped (they differ in rows and + // content here), and the 4*heads shift is applied from o_proj on in every layer, not only at the head. + const std::string nosink = fconst(true); + check(int64_t(nosink.size()) + 2 * 4 * 2 == int64_t(good.size()), "no-sink fixture: 8 B shorter per layer"); + const fs::path dir = build("b32-nosink", nosink, true); + const std::string out = out_of("nosink"); + fs::remove_all(out); + pi::Report r; + std::string said; + try { + const pi::Plan p = pi::plan(dir.string()); + check(p.block32_layers && p.block32_form18 && p.block32_note.empty() && p.refused().empty(), + "all-block-32, 1.8 form: the plan takes it (" + p.block32_note + ")"); + r = pi::import_pack(dir.string(), out, pi::Options{}); + } catch (const std::exception& e) { + said = e.what(); + } + check(said.empty() && r.refused.empty(), "all-block-32, 1.8 form: imports (" + said + ")"); + check(r.stride == (int64_t(good.size()) - (64 + 2 * H) - pi::block32_unit_bytes(H)) / LAYERS - 4 * 2, + "all-block-32, 1.8 form: the stride is the 1.9 one less 4 x heads (" + + std::to_string(r.stride) + ")"); + if (fs::exists(out + "/model.q4nx")) { + const json tbl = container_table(out + "/model.q4nx"); + const std::vector blob = container_payload(out + "/model.q4nx"); + for (int l = 0; l < LAYERS; ++l) + for (int i = 0; i < NPROJ; ++i) { + const std::string name = "model.layers." + std::to_string(l) + "." + pname[i] + ".weight"; + const int cols = std::min(pcol[i], onebit::npu::q4nx::kCols); + const std::vector got = tile0(tbl, blob, name, cols); + const double c = got.empty() ? 0.0 : ncorr(got, head_of(mats[size_t(l * NPROJ + i)], pcol[i], cols)); + check(c > 0.99, "all-block-32, 1.8 form: " + name + " is its own matrix (corr " + + std::to_string(c) + ")"); + } + const std::vector hd = tile0(tbl, blob, "lm_head.weight", H); + check(!hd.empty() && ncorr(hd, head_of(emb, H, H)) > 0.99, + "all-block-32, 1.8 form: lm_head is the tail's head"); + } else { + check(false, "all-block-32, 1.8 form: the container was written"); + } + fs::remove_all(out); + fs::remove_all(dir); + } // what a refused import says, and that it wrote no container const auto refused = [&](const char* tag, const std::string& fc) { const fs::path dir = build(tag, fc, true);