diff --git a/docs/npu.md b/docs/npu.md index 5337e7d9c..92527a0de 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1249,6 +1249,50 @@ layout checks are all there is, and the note says the values are not verified. M equals `bf16(-m/8)` of its 32-column embedding block, `m` being the block's largest-magnitude weight, and the read head correlates 0.996 with the embedding. +**A pack whose every MatMulNBits is `block_size 32` keeps its layers in those units too** +(engine#1503). `SmolLM2-135M-Instruct_rai_1.9.0_npu` declares block 32 on all 151 ops, and its +*raw* Token `.fconst` (the file as shipped, not the 1.8-form copy the block-128 path makes) is +`layers` blocks of one fixed shape and then the tail. With `H` = hidden, `Nq` = heads x head_dim, +`Nkv` = KV heads x head_dim, `IM` = intermediate, `unit(K) = 2,560 + 2 x (K/32) x 640` (the head's +unit above) and `mlp_unit(K) = 4,864 + 2 x (K/32) x 1,216`: + +| section | bytes | holds | +|---|---|---| +| envelope | `64 + 2H` | bf16 `rms_norm_eps`, zeros to 64, `input_layernorm` bf16[H] | +| q, k, v | `(Nq/64 + 2 x Nkv/64) x unit(H)` | `q_proj` [Nq][H], `k_proj` [Nkv][H], `v_proj` [Nkv][H] | +| — | `4 x heads` | not a weight; skipped | +| o | `(H/64) x unit(Nq)` | `o_proj` [H][Nq] | +| envelope | `64 + 2H` | epsilon, zeros, `post_attention_layernorm` bf16[H] | +| gate + up | `(IM/64) x mlp_unit(H)` | `gate_proj` and `up_proj`, [IM][H] each | +| — | 64 | skipped, not interpreted (`01` then zeros on the measured pack) | +| down | `(H/64) x unit(IM)` | `down_proj` [H][IM] | + +The tail is the same envelope with the final norm, then `ceil(vocab/64)` head units at K = `H`, +then zeros. On SmolLM2-135M the block is 2,360,292 B (q at 1,216, k at 231,616, v at 308,416, o at +385,252, the second envelope at 615,652, gate/up at 616,868, down at 1,784,292), the head is at +70,809,976 and the file is exactly `30 x 2,360,292 + 1,216 + 768 x 25,600 = 90,470,776` B. + +The **gate/up unit** is the head's unit with 64 row *slots* per strip instead of 32 rows: 4,864 +zero bytes, then `2 x (K/32)` strips of 1,216 B. Strip `s` holds K block `s/2`; slots 0..31 are +`gate_proj` rows `32 x (s%2) .. +31` of the unit and slots 32..63 are the same rows of `up_proj`. +A strip is 1,024 B of 4-bit codes (low nibble first; position `p` is slot `(p/256) x 8 + p%8`, +column `(p/8) % 32`), 32 B of zero-point nibbles, 32 zero bytes, and 64 signed bf16 scales. The +code order was measured against the public checkpoint; that zero point and scale are indexed by +slot is assumed by analogy with the 640-byte strip, and is tested by the imported container's +gate/up still correlating with the checkpoint once they are applied. + +`npu-pack import` takes this path when every op the pack's `cache/*_meta.json` declares is block 32, +at least one of them is a layer projection, and the pack has a Token `.fconst` (a pack that +declares only its head at 32 is the case above). Nothing is located by search: the offsets are +arithmetic on the geometry, and the file has to agree with them — at least +`layers x stride + 64 + 2H + ceil(vocab/64) x unit(H)` bytes, the pack's epsilon and zeros at every +envelope, zero unit headers and strip gaps, finite scales, zeros after the head — or the tensors +are refused by name with the offset. The container gets the same tensors in the same q4_1f tiles as +a block-128 import; this layout has no bias field, so no bias tensor is written. There is no +decode->encode round trip on this path (no encoder for these units exists), so the import's +`attention:` and `MLP:` round-trip lines are replaced by a `block 32:` line. Measured on one pack +only, where `Nq == H`; `o_proj`'s K being `Nq` is the declared shape, not a measurement. + The norms that *are* imported sit at the documented offsets, each confirmed by the epsilon that opens its layer block: **48/48 anchors found**. `tests/npu_pack_import_test` (registered as `npu_pack_import`, and as `npu_pack_import_oga` when `-DONEBIT_OGA_PACK=` is given) checks diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 2c06dfcf2..ca248267b 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -2562,14 +2562,104 @@ static TailHeadLocate locate_tail_head(const Fconst& fc, int64_t tail_off, int64 return r; } -// engine#1740: the block-32 tail head. Layout and its measurement are in pack_import.h. -Block32Head read_block32_head(const uint8_t* p, int64_t avail, int64_t V, int64_t K) { +// engine#1740 / engine#1503: the block-32 units. Layouts and their measurement are in pack_import.h. +namespace { +bool b32_all_zero(const uint8_t* b, int64_t n) { + for (int64_t i = 0; i < n; ++i) + if (b[i] != 0) return false; + return true; +} +float b32_bf16(const uint8_t* b) { + const uint32_t wide = uint32_t(uint16_t(b[0]) | uint16_t(uint16_t(b[1]) << 8)) << 16; + float v; + std::memcpy(&v, &wide, 4); + return v; +} +void b32_q_init(Q4& q, int64_t rows, int64_t K) { + q.N = int(rows); + q.K = int(K); + q.block = 32; + q.codes.assign(size_t(rows) * size_t(K), 0); + q.scale.assign(size_t(rows) * size_t(K / 32), 1.0f); + q.zero.assign(size_t(rows) * size_t(K / 32), 0.0f); +} +// One row's 32 codes of one K block out of a strip's code field. `slot` is the row's index inside the strip (0..31 +// in a 640-byte strip, 0..63 in a gate/up one): position `(slot/8)*256 + col*8 + slot%8`, low nibble first. A negative +// scale is folded into the codes, so the stored scale is positive: (code - zp) * sc == ((15-code) - (15-zp)) * -sc. +void b32_row(Q4& q, int64_t row, int64_t kblock, const uint8_t* codes, int slot, int zp, float sc) { + if (sc == 0.0f) return; // an all-zero block: codes 0, scale 1, zero 0 decode to 0 + const bool flip = sc < 0; + const float mag = flip ? -sc : sc; + const size_t g = size_t(row) * size_t(q.K / 32) + size_t(kblock); + q.scale[g] = mag; + q.zero[g] = -float(flip ? 15 - zp : zp) * mag; + const int base = (slot / 8) * 256 + (slot % 8); + uint8_t* dst = q.codes.data() + size_t(row) * size_t(q.K) + size_t(kblock) * 32; + for (int c = 0; c < 32; ++c) { + const int pos = base + c * 8; + const int code = (codes[pos / 2] >> (4 * (pos & 1))) & 15; + dst[c] = uint8_t(flip ? 15 - code : code); + } +} +} // namespace + +Block32Units read_block32_units(const uint8_t* p, int64_t avail, int64_t rows, int64_t K, const char* what, + int64_t file_off) { + Block32Units h; + const std::string tag = std::string("block-32 ") + what; + if (rows <= 0 || K <= 0 || K % 32 != 0) { + h.why = "a " + tag + " needs K to be a multiple of 32 (K " + std::to_string(K) + ")"; + return h; + } + const int64_t kb = K / 32, strips = 2 * kb, unit = block32_unit_bytes(K); + h.units = (rows + kB32UnitRows - 1) / kB32UnitRows; + h.bytes = h.units * unit; + if (avail < h.bytes) { + h.why = "the .fconst has " + std::to_string(avail) + " B at " + std::to_string(file_off) + " B, and a " + tag + + " of " + std::to_string(h.units) + " units x " + std::to_string(unit) + " B needs " + + std::to_string(h.bytes); + return h; + } + b32_q_init(h.q, h.units * kB32UnitRows, K); + for (int64_t u = 0; u < h.units; ++u) { + const uint8_t* up = p + u * unit; + const std::string where = tag + " unit " + std::to_string(u) + " (at " + std::to_string(file_off + u * unit) + + " B)"; + if (!b32_all_zero(up, kB32UnitHeader)) { + h.why = where + " does not open with " + std::to_string(kB32UnitHeader) + + " zero bytes, so this is not the layout measured"; + return h; + } + for (int64_t s = 0; s < strips; ++s) { + const uint8_t* sp = up + kB32UnitHeader + s * kB32StripBytes; + if (!b32_all_zero(sp + 528, 48)) { + h.why = where + " strip " + std::to_string(s) + + " has non-zero bytes between its zero points and its scales, so this is not the layout measured"; + return h; + } + const int64_t row0 = u * kB32UnitRows + 32 * (s % 2); + for (int r = 0; r < 32; ++r) { + const float sc = b32_bf16(sp + 576 + 2 * r); + if (!std::isfinite(sc)) { + h.why = where + " strip " + std::to_string(s) + " row " + std::to_string(r) + + " has a scale that is not finite"; + return h; + } + b32_row(h.q, row0 + r, s / 2, sp, r, (sp[512 + r / 2] >> (4 * (r & 1))) & 15, sc); + } + } + } + h.ok = true; + return h; +} + +Block32Head read_block32_head(const uint8_t* p, int64_t avail, int64_t V, int64_t K, int64_t file_off) { Block32Head h; if (V <= 0 || K <= 0 || K % 32 != 0) { h.why = "a block-32 head needs K to be a multiple of 32 (K " + std::to_string(K) + ")"; return h; } - const int64_t kb = K / 32, strips = 2 * kb, unit = block32_unit_bytes(K); + const int64_t unit = block32_unit_bytes(K); h.units = (V + kB32UnitRows - 1) / kB32UnitRows; h.bytes = h.units * unit; if (avail < h.bytes) { @@ -2585,58 +2675,60 @@ Block32Head read_block32_head(const uint8_t* p, int64_t avail, int64_t V, int64_ return h; } h.padding = avail - h.bytes; - const auto all_zero = [](const uint8_t* b, int64_t n) { - for (int64_t i = 0; i < n; ++i) - if (b[i] != 0) return false; - return true; - }; - Q4& q = h.q; - q.N = int(h.units * kB32UnitRows); - q.K = int(K); - q.block = 32; - q.codes.assign(size_t(q.N) * size_t(K), 0); - q.scale.assign(size_t(q.N) * size_t(kb), 1.0f); - q.zero.assign(size_t(q.N) * size_t(kb), 0.0f); + Block32Units u = read_block32_units(p, h.bytes, V, K, "head", file_off); + if (!u.ok) { + h.why = u.why; + return h; + } + h.q = std::move(u.q); + h.ok = true; + return h; +} + +Block32GateUp read_block32_gate_up(const uint8_t* p, int64_t avail, int64_t rows, int64_t K, int64_t file_off) { + Block32GateUp h; + if (rows <= 0 || K <= 0 || K % 32 != 0) { + h.why = "a block-32 gate/up needs K to be a multiple of 32 (K " + std::to_string(K) + ")"; + return h; + } + const int64_t kb = K / 32, strips = 2 * kb, unit = block32_mlp_unit_bytes(K); + h.units = (rows + kB32UnitRows - 1) / kB32UnitRows; + h.bytes = h.units * unit; + if (avail < h.bytes) { + h.why = "the .fconst has " + std::to_string(avail) + " B at " + std::to_string(file_off) + + " B, and a block-32 gate/up of " + std::to_string(h.units) + " units x " + std::to_string(unit) + + " B needs " + std::to_string(h.bytes); + return h; + } + b32_q_init(h.gate, h.units * kB32UnitRows, K); + b32_q_init(h.up, h.units * kB32UnitRows, K); for (int64_t u = 0; u < h.units; ++u) { const uint8_t* up = p + u * unit; - if (!all_zero(up, kB32UnitHeader)) { - h.why = "block-32 head unit " + std::to_string(u) + " does not open with " + - std::to_string(kB32UnitHeader) + " zero bytes, so this is not the layout measured"; + const std::string where = + "block-32 gate/up unit " + std::to_string(u) + " (at " + std::to_string(file_off + u * unit) + " B)"; + if (!b32_all_zero(up, kB32MlpUnitHeader)) { + h.why = where + " does not open with " + std::to_string(kB32MlpUnitHeader) + + " zero bytes, so this is not the layout measured"; return h; } for (int64_t s = 0; s < strips; ++s) { - const uint8_t* sp = up + kB32UnitHeader + s * kB32StripBytes; - if (!all_zero(sp + 528, 48)) { - h.why = "block-32 head unit " + std::to_string(u) + " strip " + std::to_string(s) + + const uint8_t* sp = up + kB32MlpUnitHeader + s * kB32MlpStripBytes; + if (!b32_all_zero(sp + 1056, 32)) { + h.why = where + " strip " + std::to_string(s) + " has non-zero bytes between its zero points and its scales, so this is not the layout measured"; return h; } - const int64_t col0 = (s / 2) * 32, row0 = u * kB32UnitRows + 32 * (s % 2); - for (int r = 0; r < 32; ++r) { - const uint16_t bits = uint16_t(sp[576 + 2 * r]) | uint16_t(uint16_t(sp[577 + 2 * r]) << 8); - const uint32_t wide = uint32_t(bits) << 16; - float sc; - std::memcpy(&sc, &wide, 4); + const int64_t row0 = u * kB32UnitRows + 32 * (s % 2); + for (int slot = 0; slot < 64; ++slot) { + const float sc = b32_bf16(sp + 1088 + 2 * slot); if (!std::isfinite(sc)) { - h.why = "block-32 head unit " + std::to_string(u) + " strip " + std::to_string(s) + " row " + - std::to_string(r) + " has a scale that is not finite"; + h.why = where + " strip " + std::to_string(s) + " slot " + std::to_string(slot) + + " has a scale that is not finite"; return h; } - const int zp = (sp[512 + r / 2] >> (4 * (r & 1))) & 15; - const bool flip = sc < 0; - const size_t g = size_t(row0 + r) * size_t(kb) + size_t(s / 2); - if (sc == 0.0f) continue; // an all-zero block: codes 0, scale 1, zero 0 decode to 0 - const float mag = flip ? -sc : sc; - q.scale[g] = mag; - q.zero[g] = -float(flip ? 15 - zp : zp) * mag; - // position p = band*256 + col*8 + (r%8) holds row band*8 + r%8, column col - const int base = (r / 8) * 256 + (r % 8); - uint8_t* dst = q.codes.data() + size_t(row0 + r) * size_t(K) + size_t(col0); - for (int c = 0; c < 32; ++c) { - const int pos = base + c * 8; - const int code = (sp[pos / 2] >> (4 * (pos & 1))) & 15; - dst[c] = uint8_t(flip ? 15 - code : code); - } + // slots 0..31 are gate rows, 32..63 the same rows of up; zero point and scale by slot (assumed) + b32_row(slot < 32 ? h.gate : h.up, row0 + slot % 32, s / 2, sp, slot, + (sp[1024 + slot / 2] >> (4 * (slot & 1))) & 15, sc); } } } @@ -2644,6 +2736,70 @@ Block32Head read_block32_head(const uint8_t* p, int64_t avail, int64_t V, int64_ return h; } +Block32Layout block32_layout(const Dims& d) { + Block32Layout l; + const int64_t H = d.hidden, IM = d.intermediate, Nq = int64_t(d.heads) * d.head_dim, + Nkv = int64_t(d.kv_heads) * d.head_dim, V = d.vocab; + if (H <= 0 || IM <= 0 || Nq <= 0 || Nkv <= 0 || V <= 0 || d.layers <= 0) { + l.why = "the all-block-32 layout needs hidden, intermediate, heads, kv heads, head_dim, vocab and layers, and " + "the pack's geometry leaves one of them 0"; + return l; + } + for (const auto& [name, v] : {std::pair{"hidden", H}, + std::pair{"intermediate", IM}, + std::pair{"heads*head_dim", Nq}, + std::pair{"kv_heads*head_dim", Nkv}}) + if (v % kB32UnitRows != 0) { + l.why = std::string("the all-block-32 layout is 64-row units of 32-column blocks, and ") + name + " " + + std::to_string(v) + " is not a multiple of 64"; + return l; + } + l.env = kNormOff + 2 * H; + l.q_off = l.env; + l.k_off = l.q_off + (Nq / kB32UnitRows) * block32_unit_bytes(H); + l.v_off = l.k_off + (Nkv / kB32UnitRows) * block32_unit_bytes(H); + l.sink_off = l.v_off + (Nkv / kB32UnitRows) * block32_unit_bytes(H); + l.o_off = l.sink_off + 4 * int64_t(d.heads); + l.post_off = l.o_off + (H / kB32UnitRows) * block32_unit_bytes(Nq); + l.gu_off = l.post_off + l.env; + l.hdr_off = l.gu_off + (IM / kB32UnitRows) * block32_mlp_unit_bytes(H); + l.down_off = l.hdr_off + 64; + l.stride = l.down_off + (H / kB32UnitRows) * block32_unit_bytes(IM); + l.tail_off = int64_t(d.layers) * l.stride; + l.head_off = l.tail_off + l.env; + l.head_units = (V + kB32UnitRows - 1) / kB32UnitRows; + l.min_size = l.head_off + l.head_units * block32_unit_bytes(H); + return l; +} + +std::string block32_file_refusal(const uint8_t* map, int64_t size, const Dims& d, const Block32Layout& lay) { + if (!lay.why.empty()) return lay.why; + if (size < lay.min_size) + return "the Token .fconst is " + std::to_string(size) + " B, and the all-block-32 layout of this geometry is " + + std::to_string(d.layers) + " layer blocks x " + std::to_string(lay.stride) + " B + a " + + std::to_string(lay.env) + " B envelope + " + std::to_string(lay.head_units) + " head units x " + + std::to_string(block32_unit_bytes(d.hidden)) + " B = " + std::to_string(lay.min_size) + + " B (plus zero padding), so the file is not this layout"; + const auto envelope = [&](int64_t off, const std::string& what) -> std::string { + if (map[off] != uint8_t(d.eps_bits & 0xFF) || map[off + 1] != uint8_t(d.eps_bits >> 8)) + return "no RMSNorm epsilon at " + std::to_string(off) + " B, where the all-block-32 layout puts the " + + what + " envelope, so the file is not this layout"; + for (int64_t i = 2; i < kNormOff; ++i) + if (map[off + i] != 0) + return "the " + what + " envelope at " + std::to_string(off) + " B is not zero between its epsilon " + "and its norm (byte " + std::to_string(off + i) + "), so the file is not this layout"; + return {}; + }; + for (int L = 0; L <= d.layers; ++L) { + const int64_t base = int64_t(L) * lay.stride; + const std::string name = L == d.layers ? std::string("tail") : "layer " + std::to_string(L); + std::string why = envelope(base, name + (L == d.layers ? "" : " input_layernorm")); + if (why.empty() && L < d.layers) why = envelope(base + lay.post_off, name + " post_attention_layernorm"); + if (!why.empty()) return why; + } + return {}; +} + // engine#1740: a self-test for the SEARCH, not the arithmetic. `plan_tail_head` is pinned by the host test; this pins // the part that can go silently wrong -- the candidate loop, best/runner-up tracking, the kTieCorr cut, the ambiguity // refusal and the untied-with-padding refusal -- using a synthetic head built through encode_tile, so it needs no pack, @@ -2913,6 +3069,17 @@ std::string Plan::manifest() const { {"o_proj_k", o_k}, {"qk_norm", knorm_off != 0}, {"note", attn_note}}; + // engine#1503: an all-block-32 pack is read with its own layout, so the block-128 measurements above are 0 for it; + // this says which layout was used and what its arithmetic gives. Absent for every other pack. + if (block32_layers) { + const Block32Layout lay = block32_layout(dims); + j["block32_layers"] = {{"stride", lay.stride}, + {"post_attention_envelope", lay.post_off}, + {"head_offset", lay.head_off}, + {"head_units", lay.head_units}, + {"bytes", lay.min_size}, + {"note", block32_note}}; + } // What the *graph* carries, in the graph's own terms. A pack's genre is the first question every sweep row asks, // and until now it could only be answered by an external tool: the rai_1.3/1.6/1.7.1 line ships its 4-bit weights // as ordinary ONNX initializers (pi-inferzo: 25/25 real qweights, UINT8 [1152,7,64], location model_jit.onnx.data) @@ -3140,7 +3307,22 @@ Plan plan(const std::string& pack_dir) { // The Token .fconst, found once (see find_token_fconst) and used by the attention block below. Empty means this // pack keeps no weights there -- which for the rai_1.3/1.6/1.7.1 line is the normal case, not a fault. std::string tok_why; - std::string tok_path = materialise_token_fconst(pack_dir, p.dims); + const DeclaredMap declared = read_declared(pack_dir); + // engine#1503: a pack whose every declared MatMulNBits is block 32 -- layers included, not only the head -- is read + // from its RAW .fconst with the all-block-32 layout (pack_import.h, Block32Layout). Decided here, before the + // canonical copy is made, because that copy is the 1.8 form of a block-128 file and is not what this path reads. + // A pack that declares only its head (the block-32-head fixtures, and nothing measured) is not this class. + { + bool any = false, all32 = true, layer = false; + for (const auto& [proj, dec] : declared) { + if (!dec.block) continue; + any = true; + if (dec.block != 32) all32 = false; + if (proj.rfind("lm_head", 0) != 0) layer = true; + } + p.block32_layers = any && all32 && layer && !find_token_fconst(pack_dir).empty(); + } + std::string tok_path = p.block32_layers ? find_token_fconst(pack_dir) : materialise_token_fconst(pack_dir, p.dims); // engine#1671 / #1687: a pack whose graph names `*.qweight.empty` placeholders whose `*.packed` siblings carry no // data has no weight for those ops IN THE GRAPH. With a Token .fconst that is the normal shape of a 1.9 pack -- its // graph is all placeholders and the weights are in the .fconst -- so the walker drops them and the import proceeds. @@ -3198,6 +3380,17 @@ Plan plan(const std::string& pack_dir) { {embed->dims.at(0), embed->dims.at(1)})); } + // engine#1503: the all-block-32 layout against the file, once the vocab is known (the head's unit count needs it). + if (p.block32_layers) { + const Block32Layout lay = block32_layout(p.dims); + if (!lay.why.empty()) { + p.block32_note = lay.why; + } else { + Fconst fc = open_fconst(tok_path); + p.block32_note = block32_file_refusal(fc.map, int64_t(fc.size), p.dims, lay); + unmap_file(fc.map, fc.size); + } + } // Only the node whose geometry IS the head (N = V, K = H) is the head. This loop used to take the FIRST MatMulNBits and // `break`, so on a pack whose first matmul is a projection the plan reported that projection's shape as // `lm_head.weight` -- [1152, 896], a fused qkv, instead of [151936, 896] on Qwen2.5-0.5B-Instruct 1.9 (engine#1671). @@ -3218,14 +3411,16 @@ Plan plan(const std::string& pack_dir) { // there if the tail's arithmetic does not come out, so the plan does not refuse it here. Keyed on "did a graph // head get written", not "is the graph free of matmuls": a graph of projections only is not a pack without a head // (engine#1671). - p.tensors.push_back(make("lm_head.weight", "fconst:tail", "q4_1", {d.vocab, d.hidden})); + auto x = make("lm_head.weight", "fconst:tail", "q4_1", {d.vocab, d.hidden}); + x.refusal = p.block32_note; // empty unless an all-block-32 pack's file does not fit its layout + p.tensors.push_back(std::move(x)); } // norms: the layer blocks are located by their epsilon anchors at import time, so the offsets work // for any hidden size (engine#734). plan() cannot prove that without reading the file, so the // refusal here is empty and import_pack reports what the anchors actually found. const std::string norm_source = "fconst:Llm_Token_Token"; - const std::string norm_refusal; + const std::string norm_refusal = p.block32_note; // empty except on an all-block-32 pack whose file does not fit const int64_t H = d.hidden; // the geometry's hidden; the embedding, when the main graph has none, must match it // Genre A: the norms are ordinary F16 initializers named after themselves, with real offsets -- not the // .fconst tail envelope -- so the refusal names the initializer and where it sits. @@ -3265,6 +3460,7 @@ Plan plan(const std::string& pack_dir) { // and the norm at +64, and where both .fconst files exist the two copies are byte-identical. The // position is confirmed at import time by the epsilon marker plus that cross-check. auto fin = make("model.norm.weight", norm_source, "BF16", {H}); + fin.refusal = p.block32_note; p.tensors.push_back(std::move(fin)); // projections: the attention payload (q/k/v/o) decodes tile by tile and the MLP section group by @@ -3281,6 +3477,10 @@ Plan plan(const std::string& pack_dir) { if (tok.empty()) { p.attn_note = tok_why; p.mlp_note = p.attn_note; + } else if (p.block32_layers) { + // the all-block-32 layout is arithmetic on the geometry, checked against the file above + p.attn_note = p.block32_note; + p.mlp_note = p.block32_note; } else if (tile_bytes == 0) { p.attn_note = "hidden " + std::to_string(H) + " is not a multiple of 128, so the attention " "payload is not 64-row tiles of this shape"; @@ -3450,7 +3650,6 @@ Plan plan(const std::string& pack_dir) { {"mlp.gate_proj", d.intermediate, d.hidden}, {"mlp.up_proj", d.intermediate, d.hidden}, {"mlp.down_proj", d.hidden, d.intermediate}}; - const DeclaredMap declared = read_declared(pack_dir); // Block size gate (engine#1410 §3 of FACTS-fconst-layout): the compiler states `block_size` per MatMulNBits, and // this importer's tile geometry is the block-128 one -- a group is 2*(K/block)+2 subvolumes, so a block-32 pack // has every offset in a different place. Refusing by number beats parsing it at the 1.8 geometry, which is what @@ -3488,7 +3687,12 @@ Plan plan(const std::string& pack_dir) { // calling the writer here cost a second 278 MiB rewrite per command. const bool fconst_weights = !find_token_fconst(pack_dir).empty(); if (fconst_weights && !blocks.empty() && !(blocks.size() == 1 && blocks.begin()->first == kKBlock)) { - if (only_the_head) { + if (p.block32_layers) { + // engine#1503: every op is block 32 and the pack is read with the all-block-32 layout, which was + // checked against the file where `block32_layers` was decided; a file that does not fit is refused + // tensor by tensor with `block32_note`. + if (const auto it = declared.find("lm_head"); it != declared.end()) p.head_block = it->second.block; + } else if (only_the_head) { p.head_block = declared.at(off_block[0].substr(0, off_block[0].find(" @ "))).block; } else { // Name a few and count the rest: SmolLM2-135M has 151 off-block ops, and listing every one of them made @@ -3505,7 +3709,9 @@ Plan plan(const std::string& pack_dir) { }() + ", and this importer decodes the block-" + std::to_string(kKBlock) + " tile layout only: at a different block every subvolume offset in the .fconst moves, so the " "weights would be read from the wrong place, not merely mis-scaled. The off-block tensor(s) are " + - list + "; only the LM head could be tolerated, because the layers come before it in the tail."); + list + "; only the LM head could be tolerated, because the layers come before it in the tail. (The " + "one other layout this importer reads is the all-block-32 one, and only for a pack that declares " + "32 on every op.)"); } } if (!bits.empty() && !(bits.size() == 1 && bits.begin()->first == 4)) @@ -4604,6 +4810,175 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons break; } + // engine#1740: read a block-32 head whose first unit is at `head_off` of the Token .fconst and write it as + // lm_head.weight. Returns why it was NOT taken (empty when it was). On a tied pack the read has to correlate with + // the embedding above kTieCorr; on an untied one the structural checks are all there is, which the note states. + // Shared by the mixed pack (layers at 128, head at 32) and the all-block-32 pack (engine#1503). + auto take_block32_head = [&](const Fconst& fc, int64_t head_off) -> std::string { + std::string why; + Block32Head h32 = read_block32_head(fc.map + head_off, int64_t(fc.size) - head_off, V, H, head_off); + const bool is_tied = tie_declared && tied; + double c32 = 0.0; + if (h32.ok && embed_tbl.ok()) c32 = corr(h32.q, embed_tbl, int(V)); + if (!h32.ok) { + why = h32.why; + } else if (is_tied && !(c32 > kTieCorr)) { + char b[160]; + std::snprintf(b, sizeof b, + "the tail's block-32 head decodes but correlates %.6f with the embedding of " + "a pack that declares them tied", c32); + why = b; + } else { + Q4& q = h32.q; + std::fprintf(stderr, + " lm head: block 32, %lld unit(s) of %lld B at %lld B, %lld B of zeros after " + "them%s\n", + (long long)h32.units, (long long)block32_unit_bytes(H), + (long long)head_off, (long long)h32.padding, + is_tied ? " (tied: verified by correlation with the pack's embedding)" + : " (untied: layout checked, values not verified)"); + if (embed_tbl.ok()) { + r.lmhead_corr = c32; + { + double sum_abs = 0, sum_ref = 0; + for (int n = 0; n < int(V); ++n) + for (int k = 0; k < q.K; ++k) { + const size_t gg = size_t(n) * (q.K / 32) + size_t(k) / 32; + const double x = double(q.codes[size_t(n) * q.K + k]) * q.scale[gg] + q.zero[gg]; + const double e = embed_at(embed_tbl, n, k); + sum_abs += std::fabs(x - e); + sum_ref += std::fabs(e); + } + if (sum_ref > 0) r.lmhead_rel = sum_abs / sum_ref; + } + r.selfcheck_reference = true; + r.selfcheck_tied = c32 > kTieCorr; + double sxy = 0, sx = 0, sy = 0, sxx = 0, syy = 0; + const int nsh = std::min(2048, int(V) - 1); + for (int rr = 0; rr < nsh; ++rr) + for (int k = 0; k < q.K; ++k) { + const size_t gg = size_t(rr) * (q.K / 32) + size_t(k) / 32; + const double x = double(q.codes[size_t(rr) * q.K + k]) * q.scale[gg] + q.zero[gg]; + const double y = embed_at(embed_tbl, rr + 1, k); + sxy += x * y; sx += x; sy += y; sxx += x * x; syy += y * y; + } + const double m = double(nsh) * q.K; + const double cx = sxx - sx * sx / m, cy = syy - sy * sy / m; + if (m > 0 && cx > 1e-12 && cy > 1e-12) + r.ctrl_corr_rowshift = (sxy - sx * sy / m) / std::sqrt(cx * cy); + } + std::vector tiled; + long long pinexact = 0; + tile_q4_1_f32(q, tiled, &pinexact); + r.codes_total = q.codes.size(); + r.zero_exact_pct = 100.0 * double(q.scale.size() - pinexact) / double(q.scale.size()); + const int64_t head_tiles = int64_t(tiled.size() / kImportTileBytes); + r.tiles += head_tiles; + w.add("lm_head.weight", "I8", {head_tiles, kImportTileBytes}, std::move(tiled)); + ++r.tensors; + head_written = true; + r.head_read_note = + std::string("lm_head was read from the .fconst tail's block 32 head (") + + std::to_string(h32.units) + " units of " + std::to_string(block32_unit_bytes(H)) + + " B)" + (is_tied ? ", and it reproduces model.embed_tokens" + : "; the pack is untied, so its values are not verified against " + "anything in the pack"); + } + return why; + }; + if (p.block32_layers) { + // engine#1503: every MatMulNBits of this pack is block 32. The RAW Token .fconst is read with the layout in + // pack_import.h (Block32Layout); the offsets are arithmetic on the geometry, and plan() has already checked the + // file's size and every envelope's epsilon against them. The tensors written are the ones the block-128 walk + // below writes, in the same form. This layout has no bias field, so no bias tensor is written. + std::vector late; + if (p.block32_note.empty()) { + const Block32Layout lay = block32_layout(p.dims); + Fconst fc = open_fconst(find_token_fconst(pack_dir)); + // the file is re-checked on this mapping: plan() looked at its own + if (const std::string why = block32_file_refusal(fc.map, int64_t(fc.size), p.dims, lay); !why.empty()) + throw std::runtime_error(fc.path + ": " + why); + r.stride = lay.stride; + r.second_header = lay.post_off; + if (!head_written) { + const std::string why = take_block32_head(fc, lay.head_off); + if (!why.empty()) late.push_back("lm_head.weight: " + why); + else r.block32_units += lay.head_units; + } + auto add_norm = [&](int64_t off, const std::string& name) { + w.add(name, "BF16", {H}, as_bf16(at(fc, off, H * 2), kBF16, size_t(H))); + ++r.tensors; + }; + for (int L = 0; L < p.dims.layers; ++L) { + const int64_t base = int64_t(L) * lay.stride; + r.norm_anchors_total += 2; + r.norm_anchors_ok += int(eps_at(fc, base, p.dims.eps_bits)) + + int(eps_at(fc, base + lay.post_off, p.dims.eps_bits)); + add_norm(base + kNormOff, "model.layers." + std::to_string(L) + ".input_layernorm.weight"); + add_norm(base + lay.post_off + kNormOff, + "model.layers." + std::to_string(L) + ".post_attention_layernorm.weight"); + } + add_norm(lay.tail_off + kNormOff, "model.norm.weight"); + r.final_norm_imported = true; + if (!find_one(pack_dir, "dd_metastate_Llm_Prefill_", ".fconst").empty()) + r.fconst_note = "the pack ships a prefill .fconst, and its copy of the final norm is not compared on " + "the all-block-32 path"; + const int64_t q_out = int64_t(p.dims.heads) * p.dims.head_dim, + kv_out = int64_t(p.dims.kv_heads) * p.dims.head_dim, IM = p.dims.intermediate; + long long attn_inexact = 0, attn_groups = 0, mlp_inexact = 0, mlp_groups = 0; + auto emit = [&](Q4& q, const std::string& name, bool mlp) { + std::vector tiled; + long long pinexact = 0; + tile_q4_1_f32(q, tiled, &pinexact); + (mlp ? mlp_inexact : attn_inexact) += pinexact; + (mlp ? mlp_groups : attn_groups) += (long long)q.scale.size(); + const int64_t nt = int64_t(tiled.size() / kImportTileBytes); + r.tiles += nt; + w.add(name + ".weight", "I8", {nt, kImportTileBytes}, std::move(tiled)); + ++r.tensors; + ++(mlp ? r.mlp_projections : r.attn_projections); + }; + struct Slot { const char* proj; int64_t off, rows, k; bool mlp; }; + const Slot slots[] = {{"self_attn.q_proj", lay.q_off, q_out, H, false}, + {"self_attn.k_proj", lay.k_off, kv_out, H, false}, + {"self_attn.v_proj", lay.v_off, kv_out, H, false}, + {"self_attn.o_proj", lay.o_off, H, q_out, false}, + {"mlp.down_proj", lay.down_off, H, IM, true}}; + for (int L = 0; L < p.dims.layers; ++L) { + const int64_t base = int64_t(L) * lay.stride; + const std::string layer = "model.layers." + std::to_string(L) + "."; + for (const auto& sl : slots) { + Block32Units u = read_block32_units(fc.map + base + sl.off, int64_t(fc.size) - base - sl.off, + sl.rows, sl.k, sl.proj, base + sl.off); + if (!u.ok) { + late.push_back(layer + sl.proj + ".weight: " + u.why); + continue; + } + r.block32_units += u.units; + emit(u.q, layer + sl.proj, sl.mlp); + } + Block32GateUp gu = read_block32_gate_up(fc.map + base + lay.gu_off, + int64_t(fc.size) - base - lay.gu_off, IM, H, base + lay.gu_off); + if (!gu.ok) { + late.push_back(layer + "mlp.gate_proj.weight: " + gu.why); + late.push_back(layer + "mlp.up_proj.weight: " + gu.why); + continue; + } + r.block32_units += gu.units; + emit(gu.gate, layer + "mlp.gate_proj", true); + emit(gu.up, layer + "mlp.up_proj", true); + } + r.attn_zero_inexact_pct = attn_groups ? 100.0 * double(attn_inexact) / double(attn_groups) : 0.0; + r.mlp_zero_inexact_pct = mlp_groups ? 100.0 * double(mlp_inexact) / double(mlp_groups) : 0.0; + unmap_file(fc.map, fc.size); + } else { + r.fconst_note = p.block32_note; // every .fconst tensor is already refused by name with this + } + if (!late.empty()) { + for (auto& s : late) r.refused.push_back(std::move(s)); + if (!opt.allow_incomplete) return r; // nothing has been written yet + } + } else { std::string path_why; const std::string path = materialise_token_fconst(pack_dir, p.dims); @@ -4646,76 +5021,8 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons if (tail_off + env > int64_t(fc.size)) { head32_why = "the .fconst ends before the tail's envelope"; } else { - Block32Head h32 = read_block32_head(fc.map + tail_off + env, int64_t(fc.size) - tail_off - env, - V, H); - const bool is_tied = tie_declared && tied; - double c32 = 0.0; - if (h32.ok && embed_tbl.ok()) c32 = corr(h32.q, embed_tbl, int(V)); - if (!h32.ok) { - head32_why = h32.why; - } else if (is_tied && !(c32 > kTieCorr)) { - char b[160]; - std::snprintf(b, sizeof b, - "the tail's block-32 head decodes but correlates %.6f with the embedding of " - "a pack that declares them tied", c32); - head32_why = b; - } else { - Q4& q = h32.q; - std::fprintf(stderr, - " lm head: block 32, %lld unit(s) of %lld B at %lld B, %lld B of zeros after " - "them%s\n", - (long long)h32.units, (long long)block32_unit_bytes(H), - (long long)(tail_off + env), (long long)h32.padding, - is_tied ? " (tied: verified by correlation with the pack's embedding)" - : " (untied: layout checked, values not verified)"); - if (embed_tbl.ok()) { - r.lmhead_corr = c32; - { - double sum_abs = 0, sum_ref = 0; - for (int n = 0; n < int(V); ++n) - for (int k = 0; k < q.K; ++k) { - const size_t gg = size_t(n) * (q.K / 32) + size_t(k) / 32; - const double x = double(q.codes[size_t(n) * q.K + k]) * q.scale[gg] + q.zero[gg]; - const double e = embed_at(embed_tbl, n, k); - sum_abs += std::fabs(x - e); - sum_ref += std::fabs(e); - } - if (sum_ref > 0) r.lmhead_rel = sum_abs / sum_ref; - } - r.selfcheck_reference = true; - r.selfcheck_tied = c32 > kTieCorr; - double sxy = 0, sx = 0, sy = 0, sxx = 0, syy = 0; - const int nsh = std::min(2048, int(V) - 1); - for (int rr = 0; rr < nsh; ++rr) - for (int k = 0; k < q.K; ++k) { - const size_t gg = size_t(rr) * (q.K / 32) + size_t(k) / 32; - const double x = double(q.codes[size_t(rr) * q.K + k]) * q.scale[gg] + q.zero[gg]; - const double y = embed_at(embed_tbl, rr + 1, k); - sxy += x * y; sx += x; sy += y; sxx += x * x; syy += y * y; - } - const double m = double(nsh) * q.K; - const double cx = sxx - sx * sx / m, cy = syy - sy * sy / m; - if (m > 0 && cx > 1e-12 && cy > 1e-12) - r.ctrl_corr_rowshift = (sxy - sx * sy / m) / std::sqrt(cx * cy); - } - std::vector tiled; - long long pinexact = 0; - tile_q4_1_f32(q, tiled, &pinexact); - r.codes_total = q.codes.size(); - r.zero_exact_pct = 100.0 * double(q.scale.size() - pinexact) / double(q.scale.size()); - const int64_t head_tiles = int64_t(tiled.size() / kImportTileBytes); - r.tiles += head_tiles; - w.add("lm_head.weight", "I8", {head_tiles, kImportTileBytes}, std::move(tiled)); - ++r.tensors; - head_written = true; - head32_read = true; - r.head_read_note = - std::string("lm_head was read from the .fconst tail's block 32 head (") + - std::to_string(h32.units) + " units of " + std::to_string(block32_unit_bytes(H)) + - " B)" + (is_tied ? ", and it reproduces model.embed_tokens" - : "; the pack is untied, so its values are not verified against " - "anything in the pack"); - } + head32_why = take_block32_head(fc, tail_off + env); + head32_read = head32_why.empty(); } } if (head32_read) { @@ -5523,9 +5830,12 @@ int run(int argc, char** argv) { r.mlp_groups_ok, r.mlp_groups_total, r.mlp_projections, 100.0 - r.mlp_zero_inexact_pct); if (r.stride) - std::printf(" fconst: stride %lld, second header %lld (measured from the eps anchors); " + std::printf(" fconst: stride %lld, second header %lld (%s); " "%d/%d RMSNorm anchors found, final norm %s\n", - (long long)r.stride, (long long)r.second_header, r.norm_anchors_ok, + (long long)r.stride, (long long)r.second_header, + r.block32_units ? "computed from the geometry, checked at the eps anchors" + : "measured from the eps anchors", + r.norm_anchors_ok, r.norm_anchors_total, !r.final_norm_imported ? "REFUSED (no readable tail envelope)" @@ -5533,6 +5843,11 @@ int run(int argc, char** argv) { ? "identical in both .fconst tails" : "taken from the Token_Token tail -- the two copies DISAGREE") : "taken from the tail envelope (the pack ships no prefill copy)")); + if (r.block32_units) + std::printf(" block 32: every MatMulNBits is block 32: %d attention + %d MLP projections and the head read " + "from %lld units of the raw .fconst (zero fields, finite scales and epsilon anchors checked; this " + "path has no decode->encode round trip)\n", + r.attn_projections, r.mlp_projections, r.block32_units); if (!r.fconst_note.empty()) std::printf(" fconst: %s\n", r.fconst_note.c_str()); if (!r.refused.empty()) std::printf(" incomplete: %zu tensors, listed in %s/model.q4nx\n", r.refused.size(), out.c_str()); diff --git a/npu/pack_import.h b/npu/pack_import.h index 775eed018..48140ecd2 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -183,6 +183,12 @@ struct Plan { // .fconst tile geometry assumes. Non-zero means the tail's head tiles cannot be walked (engine#1503): a tied pack // takes the head from its embedding instead, an untied one is refused by name. int64_t head_block = 0; + // engine#1503: every MatMulNBits the pack declares is block 32 and at least one of them is a layer projection, and + // the pack has a Token .fconst. Such a pack is read with the all-block-32 layout (Block32Layout below) from the + // raw .fconst; the block-128 walk and the canonical copy are not used. `block32_note` is why that file does not + // fit the layout (empty when it does), and is the refusal every .fconst tensor of the plan then carries. + bool block32_layers = false; + std::string block32_note; std::string pack; // *_rai_1.8.0_npu_4K, from the directory name Dims dims; std::vector tensors; @@ -442,6 +448,10 @@ struct Report { long long mlp_groups_ok = 0, mlp_groups_total = 0; int mlp_projections = 0; double mlp_zero_inexact_pct = 0; + + // engine#1503: the all-block-32 path. Units read (640-byte-strip units plus gate/up units, head included). There + // is no decode->encode round trip on this path, so the two counters above stay 0 for such a pack. + long long block32_units = 0; }; Report import_pack(const std::string& pack_dir, const std::string& out_dir, const Options&); @@ -490,8 +500,81 @@ struct Block32Head { int64_t padding = 0; // zero bytes after the last unit std::string why; // set when !ok }; -// `p` points at the first unit (tail_off + envelope), `avail` is the bytes from there to the end of the file. -Block32Head read_block32_head(const uint8_t* p, int64_t avail, int64_t V, int64_t K); +// `p` points at the first unit (tail_off + envelope), `avail` is the bytes from there to the end of the file, and +// `file_off` is where `p` sits in the file (only used to put an offset in a refusal). +Block32Head read_block32_head(const uint8_t* p, int64_t avail, int64_t V, int64_t K, int64_t file_off = 0); + +// The same units for any matrix (engine#1503): `ceil(rows/64)` units of `block32_unit_bytes(K)` starting at `p`. This +// is the head reader without its tail rule (nothing is required of the bytes after the last unit), and it is what the +// all-block-32 layer block's q/k/v/o/down are read with. `what` names the tensor in a refusal. Same checks: the bytes +// fit, every unit header and strip gap is zero, every scale is finite; same normalisation (positive scales). +struct Block32Units { + bool ok = false; + Q4 q; // N = units x 64 (>= rows), K, block 32 + int64_t units = 0; + int64_t bytes = 0; // units x block32_unit_bytes(K) + std::string why; // set when !ok +}; +Block32Units read_block32_units(const uint8_t* p, int64_t avail, int64_t rows, int64_t K, const char* what = "head", + int64_t file_off = 0); + +// The gate/up unit of an all-block-32 pack (engine#1503): gate_proj and up_proj share one run of 64-row units, each +// `4,864 + 2*(K/32)*1,216` bytes. +// +// Measured on `SmolLM2-135M-Instruct_rai_1.9.0_npu` (K 576, intermediate 1,536: 24 units of 48,640 B per layer) +// against the public checkpoint's gate_proj / up_proj: +// unit : 4,864 B of zeros, then 2*(K/32) strips of 1,216 B +// strip s: K block `s/2` (32 columns), 64 row SLOTS: slots 0..31 are GATE rows `32*(s%2) .. +31` of the unit, +// slots 32..63 are UP rows with the same numbers +// +0 1,024 B of 4-bit codes, low nibble first; code position p (0..2047) is slot `(p/256)*8 + p%8`, column +// `(p/8)%32` -- the head strip's order over eight 8-slot bands instead of four +// +1024 32 B of zero points, one nibble per slot +// +1056 32 B of zeros +// +1088 64 bf16 scales, one per slot, signed +// The code order is the measured part (0.98 .. 0.996 per 32-code block against the reference). That zero point and +// scale are indexed by slot is ASSUMED by analogy with the 640-byte strip; the import of the measured pack is what +// tests it (a container gate/up that still correlates with the checkpoint after the zero points and scales are +// applied). Returned as two `Q4`s with the same normalisation and the same refusals as the unit reader. +constexpr int64_t kB32MlpUnitHeader = 4864, kB32MlpStripBytes = 1216; +inline int64_t block32_mlp_unit_bytes(int64_t K) { return kB32MlpUnitHeader + 2 * (K / 32) * kB32MlpStripBytes; } +struct Block32GateUp { + bool ok = false; + Q4 gate, up; // each N = units x 64, K, block 32 + int64_t units = 0; + int64_t bytes = 0; // units x block32_mlp_unit_bytes(K) + std::string why; // set when !ok +}; +Block32GateUp read_block32_gate_up(const uint8_t* p, int64_t avail, int64_t rows, int64_t K, int64_t file_off = 0); + +// The layer block of a pack whose EVERY MatMulNBits is block 32 (engine#1503), as offsets from the block's start. +// Read from the RAW 1.9 Token .fconst (not the canonical 1.8-form copy). With H = hidden, Nq = heads*head_dim, +// Nkv = kv_heads*head_dim, IM = intermediate, unit = block32_unit_bytes, mlp_unit = block32_mlp_unit_bytes: +// +0 64 + 2*H envelope: bf16 rms_norm_eps, zeros to 64, input_layernorm bf16[H] +// q_off (Nq/64) x unit(H) q_proj [Nq][H] +// k_off (Nkv/64) x unit(H) k_proj [Nkv][H] +// v_off (Nkv/64) x unit(H) v_proj [Nkv][H] +// sink_off 4*heads bytes not a weight; skipped (int32 0xFF7F0000 x heads on the measured pack) +// o_off (H/64) x unit(Nq) o_proj [H][Nq] +// post_off 64 + 2*H envelope: eps, zeros, post_attention_layernorm bf16[H] +// gu_off (IM/64) x mlp_unit(H) gate_proj + up_proj, [IM][H] each +// hdr_off 64 bytes skipped, content not interpreted (01 then 63 zeros on the measured pack) +// down_off (H/64) x unit(IM) down_proj [H][IM] +// stride +// and the tail at `layers x stride`: the same envelope with the final norm, `ceil(V/64)` head units at K = H, zeros. +// Measured on SmolLM2-135M-Instruct 1.9 only (stride 2,360,292; file 90,470,776 B), where Nq == H -- so o_proj's K +// being Nq rather than H is the declared shape, not a measurement. There is no bias field in this layout. +struct Block32Layout { + std::string why; // non-empty: this geometry does not fit the layout (a width that is not whole units) + int64_t env = 0, q_off = 0, k_off = 0, v_off = 0, sink_off = 0, o_off = 0, post_off = 0, gu_off = 0, hdr_off = 0, + down_off = 0, stride = 0; + int64_t tail_off = 0, head_off = 0, head_units = 0; + int64_t min_size = 0; // head_off + head_units x unit(H); the file is this plus zero padding +}; +Block32Layout block32_layout(const Dims& d); +// Empty when a mapped file fits `lay`: it is at least `min_size` long, and every envelope (two per layer and the +// tail's) opens with the pack's epsilon followed by zeros up to the norm. Otherwise the reason, with the offset. The +// zero padding after the head is the head reader's check. +std::string block32_file_refusal(const uint8_t* map, int64_t size, const Dims& d, const Block32Layout& lay); // The 1.9 external-data tile (engine#1503): the `.fconst` tile's subvolume written densely per tensor. // diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index a2656f574..088cf8052 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -2156,6 +2156,508 @@ void test_block32_tail_head(const std::string& root) { } } +// ── engine#1503: a pack whose EVERY MatMulNBits is block 32 ──────────────────────────────────────────────────────── +// The gate/up unit, written from its description (pack_import.h, above read_block32_gate_up) and NOT through the +// reader: 64-row units of 4,864 zero bytes + 2*(K/32) strips of 1,216 B; strip s is K block s/2 and holds 64 slots, +// slots 0..31 = gate rows 32*(s%2)..+31, slots 32..63 = the same rows of up; code position +// `(slot/8)*256 + c*8 + slot%8`, low nibble first; 32 B of zero-point nibbles (8) by slot; 32 zero bytes; 64 signed +// bf16 scales by slot, `-m/8` as in encode_block32_units. +static std::string encode_block32_gate_up(const std::vector& G, const std::vector& U, int rows, int K) { + const int kb = K / 32, units = rows / 64; + const size_t unit = size_t(4864 + 2 * kb * 1216); + std::string out(size_t(units) * unit, '\0'); + for (int u = 0; u < units; ++u) + for (int s = 0; s < 2 * kb; ++s) { + char* sp = out.data() + size_t(u) * unit + 4864 + size_t(s) * 1216; + for (int slot = 0; slot < 64; ++slot) { + const std::vector& M = slot < 32 ? G : U; + const float* w = M.data() + size_t(u * 64 + 32 * (s % 2) + slot % 32) * size_t(K) + size_t((s / 2) * 32); + float m = 0; + for (int c = 0; c < 32; ++c) + if (std::fabs(w[c]) > std::fabs(m)) m = w[c]; + const uint16_t bits = pi::bf16_bits_of(-m / 8.0f); + const uint32_t wide = uint32_t(bits) << 16; + float sc; + std::memcpy(&sc, &wide, 4); + sp[1088 + 2 * slot] = char(bits & 0xFF); + sp[1089 + 2 * slot] = char(bits >> 8); + sp[1024 + slot / 2] = char(uint8_t(sp[1024 + slot / 2]) | uint8_t(8 << (4 * (slot & 1)))); + for (int c = 0; c < 32; ++c) { + const int code = sc == 0 ? 8 : std::min(15, std::max(0, int(std::lround(w[c] / sc)) + 8)); + const int pos = (slot / 8) * 256 + c * 8 + slot % 8; + sp[pos / 2] = char(uint8_t(sp[pos / 2]) | uint8_t(code << (4 * (pos & 1)))); + } + } + } + return out; +} + +void test_block32_layers(const std::string& root) { + const auto ncorr = [](const std::vector& a, const std::vector& b) { + double num = 0, da = 0, db = 0, sa = 0, sb = 0; + const double n = double(a.size()); + for (size_t i = 0; i < a.size(); ++i) { + num += double(a[i]) * b[i], da += double(a[i]) * a[i], db += double(b[i]) * b[i]; + sa += a[i], sb += b[i]; + } + const double den = (n * da - sa * sa) * (n * db - sb * sb); + return den > 0 ? (n * num - sa * sb) / std::sqrt(den) : 0.0; + }; + const auto value = [](const pi::Q4& q, int row, int col) { + const size_t g = size_t(row) * size_t(q.K / 32) + size_t(col / 32); + return double(q.codes[size_t(row) * size_t(q.K) + col]) * q.scale[g] + q.zero[g]; + }; + // a deterministic matrix per (seed): values in about [-2, 2], unrelated between seeds + const auto matrix = [](int rows, int cols, uint32_t seed) { + std::vector m(size_t(rows) * size_t(cols)); + uint32_t x = seed * 2654435761u + 12345u; + for (auto& v : m) { + x = x * 1664525u + 1013904223u; + v = float(int((x >> 8) & 0xFFFF) - 32768) / 16384.0f; + } + return m; + }; + + { // arm 1: the unit reader with ROWS as a parameter, typed bytes (K 64: one unit is 5,120 B) + std::vector u(2 * 5120, 0); + uint8_t* sp = u.data() + 5120 + 2560 + 2 * 640; // unit 1, strip 2: K block 1, rows 0..31 of the unit + const uint16_t bits = pi::bf16_bits_of(-0.5f); + sp[576 + 2 * 20] = uint8_t(bits & 0xFF); // row 20's scale + sp[577 + 2 * 20] = uint8_t(bits >> 8); + sp[512 + 10] = 0x05; // row 20's zero point: low nibble of byte 10 + sp[(2 * 256 + 3 * 8 + 4) / 2] |= 0x09; // row 20 (band 2, r%8 4), column 3: position 540, low nibble + pi::Block32Units r = pi::read_block32_units(u.data(), int64_t(u.size()), 128, 64, "q_proj", 7000); + check(r.ok && r.units == 2 && r.bytes == 10240 && r.q.N == 128 && r.q.K == 64 && r.q.block == 32, + "block-32 units: 128 rows of K 64 are two units (" + r.why + ")"); + if (r.ok) { + check(value(r.q, 64 + 20, 32 + 3) == -2.0, + "block-32 units: unit 1 row 20 col 35 = (9 - 5) x -0.5 (" + std::to_string(value(r.q, 84, 35)) + ")"); + check(value(r.q, 84, 34) == 2.5, "block-32 units: its neighbour is code 0, (0 - 5) x -0.5"); + check(r.q.scale[size_t(84) * 2 + 1] == 0.5f, "block-32 units: the stored scale is positive"); + check(value(r.q, 20, 35) == 0.0 && value(r.q, 84, 3) == 0.0, + "block-32 units: the same position in unit 0 and in K block 0 is untouched"); + } + check(pi::read_block32_units(u.data(), int64_t(u.size()), 65, 64).units == 2, + "block-32 units: 65 rows round up to two units"); + const pi::Block32Units one = pi::read_block32_units(u.data(), int64_t(u.size()), 64, 64); + check(one.ok && one.units == 1 && one.q.N == 64, + "block-32 units: 64 rows read one unit and nothing is required of the bytes after it"); + const pi::Block32Units sh = pi::read_block32_units(u.data(), 10239, 128, 64, "q_proj", 7000); + check(!sh.ok && sh.why.find("q_proj") != std::string::npos && sh.why.find("7000") != std::string::npos, + "block-32 units: a buffer one byte short is refused, naming the tensor and the offset (" + sh.why + ")"); + std::vector hdr = u; + hdr[5120 + 9] = 1; + const pi::Block32Units hr = pi::read_block32_units(hdr.data(), int64_t(hdr.size()), 128, 64, "q_proj", 7000); + check(!hr.ok && hr.why.find("12120 B") != std::string::npos, + "block-32 units: a non-zero unit header is refused with the unit's offset, 7000 + 5120 (" + hr.why + ")"); + std::vector nan = u; + nan[2560 + 576] = 0xC0, nan[2560 + 577] = 0x7F; // bf16 quiet NaN in unit 0, strip 0, row 0 + check(!pi::read_block32_units(nan.data(), int64_t(nan.size()), 128, 64).ok, + "block-32 units: a scale that is not finite is refused"); + } + + { // arm 2: the gate/up reader, typed bytes (K 32: one unit is 4,864 + 2 x 1,216 = 7,296 B) + check(pi::block32_mlp_unit_bytes(32) == 7296 && pi::block32_mlp_unit_bytes(576) == 48640, + "block-32 gate/up: the unit is 4,864 + 2*(K/32)*1,216 B (48,640 at K 576)"); + std::vector u(7296, 0); + uint8_t* sp = u.data() + 4864 + 1216; // strip 1: K block 0, rows 32..63 + const auto put_scale = [&](int slot, float v) { + const uint16_t bits = pi::bf16_bits_of(v); + sp[1088 + 2 * slot] = uint8_t(bits & 0xFF); + sp[1089 + 2 * slot] = uint8_t(bits >> 8); + }; + put_scale(5, 0.5f); // slot 5: GATE row 32 + 5 + put_scale(41, -0.25f); // slot 41: UP row 32 + 9 + sp[1024 + 2] = 0x80; // slot 5's zero point (high nibble of byte 2) = 8 + sp[1024 + 20] = 0x60; // slot 41's zero point (high nibble of byte 20) = 6 + sp[(0 * 256 + 7 * 8 + 5) / 2] |= 0xB0; // slot 5 (band 0, 5), column 7: position 61, high nibble + sp[(5 * 256 + 31 * 8 + 1) / 2] |= 0x30; // slot 41 (band 5, 1), column 31: position 1529, high nibble + pi::Block32GateUp h = pi::read_block32_gate_up(u.data(), int64_t(u.size()), 64, 32, 900); + check(h.ok && h.units == 1 && h.bytes == 7296 && h.gate.N == 64 && h.up.N == 64 && h.gate.K == 32 && + h.up.block == 32, + "block-32 gate/up: one unit of K 32 reads (" + h.why + ")"); + if (h.ok) { + check(value(h.gate, 37, 7) == 1.5, + "block-32 gate/up: slot 5 is GATE row 37, (11 - 8) x 0.5 (" + std::to_string(value(h.gate, 37, 7)) + ")"); + check(value(h.gate, 37, 6) == -4.0, "block-32 gate/up: its neighbour is code 0, (0 - 8) x 0.5"); + check(value(h.up, 37, 7) == 0.0, "block-32 gate/up: and UP row 37 holds nothing"); + check(value(h.up, 41, 31) == 0.75, + "block-32 gate/up: slot 41 is UP row 41, a NEGATIVE scale, (3 - 6) x -0.25 (" + + std::to_string(value(h.up, 41, 31)) + ")"); + check(value(h.up, 41, 0) == 1.5, "block-32 gate/up: its code 0, (0 - 6) x -0.25"); + check(h.up.scale[41] == 0.25f, "block-32 gate/up: the stored scale is positive"); + check(value(h.gate, 41, 31) == 0.0 && value(h.gate, 5, 7) == 0.0, + "block-32 gate/up: GATE row 41 and the other strip's row 5 hold nothing"); + } + const pi::Block32GateUp sh = pi::read_block32_gate_up(u.data(), 7295, 64, 32, 900); + check(!sh.ok && sh.why.find("900") != std::string::npos, + "block-32 gate/up: a buffer one byte short is refused with the offset (" + sh.why + ")"); + std::vector hdr = u; + hdr[4863] = 1; + check(!pi::read_block32_gate_up(hdr.data(), int64_t(hdr.size()), 64, 32).ok, + "block-32 gate/up: a non-zero unit header is refused"); + std::vector gap = u; + gap[4864 + 1216 + 1056 + 31] = 1; + check(!pi::read_block32_gate_up(gap.data(), int64_t(gap.size()), 64, 32).ok, + "block-32 gate/up: a non-zero byte between the zero points and the scales is refused"); + std::vector inf = u; + inf[4864 + 1088 + 2 * 63] = 0x80, inf[4864 + 1089 + 2 * 63] = 0x7F; // bf16 +inf in strip 0, slot 63 + check(!pi::read_block32_gate_up(inf.data(), int64_t(inf.size()), 64, 32).ok, + "block-32 gate/up: a scale that is not finite is refused"); + check(!pi::read_block32_gate_up(u.data(), int64_t(u.size()), 64, 48).ok, "block-32 gate/up: K 48 is refused"); + } + + { // arm 3: the description's encoder against the reader, two unrelated matrices over two units and three K blocks + const int rows = 128, K = 96; + const std::vector G = matrix(rows, K, 1), U = matrix(rows, K, 2); + const std::string enc = encode_block32_gate_up(G, U, rows, K); + const pi::Block32GateUp h = + pi::read_block32_gate_up(reinterpret_cast(enc.data()), int64_t(enc.size()), rows, K); + check(h.ok, "block-32 gate/up: the encoded stream reads (" + h.why + ")"); + if (h.ok) { + std::vector g(G.size()), up(U.size()); + for (int r = 0; r < rows; ++r) + for (int c = 0; c < K; ++c) { + g[size_t(r) * K + c] = float(value(h.gate, r, c)); + up[size_t(r) * K + c] = float(value(h.up, r, c)); + } + check(ncorr(g, G) > 0.99 && ncorr(up, U) > 0.99, + "block-32 gate/up: gate and up come back (corr " + std::to_string(ncorr(g, G)) + " / " + + std::to_string(ncorr(up, U)) + ")"); + check(std::fabs(ncorr(g, U)) < 0.3 && std::fabs(ncorr(up, G)) < 0.3, + "block-32 gate/up: and not swapped (corr " + std::to_string(ncorr(g, U)) + " / " + + std::to_string(ncorr(up, G)) + ")"); + } + } + + { // arm 4: the layer block's arithmetic at the one measured geometry (SmolLM2-135M-Instruct 1.9) + pi::Dims d; + d.hidden = 576, d.intermediate = 1536, d.layers = 30, d.heads = 9, d.kv_heads = 3, d.head_dim = 64; + d.vocab = 49152; + const pi::Block32Layout l = pi::block32_layout(d); + check(l.why.empty() && l.q_off == 1216 && l.k_off == 231616 && l.v_off == 308416 && l.sink_off == 385216 && + l.o_off == 385252 && l.post_off == 615652 && l.gu_off == 616868 && l.hdr_off == 1784228 && + l.down_off == 1784292 && l.stride == 2360292, + "block-32 layout: SmolLM2-135M's layer block, stride 2,360,292 (" + l.why + " " + + std::to_string(l.stride) + ")"); + check(l.head_off == 70809976 && l.head_units == 768 && l.min_size == 90470776, + "block-32 layout: its head at 70,809,976 and the file's 90,470,776 B (" + std::to_string(l.min_size) + ")"); + d.intermediate = 1500; + check(pi::block32_layout(d).why.find("intermediate 1500") != std::string::npos, + "block-32 layout: a width that is not whole 64-row units is refused by name"); + } + + // arm 5: a crafted all-block-32 pack through import_pack(). hidden 128, intermediate 256, 2 heads x 64, 1 KV head, + // 2 layers, vocab 64, tied. Every matrix is its own pseudo-random one, so a tensor read from another's offset + // cannot correlate with what the container is compared against. + const int H = 128, IM = 256, LAYERS = 2, V = 64, NQ = 128, NKV = 64; + const int R = onebit::npu::q4nx::kRows; + enum { Q, K_, V_, O, G, U, D, NPROJ }; + const char* const pname[NPROJ] = {"self_attn.q_proj", "self_attn.k_proj", "self_attn.v_proj", "self_attn.o_proj", + "mlp.gate_proj", "mlp.up_proj", "mlp.down_proj"}; + const int prow[NPROJ] = {NQ, NKV, NKV, H, IM, IM, H}, pcol[NPROJ] = {H, H, H, NQ, H, H, IM}; + std::vector> mats; + for (int l = 0; l < LAYERS; ++l) + for (int i = 0; i < NPROJ; ++i) mats.push_back(matrix(prow[i], pcol[i], uint32_t(100 + l * NPROJ + i))); + const std::vector emb = matrix(V, H, 7); + const auto norm_bytes = [&](int which) { // bf16 [H], a different ramp per norm + std::string b(size_t(H) * 2, '\0'); + for (int i = 0; i < H; ++i) { + const uint16_t bits = pi::bf16_bits_of(float(which) + float(i) / 256.0f); + b[2 * i] = char(bits & 0xFF), b[2 * i + 1] = char(bits >> 8); + } + return b; + }; + const auto envelope = [&](int which) { + std::string e(64, '\0'); + e[0] = char(0x86), e[1] = char(0x35); // bf16 1e-06 + return e + norm_bytes(which); + }; + const auto fconst = [&] { + std::string fc; + for (int l = 0; l < LAYERS; ++l) { + const auto& m = [&](int i) -> const std::vector& { return mats[size_t(l * NPROJ + i)]; }; + fc += envelope(10 * l + 1); + fc += encode_block32_units(m(Q), NQ, H) + encode_block32_units(m(K_), NKV, H) + + encode_block32_units(m(V_), NKV, H); + fc += std::string(4 * 2, '\x7f'); // 4 x heads bytes that are not a weight (their content is not read) + fc += encode_block32_units(m(O), H, NQ); + fc += envelope(10 * l + 2); + fc += encode_block32_gate_up(m(G), m(U), IM, H); + fc += std::string(1, '\x01') + std::string(63, '\0'); + fc += encode_block32_units(m(D), H, IM); + } + fc += envelope(99); + fc += encode_block32_units(emb, V, H); + return fc; + }; + const std::string good = fconst(); + { + pi::Dims d; + d.hidden = H, d.intermediate = IM, d.layers = LAYERS, d.heads = 2, d.kv_heads = 1, d.head_dim = 64, d.vocab = V; + check(pi::block32_layout(d).min_size == int64_t(good.size()), + "all-block-32 fixture: the stream built from the description is the layout's size (" + + std::to_string(good.size()) + ")"); + } + // `layer_block` / `head_block` are what cache/ops_meta.json DECLARES; the .fconst bytes are the caller's. + const auto build = [&](const char* tag, const std::string& fc, bool layers_declared, int layer_block = 32, + int head_block = 32) { + const fs::path dir = fs::path(root) / tag; + fs::remove_all(dir); + fs::create_directories(dir / "cache"); + // the 1.9 spelling of the Token file: this path reads the raw file, and it must not be mistaken for a 1.8 one + put_file((dir / "dd_metastate_Llm_Token_aie2_token_crafted.fconst").string(), fc); + put_file((dir / "config.json").string(), + R"({"model_type":"llama","hidden_size":128,"intermediate_size":256,"num_hidden_layers":2,)" + R"("num_attention_heads":2,"num_key_value_heads":1,"head_dim":64,"vocab_size":64,)" + R"("rope_theta":10000.0,"tie_word_embeddings":true,"rms_norm_eps":1e-06})"); + std::string data(size_t(V) * size_t(H) * 2, '\0'); + for (size_t i = 0; i < emb.size(); ++i) { + const uint16_t bits = pi::bf16_bits_of(emb[i]); + data[2 * i] = char(bits & 0xFF); + data[2 * i + 1] = char(bits >> 8); + } + put_file((dir / "model.onnx.data").string(), data); + const auto op_at = [](const std::string& name, int n, int k, int block) { + return R"({"type":"MladfMatMul","name":")" + name + R"(.MatMulNBits","attrs":{"N":{"value":[)" + + std::to_string(n) + R"(]},"K":{"value":[)" + std::to_string(k) + R"(]},"block_size":{"value":[)" + + std::to_string(block) + R"(]},"bits":{"value":[4]}}})"; + }; + const auto op = [&](const std::string& name, int n, int k) { return op_at(name, n, k, layer_block); }; + std::string ops = op_at("lm_head", V, H, head_block); + if (layers_declared) + for (int l = 0; l < LAYERS; ++l) { + const std::string a = "model.layers." + std::to_string(l) + ".attn.", + m = "model.layers." + std::to_string(l) + ".mlp."; + ops += "," + op(a + "q_proj", NQ, H) + "," + op(a + "k_proj", NKV, H) + "," + op(a + "v_proj", NKV, H) + + "," + op(a + "o_proj", H, NQ) + "," + op(m + "down_proj", H, IM); + } + put_file((dir / "cache" / "ops_meta.json").string(), R"({"op_list":[)" + ops + R"(],"tensor_map":{}})"); + put_file((dir / "model.onnx").string(), + pb::model({pb::tensor("model.embed_tokens.weight", {V, H}, "model.onnx.data", 0, + uint64_t(V) * uint64_t(H) * 2, 16), + pb::placeholder("lm_head.weight.MatMulNBits.qweight"), + pb::placeholder("lm_head.weight.MatMulNBits.scales"), + pb::placeholder("lm_head.weight.MatMulNBits.zeros")}, + {})); + return dir; + }; + const auto out_of = [](const char* tag) { + return (fs::temp_directory_path() / + (std::string("oga_pack_import_b32layers_") + tag + "." + std::to_string(::getpid()))).string(); + }; + // the first `cols` columns of rows 0..R-1 of a container tensor, from its own bytes (tile 0) + const auto tile0 = [&](const json& tbl, const std::vector& blob, const std::string& name, int cols) { + std::vector w(size_t(R) * onebit::npu::q4nx::kCols, 0.f), out(size_t(R) * size_t(cols), 0.f); + if (!tbl.contains(name)) return std::vector(); + const auto offs = tbl[name]["data_offsets"].get>(); + onebit::npu::q4nx::dequant_tile(onebit::npu::q4nx::Chunk::Q4_1F, blob.data() + offs[0], w.data(), + onebit::npu::q4nx::kCols); + for (int r = 0; r < R; ++r) + for (int c = 0; c < cols; ++c) out[size_t(r) * cols + c] = w[size_t(r) * onebit::npu::q4nx::kCols + c]; + return out; + }; + const auto head_of = [&](const std::vector& m, int cols_total, int cols) { // the same window of a source + std::vector out(size_t(R) * size_t(cols)); + for (int r = 0; r < R; ++r) + for (int c = 0; c < cols; ++c) out[size_t(r) * cols + c] = m[size_t(r) * cols_total + c]; + return out; + }; + + { // 5a: the pack imports, complete, and every container tensor is the matrix the description put at its offset + const fs::path dir = build("b32-layers", good, true); + const std::string out = out_of("good"); + fs::remove_all(out); + pi::Report r; + std::string said; + try { + const pi::Plan p = pi::plan(dir.string()); + check(p.block32_layers && p.block32_note.empty() && p.refused().empty(), + "all-block-32: the plan takes the block-32 layout and refuses nothing (" + p.block32_note + ")"); + r = pi::import_pack(dir.string(), out, pi::Options{}); + } catch (const std::exception& e) { + said = e.what(); + } + check(said.empty() && r.refused.empty(), + "all-block-32: a crafted pack imports with nothing refused (" + said + + (r.refused.empty() ? "" : r.refused.front()) + ")"); + check(r.attn_projections == 4 * LAYERS && r.mlp_projections == 3 * LAYERS && r.attn_bias_tensors == 0, + "all-block-32: 4 attention + 3 MLP projections per layer, no bias tensor"); + check(r.stride == (int64_t(good.size()) - (64 + 2 * H) - pi::block32_unit_bytes(H)) / LAYERS, + "all-block-32: the report carries the stride (" + std::to_string(r.stride) + ")"); + check(r.selfcheck_tied && r.lmhead_corr > 0.99 && r.head_read_note.find("block 32") != std::string::npos, + "all-block-32: the head is read and verified against the embedding (corr " + + std::to_string(r.lmhead_corr) + ")"); + if (fs::exists(out + "/model.q4nx")) { + const json tbl = container_table(out + "/model.q4nx"); + const std::vector blob = container_payload(out + "/model.q4nx"); + for (int l = 0; l < LAYERS; ++l) + for (int i = 0; i < NPROJ; ++i) { + const std::string name = "model.layers." + std::to_string(l) + "." + pname[i] + ".weight"; + const int cols = std::min(pcol[i], onebit::npu::q4nx::kCols); + const std::vector got = tile0(tbl, blob, name, cols); + const double c = got.empty() ? 0.0 : ncorr(got, head_of(mats[size_t(l * NPROJ + i)], pcol[i], cols)); + check(c > 0.99, "all-block-32: container " + name + " is its own matrix (corr " + + std::to_string(c) + ")"); + // the control: the matrix one slot along (gate <-> up among them) must not match + const int j = (i == G) ? U : (i == U) ? G : -1; + if (j >= 0 && !got.empty()) { + const double x = ncorr(got, head_of(mats[size_t(l * NPROJ + j)], pcol[j], cols)); + check(std::fabs(x) < 0.3, "all-block-32: container " + name + " is not the other MLP half " + "(corr " + std::to_string(x) + ")"); + } + } + const std::vector hd = tile0(tbl, blob, "lm_head.weight", H); + check(!hd.empty() && ncorr(hd, head_of(emb, H, H)) > 0.99, "all-block-32: container lm_head is the tail's head"); + const auto norm_is = [&](const std::string& name, int which) { + if (!tbl.contains(name)) return false; + const auto offs = tbl[name]["data_offsets"].get>(); + const std::string want = norm_bytes(which); + return offs[1] - offs[0] == want.size() && std::memcmp(blob.data() + offs[0], want.data(), want.size()) == 0; + }; + check(norm_is("model.layers.0.input_layernorm.weight", 1) && + norm_is("model.layers.1.input_layernorm.weight", 11) && + norm_is("model.layers.0.post_attention_layernorm.weight", 2) && + norm_is("model.layers.1.post_attention_layernorm.weight", 12) && norm_is("model.norm.weight", 99), + "all-block-32: the five norms are the envelopes' bytes"); + check(!tbl.contains("model.layers.0.self_attn.q_proj.bias"), "all-block-32: no bias tensor is invented"); + } else { + check(false, "all-block-32: the container was written"); + } + fs::remove_all(out); + fs::remove_all(dir); + } + // what a refused import says, and that it wrote no container + const auto refused = [&](const char* tag, const std::string& fc) { + const fs::path dir = build(tag, fc, true); + const std::string out = out_of(tag); + fs::remove_all(out); + std::string said; + try { + const pi::Report r = pi::import_pack(dir.string(), out, pi::Options{}); + for (const auto& w : r.refused) said += w + " | "; + } catch (const std::exception& e) { + said = e.what(); + } + if (fs::exists(out + "/model.q4nx")) said = "WROTE A CONTAINER " + said; + fs::remove_all(out); + fs::remove_all(dir); + return said; + }; + { // 5b: a file one byte short of the layout is refused with the arithmetic + const std::string said = refused("b32-layers-short", good.substr(0, good.size() - 1)); + check(said.find("WROTE") == std::string::npos && said.find("all-block-32 layout") != std::string::npos && + said.find(std::to_string(good.size())) != std::string::npos, + "all-block-32: a short file is refused with the size the layout needs (" + said.substr(0, 300) + ")"); + } + { // 5c: layer 1's post-attention envelope without its epsilon + pi::Dims d; + d.hidden = H, d.intermediate = IM, d.layers = LAYERS, d.heads = 2, d.kv_heads = 1, d.head_dim = 64, d.vocab = V; + const pi::Block32Layout l = pi::block32_layout(d); + std::string bad = good; + bad[size_t(l.stride + l.post_off)] = 0; + const std::string said = refused("b32-layers-eps", bad); + check(said.find("WROTE") == std::string::npos && said.find("no RMSNorm epsilon at " + + std::to_string(l.stride + l.post_off)) != + std::string::npos, + "all-block-32: a missing epsilon is refused with its offset (" + said.substr(0, 300) + ")"); + // 5d: a non-zero byte in a gate/up unit header of layer 1 -> both halves refused by name + bad = good; + bad[size_t(l.stride + l.gu_off + pi::block32_mlp_unit_bytes(H) + 17)] = 1; + const std::string s2 = refused("b32-layers-gu", bad); + check(s2.find("WROTE") == std::string::npos && + s2.find("model.layers.1.mlp.gate_proj.weight: block-32 gate/up unit 1") != std::string::npos && + s2.find("model.layers.1.mlp.up_proj.weight") != std::string::npos && + s2.find("model.layers.0") == std::string::npos, + "all-block-32: a non-zero gate/up unit header refuses that layer's gate and up by name (" + + s2.substr(0, 300) + ")"); + // 5e: trailing junk after the head is the head's refusal + const std::string s3 = refused("b32-layers-junk", good + std::string(99, '\0') + "x"); + check(s3.find("WROTE") == std::string::npos && s3.find("lm_head.weight") != std::string::npos && + s3.find("not zero after") != std::string::npos, + "all-block-32: a non-zero byte after the head refuses the head (" + s3.substr(0, 300) + ")"); + } + { // 5f: zero padding after the head is accepted (the measured mixed pack has 110 MB of it) + const fs::path dir = build("b32-layers-pad", good + std::string(5000, '\0'), true); + const std::string out = out_of("pad"); + fs::remove_all(out); + std::string said; + try { + const pi::Report r = pi::import_pack(dir.string(), out, pi::Options{}); + for (const auto& w : r.refused) said += w + " | "; + } catch (const std::exception& e) { + said = e.what(); + } + check(said.empty() && fs::exists(out + "/model.q4nx"), "all-block-32: zero padding after the head imports (" + + said.substr(0, 200) + ")"); + fs::remove_all(out); + fs::remove_all(dir); + } + { // 5g: the same pack declaring ONLY its head at block 32 is not this class (it is the mixed pack's path) + const fs::path dir = build("b32-layers-headonly", good, false); + bool block32 = true; + try { + block32 = pi::plan(dir.string()).block32_layers; + } catch (const std::exception&) { + block32 = false; + } + check(!block32, "all-block-32: a pack that declares only lm_head at block 32 does not take the all-block-32 path"); + fs::remove_all(dir); + } + // 5h..5k: the path is chosen by what the pack DECLARES, one arm per declaration. The bytes are the same good + // all-block-32 file every time, so the only thing that changes is cache/ops_meta.json -- and a declaration this + // importer has no layout for must be refused with its number, never read at 32 or at 128 by default. + const auto declared_as = [&](const char* tag, int layer_block, int head_block) { + const fs::path dir = build(tag, good, true, layer_block, head_block); + const std::string out = out_of(tag); + fs::remove_all(out); + std::string said; + bool block32 = false; + try { + block32 = pi::plan(dir.string()).block32_layers; + const pi::Report r = pi::import_pack(dir.string(), out, pi::Options{}); + for (const auto& w : r.refused) said += w + " | "; + } catch (const std::exception& e) { + said = e.what(); + } + if (block32) said = "TOOK THE ALL-BLOCK-32 PATH " + said; + if (fs::exists(out + "/model.q4nx")) said = "WROTE A CONTAINER " + said; + fs::remove_all(out); + fs::remove_all(dir); + return said; + }; + for (const int b : {3, 64, 256}) { // 5h: every op at a block nobody has measured + const std::string said = declared_as(("b32-layers-all" + std::to_string(b)).c_str(), b, b); + check(said.find("WROTE") == std::string::npos && said.find("TOOK") == std::string::npos && + said.find("block size along K is " + std::to_string(b) + " (11 ops)") != std::string::npos && + said.find("lm_head @ " + std::to_string(b)) != std::string::npos, + "block declaration: every op at " + std::to_string(b) + " is refused with that number (" + + said.substr(0, 200) + ")"); + } + { // 5i: layers at 32 under a block-128 head is not the all-block-32 class, and is refused by tensor + const std::string said = declared_as("b32-layers-head128", 32, 128); + check(said.find("WROTE") == std::string::npos && said.find("TOOK") == std::string::npos && + said.find("128 (1 ops)") != std::string::npos && said.find("32 (10 ops)") != std::string::npos && + said.find("lm_head @") == std::string::npos, + "block declaration: layers at 32 with the head at 128 is refused, naming the layers (" + + said.substr(0, 260) + ")"); + } + { // 5j: layers at 32 with the head at 64 -- one unmeasured op is enough to leave the class + const std::string said = declared_as("b32-layers-head64", 32, 64); + check(said.find("WROTE") == std::string::npos && said.find("TOOK") == std::string::npos && + said.find("64 (1 ops)") != std::string::npos, + "block declaration: layers at 32 with the head at 64 is refused (" + said.substr(0, 260) + ")"); + } + { // 5k: the same ops all declared at 128 stay on the block-128 walk (which then refuses THESE bytes, because + // they are a block-32 file: the point is that the declaration, not a default, picked the path) + const std::string said = declared_as("b32-layers-all128", 128, 128); + check(said.find("WROTE") == std::string::npos && said.find("TOOK") == std::string::npos && + said.find("block size along K") == std::string::npos && !said.empty(), + "block declaration: every op at 128 does not take the all-block-32 path or the block gate (" + + said.substr(0, 260) + ")"); + } +} + // ── the final norm out of the tail envelope, on a crafted pack (engine#844) ────────────────── // The real-pack leg only covers this where a pack is supplied, and three of the four packs on the // box ship no prefill program, so the import's tail-envelope read had no coverage in the default @@ -4880,6 +5382,7 @@ int main(int argc, char** argv) { test_post_attn_norm_header(golden_dir.empty() ? "." : golden_dir); test_tied_block32_head(golden_dir.empty() ? "." : golden_dir); test_block32_tail_head(golden_dir.empty() ? "." : golden_dir); + test_block32_layers(golden_dir.empty() ? "." : golden_dir); test_final_norm_envelope(golden_dir.empty() ? "." : golden_dir); test_import_fp32_params(golden_dir.empty() ? "." : golden_dir); test_text_artifacts(golden_dir.empty() ? "." : golden_dir);