diff --git a/docs/npu.md b/docs/npu.md
index 5337e7d9c..92527a0de 100644
--- a/docs/npu.md
+++ b/docs/npu.md
@@ -1249,6 +1249,50 @@ layout checks are all there is, and the note says the values are not verified. M
equals `bf16(-m/8)` of its 32-column embedding block, `m` being the block's largest-magnitude
weight, and the read head correlates 0.996 with the embedding.
+**A pack whose every MatMulNBits is `block_size 32` keeps its layers in those units too**
+(engine#1503). `SmolLM2-135M-Instruct_rai_1.9.0_npu` declares block 32 on all 151 ops, and its
+*raw* Token `.fconst` (the file as shipped, not the 1.8-form copy the block-128 path makes) is
+`layers` blocks of one fixed shape and then the tail. With `H` = hidden, `Nq` = heads x head_dim,
+`Nkv` = KV heads x head_dim, `IM` = intermediate, `unit(K) = 2,560 + 2 x (K/32) x 640` (the head's
+unit above) and `mlp_unit(K) = 4,864 + 2 x (K/32) x 1,216`:
+
+| section | bytes | holds |
+|---|---|---|
+| envelope | `64 + 2H` | bf16 `rms_norm_eps`, zeros to 64, `input_layernorm` bf16[H] |
+| q, k, v | `(Nq/64 + 2 x Nkv/64) x unit(H)` | `q_proj` [Nq][H], `k_proj` [Nkv][H], `v_proj` [Nkv][H] |
+| — | `4 x heads` | not a weight; skipped |
+| o | `(H/64) x unit(Nq)` | `o_proj` [H][Nq] |
+| envelope | `64 + 2H` | epsilon, zeros, `post_attention_layernorm` bf16[H] |
+| gate + up | `(IM/64) x mlp_unit(H)` | `gate_proj` and `up_proj`, [IM][H] each |
+| — | 64 | skipped, not interpreted (`01` then zeros on the measured pack) |
+| down | `(H/64) x unit(IM)` | `down_proj` [H][IM] |
+
+The tail is the same envelope with the final norm, then `ceil(vocab/64)` head units at K = `H`,
+then zeros. On SmolLM2-135M the block is 2,360,292 B (q at 1,216, k at 231,616, v at 308,416, o at
+385,252, the second envelope at 615,652, gate/up at 616,868, down at 1,784,292), the head is at
+70,809,976 and the file is exactly `30 x 2,360,292 + 1,216 + 768 x 25,600 = 90,470,776` B.
+
+The **gate/up unit** is the head's unit with 64 row *slots* per strip instead of 32 rows: 4,864
+zero bytes, then `2 x (K/32)` strips of 1,216 B. Strip `s` holds K block `s/2`; slots 0..31 are
+`gate_proj` rows `32 x (s%2) .. +31` of the unit and slots 32..63 are the same rows of `up_proj`.
+A strip is 1,024 B of 4-bit codes (low nibble first; position `p` is slot `(p/256) x 8 + p%8`,
+column `(p/8) % 32`), 32 B of zero-point nibbles, 32 zero bytes, and 64 signed bf16 scales. The
+code order was measured against the public checkpoint; that zero point and scale are indexed by
+slot is assumed by analogy with the 640-byte strip, and is tested by the imported container's
+gate/up still correlating with the checkpoint once they are applied.
+
+`npu-pack import` takes this path when every op the pack's `cache/*_meta.json` declares is block 32,
+at least one of them is a layer projection, and the pack has a Token `.fconst` (a pack that
+declares only its head at 32 is the case above). Nothing is located by search: the offsets are
+arithmetic on the geometry, and the file has to agree with them — at least
+`layers x stride + 64 + 2H + ceil(vocab/64) x unit(H)` bytes, the pack's epsilon and zeros at every
+envelope, zero unit headers and strip gaps, finite scales, zeros after the head — or the tensors
+are refused by name with the offset. The container gets the same tensors in the same q4_1f tiles as
+a block-128 import; this layout has no bias field, so no bias tensor is written. There is no
+decode->encode round trip on this path (no encoder for these units exists), so the import's
+`attention:` and `MLP:` round-trip lines are replaced by a `block 32:` line. Measured on one pack
+only, where `Nq == H`; `o_proj`'s K being `Nq` is the declared shape, not a measurement.
+
The norms that *are* imported sit at the documented offsets, each confirmed by the epsilon that
opens its layer block: **48/48 anchors found**. `tests/npu_pack_import_test` (registered as
`npu_pack_import`, and as `npu_pack_import_oga` when `-DONEBIT_OGA_PACK=
` is given) checks
diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp
index 2c06dfcf2..ca248267b 100644
--- a/npu/pack_import.cpp
+++ b/npu/pack_import.cpp
@@ -2562,14 +2562,104 @@ static TailHeadLocate locate_tail_head(const Fconst& fc, int64_t tail_off, int64
return r;
}
-// engine#1740: the block-32 tail head. Layout and its measurement are in pack_import.h.
-Block32Head read_block32_head(const uint8_t* p, int64_t avail, int64_t V, int64_t K) {
+// engine#1740 / engine#1503: the block-32 units. Layouts and their measurement are in pack_import.h.
+namespace {
+bool b32_all_zero(const uint8_t* b, int64_t n) {
+ for (int64_t i = 0; i < n; ++i)
+ if (b[i] != 0) return false;
+ return true;
+}
+float b32_bf16(const uint8_t* b) {
+ const uint32_t wide = uint32_t(uint16_t(b[0]) | uint16_t(uint16_t(b[1]) << 8)) << 16;
+ float v;
+ std::memcpy(&v, &wide, 4);
+ return v;
+}
+void b32_q_init(Q4& q, int64_t rows, int64_t K) {
+ q.N = int(rows);
+ q.K = int(K);
+ q.block = 32;
+ q.codes.assign(size_t(rows) * size_t(K), 0);
+ q.scale.assign(size_t(rows) * size_t(K / 32), 1.0f);
+ q.zero.assign(size_t(rows) * size_t(K / 32), 0.0f);
+}
+// One row's 32 codes of one K block out of a strip's code field. `slot` is the row's index inside the strip (0..31
+// in a 640-byte strip, 0..63 in a gate/up one): position `(slot/8)*256 + col*8 + slot%8`, low nibble first. A negative
+// scale is folded into the codes, so the stored scale is positive: (code - zp) * sc == ((15-code) - (15-zp)) * -sc.
+void b32_row(Q4& q, int64_t row, int64_t kblock, const uint8_t* codes, int slot, int zp, float sc) {
+ if (sc == 0.0f) return; // an all-zero block: codes 0, scale 1, zero 0 decode to 0
+ const bool flip = sc < 0;
+ const float mag = flip ? -sc : sc;
+ const size_t g = size_t(row) * size_t(q.K / 32) + size_t(kblock);
+ q.scale[g] = mag;
+ q.zero[g] = -float(flip ? 15 - zp : zp) * mag;
+ const int base = (slot / 8) * 256 + (slot % 8);
+ uint8_t* dst = q.codes.data() + size_t(row) * size_t(q.K) + size_t(kblock) * 32;
+ for (int c = 0; c < 32; ++c) {
+ const int pos = base + c * 8;
+ const int code = (codes[pos / 2] >> (4 * (pos & 1))) & 15;
+ dst[c] = uint8_t(flip ? 15 - code : code);
+ }
+}
+} // namespace
+
+Block32Units read_block32_units(const uint8_t* p, int64_t avail, int64_t rows, int64_t K, const char* what,
+ int64_t file_off) {
+ Block32Units h;
+ const std::string tag = std::string("block-32 ") + what;
+ if (rows <= 0 || K <= 0 || K % 32 != 0) {
+ h.why = "a " + tag + " needs K to be a multiple of 32 (K " + std::to_string(K) + ")";
+ return h;
+ }
+ const int64_t kb = K / 32, strips = 2 * kb, unit = block32_unit_bytes(K);
+ h.units = (rows + kB32UnitRows - 1) / kB32UnitRows;
+ h.bytes = h.units * unit;
+ if (avail < h.bytes) {
+ h.why = "the .fconst has " + std::to_string(avail) + " B at " + std::to_string(file_off) + " B, and a " + tag +
+ " of " + std::to_string(h.units) + " units x " + std::to_string(unit) + " B needs " +
+ std::to_string(h.bytes);
+ return h;
+ }
+ b32_q_init(h.q, h.units * kB32UnitRows, K);
+ for (int64_t u = 0; u < h.units; ++u) {
+ const uint8_t* up = p + u * unit;
+ const std::string where = tag + " unit " + std::to_string(u) + " (at " + std::to_string(file_off + u * unit) +
+ " B)";
+ if (!b32_all_zero(up, kB32UnitHeader)) {
+ h.why = where + " does not open with " + std::to_string(kB32UnitHeader) +
+ " zero bytes, so this is not the layout measured";
+ return h;
+ }
+ for (int64_t s = 0; s < strips; ++s) {
+ const uint8_t* sp = up + kB32UnitHeader + s * kB32StripBytes;
+ if (!b32_all_zero(sp + 528, 48)) {
+ h.why = where + " strip " + std::to_string(s) +
+ " has non-zero bytes between its zero points and its scales, so this is not the layout measured";
+ return h;
+ }
+ const int64_t row0 = u * kB32UnitRows + 32 * (s % 2);
+ for (int r = 0; r < 32; ++r) {
+ const float sc = b32_bf16(sp + 576 + 2 * r);
+ if (!std::isfinite(sc)) {
+ h.why = where + " strip " + std::to_string(s) + " row " + std::to_string(r) +
+ " has a scale that is not finite";
+ return h;
+ }
+ b32_row(h.q, row0 + r, s / 2, sp, r, (sp[512 + r / 2] >> (4 * (r & 1))) & 15, sc);
+ }
+ }
+ }
+ h.ok = true;
+ return h;
+}
+
+Block32Head read_block32_head(const uint8_t* p, int64_t avail, int64_t V, int64_t K, int64_t file_off) {
Block32Head h;
if (V <= 0 || K <= 0 || K % 32 != 0) {
h.why = "a block-32 head needs K to be a multiple of 32 (K " + std::to_string(K) + ")";
return h;
}
- const int64_t kb = K / 32, strips = 2 * kb, unit = block32_unit_bytes(K);
+ const int64_t unit = block32_unit_bytes(K);
h.units = (V + kB32UnitRows - 1) / kB32UnitRows;
h.bytes = h.units * unit;
if (avail < h.bytes) {
@@ -2585,58 +2675,60 @@ Block32Head read_block32_head(const uint8_t* p, int64_t avail, int64_t V, int64_
return h;
}
h.padding = avail - h.bytes;
- const auto all_zero = [](const uint8_t* b, int64_t n) {
- for (int64_t i = 0; i < n; ++i)
- if (b[i] != 0) return false;
- return true;
- };
- Q4& q = h.q;
- q.N = int(h.units * kB32UnitRows);
- q.K = int(K);
- q.block = 32;
- q.codes.assign(size_t(q.N) * size_t(K), 0);
- q.scale.assign(size_t(q.N) * size_t(kb), 1.0f);
- q.zero.assign(size_t(q.N) * size_t(kb), 0.0f);
+ Block32Units u = read_block32_units(p, h.bytes, V, K, "head", file_off);
+ if (!u.ok) {
+ h.why = u.why;
+ return h;
+ }
+ h.q = std::move(u.q);
+ h.ok = true;
+ return h;
+}
+
+Block32GateUp read_block32_gate_up(const uint8_t* p, int64_t avail, int64_t rows, int64_t K, int64_t file_off) {
+ Block32GateUp h;
+ if (rows <= 0 || K <= 0 || K % 32 != 0) {
+ h.why = "a block-32 gate/up needs K to be a multiple of 32 (K " + std::to_string(K) + ")";
+ return h;
+ }
+ const int64_t kb = K / 32, strips = 2 * kb, unit = block32_mlp_unit_bytes(K);
+ h.units = (rows + kB32UnitRows - 1) / kB32UnitRows;
+ h.bytes = h.units * unit;
+ if (avail < h.bytes) {
+ h.why = "the .fconst has " + std::to_string(avail) + " B at " + std::to_string(file_off) +
+ " B, and a block-32 gate/up of " + std::to_string(h.units) + " units x " + std::to_string(unit) +
+ " B needs " + std::to_string(h.bytes);
+ return h;
+ }
+ b32_q_init(h.gate, h.units * kB32UnitRows, K);
+ b32_q_init(h.up, h.units * kB32UnitRows, K);
for (int64_t u = 0; u < h.units; ++u) {
const uint8_t* up = p + u * unit;
- if (!all_zero(up, kB32UnitHeader)) {
- h.why = "block-32 head unit " + std::to_string(u) + " does not open with " +
- std::to_string(kB32UnitHeader) + " zero bytes, so this is not the layout measured";
+ const std::string where =
+ "block-32 gate/up unit " + std::to_string(u) + " (at " + std::to_string(file_off + u * unit) + " B)";
+ if (!b32_all_zero(up, kB32MlpUnitHeader)) {
+ h.why = where + " does not open with " + std::to_string(kB32MlpUnitHeader) +
+ " zero bytes, so this is not the layout measured";
return h;
}
for (int64_t s = 0; s < strips; ++s) {
- const uint8_t* sp = up + kB32UnitHeader + s * kB32StripBytes;
- if (!all_zero(sp + 528, 48)) {
- h.why = "block-32 head unit " + std::to_string(u) + " strip " + std::to_string(s) +
+ const uint8_t* sp = up + kB32MlpUnitHeader + s * kB32MlpStripBytes;
+ if (!b32_all_zero(sp + 1056, 32)) {
+ h.why = where + " strip " + std::to_string(s) +
" has non-zero bytes between its zero points and its scales, so this is not the layout measured";
return h;
}
- const int64_t col0 = (s / 2) * 32, row0 = u * kB32UnitRows + 32 * (s % 2);
- for (int r = 0; r < 32; ++r) {
- const uint16_t bits = uint16_t(sp[576 + 2 * r]) | uint16_t(uint16_t(sp[577 + 2 * r]) << 8);
- const uint32_t wide = uint32_t(bits) << 16;
- float sc;
- std::memcpy(&sc, &wide, 4);
+ const int64_t row0 = u * kB32UnitRows + 32 * (s % 2);
+ for (int slot = 0; slot < 64; ++slot) {
+ const float sc = b32_bf16(sp + 1088 + 2 * slot);
if (!std::isfinite(sc)) {
- h.why = "block-32 head unit " + std::to_string(u) + " strip " + std::to_string(s) + " row " +
- std::to_string(r) + " has a scale that is not finite";
+ h.why = where + " strip " + std::to_string(s) + " slot " + std::to_string(slot) +
+ " has a scale that is not finite";
return h;
}
- const int zp = (sp[512 + r / 2] >> (4 * (r & 1))) & 15;
- const bool flip = sc < 0;
- const size_t g = size_t(row0 + r) * size_t(kb) + size_t(s / 2);
- if (sc == 0.0f) continue; // an all-zero block: codes 0, scale 1, zero 0 decode to 0
- const float mag = flip ? -sc : sc;
- q.scale[g] = mag;
- q.zero[g] = -float(flip ? 15 - zp : zp) * mag;
- // position p = band*256 + col*8 + (r%8) holds row band*8 + r%8, column col
- const int base = (r / 8) * 256 + (r % 8);
- uint8_t* dst = q.codes.data() + size_t(row0 + r) * size_t(K) + size_t(col0);
- for (int c = 0; c < 32; ++c) {
- const int pos = base + c * 8;
- const int code = (sp[pos / 2] >> (4 * (pos & 1))) & 15;
- dst[c] = uint8_t(flip ? 15 - code : code);
- }
+ // slots 0..31 are gate rows, 32..63 the same rows of up; zero point and scale by slot (assumed)
+ b32_row(slot < 32 ? h.gate : h.up, row0 + slot % 32, s / 2, sp, slot,
+ (sp[1024 + slot / 2] >> (4 * (slot & 1))) & 15, sc);
}
}
}
@@ -2644,6 +2736,70 @@ Block32Head read_block32_head(const uint8_t* p, int64_t avail, int64_t V, int64_
return h;
}
+Block32Layout block32_layout(const Dims& d) {
+ Block32Layout l;
+ const int64_t H = d.hidden, IM = d.intermediate, Nq = int64_t(d.heads) * d.head_dim,
+ Nkv = int64_t(d.kv_heads) * d.head_dim, V = d.vocab;
+ if (H <= 0 || IM <= 0 || Nq <= 0 || Nkv <= 0 || V <= 0 || d.layers <= 0) {
+ l.why = "the all-block-32 layout needs hidden, intermediate, heads, kv heads, head_dim, vocab and layers, and "
+ "the pack's geometry leaves one of them 0";
+ return l;
+ }
+ for (const auto& [name, v] : {std::pair{"hidden", H},
+ std::pair{"intermediate", IM},
+ std::pair{"heads*head_dim", Nq},
+ std::pair{"kv_heads*head_dim", Nkv}})
+ if (v % kB32UnitRows != 0) {
+ l.why = std::string("the all-block-32 layout is 64-row units of 32-column blocks, and ") + name + " " +
+ std::to_string(v) + " is not a multiple of 64";
+ return l;
+ }
+ l.env = kNormOff + 2 * H;
+ l.q_off = l.env;
+ l.k_off = l.q_off + (Nq / kB32UnitRows) * block32_unit_bytes(H);
+ l.v_off = l.k_off + (Nkv / kB32UnitRows) * block32_unit_bytes(H);
+ l.sink_off = l.v_off + (Nkv / kB32UnitRows) * block32_unit_bytes(H);
+ l.o_off = l.sink_off + 4 * int64_t(d.heads);
+ l.post_off = l.o_off + (H / kB32UnitRows) * block32_unit_bytes(Nq);
+ l.gu_off = l.post_off + l.env;
+ l.hdr_off = l.gu_off + (IM / kB32UnitRows) * block32_mlp_unit_bytes(H);
+ l.down_off = l.hdr_off + 64;
+ l.stride = l.down_off + (H / kB32UnitRows) * block32_unit_bytes(IM);
+ l.tail_off = int64_t(d.layers) * l.stride;
+ l.head_off = l.tail_off + l.env;
+ l.head_units = (V + kB32UnitRows - 1) / kB32UnitRows;
+ l.min_size = l.head_off + l.head_units * block32_unit_bytes(H);
+ return l;
+}
+
+std::string block32_file_refusal(const uint8_t* map, int64_t size, const Dims& d, const Block32Layout& lay) {
+ if (!lay.why.empty()) return lay.why;
+ if (size < lay.min_size)
+ return "the Token .fconst is " + std::to_string(size) + " B, and the all-block-32 layout of this geometry is " +
+ std::to_string(d.layers) + " layer blocks x " + std::to_string(lay.stride) + " B + a " +
+ std::to_string(lay.env) + " B envelope + " + std::to_string(lay.head_units) + " head units x " +
+ std::to_string(block32_unit_bytes(d.hidden)) + " B = " + std::to_string(lay.min_size) +
+ " B (plus zero padding), so the file is not this layout";
+ const auto envelope = [&](int64_t off, const std::string& what) -> std::string {
+ if (map[off] != uint8_t(d.eps_bits & 0xFF) || map[off + 1] != uint8_t(d.eps_bits >> 8))
+ return "no RMSNorm epsilon at " + std::to_string(off) + " B, where the all-block-32 layout puts the " +
+ what + " envelope, so the file is not this layout";
+ for (int64_t i = 2; i < kNormOff; ++i)
+ if (map[off + i] != 0)
+ return "the " + what + " envelope at " + std::to_string(off) + " B is not zero between its epsilon "
+ "and its norm (byte " + std::to_string(off + i) + "), so the file is not this layout";
+ return {};
+ };
+ for (int L = 0; L <= d.layers; ++L) {
+ const int64_t base = int64_t(L) * lay.stride;
+ const std::string name = L == d.layers ? std::string("tail") : "layer " + std::to_string(L);
+ std::string why = envelope(base, name + (L == d.layers ? "" : " input_layernorm"));
+ if (why.empty() && L < d.layers) why = envelope(base + lay.post_off, name + " post_attention_layernorm");
+ if (!why.empty()) return why;
+ }
+ return {};
+}
+
// engine#1740: a self-test for the SEARCH, not the arithmetic. `plan_tail_head` is pinned by the host test; this pins
// the part that can go silently wrong -- the candidate loop, best/runner-up tracking, the kTieCorr cut, the ambiguity
// refusal and the untied-with-padding refusal -- using a synthetic head built through encode_tile, so it needs no pack,
@@ -2913,6 +3069,17 @@ std::string Plan::manifest() const {
{"o_proj_k", o_k},
{"qk_norm", knorm_off != 0},
{"note", attn_note}};
+ // engine#1503: an all-block-32 pack is read with its own layout, so the block-128 measurements above are 0 for it;
+ // this says which layout was used and what its arithmetic gives. Absent for every other pack.
+ if (block32_layers) {
+ const Block32Layout lay = block32_layout(dims);
+ j["block32_layers"] = {{"stride", lay.stride},
+ {"post_attention_envelope", lay.post_off},
+ {"head_offset", lay.head_off},
+ {"head_units", lay.head_units},
+ {"bytes", lay.min_size},
+ {"note", block32_note}};
+ }
// What the *graph* carries, in the graph's own terms. A pack's genre is the first question every sweep row asks,
// and until now it could only be answered by an external tool: the rai_1.3/1.6/1.7.1 line ships its 4-bit weights
// as ordinary ONNX initializers (pi-inferzo: 25/25 real qweights, UINT8 [1152,7,64], location model_jit.onnx.data)
@@ -3140,7 +3307,22 @@ Plan plan(const std::string& pack_dir) {
// The Token .fconst, found once (see find_token_fconst) and used by the attention block below. Empty means this
// pack keeps no weights there -- which for the rai_1.3/1.6/1.7.1 line is the normal case, not a fault.
std::string tok_why;
- std::string tok_path = materialise_token_fconst(pack_dir, p.dims);
+ const DeclaredMap declared = read_declared(pack_dir);
+ // engine#1503: a pack whose every declared MatMulNBits is block 32 -- layers included, not only the head -- is read
+ // from its RAW .fconst with the all-block-32 layout (pack_import.h, Block32Layout). Decided here, before the
+ // canonical copy is made, because that copy is the 1.8 form of a block-128 file and is not what this path reads.
+ // A pack that declares only its head (the block-32-head fixtures, and nothing measured) is not this class.
+ {
+ bool any = false, all32 = true, layer = false;
+ for (const auto& [proj, dec] : declared) {
+ if (!dec.block) continue;
+ any = true;
+ if (dec.block != 32) all32 = false;
+ if (proj.rfind("lm_head", 0) != 0) layer = true;
+ }
+ p.block32_layers = any && all32 && layer && !find_token_fconst(pack_dir).empty();
+ }
+ std::string tok_path = p.block32_layers ? find_token_fconst(pack_dir) : materialise_token_fconst(pack_dir, p.dims);
// engine#1671 / #1687: a pack whose graph names `*.qweight.empty` placeholders whose `*.packed` siblings carry no
// data has no weight for those ops IN THE GRAPH. With a Token .fconst that is the normal shape of a 1.9 pack -- its
// graph is all placeholders and the weights are in the .fconst -- so the walker drops them and the import proceeds.
@@ -3198,6 +3380,17 @@ Plan plan(const std::string& pack_dir) {
{embed->dims.at(0), embed->dims.at(1)}));
}
+ // engine#1503: the all-block-32 layout against the file, once the vocab is known (the head's unit count needs it).
+ if (p.block32_layers) {
+ const Block32Layout lay = block32_layout(p.dims);
+ if (!lay.why.empty()) {
+ p.block32_note = lay.why;
+ } else {
+ Fconst fc = open_fconst(tok_path);
+ p.block32_note = block32_file_refusal(fc.map, int64_t(fc.size), p.dims, lay);
+ unmap_file(fc.map, fc.size);
+ }
+ }
// Only the node whose geometry IS the head (N = V, K = H) is the head. This loop used to take the FIRST MatMulNBits and
// `break`, so on a pack whose first matmul is a projection the plan reported that projection's shape as
// `lm_head.weight` -- [1152, 896], a fused qkv, instead of [151936, 896] on Qwen2.5-0.5B-Instruct 1.9 (engine#1671).
@@ -3218,14 +3411,16 @@ Plan plan(const std::string& pack_dir) {
// there if the tail's arithmetic does not come out, so the plan does not refuse it here. Keyed on "did a graph
// head get written", not "is the graph free of matmuls": a graph of projections only is not a pack without a head
// (engine#1671).
- p.tensors.push_back(make("lm_head.weight", "fconst:tail", "q4_1", {d.vocab, d.hidden}));
+ auto x = make("lm_head.weight", "fconst:tail", "q4_1", {d.vocab, d.hidden});
+ x.refusal = p.block32_note; // empty unless an all-block-32 pack's file does not fit its layout
+ p.tensors.push_back(std::move(x));
}
// norms: the layer blocks are located by their epsilon anchors at import time, so the offsets work
// for any hidden size (engine#734). plan() cannot prove that without reading the file, so the
// refusal here is empty and import_pack reports what the anchors actually found.
const std::string norm_source = "fconst:Llm_Token_Token";
- const std::string norm_refusal;
+ const std::string norm_refusal = p.block32_note; // empty except on an all-block-32 pack whose file does not fit
const int64_t H = d.hidden; // the geometry's hidden; the embedding, when the main graph has none, must match it
// Genre A: the norms are ordinary F16 initializers named after themselves, with real offsets -- not the
// .fconst tail envelope -- so the refusal names the initializer and where it sits.
@@ -3265,6 +3460,7 @@ Plan plan(const std::string& pack_dir) {
// and the norm at +64, and where both .fconst files exist the two copies are byte-identical. The
// position is confirmed at import time by the epsilon marker plus that cross-check.
auto fin = make("model.norm.weight", norm_source, "BF16", {H});
+ fin.refusal = p.block32_note;
p.tensors.push_back(std::move(fin));
// projections: the attention payload (q/k/v/o) decodes tile by tile and the MLP section group by
@@ -3281,6 +3477,10 @@ Plan plan(const std::string& pack_dir) {
if (tok.empty()) {
p.attn_note = tok_why;
p.mlp_note = p.attn_note;
+ } else if (p.block32_layers) {
+ // the all-block-32 layout is arithmetic on the geometry, checked against the file above
+ p.attn_note = p.block32_note;
+ p.mlp_note = p.block32_note;
} else if (tile_bytes == 0) {
p.attn_note = "hidden " + std::to_string(H) + " is not a multiple of 128, so the attention "
"payload is not 64-row tiles of this shape";
@@ -3450,7 +3650,6 @@ Plan plan(const std::string& pack_dir) {
{"mlp.gate_proj", d.intermediate, d.hidden},
{"mlp.up_proj", d.intermediate, d.hidden},
{"mlp.down_proj", d.hidden, d.intermediate}};
- const DeclaredMap declared = read_declared(pack_dir);
// Block size gate (engine#1410 §3 of FACTS-fconst-layout): the compiler states `block_size` per MatMulNBits, and
// this importer's tile geometry is the block-128 one -- a group is 2*(K/block)+2 subvolumes, so a block-32 pack
// has every offset in a different place. Refusing by number beats parsing it at the 1.8 geometry, which is what
@@ -3488,7 +3687,12 @@ Plan plan(const std::string& pack_dir) {
// calling the writer here cost a second 278 MiB rewrite per command.
const bool fconst_weights = !find_token_fconst(pack_dir).empty();
if (fconst_weights && !blocks.empty() && !(blocks.size() == 1 && blocks.begin()->first == kKBlock)) {
- if (only_the_head) {
+ if (p.block32_layers) {
+ // engine#1503: every op is block 32 and the pack is read with the all-block-32 layout, which was
+ // checked against the file where `block32_layers` was decided; a file that does not fit is refused
+ // tensor by tensor with `block32_note`.
+ if (const auto it = declared.find("lm_head"); it != declared.end()) p.head_block = it->second.block;
+ } else if (only_the_head) {
p.head_block = declared.at(off_block[0].substr(0, off_block[0].find(" @ "))).block;
} else {
// Name a few and count the rest: SmolLM2-135M has 151 off-block ops, and listing every one of them made
@@ -3505,7 +3709,9 @@ Plan plan(const std::string& pack_dir) {
}() + ", and this importer decodes the block-" + std::to_string(kKBlock) +
" tile layout only: at a different block every subvolume offset in the .fconst moves, so the "
"weights would be read from the wrong place, not merely mis-scaled. The off-block tensor(s) are " +
- list + "; only the LM head could be tolerated, because the layers come before it in the tail.");
+ list + "; only the LM head could be tolerated, because the layers come before it in the tail. (The "
+ "one other layout this importer reads is the all-block-32 one, and only for a pack that declares "
+ "32 on every op.)");
}
}
if (!bits.empty() && !(bits.size() == 1 && bits.begin()->first == 4))
@@ -4604,6 +4810,175 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons
break;
}
+ // engine#1740: read a block-32 head whose first unit is at `head_off` of the Token .fconst and write it as
+ // lm_head.weight. Returns why it was NOT taken (empty when it was). On a tied pack the read has to correlate with
+ // the embedding above kTieCorr; on an untied one the structural checks are all there is, which the note states.
+ // Shared by the mixed pack (layers at 128, head at 32) and the all-block-32 pack (engine#1503).
+ auto take_block32_head = [&](const Fconst& fc, int64_t head_off) -> std::string {
+ std::string why;
+ Block32Head h32 = read_block32_head(fc.map + head_off, int64_t(fc.size) - head_off, V, H, head_off);
+ const bool is_tied = tie_declared && tied;
+ double c32 = 0.0;
+ if (h32.ok && embed_tbl.ok()) c32 = corr(h32.q, embed_tbl, int(V));
+ if (!h32.ok) {
+ why = h32.why;
+ } else if (is_tied && !(c32 > kTieCorr)) {
+ char b[160];
+ std::snprintf(b, sizeof b,
+ "the tail's block-32 head decodes but correlates %.6f with the embedding of "
+ "a pack that declares them tied", c32);
+ why = b;
+ } else {
+ Q4& q = h32.q;
+ std::fprintf(stderr,
+ " lm head: block 32, %lld unit(s) of %lld B at %lld B, %lld B of zeros after "
+ "them%s\n",
+ (long long)h32.units, (long long)block32_unit_bytes(H),
+ (long long)head_off, (long long)h32.padding,
+ is_tied ? " (tied: verified by correlation with the pack's embedding)"
+ : " (untied: layout checked, values not verified)");
+ if (embed_tbl.ok()) {
+ r.lmhead_corr = c32;
+ {
+ double sum_abs = 0, sum_ref = 0;
+ for (int n = 0; n < int(V); ++n)
+ for (int k = 0; k < q.K; ++k) {
+ const size_t gg = size_t(n) * (q.K / 32) + size_t(k) / 32;
+ const double x = double(q.codes[size_t(n) * q.K + k]) * q.scale[gg] + q.zero[gg];
+ const double e = embed_at(embed_tbl, n, k);
+ sum_abs += std::fabs(x - e);
+ sum_ref += std::fabs(e);
+ }
+ if (sum_ref > 0) r.lmhead_rel = sum_abs / sum_ref;
+ }
+ r.selfcheck_reference = true;
+ r.selfcheck_tied = c32 > kTieCorr;
+ double sxy = 0, sx = 0, sy = 0, sxx = 0, syy = 0;
+ const int nsh = std::min(2048, int(V) - 1);
+ for (int rr = 0; rr < nsh; ++rr)
+ for (int k = 0; k < q.K; ++k) {
+ const size_t gg = size_t(rr) * (q.K / 32) + size_t(k) / 32;
+ const double x = double(q.codes[size_t(rr) * q.K + k]) * q.scale[gg] + q.zero[gg];
+ const double y = embed_at(embed_tbl, rr + 1, k);
+ sxy += x * y; sx += x; sy += y; sxx += x * x; syy += y * y;
+ }
+ const double m = double(nsh) * q.K;
+ const double cx = sxx - sx * sx / m, cy = syy - sy * sy / m;
+ if (m > 0 && cx > 1e-12 && cy > 1e-12)
+ r.ctrl_corr_rowshift = (sxy - sx * sy / m) / std::sqrt(cx * cy);
+ }
+ std::vector tiled;
+ long long pinexact = 0;
+ tile_q4_1_f32(q, tiled, &pinexact);
+ r.codes_total = q.codes.size();
+ r.zero_exact_pct = 100.0 * double(q.scale.size() - pinexact) / double(q.scale.size());
+ const int64_t head_tiles = int64_t(tiled.size() / kImportTileBytes);
+ r.tiles += head_tiles;
+ w.add("lm_head.weight", "I8", {head_tiles, kImportTileBytes}, std::move(tiled));
+ ++r.tensors;
+ head_written = true;
+ r.head_read_note =
+ std::string("lm_head was read from the .fconst tail's block 32 head (") +
+ std::to_string(h32.units) + " units of " + std::to_string(block32_unit_bytes(H)) +
+ " B)" + (is_tied ? ", and it reproduces model.embed_tokens"
+ : "; the pack is untied, so its values are not verified against "
+ "anything in the pack");
+ }
+ return why;
+ };
+ if (p.block32_layers) {
+ // engine#1503: every MatMulNBits of this pack is block 32. The RAW Token .fconst is read with the layout in
+ // pack_import.h (Block32Layout); the offsets are arithmetic on the geometry, and plan() has already checked the
+ // file's size and every envelope's epsilon against them. The tensors written are the ones the block-128 walk
+ // below writes, in the same form. This layout has no bias field, so no bias tensor is written.
+ std::vector late;
+ if (p.block32_note.empty()) {
+ const Block32Layout lay = block32_layout(p.dims);
+ Fconst fc = open_fconst(find_token_fconst(pack_dir));
+ // the file is re-checked on this mapping: plan() looked at its own
+ if (const std::string why = block32_file_refusal(fc.map, int64_t(fc.size), p.dims, lay); !why.empty())
+ throw std::runtime_error(fc.path + ": " + why);
+ r.stride = lay.stride;
+ r.second_header = lay.post_off;
+ if (!head_written) {
+ const std::string why = take_block32_head(fc, lay.head_off);
+ if (!why.empty()) late.push_back("lm_head.weight: " + why);
+ else r.block32_units += lay.head_units;
+ }
+ auto add_norm = [&](int64_t off, const std::string& name) {
+ w.add(name, "BF16", {H}, as_bf16(at(fc, off, H * 2), kBF16, size_t(H)));
+ ++r.tensors;
+ };
+ for (int L = 0; L < p.dims.layers; ++L) {
+ const int64_t base = int64_t(L) * lay.stride;
+ r.norm_anchors_total += 2;
+ r.norm_anchors_ok += int(eps_at(fc, base, p.dims.eps_bits)) +
+ int(eps_at(fc, base + lay.post_off, p.dims.eps_bits));
+ add_norm(base + kNormOff, "model.layers." + std::to_string(L) + ".input_layernorm.weight");
+ add_norm(base + lay.post_off + kNormOff,
+ "model.layers." + std::to_string(L) + ".post_attention_layernorm.weight");
+ }
+ add_norm(lay.tail_off + kNormOff, "model.norm.weight");
+ r.final_norm_imported = true;
+ if (!find_one(pack_dir, "dd_metastate_Llm_Prefill_", ".fconst").empty())
+ r.fconst_note = "the pack ships a prefill .fconst, and its copy of the final norm is not compared on "
+ "the all-block-32 path";
+ const int64_t q_out = int64_t(p.dims.heads) * p.dims.head_dim,
+ kv_out = int64_t(p.dims.kv_heads) * p.dims.head_dim, IM = p.dims.intermediate;
+ long long attn_inexact = 0, attn_groups = 0, mlp_inexact = 0, mlp_groups = 0;
+ auto emit = [&](Q4& q, const std::string& name, bool mlp) {
+ std::vector tiled;
+ long long pinexact = 0;
+ tile_q4_1_f32(q, tiled, &pinexact);
+ (mlp ? mlp_inexact : attn_inexact) += pinexact;
+ (mlp ? mlp_groups : attn_groups) += (long long)q.scale.size();
+ const int64_t nt = int64_t(tiled.size() / kImportTileBytes);
+ r.tiles += nt;
+ w.add(name + ".weight", "I8", {nt, kImportTileBytes}, std::move(tiled));
+ ++r.tensors;
+ ++(mlp ? r.mlp_projections : r.attn_projections);
+ };
+ struct Slot { const char* proj; int64_t off, rows, k; bool mlp; };
+ const Slot slots[] = {{"self_attn.q_proj", lay.q_off, q_out, H, false},
+ {"self_attn.k_proj", lay.k_off, kv_out, H, false},
+ {"self_attn.v_proj", lay.v_off, kv_out, H, false},
+ {"self_attn.o_proj", lay.o_off, H, q_out, false},
+ {"mlp.down_proj", lay.down_off, H, IM, true}};
+ for (int L = 0; L < p.dims.layers; ++L) {
+ const int64_t base = int64_t(L) * lay.stride;
+ const std::string layer = "model.layers." + std::to_string(L) + ".";
+ for (const auto& sl : slots) {
+ Block32Units u = read_block32_units(fc.map + base + sl.off, int64_t(fc.size) - base - sl.off,
+ sl.rows, sl.k, sl.proj, base + sl.off);
+ if (!u.ok) {
+ late.push_back(layer + sl.proj + ".weight: " + u.why);
+ continue;
+ }
+ r.block32_units += u.units;
+ emit(u.q, layer + sl.proj, sl.mlp);
+ }
+ Block32GateUp gu = read_block32_gate_up(fc.map + base + lay.gu_off,
+ int64_t(fc.size) - base - lay.gu_off, IM, H, base + lay.gu_off);
+ if (!gu.ok) {
+ late.push_back(layer + "mlp.gate_proj.weight: " + gu.why);
+ late.push_back(layer + "mlp.up_proj.weight: " + gu.why);
+ continue;
+ }
+ r.block32_units += gu.units;
+ emit(gu.gate, layer + "mlp.gate_proj", true);
+ emit(gu.up, layer + "mlp.up_proj", true);
+ }
+ r.attn_zero_inexact_pct = attn_groups ? 100.0 * double(attn_inexact) / double(attn_groups) : 0.0;
+ r.mlp_zero_inexact_pct = mlp_groups ? 100.0 * double(mlp_inexact) / double(mlp_groups) : 0.0;
+ unmap_file(fc.map, fc.size);
+ } else {
+ r.fconst_note = p.block32_note; // every .fconst tensor is already refused by name with this
+ }
+ if (!late.empty()) {
+ for (auto& s : late) r.refused.push_back(std::move(s));
+ if (!opt.allow_incomplete) return r; // nothing has been written yet
+ }
+ } else
{
std::string path_why;
const std::string path = materialise_token_fconst(pack_dir, p.dims);
@@ -4646,76 +5021,8 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons
if (tail_off + env > int64_t(fc.size)) {
head32_why = "the .fconst ends before the tail's envelope";
} else {
- Block32Head h32 = read_block32_head(fc.map + tail_off + env, int64_t(fc.size) - tail_off - env,
- V, H);
- const bool is_tied = tie_declared && tied;
- double c32 = 0.0;
- if (h32.ok && embed_tbl.ok()) c32 = corr(h32.q, embed_tbl, int(V));
- if (!h32.ok) {
- head32_why = h32.why;
- } else if (is_tied && !(c32 > kTieCorr)) {
- char b[160];
- std::snprintf(b, sizeof b,
- "the tail's block-32 head decodes but correlates %.6f with the embedding of "
- "a pack that declares them tied", c32);
- head32_why = b;
- } else {
- Q4& q = h32.q;
- std::fprintf(stderr,
- " lm head: block 32, %lld unit(s) of %lld B at %lld B, %lld B of zeros after "
- "them%s\n",
- (long long)h32.units, (long long)block32_unit_bytes(H),
- (long long)(tail_off + env), (long long)h32.padding,
- is_tied ? " (tied: verified by correlation with the pack's embedding)"
- : " (untied: layout checked, values not verified)");
- if (embed_tbl.ok()) {
- r.lmhead_corr = c32;
- {
- double sum_abs = 0, sum_ref = 0;
- for (int n = 0; n < int(V); ++n)
- for (int k = 0; k < q.K; ++k) {
- const size_t gg = size_t(n) * (q.K / 32) + size_t(k) / 32;
- const double x = double(q.codes[size_t(n) * q.K + k]) * q.scale[gg] + q.zero[gg];
- const double e = embed_at(embed_tbl, n, k);
- sum_abs += std::fabs(x - e);
- sum_ref += std::fabs(e);
- }
- if (sum_ref > 0) r.lmhead_rel = sum_abs / sum_ref;
- }
- r.selfcheck_reference = true;
- r.selfcheck_tied = c32 > kTieCorr;
- double sxy = 0, sx = 0, sy = 0, sxx = 0, syy = 0;
- const int nsh = std::min(2048, int(V) - 1);
- for (int rr = 0; rr < nsh; ++rr)
- for (int k = 0; k < q.K; ++k) {
- const size_t gg = size_t(rr) * (q.K / 32) + size_t(k) / 32;
- const double x = double(q.codes[size_t(rr) * q.K + k]) * q.scale[gg] + q.zero[gg];
- const double y = embed_at(embed_tbl, rr + 1, k);
- sxy += x * y; sx += x; sy += y; sxx += x * x; syy += y * y;
- }
- const double m = double(nsh) * q.K;
- const double cx = sxx - sx * sx / m, cy = syy - sy * sy / m;
- if (m > 0 && cx > 1e-12 && cy > 1e-12)
- r.ctrl_corr_rowshift = (sxy - sx * sy / m) / std::sqrt(cx * cy);
- }
- std::vector tiled;
- long long pinexact = 0;
- tile_q4_1_f32(q, tiled, &pinexact);
- r.codes_total = q.codes.size();
- r.zero_exact_pct = 100.0 * double(q.scale.size() - pinexact) / double(q.scale.size());
- const int64_t head_tiles = int64_t(tiled.size() / kImportTileBytes);
- r.tiles += head_tiles;
- w.add("lm_head.weight", "I8", {head_tiles, kImportTileBytes}, std::move(tiled));
- ++r.tensors;
- head_written = true;
- head32_read = true;
- r.head_read_note =
- std::string("lm_head was read from the .fconst tail's block 32 head (") +
- std::to_string(h32.units) + " units of " + std::to_string(block32_unit_bytes(H)) +
- " B)" + (is_tied ? ", and it reproduces model.embed_tokens"
- : "; the pack is untied, so its values are not verified against "
- "anything in the pack");
- }
+ head32_why = take_block32_head(fc, tail_off + env);
+ head32_read = head32_why.empty();
}
}
if (head32_read) {
@@ -5523,9 +5830,12 @@ int run(int argc, char** argv) {
r.mlp_groups_ok, r.mlp_groups_total, r.mlp_projections,
100.0 - r.mlp_zero_inexact_pct);
if (r.stride)
- std::printf(" fconst: stride %lld, second header %lld (measured from the eps anchors); "
+ std::printf(" fconst: stride %lld, second header %lld (%s); "
"%d/%d RMSNorm anchors found, final norm %s\n",
- (long long)r.stride, (long long)r.second_header, r.norm_anchors_ok,
+ (long long)r.stride, (long long)r.second_header,
+ r.block32_units ? "computed from the geometry, checked at the eps anchors"
+ : "measured from the eps anchors",
+ r.norm_anchors_ok,
r.norm_anchors_total,
!r.final_norm_imported
? "REFUSED (no readable tail envelope)"
@@ -5533,6 +5843,11 @@ int run(int argc, char** argv) {
? "identical in both .fconst tails"
: "taken from the Token_Token tail -- the two copies DISAGREE")
: "taken from the tail envelope (the pack ships no prefill copy)"));
+ if (r.block32_units)
+ std::printf(" block 32: every MatMulNBits is block 32: %d attention + %d MLP projections and the head read "
+ "from %lld units of the raw .fconst (zero fields, finite scales and epsilon anchors checked; this "
+ "path has no decode->encode round trip)\n",
+ r.attn_projections, r.mlp_projections, r.block32_units);
if (!r.fconst_note.empty()) std::printf(" fconst: %s\n", r.fconst_note.c_str());
if (!r.refused.empty()) std::printf(" incomplete: %zu tensors, listed in %s/model.q4nx\n",
r.refused.size(), out.c_str());
diff --git a/npu/pack_import.h b/npu/pack_import.h
index 775eed018..48140ecd2 100644
--- a/npu/pack_import.h
+++ b/npu/pack_import.h
@@ -183,6 +183,12 @@ struct Plan {
// .fconst tile geometry assumes. Non-zero means the tail's head tiles cannot be walked (engine#1503): a tied pack
// takes the head from its embedding instead, an untied one is refused by name.
int64_t head_block = 0;
+ // engine#1503: every MatMulNBits the pack declares is block 32 and at least one of them is a layer projection, and
+ // the pack has a Token .fconst. Such a pack is read with the all-block-32 layout (Block32Layout below) from the
+ // raw .fconst; the block-128 walk and the canonical copy are not used. `block32_note` is why that file does not
+ // fit the layout (empty when it does), and is the refusal every .fconst tensor of the plan then carries.
+ bool block32_layers = false;
+ std::string block32_note;
std::string pack; // *_rai_1.8.0_npu_4K, from the directory name
Dims dims;
std::vector tensors;
@@ -442,6 +448,10 @@ struct Report {
long long mlp_groups_ok = 0, mlp_groups_total = 0;
int mlp_projections = 0;
double mlp_zero_inexact_pct = 0;
+
+ // engine#1503: the all-block-32 path. Units read (640-byte-strip units plus gate/up units, head included). There
+ // is no decode->encode round trip on this path, so the two counters above stay 0 for such a pack.
+ long long block32_units = 0;
};
Report import_pack(const std::string& pack_dir, const std::string& out_dir, const Options&);
@@ -490,8 +500,81 @@ struct Block32Head {
int64_t padding = 0; // zero bytes after the last unit
std::string why; // set when !ok
};
-// `p` points at the first unit (tail_off + envelope), `avail` is the bytes from there to the end of the file.
-Block32Head read_block32_head(const uint8_t* p, int64_t avail, int64_t V, int64_t K);
+// `p` points at the first unit (tail_off + envelope), `avail` is the bytes from there to the end of the file, and
+// `file_off` is where `p` sits in the file (only used to put an offset in a refusal).
+Block32Head read_block32_head(const uint8_t* p, int64_t avail, int64_t V, int64_t K, int64_t file_off = 0);
+
+// The same units for any matrix (engine#1503): `ceil(rows/64)` units of `block32_unit_bytes(K)` starting at `p`. This
+// is the head reader without its tail rule (nothing is required of the bytes after the last unit), and it is what the
+// all-block-32 layer block's q/k/v/o/down are read with. `what` names the tensor in a refusal. Same checks: the bytes
+// fit, every unit header and strip gap is zero, every scale is finite; same normalisation (positive scales).
+struct Block32Units {
+ bool ok = false;
+ Q4 q; // N = units x 64 (>= rows), K, block 32
+ int64_t units = 0;
+ int64_t bytes = 0; // units x block32_unit_bytes(K)
+ std::string why; // set when !ok
+};
+Block32Units read_block32_units(const uint8_t* p, int64_t avail, int64_t rows, int64_t K, const char* what = "head",
+ int64_t file_off = 0);
+
+// The gate/up unit of an all-block-32 pack (engine#1503): gate_proj and up_proj share one run of 64-row units, each
+// `4,864 + 2*(K/32)*1,216` bytes.
+//
+// Measured on `SmolLM2-135M-Instruct_rai_1.9.0_npu` (K 576, intermediate 1,536: 24 units of 48,640 B per layer)
+// against the public checkpoint's gate_proj / up_proj:
+// unit : 4,864 B of zeros, then 2*(K/32) strips of 1,216 B
+// strip s: K block `s/2` (32 columns), 64 row SLOTS: slots 0..31 are GATE rows `32*(s%2) .. +31` of the unit,
+// slots 32..63 are UP rows with the same numbers
+// +0 1,024 B of 4-bit codes, low nibble first; code position p (0..2047) is slot `(p/256)*8 + p%8`, column
+// `(p/8)%32` -- the head strip's order over eight 8-slot bands instead of four
+// +1024 32 B of zero points, one nibble per slot
+// +1056 32 B of zeros
+// +1088 64 bf16 scales, one per slot, signed
+// The code order is the measured part (0.98 .. 0.996 per 32-code block against the reference). That zero point and
+// scale are indexed by slot is ASSUMED by analogy with the 640-byte strip; the import of the measured pack is what
+// tests it (a container gate/up that still correlates with the checkpoint after the zero points and scales are
+// applied). Returned as two `Q4`s with the same normalisation and the same refusals as the unit reader.
+constexpr int64_t kB32MlpUnitHeader = 4864, kB32MlpStripBytes = 1216;
+inline int64_t block32_mlp_unit_bytes(int64_t K) { return kB32MlpUnitHeader + 2 * (K / 32) * kB32MlpStripBytes; }
+struct Block32GateUp {
+ bool ok = false;
+ Q4 gate, up; // each N = units x 64, K, block 32
+ int64_t units = 0;
+ int64_t bytes = 0; // units x block32_mlp_unit_bytes(K)
+ std::string why; // set when !ok
+};
+Block32GateUp read_block32_gate_up(const uint8_t* p, int64_t avail, int64_t rows, int64_t K, int64_t file_off = 0);
+
+// The layer block of a pack whose EVERY MatMulNBits is block 32 (engine#1503), as offsets from the block's start.
+// Read from the RAW 1.9 Token .fconst (not the canonical 1.8-form copy). With H = hidden, Nq = heads*head_dim,
+// Nkv = kv_heads*head_dim, IM = intermediate, unit = block32_unit_bytes, mlp_unit = block32_mlp_unit_bytes:
+// +0 64 + 2*H envelope: bf16 rms_norm_eps, zeros to 64, input_layernorm bf16[H]
+// q_off (Nq/64) x unit(H) q_proj [Nq][H]
+// k_off (Nkv/64) x unit(H) k_proj [Nkv][H]
+// v_off (Nkv/64) x unit(H) v_proj [Nkv][H]
+// sink_off 4*heads bytes not a weight; skipped (int32 0xFF7F0000 x heads on the measured pack)
+// o_off (H/64) x unit(Nq) o_proj [H][Nq]
+// post_off 64 + 2*H envelope: eps, zeros, post_attention_layernorm bf16[H]
+// gu_off (IM/64) x mlp_unit(H) gate_proj + up_proj, [IM][H] each
+// hdr_off 64 bytes skipped, content not interpreted (01 then 63 zeros on the measured pack)
+// down_off (H/64) x unit(IM) down_proj [H][IM]
+// stride
+// and the tail at `layers x stride`: the same envelope with the final norm, `ceil(V/64)` head units at K = H, zeros.
+// Measured on SmolLM2-135M-Instruct 1.9 only (stride 2,360,292; file 90,470,776 B), where Nq == H -- so o_proj's K
+// being Nq rather than H is the declared shape, not a measurement. There is no bias field in this layout.
+struct Block32Layout {
+ std::string why; // non-empty: this geometry does not fit the layout (a width that is not whole units)
+ int64_t env = 0, q_off = 0, k_off = 0, v_off = 0, sink_off = 0, o_off = 0, post_off = 0, gu_off = 0, hdr_off = 0,
+ down_off = 0, stride = 0;
+ int64_t tail_off = 0, head_off = 0, head_units = 0;
+ int64_t min_size = 0; // head_off + head_units x unit(H); the file is this plus zero padding
+};
+Block32Layout block32_layout(const Dims& d);
+// Empty when a mapped file fits `lay`: it is at least `min_size` long, and every envelope (two per layer and the
+// tail's) opens with the pack's epsilon followed by zeros up to the norm. Otherwise the reason, with the offset. The
+// zero padding after the head is the head reader's check.
+std::string block32_file_refusal(const uint8_t* map, int64_t size, const Dims& d, const Block32Layout& lay);
// The 1.9 external-data tile (engine#1503): the `.fconst` tile's subvolume written densely per tensor.
//
diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp
index a2656f574..088cf8052 100644
--- a/tests/npu_pack_import_test.cpp
+++ b/tests/npu_pack_import_test.cpp
@@ -2156,6 +2156,508 @@ void test_block32_tail_head(const std::string& root) {
}
}
+// ── engine#1503: a pack whose EVERY MatMulNBits is block 32 ────────────────────────────────────────────────────────
+// The gate/up unit, written from its description (pack_import.h, above read_block32_gate_up) and NOT through the
+// reader: 64-row units of 4,864 zero bytes + 2*(K/32) strips of 1,216 B; strip s is K block s/2 and holds 64 slots,
+// slots 0..31 = gate rows 32*(s%2)..+31, slots 32..63 = the same rows of up; code position
+// `(slot/8)*256 + c*8 + slot%8`, low nibble first; 32 B of zero-point nibbles (8) by slot; 32 zero bytes; 64 signed
+// bf16 scales by slot, `-m/8` as in encode_block32_units.
+static std::string encode_block32_gate_up(const std::vector& G, const std::vector& U, int rows, int K) {
+ const int kb = K / 32, units = rows / 64;
+ const size_t unit = size_t(4864 + 2 * kb * 1216);
+ std::string out(size_t(units) * unit, '\0');
+ for (int u = 0; u < units; ++u)
+ for (int s = 0; s < 2 * kb; ++s) {
+ char* sp = out.data() + size_t(u) * unit + 4864 + size_t(s) * 1216;
+ for (int slot = 0; slot < 64; ++slot) {
+ const std::vector& M = slot < 32 ? G : U;
+ const float* w = M.data() + size_t(u * 64 + 32 * (s % 2) + slot % 32) * size_t(K) + size_t((s / 2) * 32);
+ float m = 0;
+ for (int c = 0; c < 32; ++c)
+ if (std::fabs(w[c]) > std::fabs(m)) m = w[c];
+ const uint16_t bits = pi::bf16_bits_of(-m / 8.0f);
+ const uint32_t wide = uint32_t(bits) << 16;
+ float sc;
+ std::memcpy(&sc, &wide, 4);
+ sp[1088 + 2 * slot] = char(bits & 0xFF);
+ sp[1089 + 2 * slot] = char(bits >> 8);
+ sp[1024 + slot / 2] = char(uint8_t(sp[1024 + slot / 2]) | uint8_t(8 << (4 * (slot & 1))));
+ for (int c = 0; c < 32; ++c) {
+ const int code = sc == 0 ? 8 : std::min(15, std::max(0, int(std::lround(w[c] / sc)) + 8));
+ const int pos = (slot / 8) * 256 + c * 8 + slot % 8;
+ sp[pos / 2] = char(uint8_t(sp[pos / 2]) | uint8_t(code << (4 * (pos & 1))));
+ }
+ }
+ }
+ return out;
+}
+
+void test_block32_layers(const std::string& root) {
+ const auto ncorr = [](const std::vector& a, const std::vector& b) {
+ double num = 0, da = 0, db = 0, sa = 0, sb = 0;
+ const double n = double(a.size());
+ for (size_t i = 0; i < a.size(); ++i) {
+ num += double(a[i]) * b[i], da += double(a[i]) * a[i], db += double(b[i]) * b[i];
+ sa += a[i], sb += b[i];
+ }
+ const double den = (n * da - sa * sa) * (n * db - sb * sb);
+ return den > 0 ? (n * num - sa * sb) / std::sqrt(den) : 0.0;
+ };
+ const auto value = [](const pi::Q4& q, int row, int col) {
+ const size_t g = size_t(row) * size_t(q.K / 32) + size_t(col / 32);
+ return double(q.codes[size_t(row) * size_t(q.K) + col]) * q.scale[g] + q.zero[g];
+ };
+ // a deterministic matrix per (seed): values in about [-2, 2], unrelated between seeds
+ const auto matrix = [](int rows, int cols, uint32_t seed) {
+ std::vector m(size_t(rows) * size_t(cols));
+ uint32_t x = seed * 2654435761u + 12345u;
+ for (auto& v : m) {
+ x = x * 1664525u + 1013904223u;
+ v = float(int((x >> 8) & 0xFFFF) - 32768) / 16384.0f;
+ }
+ return m;
+ };
+
+ { // arm 1: the unit reader with ROWS as a parameter, typed bytes (K 64: one unit is 5,120 B)
+ std::vector u(2 * 5120, 0);
+ uint8_t* sp = u.data() + 5120 + 2560 + 2 * 640; // unit 1, strip 2: K block 1, rows 0..31 of the unit
+ const uint16_t bits = pi::bf16_bits_of(-0.5f);
+ sp[576 + 2 * 20] = uint8_t(bits & 0xFF); // row 20's scale
+ sp[577 + 2 * 20] = uint8_t(bits >> 8);
+ sp[512 + 10] = 0x05; // row 20's zero point: low nibble of byte 10
+ sp[(2 * 256 + 3 * 8 + 4) / 2] |= 0x09; // row 20 (band 2, r%8 4), column 3: position 540, low nibble
+ pi::Block32Units r = pi::read_block32_units(u.data(), int64_t(u.size()), 128, 64, "q_proj", 7000);
+ check(r.ok && r.units == 2 && r.bytes == 10240 && r.q.N == 128 && r.q.K == 64 && r.q.block == 32,
+ "block-32 units: 128 rows of K 64 are two units (" + r.why + ")");
+ if (r.ok) {
+ check(value(r.q, 64 + 20, 32 + 3) == -2.0,
+ "block-32 units: unit 1 row 20 col 35 = (9 - 5) x -0.5 (" + std::to_string(value(r.q, 84, 35)) + ")");
+ check(value(r.q, 84, 34) == 2.5, "block-32 units: its neighbour is code 0, (0 - 5) x -0.5");
+ check(r.q.scale[size_t(84) * 2 + 1] == 0.5f, "block-32 units: the stored scale is positive");
+ check(value(r.q, 20, 35) == 0.0 && value(r.q, 84, 3) == 0.0,
+ "block-32 units: the same position in unit 0 and in K block 0 is untouched");
+ }
+ check(pi::read_block32_units(u.data(), int64_t(u.size()), 65, 64).units == 2,
+ "block-32 units: 65 rows round up to two units");
+ const pi::Block32Units one = pi::read_block32_units(u.data(), int64_t(u.size()), 64, 64);
+ check(one.ok && one.units == 1 && one.q.N == 64,
+ "block-32 units: 64 rows read one unit and nothing is required of the bytes after it");
+ const pi::Block32Units sh = pi::read_block32_units(u.data(), 10239, 128, 64, "q_proj", 7000);
+ check(!sh.ok && sh.why.find("q_proj") != std::string::npos && sh.why.find("7000") != std::string::npos,
+ "block-32 units: a buffer one byte short is refused, naming the tensor and the offset (" + sh.why + ")");
+ std::vector hdr = u;
+ hdr[5120 + 9] = 1;
+ const pi::Block32Units hr = pi::read_block32_units(hdr.data(), int64_t(hdr.size()), 128, 64, "q_proj", 7000);
+ check(!hr.ok && hr.why.find("12120 B") != std::string::npos,
+ "block-32 units: a non-zero unit header is refused with the unit's offset, 7000 + 5120 (" + hr.why + ")");
+ std::vector nan = u;
+ nan[2560 + 576] = 0xC0, nan[2560 + 577] = 0x7F; // bf16 quiet NaN in unit 0, strip 0, row 0
+ check(!pi::read_block32_units(nan.data(), int64_t(nan.size()), 128, 64).ok,
+ "block-32 units: a scale that is not finite is refused");
+ }
+
+ { // arm 2: the gate/up reader, typed bytes (K 32: one unit is 4,864 + 2 x 1,216 = 7,296 B)
+ check(pi::block32_mlp_unit_bytes(32) == 7296 && pi::block32_mlp_unit_bytes(576) == 48640,
+ "block-32 gate/up: the unit is 4,864 + 2*(K/32)*1,216 B (48,640 at K 576)");
+ std::vector u(7296, 0);
+ uint8_t* sp = u.data() + 4864 + 1216; // strip 1: K block 0, rows 32..63
+ const auto put_scale = [&](int slot, float v) {
+ const uint16_t bits = pi::bf16_bits_of(v);
+ sp[1088 + 2 * slot] = uint8_t(bits & 0xFF);
+ sp[1089 + 2 * slot] = uint8_t(bits >> 8);
+ };
+ put_scale(5, 0.5f); // slot 5: GATE row 32 + 5
+ put_scale(41, -0.25f); // slot 41: UP row 32 + 9
+ sp[1024 + 2] = 0x80; // slot 5's zero point (high nibble of byte 2) = 8
+ sp[1024 + 20] = 0x60; // slot 41's zero point (high nibble of byte 20) = 6
+ sp[(0 * 256 + 7 * 8 + 5) / 2] |= 0xB0; // slot 5 (band 0, 5), column 7: position 61, high nibble
+ sp[(5 * 256 + 31 * 8 + 1) / 2] |= 0x30; // slot 41 (band 5, 1), column 31: position 1529, high nibble
+ pi::Block32GateUp h = pi::read_block32_gate_up(u.data(), int64_t(u.size()), 64, 32, 900);
+ check(h.ok && h.units == 1 && h.bytes == 7296 && h.gate.N == 64 && h.up.N == 64 && h.gate.K == 32 &&
+ h.up.block == 32,
+ "block-32 gate/up: one unit of K 32 reads (" + h.why + ")");
+ if (h.ok) {
+ check(value(h.gate, 37, 7) == 1.5,
+ "block-32 gate/up: slot 5 is GATE row 37, (11 - 8) x 0.5 (" + std::to_string(value(h.gate, 37, 7)) + ")");
+ check(value(h.gate, 37, 6) == -4.0, "block-32 gate/up: its neighbour is code 0, (0 - 8) x 0.5");
+ check(value(h.up, 37, 7) == 0.0, "block-32 gate/up: and UP row 37 holds nothing");
+ check(value(h.up, 41, 31) == 0.75,
+ "block-32 gate/up: slot 41 is UP row 41, a NEGATIVE scale, (3 - 6) x -0.25 (" +
+ std::to_string(value(h.up, 41, 31)) + ")");
+ check(value(h.up, 41, 0) == 1.5, "block-32 gate/up: its code 0, (0 - 6) x -0.25");
+ check(h.up.scale[41] == 0.25f, "block-32 gate/up: the stored scale is positive");
+ check(value(h.gate, 41, 31) == 0.0 && value(h.gate, 5, 7) == 0.0,
+ "block-32 gate/up: GATE row 41 and the other strip's row 5 hold nothing");
+ }
+ const pi::Block32GateUp sh = pi::read_block32_gate_up(u.data(), 7295, 64, 32, 900);
+ check(!sh.ok && sh.why.find("900") != std::string::npos,
+ "block-32 gate/up: a buffer one byte short is refused with the offset (" + sh.why + ")");
+ std::vector hdr = u;
+ hdr[4863] = 1;
+ check(!pi::read_block32_gate_up(hdr.data(), int64_t(hdr.size()), 64, 32).ok,
+ "block-32 gate/up: a non-zero unit header is refused");
+ std::vector gap = u;
+ gap[4864 + 1216 + 1056 + 31] = 1;
+ check(!pi::read_block32_gate_up(gap.data(), int64_t(gap.size()), 64, 32).ok,
+ "block-32 gate/up: a non-zero byte between the zero points and the scales is refused");
+ std::vector inf = u;
+ inf[4864 + 1088 + 2 * 63] = 0x80, inf[4864 + 1089 + 2 * 63] = 0x7F; // bf16 +inf in strip 0, slot 63
+ check(!pi::read_block32_gate_up(inf.data(), int64_t(inf.size()), 64, 32).ok,
+ "block-32 gate/up: a scale that is not finite is refused");
+ check(!pi::read_block32_gate_up(u.data(), int64_t(u.size()), 64, 48).ok, "block-32 gate/up: K 48 is refused");
+ }
+
+ { // arm 3: the description's encoder against the reader, two unrelated matrices over two units and three K blocks
+ const int rows = 128, K = 96;
+ const std::vector G = matrix(rows, K, 1), U = matrix(rows, K, 2);
+ const std::string enc = encode_block32_gate_up(G, U, rows, K);
+ const pi::Block32GateUp h =
+ pi::read_block32_gate_up(reinterpret_cast(enc.data()), int64_t(enc.size()), rows, K);
+ check(h.ok, "block-32 gate/up: the encoded stream reads (" + h.why + ")");
+ if (h.ok) {
+ std::vector g(G.size()), up(U.size());
+ for (int r = 0; r < rows; ++r)
+ for (int c = 0; c < K; ++c) {
+ g[size_t(r) * K + c] = float(value(h.gate, r, c));
+ up[size_t(r) * K + c] = float(value(h.up, r, c));
+ }
+ check(ncorr(g, G) > 0.99 && ncorr(up, U) > 0.99,
+ "block-32 gate/up: gate and up come back (corr " + std::to_string(ncorr(g, G)) + " / " +
+ std::to_string(ncorr(up, U)) + ")");
+ check(std::fabs(ncorr(g, U)) < 0.3 && std::fabs(ncorr(up, G)) < 0.3,
+ "block-32 gate/up: and not swapped (corr " + std::to_string(ncorr(g, U)) + " / " +
+ std::to_string(ncorr(up, G)) + ")");
+ }
+ }
+
+ { // arm 4: the layer block's arithmetic at the one measured geometry (SmolLM2-135M-Instruct 1.9)
+ pi::Dims d;
+ d.hidden = 576, d.intermediate = 1536, d.layers = 30, d.heads = 9, d.kv_heads = 3, d.head_dim = 64;
+ d.vocab = 49152;
+ const pi::Block32Layout l = pi::block32_layout(d);
+ check(l.why.empty() && l.q_off == 1216 && l.k_off == 231616 && l.v_off == 308416 && l.sink_off == 385216 &&
+ l.o_off == 385252 && l.post_off == 615652 && l.gu_off == 616868 && l.hdr_off == 1784228 &&
+ l.down_off == 1784292 && l.stride == 2360292,
+ "block-32 layout: SmolLM2-135M's layer block, stride 2,360,292 (" + l.why + " " +
+ std::to_string(l.stride) + ")");
+ check(l.head_off == 70809976 && l.head_units == 768 && l.min_size == 90470776,
+ "block-32 layout: its head at 70,809,976 and the file's 90,470,776 B (" + std::to_string(l.min_size) + ")");
+ d.intermediate = 1500;
+ check(pi::block32_layout(d).why.find("intermediate 1500") != std::string::npos,
+ "block-32 layout: a width that is not whole 64-row units is refused by name");
+ }
+
+ // arm 5: a crafted all-block-32 pack through import_pack(). hidden 128, intermediate 256, 2 heads x 64, 1 KV head,
+ // 2 layers, vocab 64, tied. Every matrix is its own pseudo-random one, so a tensor read from another's offset
+ // cannot correlate with what the container is compared against.
+ const int H = 128, IM = 256, LAYERS = 2, V = 64, NQ = 128, NKV = 64;
+ const int R = onebit::npu::q4nx::kRows;
+ enum { Q, K_, V_, O, G, U, D, NPROJ };
+ const char* const pname[NPROJ] = {"self_attn.q_proj", "self_attn.k_proj", "self_attn.v_proj", "self_attn.o_proj",
+ "mlp.gate_proj", "mlp.up_proj", "mlp.down_proj"};
+ const int prow[NPROJ] = {NQ, NKV, NKV, H, IM, IM, H}, pcol[NPROJ] = {H, H, H, NQ, H, H, IM};
+ std::vector> mats;
+ for (int l = 0; l < LAYERS; ++l)
+ for (int i = 0; i < NPROJ; ++i) mats.push_back(matrix(prow[i], pcol[i], uint32_t(100 + l * NPROJ + i)));
+ const std::vector emb = matrix(V, H, 7);
+ const auto norm_bytes = [&](int which) { // bf16 [H], a different ramp per norm
+ std::string b(size_t(H) * 2, '\0');
+ for (int i = 0; i < H; ++i) {
+ const uint16_t bits = pi::bf16_bits_of(float(which) + float(i) / 256.0f);
+ b[2 * i] = char(bits & 0xFF), b[2 * i + 1] = char(bits >> 8);
+ }
+ return b;
+ };
+ const auto envelope = [&](int which) {
+ std::string e(64, '\0');
+ e[0] = char(0x86), e[1] = char(0x35); // bf16 1e-06
+ return e + norm_bytes(which);
+ };
+ const auto fconst = [&] {
+ std::string fc;
+ for (int l = 0; l < LAYERS; ++l) {
+ const auto& m = [&](int i) -> const std::vector& { return mats[size_t(l * NPROJ + i)]; };
+ fc += envelope(10 * l + 1);
+ fc += encode_block32_units(m(Q), NQ, H) + encode_block32_units(m(K_), NKV, H) +
+ encode_block32_units(m(V_), NKV, H);
+ fc += std::string(4 * 2, '\x7f'); // 4 x heads bytes that are not a weight (their content is not read)
+ fc += encode_block32_units(m(O), H, NQ);
+ fc += envelope(10 * l + 2);
+ fc += encode_block32_gate_up(m(G), m(U), IM, H);
+ fc += std::string(1, '\x01') + std::string(63, '\0');
+ fc += encode_block32_units(m(D), H, IM);
+ }
+ fc += envelope(99);
+ fc += encode_block32_units(emb, V, H);
+ return fc;
+ };
+ const std::string good = fconst();
+ {
+ pi::Dims d;
+ d.hidden = H, d.intermediate = IM, d.layers = LAYERS, d.heads = 2, d.kv_heads = 1, d.head_dim = 64, d.vocab = V;
+ check(pi::block32_layout(d).min_size == int64_t(good.size()),
+ "all-block-32 fixture: the stream built from the description is the layout's size (" +
+ std::to_string(good.size()) + ")");
+ }
+ // `layer_block` / `head_block` are what cache/ops_meta.json DECLARES; the .fconst bytes are the caller's.
+ const auto build = [&](const char* tag, const std::string& fc, bool layers_declared, int layer_block = 32,
+ int head_block = 32) {
+ const fs::path dir = fs::path(root) / tag;
+ fs::remove_all(dir);
+ fs::create_directories(dir / "cache");
+ // the 1.9 spelling of the Token file: this path reads the raw file, and it must not be mistaken for a 1.8 one
+ put_file((dir / "dd_metastate_Llm_Token_aie2_token_crafted.fconst").string(), fc);
+ put_file((dir / "config.json").string(),
+ R"({"model_type":"llama","hidden_size":128,"intermediate_size":256,"num_hidden_layers":2,)"
+ R"("num_attention_heads":2,"num_key_value_heads":1,"head_dim":64,"vocab_size":64,)"
+ R"("rope_theta":10000.0,"tie_word_embeddings":true,"rms_norm_eps":1e-06})");
+ std::string data(size_t(V) * size_t(H) * 2, '\0');
+ for (size_t i = 0; i < emb.size(); ++i) {
+ const uint16_t bits = pi::bf16_bits_of(emb[i]);
+ data[2 * i] = char(bits & 0xFF);
+ data[2 * i + 1] = char(bits >> 8);
+ }
+ put_file((dir / "model.onnx.data").string(), data);
+ const auto op_at = [](const std::string& name, int n, int k, int block) {
+ return R"({"type":"MladfMatMul","name":")" + name + R"(.MatMulNBits","attrs":{"N":{"value":[)" +
+ std::to_string(n) + R"(]},"K":{"value":[)" + std::to_string(k) + R"(]},"block_size":{"value":[)" +
+ std::to_string(block) + R"(]},"bits":{"value":[4]}}})";
+ };
+ const auto op = [&](const std::string& name, int n, int k) { return op_at(name, n, k, layer_block); };
+ std::string ops = op_at("lm_head", V, H, head_block);
+ if (layers_declared)
+ for (int l = 0; l < LAYERS; ++l) {
+ const std::string a = "model.layers." + std::to_string(l) + ".attn.",
+ m = "model.layers." + std::to_string(l) + ".mlp.";
+ ops += "," + op(a + "q_proj", NQ, H) + "," + op(a + "k_proj", NKV, H) + "," + op(a + "v_proj", NKV, H) +
+ "," + op(a + "o_proj", H, NQ) + "," + op(m + "down_proj", H, IM);
+ }
+ put_file((dir / "cache" / "ops_meta.json").string(), R"({"op_list":[)" + ops + R"(],"tensor_map":{}})");
+ put_file((dir / "model.onnx").string(),
+ pb::model({pb::tensor("model.embed_tokens.weight", {V, H}, "model.onnx.data", 0,
+ uint64_t(V) * uint64_t(H) * 2, 16),
+ pb::placeholder("lm_head.weight.MatMulNBits.qweight"),
+ pb::placeholder("lm_head.weight.MatMulNBits.scales"),
+ pb::placeholder("lm_head.weight.MatMulNBits.zeros")},
+ {}));
+ return dir;
+ };
+ const auto out_of = [](const char* tag) {
+ return (fs::temp_directory_path() /
+ (std::string("oga_pack_import_b32layers_") + tag + "." + std::to_string(::getpid()))).string();
+ };
+ // the first `cols` columns of rows 0..R-1 of a container tensor, from its own bytes (tile 0)
+ const auto tile0 = [&](const json& tbl, const std::vector& blob, const std::string& name, int cols) {
+ std::vector w(size_t(R) * onebit::npu::q4nx::kCols, 0.f), out(size_t(R) * size_t(cols), 0.f);
+ if (!tbl.contains(name)) return std::vector();
+ const auto offs = tbl[name]["data_offsets"].get>();
+ onebit::npu::q4nx::dequant_tile(onebit::npu::q4nx::Chunk::Q4_1F, blob.data() + offs[0], w.data(),
+ onebit::npu::q4nx::kCols);
+ for (int r = 0; r < R; ++r)
+ for (int c = 0; c < cols; ++c) out[size_t(r) * cols + c] = w[size_t(r) * onebit::npu::q4nx::kCols + c];
+ return out;
+ };
+ const auto head_of = [&](const std::vector& m, int cols_total, int cols) { // the same window of a source
+ std::vector out(size_t(R) * size_t(cols));
+ for (int r = 0; r < R; ++r)
+ for (int c = 0; c < cols; ++c) out[size_t(r) * cols + c] = m[size_t(r) * cols_total + c];
+ return out;
+ };
+
+ { // 5a: the pack imports, complete, and every container tensor is the matrix the description put at its offset
+ const fs::path dir = build("b32-layers", good, true);
+ const std::string out = out_of("good");
+ fs::remove_all(out);
+ pi::Report r;
+ std::string said;
+ try {
+ const pi::Plan p = pi::plan(dir.string());
+ check(p.block32_layers && p.block32_note.empty() && p.refused().empty(),
+ "all-block-32: the plan takes the block-32 layout and refuses nothing (" + p.block32_note + ")");
+ r = pi::import_pack(dir.string(), out, pi::Options{});
+ } catch (const std::exception& e) {
+ said = e.what();
+ }
+ check(said.empty() && r.refused.empty(),
+ "all-block-32: a crafted pack imports with nothing refused (" + said +
+ (r.refused.empty() ? "" : r.refused.front()) + ")");
+ check(r.attn_projections == 4 * LAYERS && r.mlp_projections == 3 * LAYERS && r.attn_bias_tensors == 0,
+ "all-block-32: 4 attention + 3 MLP projections per layer, no bias tensor");
+ check(r.stride == (int64_t(good.size()) - (64 + 2 * H) - pi::block32_unit_bytes(H)) / LAYERS,
+ "all-block-32: the report carries the stride (" + std::to_string(r.stride) + ")");
+ check(r.selfcheck_tied && r.lmhead_corr > 0.99 && r.head_read_note.find("block 32") != std::string::npos,
+ "all-block-32: the head is read and verified against the embedding (corr " +
+ std::to_string(r.lmhead_corr) + ")");
+ if (fs::exists(out + "/model.q4nx")) {
+ const json tbl = container_table(out + "/model.q4nx");
+ const std::vector blob = container_payload(out + "/model.q4nx");
+ for (int l = 0; l < LAYERS; ++l)
+ for (int i = 0; i < NPROJ; ++i) {
+ const std::string name = "model.layers." + std::to_string(l) + "." + pname[i] + ".weight";
+ const int cols = std::min(pcol[i], onebit::npu::q4nx::kCols);
+ const std::vector got = tile0(tbl, blob, name, cols);
+ const double c = got.empty() ? 0.0 : ncorr(got, head_of(mats[size_t(l * NPROJ + i)], pcol[i], cols));
+ check(c > 0.99, "all-block-32: container " + name + " is its own matrix (corr " +
+ std::to_string(c) + ")");
+ // the control: the matrix one slot along (gate <-> up among them) must not match
+ const int j = (i == G) ? U : (i == U) ? G : -1;
+ if (j >= 0 && !got.empty()) {
+ const double x = ncorr(got, head_of(mats[size_t(l * NPROJ + j)], pcol[j], cols));
+ check(std::fabs(x) < 0.3, "all-block-32: container " + name + " is not the other MLP half "
+ "(corr " + std::to_string(x) + ")");
+ }
+ }
+ const std::vector hd = tile0(tbl, blob, "lm_head.weight", H);
+ check(!hd.empty() && ncorr(hd, head_of(emb, H, H)) > 0.99, "all-block-32: container lm_head is the tail's head");
+ const auto norm_is = [&](const std::string& name, int which) {
+ if (!tbl.contains(name)) return false;
+ const auto offs = tbl[name]["data_offsets"].get>();
+ const std::string want = norm_bytes(which);
+ return offs[1] - offs[0] == want.size() && std::memcmp(blob.data() + offs[0], want.data(), want.size()) == 0;
+ };
+ check(norm_is("model.layers.0.input_layernorm.weight", 1) &&
+ norm_is("model.layers.1.input_layernorm.weight", 11) &&
+ norm_is("model.layers.0.post_attention_layernorm.weight", 2) &&
+ norm_is("model.layers.1.post_attention_layernorm.weight", 12) && norm_is("model.norm.weight", 99),
+ "all-block-32: the five norms are the envelopes' bytes");
+ check(!tbl.contains("model.layers.0.self_attn.q_proj.bias"), "all-block-32: no bias tensor is invented");
+ } else {
+ check(false, "all-block-32: the container was written");
+ }
+ fs::remove_all(out);
+ fs::remove_all(dir);
+ }
+ // what a refused import says, and that it wrote no container
+ const auto refused = [&](const char* tag, const std::string& fc) {
+ const fs::path dir = build(tag, fc, true);
+ const std::string out = out_of(tag);
+ fs::remove_all(out);
+ std::string said;
+ try {
+ const pi::Report r = pi::import_pack(dir.string(), out, pi::Options{});
+ for (const auto& w : r.refused) said += w + " | ";
+ } catch (const std::exception& e) {
+ said = e.what();
+ }
+ if (fs::exists(out + "/model.q4nx")) said = "WROTE A CONTAINER " + said;
+ fs::remove_all(out);
+ fs::remove_all(dir);
+ return said;
+ };
+ { // 5b: a file one byte short of the layout is refused with the arithmetic
+ const std::string said = refused("b32-layers-short", good.substr(0, good.size() - 1));
+ check(said.find("WROTE") == std::string::npos && said.find("all-block-32 layout") != std::string::npos &&
+ said.find(std::to_string(good.size())) != std::string::npos,
+ "all-block-32: a short file is refused with the size the layout needs (" + said.substr(0, 300) + ")");
+ }
+ { // 5c: layer 1's post-attention envelope without its epsilon
+ pi::Dims d;
+ d.hidden = H, d.intermediate = IM, d.layers = LAYERS, d.heads = 2, d.kv_heads = 1, d.head_dim = 64, d.vocab = V;
+ const pi::Block32Layout l = pi::block32_layout(d);
+ std::string bad = good;
+ bad[size_t(l.stride + l.post_off)] = 0;
+ const std::string said = refused("b32-layers-eps", bad);
+ check(said.find("WROTE") == std::string::npos && said.find("no RMSNorm epsilon at " +
+ std::to_string(l.stride + l.post_off)) !=
+ std::string::npos,
+ "all-block-32: a missing epsilon is refused with its offset (" + said.substr(0, 300) + ")");
+ // 5d: a non-zero byte in a gate/up unit header of layer 1 -> both halves refused by name
+ bad = good;
+ bad[size_t(l.stride + l.gu_off + pi::block32_mlp_unit_bytes(H) + 17)] = 1;
+ const std::string s2 = refused("b32-layers-gu", bad);
+ check(s2.find("WROTE") == std::string::npos &&
+ s2.find("model.layers.1.mlp.gate_proj.weight: block-32 gate/up unit 1") != std::string::npos &&
+ s2.find("model.layers.1.mlp.up_proj.weight") != std::string::npos &&
+ s2.find("model.layers.0") == std::string::npos,
+ "all-block-32: a non-zero gate/up unit header refuses that layer's gate and up by name (" +
+ s2.substr(0, 300) + ")");
+ // 5e: trailing junk after the head is the head's refusal
+ const std::string s3 = refused("b32-layers-junk", good + std::string(99, '\0') + "x");
+ check(s3.find("WROTE") == std::string::npos && s3.find("lm_head.weight") != std::string::npos &&
+ s3.find("not zero after") != std::string::npos,
+ "all-block-32: a non-zero byte after the head refuses the head (" + s3.substr(0, 300) + ")");
+ }
+ { // 5f: zero padding after the head is accepted (the measured mixed pack has 110 MB of it)
+ const fs::path dir = build("b32-layers-pad", good + std::string(5000, '\0'), true);
+ const std::string out = out_of("pad");
+ fs::remove_all(out);
+ std::string said;
+ try {
+ const pi::Report r = pi::import_pack(dir.string(), out, pi::Options{});
+ for (const auto& w : r.refused) said += w + " | ";
+ } catch (const std::exception& e) {
+ said = e.what();
+ }
+ check(said.empty() && fs::exists(out + "/model.q4nx"), "all-block-32: zero padding after the head imports (" +
+ said.substr(0, 200) + ")");
+ fs::remove_all(out);
+ fs::remove_all(dir);
+ }
+ { // 5g: the same pack declaring ONLY its head at block 32 is not this class (it is the mixed pack's path)
+ const fs::path dir = build("b32-layers-headonly", good, false);
+ bool block32 = true;
+ try {
+ block32 = pi::plan(dir.string()).block32_layers;
+ } catch (const std::exception&) {
+ block32 = false;
+ }
+ check(!block32, "all-block-32: a pack that declares only lm_head at block 32 does not take the all-block-32 path");
+ fs::remove_all(dir);
+ }
+ // 5h..5k: the path is chosen by what the pack DECLARES, one arm per declaration. The bytes are the same good
+ // all-block-32 file every time, so the only thing that changes is cache/ops_meta.json -- and a declaration this
+ // importer has no layout for must be refused with its number, never read at 32 or at 128 by default.
+ const auto declared_as = [&](const char* tag, int layer_block, int head_block) {
+ const fs::path dir = build(tag, good, true, layer_block, head_block);
+ const std::string out = out_of(tag);
+ fs::remove_all(out);
+ std::string said;
+ bool block32 = false;
+ try {
+ block32 = pi::plan(dir.string()).block32_layers;
+ const pi::Report r = pi::import_pack(dir.string(), out, pi::Options{});
+ for (const auto& w : r.refused) said += w + " | ";
+ } catch (const std::exception& e) {
+ said = e.what();
+ }
+ if (block32) said = "TOOK THE ALL-BLOCK-32 PATH " + said;
+ if (fs::exists(out + "/model.q4nx")) said = "WROTE A CONTAINER " + said;
+ fs::remove_all(out);
+ fs::remove_all(dir);
+ return said;
+ };
+ for (const int b : {3, 64, 256}) { // 5h: every op at a block nobody has measured
+ const std::string said = declared_as(("b32-layers-all" + std::to_string(b)).c_str(), b, b);
+ check(said.find("WROTE") == std::string::npos && said.find("TOOK") == std::string::npos &&
+ said.find("block size along K is " + std::to_string(b) + " (11 ops)") != std::string::npos &&
+ said.find("lm_head @ " + std::to_string(b)) != std::string::npos,
+ "block declaration: every op at " + std::to_string(b) + " is refused with that number (" +
+ said.substr(0, 200) + ")");
+ }
+ { // 5i: layers at 32 under a block-128 head is not the all-block-32 class, and is refused by tensor
+ const std::string said = declared_as("b32-layers-head128", 32, 128);
+ check(said.find("WROTE") == std::string::npos && said.find("TOOK") == std::string::npos &&
+ said.find("128 (1 ops)") != std::string::npos && said.find("32 (10 ops)") != std::string::npos &&
+ said.find("lm_head @") == std::string::npos,
+ "block declaration: layers at 32 with the head at 128 is refused, naming the layers (" +
+ said.substr(0, 260) + ")");
+ }
+ { // 5j: layers at 32 with the head at 64 -- one unmeasured op is enough to leave the class
+ const std::string said = declared_as("b32-layers-head64", 32, 64);
+ check(said.find("WROTE") == std::string::npos && said.find("TOOK") == std::string::npos &&
+ said.find("64 (1 ops)") != std::string::npos,
+ "block declaration: layers at 32 with the head at 64 is refused (" + said.substr(0, 260) + ")");
+ }
+ { // 5k: the same ops all declared at 128 stay on the block-128 walk (which then refuses THESE bytes, because
+ // they are a block-32 file: the point is that the declaration, not a default, picked the path)
+ const std::string said = declared_as("b32-layers-all128", 128, 128);
+ check(said.find("WROTE") == std::string::npos && said.find("TOOK") == std::string::npos &&
+ said.find("block size along K") == std::string::npos && !said.empty(),
+ "block declaration: every op at 128 does not take the all-block-32 path or the block gate (" +
+ said.substr(0, 260) + ")");
+ }
+}
+
// ── the final norm out of the tail envelope, on a crafted pack (engine#844) ──────────────────
// The real-pack leg only covers this where a pack is supplied, and three of the four packs on the
// box ship no prefill program, so the import's tail-envelope read had no coverage in the default
@@ -4880,6 +5382,7 @@ int main(int argc, char** argv) {
test_post_attn_norm_header(golden_dir.empty() ? "." : golden_dir);
test_tied_block32_head(golden_dir.empty() ? "." : golden_dir);
test_block32_tail_head(golden_dir.empty() ? "." : golden_dir);
+ test_block32_layers(golden_dir.empty() ? "." : golden_dir);
test_final_norm_envelope(golden_dir.empty() ? "." : golden_dir);
test_import_fp32_params(golden_dir.empty() ? "." : golden_dir);
test_text_artifacts(golden_dir.empty() ? "." : golden_dir);