diff --git a/docs/npu.md b/docs/npu.md index ab1a76b58..42f539a38 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1183,6 +1183,25 @@ stored bytes equal the upstream checkpoint's `model.norm.weight` exactly. (The e "the tail envelope is `[eps][pad][448 f32]`, so the final norm's position is unknown" — was the last 1,856 bytes of the *file*, which in the Token_Token program are tile padding.) +**A head the pack declares at `block_size 32` is not in that tile stream** (engine#1740). The three +Ryzen AI 1.9 Qwen-1.5B packs quantise every layer at block 128 and `/lm_head/MatMul_Q4` at block 32, +and their tail after the envelope is 64-row *units* instead: 2,560 zero bytes, then `2 x (K/32)` +strips of 640 B. Strip `s` holds K block `s/2` for rows `32 x (s%2) .. +31` of the unit: 512 B of +4-bit codes (low nibble first; position `p` is row `(p/256) x 8 + p%8`, column `(p/8) % 32`), 16 B +of zero-point nibbles, 48 zero bytes, and 32 **signed** bf16 scales, one per row; +`w = (code - zero_point) x scale`. After the last unit the file is zero to its end (110,208,000 B on +the measured pack, which is why the block-128 arithmetic never landed there). `npu-pack import` +reads it (`read_block32_head`), folds a negative scale into its codes so the container's scales are +positive, and writes the same q4_1f tiles as any other head. The layout checks are part of the read +— unit count `ceil(vocab/64)`, the zero fields, finite scales, zeros to the end of the file — and +each failure is a refusal that says which. On a pack that declares tied embeddings the read head +also has to correlate above 0.99 with `model.embed_tokens`; if it does not, the head is made from +the embedding as before and `q4nx_head_note` says why the read was not taken. On an untied pack the +layout checks are all there is, and the note says the values are not verified. Measured on +`Qwen-2.5_1.5B_Instruct_rai_1.9.0_npu` (tied, 151,936 x 1,536, 2,374 units of 64,000 B): every scale +equals `bf16(-m/8)` of its 32-column embedding block, `m` being the block's largest-magnitude +weight, and the read head correlates 0.996 with the embedding. + The norms that *are* imported sit at the documented offsets, each confirmed by the epsilon that opens its layer block: **48/48 anchors found**. `tests/npu_pack_import_test` (registered as `npu_pack_import`, and as `npu_pack_import_oga` when `-DONEBIT_OGA_PACK=` is given) checks diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index ad46af92a..02fe8c996 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -2451,6 +2451,88 @@ static TailHeadLocate locate_tail_head(const Fconst& fc, int64_t tail_off, int64 return r; } +// engine#1740: the block-32 tail head. Layout and its measurement are in pack_import.h. +Block32Head read_block32_head(const uint8_t* p, int64_t avail, int64_t V, int64_t K) { + Block32Head h; + if (V <= 0 || K <= 0 || K % 32 != 0) { + h.why = "a block-32 head needs K to be a multiple of 32 (K " + std::to_string(K) + ")"; + return h; + } + const int64_t kb = K / 32, strips = 2 * kb, unit = block32_unit_bytes(K); + h.units = (V + kB32UnitRows - 1) / kB32UnitRows; + h.bytes = h.units * unit; + if (avail < h.bytes) { + h.why = "the .fconst tail has " + std::to_string(avail) + " B after its envelope, and a block-32 head of " + + std::to_string(h.units) + " units x " + std::to_string(unit) + " B needs " + std::to_string(h.bytes); + return h; + } + for (int64_t i = h.bytes; i < avail; ++i) + if (p[i] != 0) { + h.why = "the .fconst tail is not zero after the block-32 head's " + std::to_string(h.units) + + " units (first non-zero byte " + std::to_string(i - h.bytes) + " B past them), so the head is not " + "where this layout puts it"; + return h; + } + h.padding = avail - h.bytes; + const auto all_zero = [](const uint8_t* b, int64_t n) { + for (int64_t i = 0; i < n; ++i) + if (b[i] != 0) return false; + return true; + }; + Q4& q = h.q; + q.N = int(h.units * kB32UnitRows); + q.K = int(K); + q.block = 32; + q.codes.assign(size_t(q.N) * size_t(K), 0); + q.scale.assign(size_t(q.N) * size_t(kb), 1.0f); + q.zero.assign(size_t(q.N) * size_t(kb), 0.0f); + for (int64_t u = 0; u < h.units; ++u) { + const uint8_t* up = p + u * unit; + if (!all_zero(up, kB32UnitHeader)) { + h.why = "block-32 head unit " + std::to_string(u) + " does not open with " + + std::to_string(kB32UnitHeader) + " zero bytes, so this is not the layout measured"; + return h; + } + for (int64_t s = 0; s < strips; ++s) { + const uint8_t* sp = up + kB32UnitHeader + s * kB32StripBytes; + if (!all_zero(sp + 528, 48)) { + h.why = "block-32 head unit " + std::to_string(u) + " strip " + std::to_string(s) + + " has non-zero bytes between its zero points and its scales, so this is not the layout measured"; + return h; + } + const int64_t col0 = (s / 2) * 32, row0 = u * kB32UnitRows + 32 * (s % 2); + for (int r = 0; r < 32; ++r) { + const uint16_t bits = uint16_t(sp[576 + 2 * r]) | uint16_t(uint16_t(sp[577 + 2 * r]) << 8); + const uint32_t wide = uint32_t(bits) << 16; + float sc; + std::memcpy(&sc, &wide, 4); + if (!std::isfinite(sc)) { + h.why = "block-32 head unit " + std::to_string(u) + " strip " + std::to_string(s) + " row " + + std::to_string(r) + " has a scale that is not finite"; + return h; + } + const int zp = (sp[512 + r / 2] >> (4 * (r & 1))) & 15; + const bool flip = sc < 0; + const size_t g = size_t(row0 + r) * size_t(kb) + size_t(s / 2); + if (sc == 0.0f) continue; // an all-zero block: codes 0, scale 1, zero 0 decode to 0 + const float mag = flip ? -sc : sc; + q.scale[g] = mag; + q.zero[g] = -float(flip ? 15 - zp : zp) * mag; + // position p = band*256 + col*8 + (r%8) holds row band*8 + r%8, column col + const int base = (r / 8) * 256 + (r % 8); + uint8_t* dst = q.codes.data() + size_t(row0 + r) * size_t(K) + size_t(col0); + for (int c = 0; c < 32; ++c) { + const int pos = base + c * 8; + const int code = (sp[pos / 2] >> (4 * (pos & 1))) & 15; + dst[c] = uint8_t(flip ? 15 - code : code); + } + } + } + } + h.ok = true; + return h; +} + // engine#1740: a self-test for the SEARCH, not the arithmetic. `plan_tail_head` is pinned by the host test; this pins // the part that can go silently wrong -- the candidate loop, best/runner-up tracking, the kTieCorr cut, the ambiguity // refusal and the untied-with-padding refusal -- using a synthetic head built through encode_tile, so it needs no pack, @@ -4397,11 +4479,99 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons // writes -- and the self-check then measures the synthesis against the embedding rather than being // skipped. An untied pack is refused by name: its head is a matrix nothing else in the pack can // stand in for. - if (!(tie_declared && tied)) { + // + // engine#1740: a block-32 head is READ when the tail holds it in the measured layout (pack_import.h, + // read_block32_head). On a tied pack the read has to correlate with the embedding above kTieCorr to be + // taken -- otherwise the synthesis below stands, and says why the read did not. On an untied pack the + // structural checks are all there is (nothing in the pack can vouch for the values), which the note + // states; it is the same standing the block-128 tail head has on an untied pack. + bool head32_read = false; + std::string head32_why; + if (p.head_block == 32) { + const int64_t tail_off = int64_t(p.dims.layers) * g.stride; + const int64_t env = attn_payload_off(H); + if (tail_off + env > int64_t(fc.size)) { + head32_why = "the .fconst ends before the tail's envelope"; + } else { + Block32Head h32 = read_block32_head(fc.map + tail_off + env, int64_t(fc.size) - tail_off - env, + V, H); + const bool is_tied = tie_declared && tied; + double c32 = 0.0; + if (h32.ok && embed_tbl.ok()) c32 = corr(h32.q, embed_tbl, int(V)); + if (!h32.ok) { + head32_why = h32.why; + } else if (is_tied && !(c32 > kTieCorr)) { + char b[160]; + std::snprintf(b, sizeof b, + "the tail's block-32 head decodes but correlates %.6f with the embedding of " + "a pack that declares them tied", c32); + head32_why = b; + } else { + Q4& q = h32.q; + std::fprintf(stderr, + " lm head: block 32, %lld unit(s) of %lld B at %lld B, %lld B of zeros after " + "them%s\n", + (long long)h32.units, (long long)block32_unit_bytes(H), + (long long)(tail_off + env), (long long)h32.padding, + is_tied ? " (tied: verified by correlation with the pack's embedding)" + : " (untied: layout checked, values not verified)"); + if (embed_tbl.ok()) { + r.lmhead_corr = c32; + { + double sum_abs = 0, sum_ref = 0; + for (int n = 0; n < int(V); ++n) + for (int k = 0; k < q.K; ++k) { + const size_t gg = size_t(n) * (q.K / 32) + size_t(k) / 32; + const double x = double(q.codes[size_t(n) * q.K + k]) * q.scale[gg] + q.zero[gg]; + const double e = embed_at(embed_tbl, n, k); + sum_abs += std::fabs(x - e); + sum_ref += std::fabs(e); + } + if (sum_ref > 0) r.lmhead_rel = sum_abs / sum_ref; + } + r.selfcheck_reference = true; + r.selfcheck_tied = c32 > kTieCorr; + double sxy = 0, sx = 0, sy = 0, sxx = 0, syy = 0; + const int nsh = std::min(2048, int(V) - 1); + for (int rr = 0; rr < nsh; ++rr) + for (int k = 0; k < q.K; ++k) { + const size_t gg = size_t(rr) * (q.K / 32) + size_t(k) / 32; + const double x = double(q.codes[size_t(rr) * q.K + k]) * q.scale[gg] + q.zero[gg]; + const double y = embed_at(embed_tbl, rr + 1, k); + sxy += x * y; sx += x; sy += y; sxx += x * x; syy += y * y; + } + const double m = double(nsh) * q.K; + const double cx = sxx - sx * sx / m, cy = syy - sy * sy / m; + if (m > 0 && cx > 1e-12 && cy > 1e-12) + r.ctrl_corr_rowshift = (sxy - sx * sy / m) / std::sqrt(cx * cy); + } + std::vector tiled; + long long pinexact = 0; + tile_q4_1_f32(q, tiled, &pinexact); + r.codes_total = q.codes.size(); + r.zero_exact_pct = 100.0 * double(q.scale.size() - pinexact) / double(q.scale.size()); + const int64_t head_tiles = int64_t(tiled.size() / kImportTileBytes); + r.tiles += head_tiles; + w.add("lm_head.weight", "I8", {head_tiles, kImportTileBytes}, std::move(tiled)); + ++r.tensors; + head_written = true; + head32_read = true; + r.head_read_note = + std::string("lm_head was read from the .fconst tail's block 32 head (") + + std::to_string(h32.units) + " units of " + std::to_string(block32_unit_bytes(H)) + + " B)" + (is_tied ? ", and it reproduces model.embed_tokens" + : "; the pack is untied, so its values are not verified against " + "anything in the pack"); + } + } + } + if (head32_read) { + // nothing more to do: the pack's own head is in the container + } else if (!(tie_declared && tied)) { late.push_back("lm_head.weight: this pack quantises its head at block " + std::to_string(p.head_block) + " along K, which this importer's tile geometry cannot " "walk, and the pack does not declare tied embeddings, so the embedding cannot stand " - "in for it"); + "in for it" + (head32_why.empty() ? "" : " (" + head32_why + ")")); } else { const size_t rows_v = size_t(V), kk = size_t(H); Q4 q; @@ -4461,7 +4631,8 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons r.head_synthesized_note = "the pack quantises its head at block " + std::to_string(p.head_block) + " along K, so the tail's head tiles are not read; the pack declares tied embeddings, so " - "lm_head is the 4-bit quantisation of model.embed_tokens"; + "lm_head is the 4-bit quantisation of model.embed_tokens" + + (head32_why.empty() ? "" : " (" + head32_why + ")"); std::fprintf(stderr, "note: this pack's head is block %lld along K, which this importer cannot walk; the " "pack is tied, so the head was taken from the embedding (engine#1503)\n", @@ -5028,6 +5199,7 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons c["vocab_size"] = V; c["attention_bias"] = r.attn_bias_tensors > 0; // what the container actually carries if (!r.head_synthesized_note.empty()) c["q4nx_head_note"] = r.head_synthesized_note; + if (!r.head_read_note.empty()) c["q4nx_head_note"] = r.head_read_note; // The forward picks between "logits from the token embedding" and "logits from a separate head tensor" on this // key, and defaults to TIED when it is absent (npu/forward/src/q4nx_semantics.c). An untied pack that ships no // config.json -- every genre-A hybrid, e.g. Phi-3-mini -- therefore used to look tied, and the forward decoded diff --git a/npu/pack_import.h b/npu/pack_import.h index dcd0aa86a..dbed2c93a 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -371,6 +371,8 @@ struct Report { // walk and the pack is tied). Empty means it was read, as usual. Carried on the Report because the config JSON is // built after the .fconst block that discovers this. std::string head_synthesized_note; + // engine#1740: set when a block-32 head was READ from the .fconst tail (never together with the note above) + std::string head_read_note; // engine#1725: the per-layer embedding block the container carries (0 = none): row width per layer, and rows int64_t per_layer_P = 0, per_layer_V = 0; bool prefill_tail_seen = false; // a second copy was readable in the prefill .fconst @@ -403,6 +405,42 @@ struct Q4 { std::vector scale; // [N, K/32] std::vector zero; // [N, K/32] }; +// The 1.9 block-32 tail head (engine#1740). A pack whose `/lm_head/MatMul_Q4` is declared `block_size 32` keeps the +// head in the Token .fconst tail, after the same `64 + 2*hidden` envelope every other 1.9 tail opens with, but NOT in +// the block-128 tile: it is written in 64-row units of `2,560 + 2*(K/32)*640` bytes. +// +// Measured on `Qwen-2.5_1.5B_Instruct_rai_1.9.0_npu` (tied, V 151,936, K 1,536; unit 64,000 B, 2,374 units, then +// 110,208,000 B of zeros to EOF) against the pack's own `model.embed_tokens`: +// unit : 2,560 B of zeros, then 2*(K/32) strips of 640 B +// strip s: K block `s/2` (32 columns), rows `32*(s%2) .. +31` of the unit +// +0 512 B of 4-bit codes, low nibble first; code position p (0..1023) is row `(p/256)*8 + p%8`, column +// `(p/8)%32` -- four 8-row bands, each written column by column +// +512 16 B of zero points, one nibble per row (0x8 everywhere on the measured pack) +// +528 48 B of zeros +// +576 32 bf16 scales, one per row; SIGNED -- the measured writer stores `-m/8` where m is the block's +// largest-magnitude weight, so about half of them are negative +// w = (code - zero_point) * scale +// Every one of the 3,072 scales of a unit equals `bf16(-m/8)` of its embedding block exactly, and the decoded rows +// correlate 0.990 .. 0.997 with the embedding (4-bit), which is what fixed the field positions. The result is returned +// with positive scales (a negative one is folded into its codes: `15 - code`, `15 - zp`), as a `Q4` at block 32, rows +// padded to whole units the way the block-128 tail head is. +// +// What is checked, because a wrong region read with confidence is the failure this path exists to avoid: the unit +// count is `ceil(V/64)` and has to fit; every unit's 2,560 B and every strip's 48 B are zero; every scale is finite; +// and everything after the last unit is zero. Any of those failing is a refusal with the offset in it. +constexpr int64_t kB32UnitRows = 64, kB32UnitHeader = 2560, kB32StripBytes = 640; +inline int64_t block32_unit_bytes(int64_t K) { return kB32UnitHeader + 2 * (K / 32) * kB32StripBytes; } +struct Block32Head { + bool ok = false; + Q4 q; // N = units x 64 (>= V), K, block 32; value = code*scale + zero, scale > 0 + int64_t units = 0; + int64_t bytes = 0; // units x block32_unit_bytes(K) + int64_t padding = 0; // zero bytes after the last unit + std::string why; // set when !ok +}; +// `p` points at the first unit (tail_off + envelope), `avail` is the bytes from there to the end of the file. +Block32Head read_block32_head(const uint8_t* p, int64_t avail, int64_t V, int64_t K); + // The 1.9 external-data tile (engine#1503): the `.fconst` tile's subvolume written densely per tensor. // // Measured on `amd/phi-4_rai_1.9.0_npu` (`eager-mm.onnx.data`, 9.74 GB): every `MatMulNBitsBf` weight is a 1-D byte diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 1a91f2cf9..d6dbe17c6 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -1749,6 +1749,262 @@ void test_tied_block32_head(const std::string& root) { } } +// ── engine#1740: the block-32 head READ from the .fconst tail ────────────────────────────────────────────────────── +// The layout was measured on Qwen-2.5_1.5B_Instruct 1.9 (pack_import.h, above read_block32_head). This writes it from +// that description -- NOT through the reader -- so a reader that disagrees with the description fails here: 64-row +// units of 2,560 zero bytes + 2*(K/32) strips of 640 B; strip s is K block s/2, rows 32*(s%2)..+31; code position +// `(r/8)*256 + c*8 + r%8`, low nibble first; 16 B of zero-point nibbles (8); 48 zero bytes; 32 signed bf16 scales, +// `-m/8` with m the block's largest-magnitude weight. +static std::string encode_block32_units(const std::vector& W, int rows, int K) { + const int kb = K / 32, units = rows / 64; + const size_t unit = size_t(2560 + 2 * kb * 640); + std::string out(size_t(units) * unit, '\0'); + for (int u = 0; u < units; ++u) + for (int s = 0; s < 2 * kb; ++s) { + char* sp = out.data() + size_t(u) * unit + 2560 + size_t(s) * 640; + for (int r = 0; r < 32; ++r) { + const float* w = W.data() + size_t(u * 64 + 32 * (s % 2) + r) * size_t(K) + size_t((s / 2) * 32); + float m = 0; + for (int c = 0; c < 32; ++c) + if (std::fabs(w[c]) > std::fabs(m)) m = w[c]; + const uint16_t bits = pi::bf16_bits_of(-m / 8.0f); + const uint32_t wide = uint32_t(bits) << 16; + float sc; + std::memcpy(&sc, &wide, 4); + sp[576 + 2 * r] = char(bits & 0xFF); + sp[577 + 2 * r] = char(bits >> 8); + sp[512 + r / 2] = char(uint8_t(sp[512 + r / 2]) | uint8_t(8 << (4 * (r & 1)))); + for (int c = 0; c < 32; ++c) { + const int code = sc == 0 ? 8 : std::min(15, std::max(0, int(std::lround(w[c] / sc)) + 8)); + const int pos = (r / 8) * 256 + c * 8 + r % 8; + sp[pos / 2] = char(uint8_t(sp[pos / 2]) | uint8_t(code << (4 * (pos & 1)))); + } + } + } + return out; +} + +void test_block32_tail_head(const std::string& root) { + const int H = 128, L = 8, V = 64; + const int R = onebit::npu::q4nx::kRows; // the rows of container tile 0, which is what the arms decode + std::vector emb(size_t(V) * H), other(size_t(V) * H); + for (int r = 0; r < V; ++r) + for (int c = 0; c < H; ++c) { + emb[size_t(r) * H + c] = 0.05f * float((r * 7 + c * 13) % 97) - 2.4f; + other[size_t(r) * H + c] = 0.04f * float((r * 31 + c * 5 + (r * c) % 11) % 89) - 1.7f; // an UNRELATED head + } + // Pearson over the first `a.size()` values of both + const auto ncorr = [](const std::vector& a, const std::vector& b) { + double num = 0, da = 0, db = 0, sa = 0, sb = 0; + const double n = double(a.size()); + for (size_t i = 0; i < a.size(); ++i) { + num += double(a[i]) * b[i], da += double(a[i]) * a[i], db += double(b[i]) * b[i]; + sa += a[i], sb += b[i]; + } + const double den = (n * da - sa * sa) * (n * db - sb * sb); + return den > 0 ? (n * num - sa * sb) / std::sqrt(den) : 0.0; + }; + // The fixture's own premise, asserted: the two matrices must not resemble each other, or "reads the tail" and + // "falls back to the embedding" could not be told apart by a correlation. + check(std::fabs(ncorr(emb, other)) < 0.3, + "block-32 tail: the fixture's two heads are unrelated (corr " + std::to_string(ncorr(emb, other)) + ")"); + + const auto build = [&](bool tied, const char* tag, const std::string& tail_after_env) { + const fs::path dir = fs::path(root) / tag; + fs::remove_all(dir); + fs::create_directories(dir / "cache"); + const int64_t stride = 1 << 20, second = 262144; + std::string fc(size_t(int64_t(L) * stride + 64 + size_t(H) * 2), '\0'); + auto put_eps = [&](int64_t off) { + fc[size_t(off)] = char(0x86); + fc[size_t(off) + 1] = char(0x35); + }; + for (int l = 0; l < L; ++l) { + put_eps(int64_t(l) * stride); + put_eps(int64_t(l) * stride + second); + } + put_eps(int64_t(L) * stride); + for (int i = 0; i < H; ++i) { + const uint16_t bits = pi::bf16_bits_of(float(i + 1)); + fc[size_t(int64_t(L) * stride + 64 + 2 * i)] = char(bits & 0xFF); + fc[size_t(int64_t(L) * stride + 64 + 2 * i) + 1] = char(bits >> 8); + } + fc += tail_after_env; + put_file((dir / "dd_metastate_Llm_Token_Token_crafted.fconst").string(), fc); + put_file((dir / "config.json").string(), + std::string(R"({"model_type":"qwen2","hidden_size":128,"intermediate_size":256,"num_hidden_layers":8,)" + R"("num_attention_heads":2,"num_key_value_heads":1,"head_dim":64,"vocab_size":64,)" + R"("rope_theta":10000.0,"tie_word_embeddings":)") + + (tied ? "true" : "false") + R"(,"rms_norm_eps":1e-06})"); + std::string data(size_t(V) * size_t(H) * 2, '\0'); + for (size_t i = 0; i < emb.size(); ++i) { + const uint16_t bits = pi::bf16_bits_of(emb[i]); + data[2 * i] = char(bits & 0xFF); + data[2 * i + 1] = char(bits >> 8); + } + put_file((dir / "model.onnx.data").string(), data); + // the declaration that sets `p.head_block` (see test_tied_block32_head for why it lives in cache/) + put_file((dir / "cache" / "lm_head_meta.json").string(), + R"({"op_list":[{"type":"MladfMatMul","name":"lm_head.MatMulNBits",)" + R"("attrs":{"N":{"value":[64]},"K":{"value":[128]},"block_size":{"value":[32]},"bits":{"value":[4]}}}],)" + R"("tensor_map":{}})"); + put_file((dir / "model.onnx").string(), + pb::model({pb::tensor("model.embed_tokens.weight", {V, H}, "model.onnx.data", 0, + uint64_t(V) * uint64_t(H) * 2, 16), // 16 = BFLOAT16: the bytes ARE bf16 + pb::placeholder("lm_head.weight.MatMulNBits.qweight"), + pb::placeholder("lm_head.weight.MatMulNBits.scales"), + pb::placeholder("lm_head.weight.MatMulNBits.zeros")}, + {})); + return dir; + }; + struct Run { + pi::Report r; + std::string said, note; + bool has_head = false; + std::vector head; // the container's lm_head tile 0, decoded from its own bytes: [kRows][H] + }; + const auto run = [&](const fs::path& dir, const char* tag) { + Run x; + const std::string out = + (fs::temp_directory_path() / + (std::string("oga_pack_import_b32tail_") + tag + "." + std::to_string(::getpid()))).string(); + fs::remove_all(out); + pi::Options allow; + allow.allow_incomplete = true; // the fixture's layer payload is empty; the head gate is what is under test + try { + x.r = pi::import_pack(dir.string(), out, allow); + } catch (const std::exception& e) { + x.said = e.what(); + } + for (const auto& w : x.r.refused) + if (w.rfind("lm_head.weight", 0) == 0) x.said += " | " + w; + if (fs::exists(out + "/model.q4nx")) { + const json tbl = container_table(out + "/model.q4nx"); + x.has_head = tbl.contains("lm_head.weight"); + if (x.has_head) { + const std::vector blob = container_payload(out + "/model.q4nx"); + const auto offs = tbl["lm_head.weight"]["data_offsets"].get>(); + std::vector w(size_t(R) * onebit::npu::q4nx::kCols, 0.f); + onebit::npu::q4nx::dequant_tile(onebit::npu::q4nx::Chunk::Q4_1F, blob.data() + offs[0], w.data(), + onebit::npu::q4nx::kCols); + x.head.assign(size_t(R) * H, 0.f); + for (int r = 0; r < R; ++r) + for (int c = 0; c < H; ++c) x.head[size_t(r) * H + c] = w[size_t(r) * onebit::npu::q4nx::kCols + c]; + } + std::ifstream cf(out + "/config.json"); + if (cf.good()) { + const json cfg = json::parse(cf); + if (cfg.contains("q4nx_head_note")) x.note = cfg["q4nx_head_note"].get(); + } + } + fs::remove_all(out); + fs::remove_all(dir); + return x; + }; + + { // arm 1: UNTIED, the tail holds a head that is NOT the embedding, then padding that is not a whole unit + const Run x = run(build(false, "b32-untied-read", encode_block32_units(other, V, H) + std::string(10000, '\0')), + "untied"); + check(x.said.empty(), + "block-32 tail: an untied pack whose tail holds the head imports it (" + x.said.substr(0, 160) + ")"); + check(x.has_head && !x.head.empty(), "block-32 tail: the container carries the head"); + if (!x.head.empty()) { + const double c_tail = ncorr(x.head, other), c_emb = ncorr(x.head, emb); + check(c_tail > 0.99, + "block-32 tail: the container's head is the matrix the TAIL holds (corr " + std::to_string(c_tail) + ")"); + check(std::fabs(c_emb) < 0.3, "block-32 tail: and it is not the embedding (corr " + std::to_string(c_emb) + ")"); + } + check(x.r.head_synthesized_note.empty() && x.r.head_read_note.find("block 32") != std::string::npos && + x.r.head_read_note.find("untied") != std::string::npos, + "block-32 tail: the report says the head was read, and that an untied one is not verified (" + + x.r.head_read_note + ")"); + check(x.note == x.r.head_read_note && !x.note.empty(), "block-32 tail: config.json carries the same note"); + } + { // arm 2: TIED, the tail holds the embedding's own quantisation -> read (not synthesised), verified + const Run x = run(build(true, "b32-tied-read", encode_block32_units(emb, V, H)), "tied"); + check(x.said.empty(), "block-32 tail, tied: imports (" + x.said.substr(0, 160) + ")"); + check(x.r.head_synthesized_note.empty() && x.r.head_read_note.find("reproduces") != std::string::npos, + "block-32 tail, tied: the head is READ and the note says it reproduces the embedding (" + + x.r.head_read_note + ")"); + check(x.r.selfcheck_tied && x.r.lmhead_corr > 0.99, + "block-32 tail, tied: the self-check measures the read head (corr " + std::to_string(x.r.lmhead_corr) + ")"); + check(!x.head.empty() && ncorr(x.head, emb) > 0.99, + "block-32 tail, tied: the container's head is the embedding (corr " + + std::to_string(x.head.empty() ? 0.0 : ncorr(x.head, emb)) + ")"); + } + { // arm 3: TIED, but the tail's head is some other matrix -> the read is NOT taken, the synthesis stands, says why + const Run x = run(build(true, "b32-tied-mismatch", encode_block32_units(other, V, H)), "mismatch"); + check(x.r.head_read_note.empty() && x.r.head_synthesized_note.find("correlates") != std::string::npos, + "block-32 tail, tied mismatch: a tail head that is not the embedding is not taken (" + + x.r.head_synthesized_note.substr(0, 260) + ")"); + const double c_emb = x.head.empty() ? 0.0 : ncorr(x.head, emb), c_oth = x.head.empty() ? 1.0 : ncorr(x.head, other); + check(c_emb > 0.99 && std::fabs(c_oth) < 0.3, + "block-32 tail, tied mismatch: the container's head is the embedding's synthesis, not the tail's matrix " + "(corr " + std::to_string(c_emb) + " / " + std::to_string(c_oth) + ")"); + } + { // arm 4: UNTIED, a non-zero byte after the units -> refused by name with the reason, nothing written + std::string tail = encode_block32_units(other, V, H) + std::string(4096, '\0'); + tail[tail.size() - 100] = 1; + const Run x = run(build(false, "b32-untied-junk", tail), "junk"); + check(!x.has_head, "block-32 tail, junk after the head: no head is written"); + check(x.said.find("block 32") != std::string::npos && x.said.find("does not declare tied") != std::string::npos && + x.said.find("not zero after") != std::string::npos, + "block-32 tail, junk after the head: the refusal names the block, the missing tie and the reason (" + + x.said.substr(0, 300) + ")"); + } + { // arm 5: known answers on the reader itself, typed byte by byte (K 64: one unit is 2,560 + 4 x 640 = 5,120 B) + std::vector u(5120, 0); + uint8_t* sp = u.data() + 2560 + 3 * 640; // strip 3: K block 1, rows 32..63 + const auto put_scale = [&](int r, float v) { + const uint16_t bits = pi::bf16_bits_of(v); + sp[576 + 2 * r] = uint8_t(bits & 0xFF); + sp[577 + 2 * r] = uint8_t(bits >> 8); + }; + put_scale(9, 0.5f); + put_scale(10, -0.25f); + sp[512 + 4] = 0x80; // row 9's zero point (high nibble of byte 4) = 8 + sp[512 + 5] = 0x06; // row 10's zero point (low nibble of byte 5) = 6 + sp[(256 + 5 * 8 + 1) / 2] |= 0xB0; // row 9 (band 1, r%8 1), column 5: position 297, high nibble + sp[(256 + 31 * 8 + 2) / 2] |= 0x03; // row 10 (band 1, r%8 2), column 31: position 506, low nibble + pi::Block32Head h = pi::read_block32_head(u.data(), int64_t(u.size()), 64, 64); + check(h.ok && h.units == 1 && h.q.N == 64 && h.q.K == 64 && h.q.block == 32, + "block-32 reader: one unit of K 64 reads (" + h.why + ")"); + if (h.ok) { + const auto val = [&](int row, int col) { + const size_t g = size_t(row) * 2 + size_t(col / 32); + return double(h.q.codes[size_t(row) * 64 + col]) * h.q.scale[g] + h.q.zero[g]; + }; + check(val(41, 37) == 1.5, + "block-32 reader: (row 41, col 37) = (11 - 8) x 0.5 (" + std::to_string(val(41, 37)) + ")"); + check(val(41, 36) == -4.0, + "block-32 reader: its neighbour is code 0, (0 - 8) x 0.5 (" + std::to_string(val(41, 36)) + ")"); + check(val(42, 63) == 0.75, + "block-32 reader: a NEGATIVE scale, (3 - 6) x -0.25 (" + std::to_string(val(42, 63)) + ")"); + check(val(42, 32) == 1.5, + "block-32 reader: and its code 0, (0 - 6) x -0.25 (" + std::to_string(val(42, 32)) + ")"); + check(h.q.scale[size_t(42) * 2 + 1] == 0.25f, "block-32 reader: the stored scale is positive"); + check(val(41, 5) == 0.0 && val(0, 0) == 0.0, "block-32 reader: a block with a zero scale decodes to zero"); + } + check(!pi::read_block32_head(u.data(), 5119, 64, 64).ok, "block-32 reader: a tail one byte short is refused"); + std::vector two = u; + two.resize(5120 + 300, 0); + check(pi::read_block32_head(two.data(), int64_t(two.size()), 64, 64).padding == 300, + "block-32 reader: zeros after the last unit are padding, and counted"); + two.back() = 7; + check(!pi::read_block32_head(two.data(), int64_t(two.size()), 64, 64).ok, + "block-32 reader: a non-zero byte after the last unit is refused"); + std::vector hdr = u; + hdr[100] = 1; + check(!pi::read_block32_head(hdr.data(), int64_t(hdr.size()), 64, 64).ok, + "block-32 reader: a non-zero unit header is refused"); + std::vector gap = u; + gap[2560 + 640 + 540] = 1; + check(!pi::read_block32_head(gap.data(), int64_t(gap.size()), 64, 64).ok, + "block-32 reader: a non-zero byte between the zero points and the scales is refused"); + check(!pi::read_block32_head(u.data(), int64_t(u.size()), 64, 48).ok, "block-32 reader: K 48 is refused"); + } +} + // ── the final norm out of the tail envelope, on a crafted pack (engine#844) ────────────────── // The real-pack leg only covers this where a pack is supplied, and three of the four packs on the // box ship no prefill program, so the import's tail-envelope read had no coverage in the default @@ -4324,6 +4580,7 @@ int main(int argc, char** argv) { test_qwen3_attn_sections(); test_post_attn_norm_header(golden_dir.empty() ? "." : golden_dir); test_tied_block32_head(golden_dir.empty() ? "." : golden_dir); + test_block32_tail_head(golden_dir.empty() ? "." : golden_dir); test_final_norm_envelope(golden_dir.empty() ? "." : golden_dir); test_import_fp32_params(golden_dir.empty() ? "." : golden_dir); test_text_artifacts(golden_dir.empty() ? "." : golden_dir);