From 2b7eccc9aaa95749f01a6641d84e01102330e404 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 13:06:27 -0400 Subject: [PATCH] npu-pack: decode the external-data genre's MatMulNBitsBf blobs as AMD's packer writes them (Fixes #2561) phi-4 (1.9.0) imported with 0 gaps and answered 'void void ...' on every prompt. Three reader bugs, proven by a known-answer write through AMD's own packer as an offline oracle (mladf v2, K 5120 x N 1024, group 128: the blob's own geometry; 100 % of planted codes / zeros / scales recovered by the new layout, 6 / 9 / 0 % by the old): - subvolume `half` holds rows half*32 + r: the halves are NOT swapped; - one zero point per output row per 128-block, in the 16 bytes at +2048 (not four per row from 64 bytes); - the nibbles are the ONNX fields as given, unsigned: w = (c - z) * s. Plus rope_theta from the pack's cos_cache when config.json is empty (phi-4: 250000, corr 1.0000; default 10000). The pseudo-layer final norm landed separately in #2564. phi-4 layer-0 qkv now matches HF x the AWQ fold ratio at corr +0.992 (was 0.00), and the model answers 'Paris.', 'hydrogen ...', 'Neil Armstrong.' on --device npu. Co-Authored-By: Claude Opus 5.5 --- docs/npu.md | 18 ++++ npu/pack_import.cpp | 93 ++++++++++++++----- npu/pack_import.h | 4 + tests/npu_pack_import_test.cpp | 165 ++++++++++++++++----------------- 4 files changed, 174 insertions(+), 106 deletions(-) diff --git a/docs/npu.md b/docs/npu.md index 5b4ad0947..029f7ed5a 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1414,6 +1414,24 @@ stride the `.fconst` attention tile already carries (2,176 B subvolume = 32 rows from the node's own `N`/`K` (`gate_proj`/`up_proj` → N, `down_proj` → K), because the weight is a 1-D byte blob with no shape to read it from (engine#1503). +**How the `.packed` blob decodes (engine#2561).** It is the same 2,176-byte subvolume layout as the +[`.fconst` attention payload](#the-fconst-attention-payload), with the same rules: + +- **Codes:** the w8 interleave. +- **Zero points:** one per output row in the 16 bytes at `+2048`. +- **Scales:** 32 bf16 values at `+2112`. +- **Row order:** subvolume `2 + 2*kblock + half` holds rows `half*32 + r` of the 64-row tile. The halves are **not** swapped. +- **Values:** the nibbles are the ONNX MatMulNBits fields as given, read **unsigned**: `w = (c - z) * s`. + +This was established offline with a known-answer write. AMD's packer (`matmulnbits_pack_const_float32`, mladf v2, K 5120 x N 1024, group 128) was run on chosen weights only as an oracle; the engine neither calls nor ships it. Its output has the blob's own geometry (2,854,912 B = 16 tiles x 82 x 2,176). This layout recovers 100 % of the planted codes, zero points and scales. The earlier reading recovered 6 %, 9 % and 0 %, because it swapped the halves, read four zero points per row from 64 bytes, and sign-extended. That earlier reading had been checked only against this importer's own container. + +On `phi-4` the corrected decode reproduces HF `microsoft/phi-4` layer-0 `qkv_proj` times the pack's AWQ fold ratio (`ln_HF / ln_pack`) at corr **+0.992**, with the same row norm (14.79 vs 14.89). The old decode gave corr 0.00 and rows 49x too large, and the model answered `void void ...` on every prompt. With the fix it answers "Paris.", "hydrogen ..." and "Neil Armstrong." on `--device npu`. + +Two more facts this genre needs: + +- **Final norm:** the graph hangs it on a pseudo layer one past the last (`model.layers.40.final_norm_layernorm.weight`, equal to HF `model.norm.weight`). The importer writes it as `model.norm.weight`, not as a unit gain. +- **RoPE base:** `config.json` can be `{}`, so the base comes from the pack's own `cos_cache` initializer (`rope_cache_best_theta`). On phi-4 that adopts 250000 at corr 1.0000, where the default was 10000. + **The 1.8 hybrid line (`*_rai_1.8.0_hybrid`) is not importable, and that is a decision, not a gap still to be filled (engine#1503).** These packs pair a `model.onnx` with a `model.bin`: 1.3 GB on `Llama-3.2-1B-Instruct_rai_1.8.0_hybrid`, alongside a 542 MB `model.onnx.data` that holds only the embedding and the norms. In the graph, every `.MatMulNBits.*` initializer is a `[0]` placeholder with no bytes, and so is its `.packed` sibling (162 of 162 on that pack). The weights exist only in `model.bin`, a compiled container that the vendor's runtime reads. Writing a reader for it means decoding that container, which this engine does not do, so `npu-pack info` and `npu-pack import` refuse these packs by name before reading anything: ``` diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index c60ff711e..e479de578 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -2391,30 +2391,28 @@ Q4 decode_external_tile(const uint8_t* data, size_t bytes, int N, int K, std::st const uint8_t* zeros = subv + 2048; const uint8_t* scales = subv + 2112; for (int r = 0; r < kExtSubvRows; ++r) { - // MEASURED, not assumed: a 64-row block's two subvolume halves are stored in the order (half 1, - // half 0) -- the block's FIRST 32 rows live in the pair's SECOND subvolume. Checked row for row against - // the imported container: with `half * 32 + r` exactly one row in 64 matched (R mod 64 == 0), and with the - // swap the sampled rows match (engine#1503). - const int64_t row = tile * kExtTileRows + (1 - half) * kExtSubvRows + r; + // engine#2561, proven by a known-answer write through AMD's own packer (matmulnbits_pack_const_float32, + // mladf v2, K 5120 x N 1024, group 128: 2,854,912 B = 16 tiles x 82 subvolumes x 2,176 B, the eager + // blob's own geometry): this layout recovers 100 % of the planted codes, zero points and scales. + // Subvolume `2 + 2*kblock + half` holds output rows `half*32 + r` of the 64-row tile -- NOT swapped (the + // earlier swap was measured against this importer's own container, so it agreed with itself). + const int64_t row = tile * kExtTileRows + half * kExtSubvRows + r; const float s = bf16_to_f32(uint16_t(scales[r * 2] | (scales[r * 2 + 1] << 8))); - // four int4 zero points per row, one per 32-column group of the 128-column block - const uint8_t z0 = zeros[r * 2], z1 = zeros[r * 2 + 1]; - // Both fields are int4 in TWO'S COMPLEMENT, not 0..15 (the pack's own writer asserts -8 <= x <= 7, - // FACTS-oga-pack-weight-layout.md §7.13). Reading them unsigned is only equivalent when the code and - // its zero point happen to share a sign: for one negative and one positive the difference is off by - // 16 * scale, which is ~16 codes wide and dominates the weight. That is what left phi-4 answering - // '都都都都': every row where the two differed in sign became a near-constant matrix (engine#1503). - const int zp[4] = {int8_t(z0 << 4) >> 4, int8_t(z0) >> 4, int8_t(z1 << 4) >> 4, int8_t(z1) >> 4}; + // ONE zero point per output row per 128-column block: 16 bytes at +2048, nibble r (low nibble = even r). + // Reading 64 bytes as four zero points per row recovered 9 % of them. + const uint8_t zb = zeros[r / 2]; + const int z = (r % 2 == 0) ? int(zb & 0xF) : int(zb >> 4); for (int g = 0; g < 4; ++g) { q.scale[size_t(row) * (K / 32) + (kk - 1) * 4 + g] = s; - // The container's form is w = code*d + m with code in 0..15 and m the group MIN, so the signed - // code c becomes c + 8 and m becomes (-8 - z) * scale: (c+8)*s + (-8-z)*s == (c - z)*s exactly. - q.zero[size_t(row) * (K / 32) + (kk - 1) * 4 + g] = float(-8 - zp[g]) * s; + // The fields are the ONNX MatMulNBits nibbles as given, UNSIGNED: w = (c - z) * s. In the container's + // form w = code*d + m that is code = c, d = s, m = -z * s. (The packer stores the nibbles it is + // handed unchanged; the earlier two's-complement reading was a correction for the other two bugs.) + q.zero[size_t(row) * (K / 32) + (kk - 1) * 4 + g] = -float(z) * s; } for (int c = 0; c < 128; ++c) { - // Row-major: a row's 128 codes are 64 contiguous bytes, even column in the low nibble. - // The .fconst formula: one byte per COLUMN holds the row PAIR 2cp (low) / 2cp+1 (high), so a - // row's codes are strided by 4 through a 512-byte block -- no column pairing at all. + // w8 interleave (the packer's wh_w8_index): one byte per K column holds the output-row pair + // 2cp (low nibble) / 2cp+1 (high), strided by 4 through a 512-byte block. Row-major is kept only + // as the alternative the unit test holds to its own spec. uint8_t nib; if (row_major) { const uint8_t b = quants[r * 64 + c / 2]; @@ -2424,10 +2422,7 @@ Q4 decode_external_tile(const uint8_t* data, size_t bytes, int N, int K, std::st const uint8_t b = quants[(cp % 4) + c * 4 + (cp / 4) * 512]; nib = (r % 2 == 0) ? uint8_t(b & 0xF) : uint8_t(b >> 4); } - // signed int4 -> the container's 0..15 offset from the group minimum: c_signed = nib-16 for a - // negative nibble, then +8. A no-op would have been `nib`, which is the bug this replaces. - const int cs = (nib >= 8) ? int(nib) - 16 : int(nib); - q.codes[size_t(row) * K + (kk - 1) * 128 + c] = uint8_t(cs + 8); + q.codes[size_t(row) * K + (kk - 1) * 128 + c] = nib; } } } @@ -5063,6 +5058,32 @@ std::string kernel_set_refusal(const std::vector& file_names, const // Copies the tokenizer/chat files the container needs beside model.q4nx; defined past kTextFiles' declaration. static int copy_text_artifacts(const std::string& pack_dir, const std::string& out_dir); +double rope_cache_best_theta(const std::vector& cache, int64_t rows, int64_t half, + const std::vector& cands, double* best_theta) { + // ONE Pearson r of the cache against cos(p * theta^(-2i/d)) over the whole matrix (the per-column mean is noise + // for the flat high columns; see the .fconst path's note on Qwen3-1.7B). Invariant to a per-pack cache scale. + double best = -2.0; + if (best_theta) *best_theta = 0.0; + const int64_t pmax = std::min(rows, 4096); + const double d = double(half) * 2.0; + for (const double th : cands) { + if (!(th > 0)) continue; + double sx = 0, sy = 0, sxx = 0, syy = 0, sxy = 0, n = 0; + for (int64_t i = 0; i < half; ++i) { + const double inv = 1.0 / std::pow(th, 2.0 * double(i) / d); + for (int64_t p = 0; p < pmax; ++p) { + const double y = cache[size_t(p) * size_t(half) + size_t(i)], x = std::cos(double(p) * inv); + sx += x; sy += y; sxx += x * x; syy += y * y; sxy += x * y; n += 1; + } + } + const double cx = sxx - sx * sx / n, cy = syy - sy * sy / n; + if (cx <= 1e-12 || cy <= 1e-12) continue; + const double r = (sxy - sx * sy / n) / std::sqrt(cx * cy); + if (r == r && r > best) { best = r; if (best_theta) *best_theta = th; } + } + return best; +} + Report import_external(const std::string& pack_dir, const std::string& out_dir, const Options& opt) { const Plan p = plan(pack_dir); const std::string graph = graph_filename(pack_dir); @@ -5260,6 +5281,32 @@ Report import_external(const std::string& pack_dir, const std::string& out_dir, c["head_dim"] = p.dims.head_dim; c["vocab_size"] = p.dims.vocab; c["rope_theta"] = p.dims.rope_theta; + // engine#2561: this genre's config.json can be `{}` (phi-4), so rope_theta above is a default (10000); the graph's + // GroupQueryAttention reads a precomputed `cos_cache` initializer, which is the pack's own statement of its base. + if (onnx.index.count("cos_cache")) { + const OnnxTensor& cc = onnx.tensors[onnx.index.at("cos_cache")]; + if (cc.dims.size() == 2 && cc.dims[0] > 1 && cc.dims[1] > 0 && cc.type == kF32) { + const int64_t rows = cc.dims[0], half = cc.dims[1]; + uint64_t need = 0; + if (mul_ok(uint64_t(rows) * uint64_t(half), 4u, need)) { + std::vector cache(size_t(rows) * size_t(half)); + std::memcpy(cache.data(), bytes_of(cc, need), size_t(need)); + const std::vector cands{double(p.dims.rope_theta), 1e4, 5e4, 1e5, 2.5e5, 5e5, 1e6, 5e6, 1e7}; + double best_theta = 0.0; + const double best_corr = rope_cache_best_theta(cache, rows, half, cands, &best_theta); + if (best_corr > 0.99 && best_theta > 0.0) { + if (std::fabs(best_theta - double(p.dims.rope_theta)) > 1e-6) + std::fprintf(stderr, "note: rope_theta %.6g declared, %.6g adopted (correlation %.4f against the " + "pack's own cos_cache)\n", double(p.dims.rope_theta), best_theta, best_corr); + c["rope_theta"] = best_theta; + } else { + std::fprintf(stderr, "WARNING: no candidate rope_theta explains the pack's cos_cache (best %.4f at %.6g); " + "keeping %.6g, attention may be wrong\n", best_corr, best_theta, + double(p.dims.rope_theta)); + } + } + } + } c["q4nx_parameters"] = "fp32 scale/zero per 32-column group"; if (!opt.weight_layout.empty() && opt.weight_layout != "rm") c["q4nx_weight_layout"] = opt.weight_layout; c["q4nx_source"] = {{"kind", "oga-npu-pack external data"}, {"pack", p.pack}, diff --git a/npu/pack_import.h b/npu/pack_import.h index e320f566d..0e98b5680 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -893,6 +893,10 @@ bool external_container_tensors(const std::string& role, int nh, int nkv, int hd Q4 decode_external_tile(const uint8_t* data, size_t bytes, int N, int K, std::string& err, bool row_major = kExtQuantsRowMajor); +// engine#2561: the candidate theta whose cos(p * theta^(-2i/d)) best explains a pack's precomputed cos_cache +// ([rows, half] row-major, d = 2*half), as one Pearson r over the whole matrix; writes the winner to *best_theta. +double rope_cache_best_theta(const std::vector& cache, int64_t rows, int64_t half, + const std::vector& cands, double* best_theta); Q4 decode_matmulnbits(const uint8_t* qweight, const float* scales, const uint8_t* zeros, int N, int K, int block, bool swap_nibbles = false, bool drop_zero = false); diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 0b33f1342..ce384adb8 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -252,98 +252,96 @@ int q4_1f_tensors(const json& table) { // references. A reader that follows node inputs only reports a weightless graph for a 9.7 GB pack; one that sees the // sibling must also say whether the bytes fit the stride the .fconst tile already carries -- // length == N x (K/128 + 1) x 68 -- as a count, not an assumption. -// The external-data tile's own decode (engine#1503): a synthetic blob laid out the way the pack lays it out must come -// back exactly, under BOTH code orderings -- the decoder's constant picks one, and the test holds each to the spec it -// claims, so flipping the constant cannot silently change what the other branch means. +// The external-data tile's own decode (engine#1503, corrected by engine#2561): a synthetic blob laid out the way AMD's +// own packer lays it out must come back exactly. The layout was proven by a known-answer write through +// `matmulnbits_pack_const_float32` (mladf v2, K 5120 x N 1024, group 128): 100 % of planted codes, zero points and +// scales recovered with THIS layout, 6 % / 9 % / 0 % with the previous one (rows swapped, four zeros per row, signed). +// Both code orderings are still held to their own spec so flipping the constant cannot silently change either. void test_external_tile_decode() { const int N = 128, K = 256; // 2 tiles of 64 rows, 2 K-blocks -> 3 subvolume pairs const int64_t tiles = N / 64, per_tile = 2 * (K / 128 + 1); const int64_t bytes = tiles * per_tile * 2176; - // A code matrix and per-block scale/zero-points to encode, then to find again. std::vector codes(size_t(N) * K); for (size_t i = 0; i < codes.size(); ++i) codes[i] = uint8_t((i * 7 + i / 13) % 16); const auto bf16_bits = [](float v) { unsigned int u; std::memcpy(&u, &v, 4); - return uint16_t((u + 0x8000u + ((u >> 16) & 1u)) >> 16); // round to nearest even, as the importer does + return uint16_t((u + 0x8000u + ((u >> 16) & 1u)) >> 16); }; - for (const bool row_major : {false, true}) { - // DISTINCT scale and zero points per (tile, half, kk, r): a fixture that writes one value everywhere lets a decoder - // that reads row 0's scale for all 32 rows pass, which is exactly the addressing this test exists to pin. - const auto scale_for = [&](int64_t tile, int half, int64_t kk, int r) { - return 0.001953125f * float(1 + int(tile) * 7 + half * 5 + int(kk) * 3 + (r % 4)); // bf16-exact multiples of 2^-9 - }; - const auto zero_for = [&](int64_t tile, int half, int64_t kk, int r, int g) { - return uint8_t((int(tile) * 3 + half * 5 + int(kk) * 7 + r + g) % 16); + // DISTINCT scale and zero point per (row, k-block): a fixture with one value everywhere lets a decoder that reads + // the wrong row's parameters pass, which is exactly what the old row swap did. + const auto scale_for = [&](int64_t row, int64_t kk) { + return 0.001953125f * float(1 + int(row % 11) * 3 + int(kk) * 5); // bf16-exact multiples of 2^-9 }; - std::vector blob(size_t(bytes), 0); - for (int64_t tile = 0; tile < tiles; ++tile) - for (int half = 0; half < 2; ++half) - for (int64_t kk = 1; kk <= K / 128; ++kk) { - uint8_t* subv = blob.data() + (tile * per_tile + kk * 2 + half) * 2176; - for (int r = 0; r < 32; ++r) { - // The decoder reads the block's FIRST 32 rows from the pair's SECOND subvolume, so the fixture encodes - // its rows in that order; without this the test pins a mapping the decoder does not use (engine#1503). - const int64_t row = tile * 64 + (1 - half) * 32 + r; - const uint16_t sc = bf16_bits(scale_for(tile, half, kk, r)); - subv[2112 + r * 2] = uint8_t(sc & 0xFF); - subv[2113 + r * 2] = uint8_t(sc >> 8); - subv[2048 + r * 2] = uint8_t(zero_for(tile, half, kk, r, 0) | (zero_for(tile, half, kk, r, 1) << 4)); - subv[2049 + r * 2] = uint8_t(zero_for(tile, half, kk, r, 2) | (zero_for(tile, half, kk, r, 3) << 4)); - for (int c = 0; c < 128; ++c) { - const uint8_t nib = codes[size_t(row) * K + (kk - 1) * 128 + c]; - if (row_major) { - if (c % 2 == 0) subv[r * 64 + c / 2] = uint8_t((subv[r * 64 + c / 2] & 0xF0) | nib); - else subv[r * 64 + c / 2] = uint8_t((subv[r * 64 + c / 2] & 0x0F) | (nib << 4)); - } else { - const int cp = r / 2; // one byte per column, holding the row pair 2cp / 2cp+1 - uint8_t& b = subv[(cp % 4) + c * 4 + (cp / 4) * 512]; - if (r % 2 == 0) b = uint8_t((b & 0xF0) | nib); - else b = uint8_t((b & 0x0F) | (nib << 4)); + const auto zero_for = [&](int64_t row, int64_t kk) { return uint8_t((int(row) * 3 + int(kk) * 7) % 16); }; + for (const bool row_major : {false, true}) { + std::vector blob(size_t(bytes), 0); + for (int64_t tile = 0; tile < tiles; ++tile) + for (int half = 0; half < 2; ++half) + for (int64_t kk = 1; kk <= K / 128; ++kk) { + uint8_t* subv = blob.data() + (tile * per_tile + kk * 2 + half) * 2176; + for (int r = 0; r < 32; ++r) { + const int64_t row = tile * 64 + half * 32 + r; // NOT swapped + const uint16_t sc = bf16_bits(scale_for(row, kk)); + subv[2112 + r * 2] = uint8_t(sc & 0xFF); + subv[2113 + r * 2] = uint8_t(sc >> 8); + uint8_t& zb = subv[2048 + r / 2]; // one zero point per row, 16 bytes + zb = (r % 2 == 0) ? uint8_t((zb & 0xF0) | zero_for(row, kk)) : uint8_t((zb & 0x0F) | (zero_for(row, kk) << 4)); + for (int c = 0; c < 128; ++c) { + const uint8_t nib = codes[size_t(row) * K + (kk - 1) * 128 + c]; + if (row_major) { + if (c % 2 == 0) subv[r * 64 + c / 2] = uint8_t((subv[r * 64 + c / 2] & 0xF0) | nib); + else subv[r * 64 + c / 2] = uint8_t((subv[r * 64 + c / 2] & 0x0F) | (nib << 4)); + } else { + const int cp = r / 2; // one byte per K column, holding the row pair 2cp / 2cp+1 + uint8_t& b = subv[(cp % 4) + c * 4 + (cp / 4) * 512]; + if (r % 2 == 0) b = uint8_t((b & 0xF0) | nib); + else b = uint8_t((b & 0x0F) | (nib << 4)); + } } } } - } - std::string err; - const pi::Q4 q = pi::decode_external_tile(blob.data(), blob.size(), N, K, err, row_major); - check(err.empty() && q.N == N && q.K == K && q.block == 32, - "external tile: a blob of the measured size decodes (" + err + ")"); - bool codes_ok = q.codes.size() == codes.size(), scale_ok = true, zero_ok = true; - // The pack's codes are int4 TWO'S COMPLEMENT; the container's form is the offset from the group minimum, i.e. - // c_signed + 8. A test that expected the raw nibble passed on the unsigned bug (engine#1503). - for (size_t i = 0; codes_ok && i < codes.size(); ++i) { - const int raw = codes[i]; - const int cs = (raw >= 8) ? raw - 16 : raw; - if (q.codes[i] != uint8_t(cs + 8)) codes_ok = false; - } - // Every row's own scale and all four of its zero points, so the addressing (row r*2 in the param regions, the - // group index (kk-1)*4+g, the half and tile choice) is pinned rather than the value of one row repeated. - for (int64_t tile = 0; tile < tiles; ++tile) - for (int half = 0; half < 2; ++half) + std::string err; + const pi::Q4 q = pi::decode_external_tile(blob.data(), blob.size(), N, K, err, row_major); + check(err.empty() && q.N == N && q.K == K && q.block == 32, + "external tile: a blob of the measured size decodes (" + err + ")"); + // The nibbles are the ONNX MatMulNBits codes as given (unsigned), and they ARE the container's codes. + check(q.codes == codes, std::string("external tile: every 4-bit code comes back unchanged, ") + + (row_major ? "row-major" : "w8 interleave")); + bool scale_ok = q.scale.size() == size_t(N) * (K / 32), zero_ok = q.zero.size() == q.scale.size(); + for (int64_t row = 0; scale_ok && zero_ok && row < N; ++row) for (int64_t kk = 1; kk <= K / 128; ++kk) - for (int r = 0; r < 32; ++r) { - // The decoder reads the block's FIRST 32 rows from the pair's SECOND subvolume, so the fixture encodes - // its rows in that order; without this the test pins a mapping the decoder does not use (engine#1503). - const int64_t row = tile * 64 + (1 - half) * 32 + r; - for (int g = 0; g < 4; ++g) { - const size_t at = size_t(row) * (K / 32) + (kk - 1) * 4 + g; - if (q.scale[at] != scale_for(tile, half, kk, r)) scale_ok = false; - // and the container's zero is the offset, m = (-8 - z) * scale, for a signed int4 z - const int zraw = zero_for(tile, half, kk, r, g); - const int zs = (zraw >= 8) ? zraw - 16 : zraw; - if (q.zero[at] != float(-8 - zs) * scale_for(tile, half, kk, r)) zero_ok = false; - } + for (int g = 0; g < 4; ++g) { + const size_t at = size_t(row) * (K / 32) + (kk - 1) * 4 + g; + if (q.scale[at] != scale_for(row, kk)) scale_ok = false; + // w = (c - z) * s == c*s + (-z*s): the container's offset is -z*s + if (q.zero[at] != -float(zero_for(row, kk)) * scale_for(row, kk)) zero_ok = false; } - check(codes_ok, std::string("external tile: every 4-bit code comes back exactly, ") + (row_major ? "row-major" : "w8 interleave")); - check(scale_ok, "external tile: every row keeps its own bf16 scale, repeated over its four groups"); - check(zero_ok, "external tile: every (row, group) keeps its own int4 zero point"); - std::string err2; - (void)pi::decode_external_tile(blob.data(), blob.size() - 2176, N, K, err2); - check(!err2.empty() && err2.find("is not") != std::string::npos, - "external tile: a blob one subvolume short is refused with the arithmetic (" + err2 + ")"); + check(scale_ok, "external tile: every row keeps its own bf16 scale per 128-block, over its four groups"); + check(zero_ok, "external tile: every row keeps its own zero point per 128-block, as the offset -z*s"); + std::string err2; + (void)pi::decode_external_tile(blob.data(), blob.size() - 2176, N, K, err2); + check(!err2.empty() && err2.find("is not") != std::string::npos, + "external tile: a blob one subvolume short is refused with the arithmetic (" + err2 + ")"); } } +// engine#2561: phi-4's config.json is `{}`, so the importer reads the RoPE base from the pack's own cos_cache. A cache +// built at theta 250000 (phi-4's) must pick 250000 out of the candidates, also when the cache is scaled. +void test_rope_cache_best_theta() { + const int64_t rows = 512, half = 64; + std::vector cache(size_t(rows) * size_t(half)); + for (int64_t p = 0; p < rows; ++p) + for (int64_t i = 0; i < half; ++i) + cache[size_t(p) * half + i] = 1.875f * float(std::cos(double(p) / std::pow(250000.0, 2.0 * double(i) / 128.0))); + double th = 0.0; + const double r = pi::rope_cache_best_theta(cache, rows, half, {10000.0, 1e5, 2.5e5, 5e5, 1e6}, &th); + check(th == 250000.0 && r > 0.999, "rope cache: theta 250000 is recovered from its own (scaled) cos_cache"); + double th2 = 0.0; + (void)pi::rope_cache_best_theta(cache, rows, half, {10000.0, 1e6}, &th2); + check(th2 != 250000.0, "rope cache: without the right candidate the fit does not invent it"); +} + // The conversion half of the writer (engine#1503): the pack's `(code - zp)*scale` and the container's // `code*d + m` are different conventions, and the mapping has to reproduce the pack's own numbers. The obvious // alternative -- dequantise and re-quantise with the container's quantiser -- was measured on a real phi-4 tile and @@ -666,15 +664,15 @@ void test_external_import_success(const std::string& root) { // The fixture's own weights: w(row, col) = (code - zp) * scale, chosen so every product is exact in f32 (scales are // bf16-exact with 7 mantissa bits, |code - zp| <= 15), so the container's `code*scale + m` form must reproduce them - // bit for bit. Both nibbles are int4 TWO'S COMPLEMENT (the pack's writer asserts -8 <= x <= 7), so the reference - // sign-extends them: the generators walk all 16 nibble values, and for a code and zero of opposite sign an - // unsigned reading is off by 16 * scale (engine#1503, #1874). + // bit for bit. The nibbles are UNSIGNED, as AMD's packer stores the ONNX fields it is handed (engine#2561, which + // replaces the two's-complement reading of engine#1503/#1874); the generators walk all 16 nibble values. auto scale_of = [](int row) { return (1.0f + float(row % 128) / 128.0f) * std::ldexp(1.0f, -(row / 128 + 2)); }; - auto zp_of = [](int row, int kb, int g) { return uint8_t((row + 3 * kb + 5 * g) % 16); }; + // engine#2561: ONE zero point per row per 128-block, and the fields are the ONNX MatMulNBits nibbles as given + // (unsigned): w = (c - z) * s -- the layout AMD's own packer writes (known-answer write, 100 % recovered). + auto zp_of = [](int row, int kb) { return uint8_t((row + 3 * kb) % 16); }; auto code_of = [](int row, int col) { return uint8_t((row * 7 + col * 3) % 16); }; auto w_of = [&](int row, int col) { - auto sx = [](uint8_t v) { return v >= 8 ? int(v) - 16 : int(v); }; - return float(sx(code_of(row, col)) - sx(zp_of(row, col / 128, (col % 128) / 32))) * scale_of(row); + return float(int(code_of(row, col)) - int(zp_of(row, col / 128))) * scale_of(row); }; // Encode the blob as the pack lays it out -- the inverse of decode_external_tile() for the shipped ordering // (pi::kExtQuantsRowMajor == false: byte (cp%4) + c*4 + (cp/4)*512 holds rows 2cp/2cp+1 of column c). @@ -685,13 +683,13 @@ void test_external_import_success(const std::string& root) { for (int kk = 1; kk <= blocks; ++kk) { char* subv = &blob[size_t(tile * per_tile + kk * 2 + half) * 2176]; for (int r = 0; r < 32; ++r) { - // A 64-row block's FIRST 32 rows live in the pair's SECOND subvolume (measured on the pack, #1874). - const int row = tile * 64 + (1 - half) * 32 + r; + // Subvolume `half` holds rows half*32 + r (engine#2561: the earlier swap was circular). + const int row = tile * 64 + half * 32 + r; const uint16_t s = pi::bf16_bits_of(scale_of(row)); subv[2112 + r * 2] = char(s & 0xFF); subv[2112 + r * 2 + 1] = char(s >> 8); - subv[2048 + r * 2] = char(zp_of(row, kk - 1, 0) | (zp_of(row, kk - 1, 1) << 4)); - subv[2048 + r * 2 + 1] = char(zp_of(row, kk - 1, 2) | (zp_of(row, kk - 1, 3) << 4)); + uint8_t& zb = reinterpret_cast(subv[2048 + r / 2]); + zb = (r % 2 == 0) ? uint8_t((zb & 0xF0) | zp_of(row, kk - 1)) : uint8_t((zb & 0x0F) | (zp_of(row, kk - 1) << 4)); for (int c = 0; c < 128; ++c) { const int cp = r / 2; uint8_t& b = reinterpret_cast(subv[(cp % 4) + c * 4 + (cp / 4) * 512]); @@ -6925,6 +6923,7 @@ int main(int argc, char** argv) { try { test_bounds(); test_external_tile_decode(); + test_rope_cache_best_theta(); test_external_to_container(); test_external_roles(); test_external_container_names();