From f0f78db80e64a2fc344b02c931b6acbdec80ade2 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 03:38:26 -0400 Subject: [PATCH 1/3] tests: the external-data writer imports a crafted pack to completion, read back against the fixture (Fixes #1849) Every import_pack() arm on the 1.9 external-data genre expected a refusal, so the writer #1712 added (decode -> container q4_1 -> q/k/v split -> q4_1f tiles), the BF16 copies, and the keys #1751/#1793 added had run only against real packs. That gap was the CHANGES on #1712 that the merge went past. test_external_import_success builds a pack whose geometry the split satisfies (2 heads + 1 kv of 128, one qkv MatMulNBitsBf of N 512 x K 256, a distinct bf16 scale per row and an int4 zero per (row, block, group)), imports it, and reads model.q4nx back through q4nx::dequant_tile, which is independent of the writer: - q/k/v are three tensors of 8/4/4 q4_1f tiles, and every weight equals the fixture's own (code - zp) * scale for pack row lo + r, exactly; - the embedding is BF16 [64, 256] with every value intact; - config.json carries the geometry, and q4nx_weight_layout only for --weight-layout b8 (the arm #1840 asks for; absent at the default); - tokenizer.json is copied (#1751). Mutation-checked: swapping row pairs' scales in the writer, shifting k_proj's range by one tile, dropping the layout key, and skipping the text copy each fail an arm (exit 1). The unmodified writer passes. Fixes #1840 as well. Co-authored-by: bong-water-water-bong Co-Authored-By: Claude Opus 5.5 --- tests/npu_pack_import_test.cpp | 189 +++++++++++++++++++++++++++++++++ 1 file changed, 189 insertions(+) diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 25051340b..44c68591f 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -583,6 +583,194 @@ void test_external_packed_genre(const std::string& root) { fs::remove_all(dir2); } +// engine#1849 (the CHANGES on #1712 that the merge went past): the external-data WRITER had no success-path arm. Every +// import_pack() on the genre above expects a refusal, so the decode -> split -> tile chain, the BF16 copies and the header +// keys #1751/#1793 added were exercised only on real packs. This builds a pack whose geometry the split CAN satisfy, +// imports it to completion and reads the container back through q4nx's own decoders -- independent of the writer -- against +// values the fixture computes itself. Every row has its own scale, and every (row, block, group) its own zero point, so a +// row that lands in the wrong tensor or a q/k/v range that is off by one tile changes a weight and fails here. +void test_external_import_success(const std::string& root) { + const fs::path dir = fs::path(root) / "external-success"; + fs::remove_all(dir); + fs::create_directories(dir / "cache"); + // 2 heads + 1 kv head of 128: q 256 rows, k 128, v 128 -> one qkv matmul of N 512 at K = hidden 256. + const int H = 256, NH = 2, NKV = 1, HD = 128, V = 64; + const int N = (NH + 2 * NKV) * HD, K = H; + put_file((dir / "config.json").string(), + R"({"model_type":"phi3","hidden_size":256,"intermediate_size":512,"num_hidden_layers":1,)" + R"("num_attention_heads":2,"num_key_value_heads":1,"head_dim":128,"vocab_size":64,)" + R"("rope_theta":10000.0,"tie_word_embeddings":false})"); + put_file((dir / "genai_config.json").string(), + R"({"model":{"decoder":{"filename":"eager-mm.onnx","hidden_size":256,"num_hidden_layers":1,)" + R"("num_attention_heads":2,"num_key_value_heads":1,"head_size":128,"intermediate_size":512}}})"); + put_file((dir / "tokenizer.json").string(), R"({"model":{"type":"BPE"}})"); + + // The fixture's own weights: w(row, col) = (code - zp) * scale, chosen so every product is exact in f32 (scales are + // bf16-exact with 7 mantissa bits, codes and zero points are < 16), so the container's `code*scale + m` form must + // reproduce them bit for bit. + auto scale_of = [](int row) { return (1.0f + float(row % 128) / 128.0f) * std::ldexp(1.0f, -(row / 128 + 2)); }; + auto zp_of = [](int row, int kb, int g) { return uint8_t((row + 3 * kb + 5 * g) % 16); }; + auto code_of = [](int row, int col) { return uint8_t((row * 7 + col * 3) % 16); }; + auto w_of = [&](int row, int col) { + return (float(code_of(row, col)) - float(zp_of(row, col / 128, (col % 128) / 32))) * scale_of(row); + }; + // Encode the blob as the pack lays it out -- the inverse of decode_external_tile() for the shipped ordering + // (pi::kExtQuantsRowMajor == false: byte (cp%4) + c*4 + (cp/4)*512 holds rows 2cp/2cp+1 of column c). + const int blocks = K / 128, tiles = N / 64, per_tile = 2 * (blocks + 1); + std::string blob(size_t(tiles) * per_tile * 2176, '\0'); + for (int tile = 0; tile < tiles; ++tile) + for (int half = 0; half < 2; ++half) + for (int kk = 1; kk <= blocks; ++kk) { + char* subv = &blob[size_t(tile * per_tile + kk * 2 + half) * 2176]; + for (int r = 0; r < 32; ++r) { + const int row = tile * 64 + half * 32 + r; + const uint16_t s = pi::bf16_bits_of(scale_of(row)); + subv[2112 + r * 2] = char(s & 0xFF); + subv[2112 + r * 2 + 1] = char(s >> 8); + subv[2048 + r * 2] = char(zp_of(row, kk - 1, 0) | (zp_of(row, kk - 1, 1) << 4)); + subv[2048 + r * 2 + 1] = char(zp_of(row, kk - 1, 2) | (zp_of(row, kk - 1, 3) << 4)); + for (int c = 0; c < 128; ++c) { + const int cp = r / 2; + uint8_t& b = reinterpret_cast(subv[(cp % 4) + c * 4 + (cp / 4) * 512]); + const uint8_t nib = code_of(row, (kk - 1) * 128 + c); + b = (r % 2 == 0) ? uint8_t((b & 0xF0) | nib) : uint8_t((b & 0x0F) | (nib << 4)); + } + } + } + check(int64_t(blob.size()) == int64_t(N) * (K / 128 + 1) * 68, + "external success: the crafted blob has the genre's length N x (K/128 + 1) x 68"); + // The embedding, F32 in the pack, values exact in bf16. + std::string embed(size_t(V) * H * 4, '\0'); + for (int i = 0; i < V * H; ++i) { + const float v = float((i % 61) - 30) * 0.5f; + std::memcpy(&embed[size_t(i) * 4], &v, 4); + } + put_file((dir / "eager-mm.onnx.data").string(), embed + blob); + put_file((dir / "eager-mm.onnx").string(), + pb::model({pb::tensor("model.embed_tokens.weight", {V, H}, "eager-mm.onnx.data", 0, uint64_t(embed.size())), + pb::placeholder("model.layers.0.attn.qkv_proj.MatMulNBits.qweight.empty"), + pb::placeholder("model.layers.0.attn.qkv_proj.MatMulNBits.scales.empty"), + pb::tensor("model.layers.0.attn.qkv_proj.MatMulNBits.qweight.packed", {int64_t(blob.size())}, + "eager-mm.onnx.data", uint64_t(embed.size()), uint64_t(blob.size()))}, + {pb::matmulnbits("qkv", {"x", "/model/layers.0/attn/qkv_proj/MatMulNBits.qweight.empty", + "/model/layers.0/attn/qkv_proj/MatMulNBits.scales.empty", "y"}, + K, N, 4, 128, "MatMulNBitsBf")})); + + struct Container { + json table; + std::string data; + }; + auto read_container = [](const fs::path& out) { + Container c; + std::ifstream f(out / "model.q4nx", std::ios::binary); + uint64_t hlen = 0; + f.read(reinterpret_cast(&hlen), 8); + std::string hdr(size_t(hlen), '\0'); + f.read(hdr.data(), std::streamsize(hlen)); + while (!hdr.empty() && (hdr.back() == ' ' || hdr.back() == '\0')) hdr.pop_back(); + c.table = json::parse(hdr); + c.data.assign(std::istreambuf_iterator(f), std::istreambuf_iterator()); + return c; + }; + auto json_of = [](const fs::path& p) { + std::ifstream f(p); + return json::parse(f); + }; + + for (const std::string layout : {"", "b8"}) { + const fs::path out = dir / (layout.empty() ? "out" : "out-b8"); + fs::remove_all(out); + pi::Options opt; + opt.allow_incomplete = true; // the plan's own refusals (no .fconst, no programs) must not stop the writer + opt.weight_layout = layout; + pi::Report r; + std::string why; + try { + r = pi::import_pack(dir.string(), out.string(), opt); + } catch (const std::exception& e) { + why = e.what(); + } + const std::string tag = "external success" + std::string(layout.empty() ? "" : " (b8)") + ": "; + check(why.empty() && r.refused.empty() && fs::exists(out / "model.q4nx"), + tag + "a pack whose geometry the split satisfies imports to completion (" + why + + (r.refused.empty() ? "" : r.refused.front()) + ")"); + if (!fs::exists(out / "model.q4nx")) continue; + + // config.json: the geometry, and q4nx_weight_layout ONLY for a non-default layout (engine#1840 / #1793). + const json cfg = json_of(out / "config.json"); + check(cfg.value("num_attention_heads", 0) == NH && cfg.value("num_key_value_heads", 0) == NKV && + cfg.value("head_dim", 0) == HD && cfg.value("hidden_size", 0) == H, + tag + "config.json carries the pack's geometry"); + if (layout.empty()) + check(!cfg.contains("q4nx_weight_layout"), + tag + "a default-layout container carries no q4nx_weight_layout key (every earlier container depends " + "on that)"); + else + check(cfg.value("q4nx_weight_layout", std::string()) == "b8", + tag + "--weight-layout b8 writes q4nx_weight_layout \"b8\" into config.json"); + // The text artifacts (engine#1751): without tokenizer.json the directory is not a model directory. + check(fs::exists(out / "tokenizer.json"), tag + "tokenizer.json is copied beside the container"); + if (!layout.empty()) continue; // the weights do not depend on the layout key; check them once + + const Container c = read_container(out); + // q/k/v land as three tensors with the geometry's row ranges, and every weight is the fixture's. + struct Want { + const char* name; + int lo, rows; + }; + for (const Want& want : {Want{"model.layers.0.self_attn.q_proj.weight", 0, NH * HD}, + Want{"model.layers.0.self_attn.k_proj.weight", NH * HD, NKV * HD}, + Want{"model.layers.0.self_attn.v_proj.weight", NH * HD + NKV * HD, NKV * HD}}) { + const std::string nm = want.name; + if (!c.table.contains(nm)) { + check(false, tag + nm + " is in the container"); + continue; + } + const json& e = c.table.at(nm); + const int64_t lo = e.at("data_offsets").at(0), hi = e.at("data_offsets").at(1); + const int RT = want.rows / 32, KT = K / 256; + const size_t tb = size_t(onebit::npu::q4nx::Chunk::Q4_1F); + check(e.at("shape").at(1) == int64_t(tb) && (hi - lo) == int64_t(RT) * KT * int64_t(tb), + tag + nm + " is " + std::to_string(RT * KT) + " q4_1f tiles"); + if ((hi - lo) != int64_t(RT) * KT * int64_t(tb)) continue; + int bad = 0, first_r = -1, first_c = -1; + std::vector w(32 * 256); + for (int rt = 0; rt < RT; ++rt) + for (int kt = 0; kt < KT; ++kt) { + const auto* tile = + reinterpret_cast(c.data.data() + lo + (size_t(rt) * KT + kt) * tb); + onebit::npu::q4nx::dequant_tile(onebit::npu::q4nx::Chunk::Q4_1F, tile, w.data(), 256); + for (int r = 0; r < 32; ++r) + for (int col = 0; col < 256; ++col) { + const int row = rt * 32 + r, gcol = kt * 256 + col; + if (w[size_t(r) * 256 + col] != w_of(want.lo + row, gcol)) { + if (bad++ == 0) first_r = row, first_c = gcol; + } + } + } + check(bad == 0, tag + nm + ": every weight is the pack's row " + std::to_string(want.lo) + "+r, exactly (" + + std::to_string(bad) + " differ, first at " + std::to_string(first_r) + "," + + std::to_string(first_c) + ")"); + } + // The embedding is copied as BF16 with its shape (exact here by construction). + bool embed_ok = c.table.contains("model.embed_tokens.weight"); + if (embed_ok) { + const json& e = c.table.at("model.embed_tokens.weight"); + const int64_t lo = e.at("data_offsets").at(0), hi = e.at("data_offsets").at(1); + embed_ok = e.at("dtype") == "BF16" && e.at("shape") == json::array({V, H}) && hi - lo == int64_t(V) * H * 2; + for (int i = 0; embed_ok && i < V * H; ++i) { + uint16_t b = 0; + std::memcpy(&b, c.data.data() + lo + size_t(i) * 2, 2); + embed_ok = onebit::npu::bf16_to_f32(b) == float((i % 61) - 30) * 0.5f; + } + } + check(embed_ok, tag + "the embedding is copied as BF16 [64, 256] with every value intact"); + } + fs::remove_all(dir); + std::printf("external success: crafted external-data pack imported; q/k/v split, weights, embedding and keys hold\n"); +} + + // The two overstatements the readers must catch: declared dims larger than the external file, and a // MatMulNBits node whose N/K/block_size demand more qweight than the tensor declares. // engine#1671: a pack whose graph is all `[0]`-dims placeholders and whose bytes are in model.bin (the 1.8/hybrid @@ -3259,6 +3447,7 @@ int main(int argc, char** argv) { test_external_roles(); test_external_container_names(); test_external_packed_genre(golden_dir.empty() ? "." : golden_dir); + test_external_import_success(golden_dir.empty() ? "." : golden_dir); test_canonical_copy_reusable(); test_evict_stale_canonical_copies(); test_crafted_refusals(golden_dir.empty() ? "." : golden_dir); test_modelbin_genre_refusal(golden_dir.empty() ? "." : golden_dir); From bd8813b13c2b4e72229a22cb54c2f2514d110894 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 06:35:44 -0400 Subject: [PATCH 2/3] tests: the external-success fixture reads both int4 fields as two's complement Review at bc89e97: the reference computed (code - zp) * scale on unsigned nibbles, so it agreed with today's unsigned importer and certified the #1503 sign bug. Both fields are int4 two's complement (the pack's writer asserts -8 <= x <= 7); the reference now sign-extends them. The generators still walk all 16 nibble values, so about half the weights sit where the two readings differ by 16 * scale. On main's importer the arm now fails (32768/16384/16384 q/k/v weights differ), which is the bug. It passes once the importer sign-extends. Co-authored-by: bong-water-water-bong Co-Authored-By: Claude Opus 5.5 --- tests/npu_pack_import_test.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 0cdffab43..b72956bac 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -606,13 +606,16 @@ void test_external_import_success(const std::string& root) { put_file((dir / "tokenizer.json").string(), R"({"model":{"type":"BPE"}})"); // The fixture's own weights: w(row, col) = (code - zp) * scale, chosen so every product is exact in f32 (scales are - // bf16-exact with 7 mantissa bits, codes and zero points are < 16), so the container's `code*scale + m` form must - // reproduce them bit for bit. + // bf16-exact with 7 mantissa bits, |code - zp| <= 15), so the container's `code*scale + m` form must reproduce them + // bit for bit. Both nibbles are int4 TWO'S COMPLEMENT (the pack's writer asserts -8 <= x <= 7), so the reference + // sign-extends them: the generators walk all 16 nibble values, and for a code and zero of opposite sign an + // unsigned reading is off by 16 * scale (engine#1503, #1874). auto scale_of = [](int row) { return (1.0f + float(row % 128) / 128.0f) * std::ldexp(1.0f, -(row / 128 + 2)); }; auto zp_of = [](int row, int kb, int g) { return uint8_t((row + 3 * kb + 5 * g) % 16); }; auto code_of = [](int row, int col) { return uint8_t((row * 7 + col * 3) % 16); }; auto w_of = [&](int row, int col) { - return (float(code_of(row, col)) - float(zp_of(row, col / 128, (col % 128) / 32))) * scale_of(row); + auto sx = [](uint8_t v) { return v >= 8 ? int(v) - 16 : int(v); }; + return float(sx(code_of(row, col)) - sx(zp_of(row, col / 128, (col % 128) / 32))) * scale_of(row); }; // Encode the blob as the pack lays it out -- the inverse of decode_external_tile() for the shipped ordering // (pi::kExtQuantsRowMajor == false: byte (cp%4) + c*4 + (cp/4)*512 holds rows 2cp/2cp+1 of column c). From 13f05b67fa6f040746ad582ce3ce827414f500f0 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 08:23:22 -0400 Subject: [PATCH 3/3] tests: the external-success fixture writes a block's halves in the pack's order #1874 at b1ca08b measured that a 64-row block's first 32 rows live in the pair's second subvolume, and the decoder now reads them that way. The fixture is the inverse of the decoder, so it encodes the same order. Measured with #1874 b1ca08b plus its reverted pass-through commit 36979a5: every external-success check passes. Without 36979a5 (the zero point is converted twice), 61440/30720/30720 q/k/v weights still differ. Co-authored-by: bong-water-water-bong Co-Authored-By: Claude Opus 5.5 --- tests/npu_pack_import_test.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index b72956bac..1357c2e00 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -626,7 +626,8 @@ void test_external_import_success(const std::string& root) { for (int kk = 1; kk <= blocks; ++kk) { char* subv = &blob[size_t(tile * per_tile + kk * 2 + half) * 2176]; for (int r = 0; r < 32; ++r) { - const int row = tile * 64 + half * 32 + r; + // A 64-row block's FIRST 32 rows live in the pair's SECOND subvolume (measured on the pack, #1874). + const int row = tile * 64 + (1 - half) * 32 + r; const uint16_t s = pi::bf16_bits_of(scale_of(row)); subv[2112 + r * 2] = char(s & 0xFF); subv[2112 + r * 2 + 1] = char(s >> 8);