From 87a0dcf4b1dc4632de163a0e600cde1557d0942b Mon Sep 17 00:00:00 2001 From: pi-npufeat Date: Mon, 9 Nov 2026 22:42:40 -0400 Subject: [PATCH 1/5] npu-pack: read a THREE-input MatMulNBits, whose implicit zero point is 2^(bits-1) (engine#1503) Qwen3-VL-4B spells its projections as three-input nodes with no qzeros tensor and names them `.MatMul.weight_Q4` next to `.MatMul.weight_scales`: MatMulNBits inputs = (activations, ...attn.q_proj.MatMul.weight_Q4, ...attn.q_proj.MatMul.weight_scales) attrs = K 2560, N 4096, bits 4, block_size 32 weight_Q4 [4096, 80, 16] UINT8 N x (K/32) x 16 B (32 quants, 2 per byte) weight_scales [4096, 80] FLOAT one f32 scale per (row, 32-column block) `walk_graph` required `ins.size() >= 4`, so it recorded **no matmuls at all** for that pack: `npu-pack info` printed `matmulnbits_ops 0`, `plan()` fell through to "the graph carries no weights", and the import wrote the embedding and nothing else -- measured on the real pack as `imported 1 tensors, 0 q4_1 tiles`. Three changes, and the third is the one that decides correctness: 1. accept three inputs (data, weight, scales) and take `zeros` only when the node has it; 2. resolve both spellings -- `.MatMulNBits.qweight` (the 1.8/phi-4 form) and `.MatMul.weight_Q4` -- in the projection census and in the weight/scales lookup; 3. a node with **no qzeros** takes the implicit zero point `2^(bits-1)`, applied by `implicit_zp_zero()` after the decode. That is ONNX Runtime's own MatMulNBits rule when `qzeros` is absent, and it is corroborated inside this collection: the *embedding* component of the same family (GatherBlockQuantized, also no zeros) decodes to ORT's output at cosine 1.00000000 / max|d| 0 at zero point 8, against 0.317 at zero point 0 (engine#1725). **Measured on the real pack** (`amd/Qwen3-VL-4B-Instruct_rai_1.9.0_npu`, 2.9 GB, on disk under `~/scratch-packs`): imported 147 tensors, 47480 q4_1 tiles into ~/scratch-packs/qwen3vl4b.q4nx self-check: the head reproduces the pack's own embedding (as imported, bf16) corr 0.993981 rel_err 0.092550 controls: nibble-swapped corr -0.0699, no zero-point corr 0.3333 (both must fail) parameters: fp32 scale/zero exact for 100.00% of 32-column groups The head is tied to the embedding after all, so the pack's own reference grades the decode -- at 0.993981, with the nibble-swap control failing as it must and the **no-zero-point control at 0.3333**, which is the direct measurement that the implicit 2^(bits-1) is the right convention for this family. Test: a crafted three-input node with `[N, K/32, 16]` u8 + `[N, K/32]` f32 asserts that the walker records it (1 of 1), counts it as carrying no qzeros, and takes K/N/block from the node's attributes -- 3-input MatMulNBits: 1 matmul(s), 1 without qzeros, K 64 N 32 block 32 all npu-pack checks passed Not in this PR: the RoPE question. The same pack ships `cos_cache`/`sin_cache` that no plain theta reproduces (best 0.5918 at 3.2e6 against a declared 10000) while the forward computes RoPE from theta -- its own issue, not a reader symptom. --- npu/pack_import.cpp | 59 ++++++++++++++++++++++++++++++---- npu/pack_import.h | 5 +++ tests/npu_pack_import_test.cpp | 35 ++++++++++++++++++++ 3 files changed, 93 insertions(+), 6 deletions(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 19c171787..96aa0a705 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -200,11 +200,16 @@ void walk_graph(const uint8_t* p, size_t n, OnnxModel& m) { if (op == "MatMulNBits" || op == "MatMulNBitsBf") { if (op == "MatMulNBitsBf") ++m.ryzenai_matmuls; OnnxMatMul mm; - if (ins.size() >= 4) { + // THREE inputs is a real spelling, not a truncation: `amd/Qwen3-VL-4B-Instruct_rai_1.9.0_npu` names its + // projections `(activations, ...MatMul.weight_Q4, ...MatMul.weight_scales)` with `bits 4, block_size 32` and + // no qzeros tensor at all, and a `>= 4` test therefore recorded NO matmuls for it -- `info` printed + // matmulnbits_ops 0 and the importer wrote only the embedding (engine#1503). A missing qzeros means the + // implicit zero point, which `implicit_zp_zero()` below applies. + if (ins.size() >= 3) { mm.data = ins[0]; mm.weight = ins[1]; mm.scales = ins[2]; - mm.zeros = ins[3]; + if (ins.size() >= 4) mm.zeros = ins[3]; } if (attrs.count("K")) mm.K = attrs["K"]; if (attrs.count("N")) mm.N = attrs["N"]; @@ -1245,6 +1250,21 @@ Dims read_dims(const json& c, const std::string& source = "config.json") { // engine#1650: exported so the config reader can be tested on a pack that declares an optional // field as null, without a pack and without the vendor stack. Dims read_dims_exported(const json& c, const std::string& source) { return read_dims(c, source); } +int parse_onnx_matmul_census(const std::string& path, int* matmuls, int* without_zeros, int* K, int* N, int* block) { + const OnnxModel m = parse_onnx(path); + if (matmuls) *matmuls = int(m.matmuls.size()); + if (without_zeros) { + int n = 0; + for (const auto& mm : m.matmuls) if (mm.zeros.empty()) ++n; + *without_zeros = n; + } + if (!m.matmuls.empty()) { + if (K) *K = int(m.matmuls[0].K); + if (N) *N = int(m.matmuls[0].N); + if (block) *block = m.matmuls[0].block; + } + return int(m.matmuls.size()); +} // The 1.9 external-data genre states no FFN width anywhere a file shows it: the weights are MatMulNBitsBf nodes // whose initializers carry dims `[0]` (bytes live in a *.qweight.packed sibling), so the initializer-dimension // heuristic in plan() sees nothing. The NODE carries K and N as attributes, and walk_graph already descends into @@ -1386,6 +1406,19 @@ FconstGeom locate_fconst(const std::string& path, unsigned short eps_bits) { } +// ── The implicit zero point ─────────────────────────────────────────────────── +// A quantised matmul with no `qzeros` input takes an implicit zero point of 2^(bits-1) -- ONNX Runtime's own +// MatMulNBits does exactly that when the tensor is absent, and it is corroborated in this collection: the *embedding* +// component of the same family (GatherBlockQuantized, also no zeros) decodes to ONNX Runtime's output with cosine +// 1.00000000 and max|d| 0 at zero point 8, against 0.317 at zero point 0 (measured on +// amd/Qwen3-VL-4B-Instruct_rai_1.9.0_npu, engine#1725/#1661). `decode_matmulnbits` folds the zero into `q.zero`, so +// this subtracts the implicit one the same way: zero term = -(2^(bits-1)) * scale. +void implicit_zp_zero(Q4& q, int bits) { + if (bits <= 0 || q.scale.empty() || q.scale.size() != q.zero.size()) return; + const float zp = float(1 << (bits - 1)); + for (size_t i = 0; i < q.zero.size(); ++i) q.zero[i] -= zp * q.scale[i]; +} + // ── MatMulNBits → (codes, scale, zero) per 32-column group ─────────────────── Q4 decode_matmulnbits(const uint8_t* qweight, const float* scales, const uint8_t* zeros, int N, int K, int block, bool swap_nibbles, bool drop_zero) { @@ -2135,9 +2168,14 @@ Plan plan(const std::string& pack_dir) { std::set graph_projs; if (graph_weights) { for (const auto& t : onnx.tensors) { - const size_t pos = t.name.find(".MatMulNBits."); - if (pos == std::string::npos || t.dims.empty()) continue; // 0-dim placeholders sit beside them - graph_projs.insert(t.name.substr(0, pos)); + if (t.dims.empty()) continue; // 0-dim placeholders sit beside them + // Two spellings of the same thing: `.MatMulNBits.qweight` (the 1.8/phi-4 form) and + // `.MatMul.weight_Q4` (Qwen3-VL-4B's, three-input nodes, engine#1503). + size_t pos = t.name.find(".MatMulNBits."); + if (pos != std::string::npos) { graph_projs.insert(t.name.substr(0, pos)); continue; } + const std::string q4 = ".MatMul.weight_Q4"; + pos = t.name.find(q4); + if (pos != std::string::npos) graph_projs.insert(t.name.substr(0, pos)); } } const OnnxTensor* embed = nullptr; @@ -3160,8 +3198,15 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons // Decode rows [row0, row0+rows) of a projection's MatMulNBits weight into q4_1 tiles. auto decode_rows = [&](const std::string& proj, int row0, int rows, const std::string& out_name, int k_override) -> bool { + // The two spellings of a quantised projection (engine#1503): the 1.8/phi-4 form carries a qweight and a + // scales sibling; Qwen3-VL-4B names them `.MatMul.weight_Q4` / `.MatMul.weight_scales` on a + // three-input node with no qzeros (implicit zero point, applied below). const OnnxTensor* qw = init_of(proj + ".MatMulNBits.qweight"); const OnnxTensor* sc = init_of(proj + ".MatMulNBits.scales"); + if (!qw || !sc) { + qw = init_of(proj + "MatMul.weight_Q4"); + sc = init_of(proj + "MatMul.weight_scales"); + } if (!qw || !sc || qw->dims.size() != 3 || sc->dims.empty()) return false; const int N = int(qw->dims[0]); const int block = int(qw->dims[2]) * 2; // 64 B holds 128 four-bit codes @@ -3311,7 +3356,9 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons const uint8_t* zp = have_zp ? bytes_of(onnx.tensors[zi->second], zp_need) : nullptr; uint64_t qw_need = 0; // N * K/block * block/2 if (!mul_ok(groups_n, uint64_t(mm.block) / 2, qw_need)) throw std::runtime_error("lm_head: qweight length overflows"); - const Q4 q = decode_matmulnbits(bytes_of(wt, qw_need), scales.data(), zp, int(mm.N), int(mm.K), mm.block); + Q4 q = decode_matmulnbits(bytes_of(wt, qw_need), scales.data(), zp, int(mm.N), int(mm.K), mm.block); + // no qzeros tensor -> the implicit zero point (engine#1503: Qwen3-VL-4B's spelling, three-input nodes) + if (!have_zp) implicit_zp_zero(q, mm.bits); std::vector tiled; long long pinexact = 0; diff --git a/npu/pack_import.h b/npu/pack_import.h index 1a4f96da4..05df83517 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -396,6 +396,11 @@ unsigned short bf16_bits_of(float v); // the default rather than throw. Exported so tests/npu_pack_import_test.cpp can pin it without a // pack; `source` names where the geometry came from, for the refusal messages. Dims read_dims_exported(const nlohmann::json& c, const std::string& source); +// engine#1503: what the graph walker RECORDED, without exposing its model type. A three-input MatMulNBits (no qzeros, +// weights spelled `.MatMul.weight_Q4`) used to be dropped entirely, which is why Qwen3-VL-4B imported one tensor. +// Returns the matmul count; `without_zeros` counts those that carry no qzeros tensor (the implicit-zero-point case), and +// the last three report the first matmul's own K/N/block attributes. +int parse_onnx_matmul_census(const std::string& path, int* matmuls, int* without_zeros, int* K, int* N, int* block); // engine#1638: the geometry as read, so a test can check WHICH source a number came from. nlohmann::json load_geometry_exported(const std::string& pack_dir, std::string* source); FconstGeom locate_fconst(const std::string& path, unsigned short eps_bits = 0x3586); diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index a639cc241..9fb728905 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -2175,6 +2175,40 @@ void test_text_config_geometry(const std::string& root) { fs::remove_all(dir); } + +// engine#1503: a THREE-input MatMulNBits with no qzeros, weights spelled `.MatMul.weight_Q4` + +// `.weight_scales` -- Qwen3-VL-4B's form. `walk_graph` used to require four inputs, so it recorded NO matmuls for that +// pack: `info` printed matmulnbits_ops 0 and the import wrote the embedding only ("imported 1 tensors, 0 q4_1 tiles"). +// On the real pack the fix yields 147 tensors / 47,480 tiles and the pack's own self-check reads corr 0.993981 against its +// embedding, with the no-zero-point control at 0.3333 -- this arm pins the mechanism without the 3 GB pack. +void test_three_input_matmulnbits(const std::string& root) { + const fs::path dir = fs::path(root) / "mm3"; + fs::remove_all(dir); + fs::create_directories(dir); + const int64_t H = 64, N = 32, block = 32, groups = H / block; + std::vector qw(size_t(N) * size_t(groups) * size_t(block / 2), 0x21); // codes 1/2 alternating + std::vector sc(size_t(N) * size_t(groups), 0.01f); + std::string data(reinterpret_cast(qw.data()), qw.size()); + data.append(reinterpret_cast(sc.data()), sc.size() * sizeof(float)); + put_file((dir / "model.onnx.data").string(), data); + const auto q4 = pb::tensor("model.layers.0.attn.q_proj.MatMul.weight_Q4", {N, groups, block / 2}, + "model.onnx.data", 0, qw.size(), 2); + const auto scl = pb::tensor("model.layers.0.attn.q_proj.MatMul.weight_scales", {N, groups}, + "model.onnx.data", qw.size(), sc.size() * sizeof(float), 1); + const auto node = pb::matmulnbits("q", {"x", "model.layers.0.attn.q_proj.MatMul.weight_Q4", + "model.layers.0.attn.q_proj.MatMul.weight_scales"}, H, N, 4, block); + put_file((dir / "model.onnx").string(), pb::model({q4, scl}, {node})); + int mm = -1, noz = -1, K = 0, Nn = 0, blk = 0; + pi::parse_onnx_matmul_census((dir / "model.onnx").string(), &mm, &noz, &K, &Nn, &blk); + check(mm == 1, "3-input MatMulNBits: the walker records the node (" + std::to_string(mm) + " of 1)"); + check(noz == 1, "3-input MatMulNBits: it is counted as carrying no qzeros, so the implicit zero point applies"); + check(K == H && Nn == N && blk == block, + "3-input MatMulNBits: K/N/block come from the node's own attributes (" + std::to_string(K) + "/" + + std::to_string(Nn) + "/" + std::to_string(blk) + ")"); + std::printf(" 3-input MatMulNBits: %d matmul(s), %d without qzeros, K %d N %d block %d\n", mm, noz, K, Nn, blk); + fs::remove_all(dir); +} + void test_embedding_component(const std::string& root) { const fs::path dir = fs::path(root) / "embedcomp"; fs::remove_all(dir); @@ -2939,6 +2973,7 @@ int main(int argc, char** argv) { test_attn_payload_off(); test_text_config_geometry(golden_dir.empty() ? "." : golden_dir); test_hybrid_pack_import(golden_dir.empty() ? "." : golden_dir); + test_three_input_matmulnbits(golden_dir.empty() ? "." : golden_dir); test_embedding_component(golden_dir.empty() ? "." : golden_dir); test_fconst19_rule_table(); test_selfcheck_reads_bf16(); From 8130e6391742799cd40f324e8ce561f34e481bf1 Mon Sep 17 00:00:00 2001 From: pi-npufeat Date: Mon, 9 Nov 2026 22:51:31 -0400 Subject: [PATCH 2/5] npu-pack: read rope_theta from its nested rope_parameters block (engine#1503) Qwen3-VL-4B declares its rotary base ONE LEVEL DEEPER than any spelling this reader knew: text_config.rope_parameters = { mrope_interleaved: true, mrope_section: [24,20,20], rope_theta: 5000000, rope_type: default } with no `rope_theta` at the top level and none at `text_config.rope_theta`, so `read_dims`' 10000 default applied and the importer said what that costs: WARNING: this pack ships precomputed RoPE caches that no plain theta explains (best 0.5918 at 3.2e+06, declared 10000) **The caches are not unexplained -- the theta was not read.** Measured against the pack's own `cos_cache` (262144 x 64, external data, row 1 of position 1): theta 10000: max|cos - expected| 0.088090 theta 5000000: max|cos - expected| 0.000000 <- the pack's own value, exactly theta 3200000: max|cos - expected| 0.005052 So this fixes a **wrong-attention** bug, not a cosmetic warning: 10000 against 5e6 changes every rotary angle. The lift has the same shape and the same guards as the `text_config` one next to it: gaps only, never overriding, and the source string names the block it came from (`+ config.json rope_parameters`). With it, the importer's own check passes: note: rope_theta 5e+06 verified against the pack's cos_cache (corr 1.0000) dims.rope_theta = 5000000.0 Test: a crafted config with the nested block asserts theta 5000000 and the source naming, plus the never-override arm (a top-level 10000 wins over the nested 5e6) -- rope_parameters: theta 5000000 nested, top-level still wins all npu-pack checks passed --- npu/pack_import.cpp | 16 ++++++++++++++++ tests/npu_pack_import_test.cpp | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 96aa0a705..872280248 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -1138,6 +1138,22 @@ json load_geometry(const std::string& pack_dir, std::string* source) { ++from_text_config; } } + // `rope_theta` (and the rotary block's other keys) can be ONE LEVEL DEEPER still: Qwen3-VL-4B declares + // `text_config.rope_parameters = {mrope_interleaved: true, mrope_section: [24,20,20], rope_theta: 5000000, + // rope_type: default}` with no `rope_theta` anywhere else, so this reader fell back to its 10000 default and the + // importer warned that the pack's cos/sin caches "no plain theta explains (best 0.5918 at 3.2e+06, declared 10000)". + // Measured against the pack's own cos_cache (262144 x 64, external): **theta 5e6 reproduces row 1 to max|d| + // 0.000000**, 10000 is off by 0.088 and 3.2e6 by 0.005 -- so the caches are not unexplained at all, the theta was + // simply not read (engine#1503). Gaps only, never overriding, same rule as the lift above. + int from_rope_params = 0; + if (cfg.contains("rope_parameters") && cfg["rope_parameters"].is_object()) { + for (auto it = cfg["rope_parameters"].begin(); it != cfg["rope_parameters"].end(); ++it) { + if (cfg.contains(it.key()) || it.value().is_null()) continue; + cfg[it.key()] = it.value(); + ++from_rope_params; + } + } + if (from_rope_params > 0 && source) *source += " + config.json rope_parameters"; if (from_text_config > 0 && source) *source += " + config.json text_config"; if (fill_from_model_config_impl(pack_dir, cfg) > 0 && source) *source += " + model_config.json (text_config)"; return cfg; diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 9fb728905..d9bbbd82a 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -2181,6 +2181,37 @@ void test_text_config_geometry(const std::string& root) { // pack: `info` printed matmulnbits_ops 0 and the import wrote the embedding only ("imported 1 tensors, 0 q4_1 tiles"). // On the real pack the fix yields 147 tensors / 47,480 tiles and the pack's own self-check reads corr 0.993981 against its // embedding, with the no-zero-point control at 0.3333 -- this arm pins the mechanism without the 3 GB pack. +// engine#1503: `rope_theta` nested one level deeper still — `text_config.rope_parameters.rope_theta` — which is how +// Qwen3-VL-4B declares 5,000,000 with no `rope_theta` anywhere else. The reader's 10000 default made the importer warn that +// the pack's cos/sin caches "no plain theta explains"; measured against the pack's own cos_cache (262144 x 64), theta 5e6 +// reproduces row 1 to max|d| 0.000000 while 10000 is off by 0.088. This pins the lift and the never-override rule. +void test_rope_parameters_theta(const std::string& root) { + const fs::path dir = fs::path(root) / "ropeparams"; + fs::remove_all(dir); + fs::create_directories(dir); + put_file((dir / "config.json").string(), + R"({"model_type":"qwen3_vl","text_config":{"hidden_size":64,"num_hidden_layers":1,)" + R"("num_attention_heads":2,"num_key_value_heads":1,"head_dim":32,"intermediate_size":128,)" + R"("vocab_size":32,"rope_parameters":{"mrope_interleaved":true,"mrope_section":[24,20,20],)" + R"("rope_theta":5000000,"rope_type":"default"}}})"); + std::string src; + const nlohmann::json g = pi::load_geometry_exported(dir.string(), &src); + const pi::Dims d = pi::read_dims_exported(g, src); + check(d.rope_theta == 5000000.0f, + "rope_parameters: rope_theta comes from the nested block (" + std::to_string(d.rope_theta) + ")"); + check(src.find("rope_parameters") != std::string::npos, "rope_parameters: and the source says so (" + src + ")"); + // a top-level rope_theta still wins + put_file((dir / "config.json").string(), + R"({"model_type":"qwen3_vl","rope_theta":10000.0,"text_config":{"hidden_size":64,)" + R"("num_hidden_layers":1,"num_attention_heads":2,"head_dim":32,"intermediate_size":128,"vocab_size":32,)" + R"("rope_parameters":{"rope_theta":5000000}}})"); + const nlohmann::json g2 = pi::load_geometry_exported(dir.string(), &src); + check(pi::read_dims_exported(g2, src).rope_theta == 10000.0f, + "rope_parameters: a top-level rope_theta is NOT overridden by the nested one"); + std::printf(" rope_parameters: theta %.0f nested, top-level still wins\n", (double)d.rope_theta); + fs::remove_all(dir); +} + void test_three_input_matmulnbits(const std::string& root) { const fs::path dir = fs::path(root) / "mm3"; fs::remove_all(dir); @@ -2973,6 +3004,7 @@ int main(int argc, char** argv) { test_attn_payload_off(); test_text_config_geometry(golden_dir.empty() ? "." : golden_dir); test_hybrid_pack_import(golden_dir.empty() ? "." : golden_dir); + test_rope_parameters_theta(golden_dir.empty() ? "." : golden_dir); test_three_input_matmulnbits(golden_dir.empty() ? "." : golden_dir); test_embedding_component(golden_dir.empty() ? "." : golden_dir); test_fconst19_rule_table(); From c9fca4d4f1dba569b8bd71a788e74fc409315d06 Mon Sep 17 00:00:00 2001 From: pi-npufeat Date: Mon, 9 Nov 2026 23:14:05 -0400 Subject: [PATCH 3/5] npu-pack: the weight lookup normalises the projection name's trailing dot (engine#1503) `decode_rows()` builds a weight name as `proj + ".MatMulNBits.qweight"`, which assumes `proj` ends WITHOUT a dot -- true when it comes from a role name, false when it comes from a plan source string (`graph:model.layers.N.attn.q_proj`). With no dot the three-input spelling was looked up as `...q_projMatMul.weight_Q4`, never found, and the import silently wrote only what it could decode: before: imported 147 tensors, 47480 q4_1 tiles <- 36 layers x 4 norms + embedding + head after : imported 399 tensors, 491000 q4_1 tiles <- + 36 layers x 7 projections The pack then served on the device for the first time (all four `kernels-4b-b8` ELFs resolved, prompt 1.37 s, 8 tokens in 4.90 s), and its answer was **garbage** -- which is the next question, not this one: the container is now complete, so a host-GEMM run of the same prompt separates "our weights/rotary" from "the kernels". Refs #1790, which fixed the node spelling, the implicit zero point and the census; this is the last link between the plan and the writer. --- npu/pack_import.cpp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 872280248..8eb3886fb 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -3217,12 +3217,20 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons // The two spellings of a quantised projection (engine#1503): the 1.8/phi-4 form carries a qweight and a // scales sibling; Qwen3-VL-4B names them `.MatMul.weight_Q4` / `.MatMul.weight_scales` on a // three-input node with no qzeros (implicit zero point, applied below). - const OnnxTensor* qw = init_of(proj + ".MatMulNBits.qweight"); - const OnnxTensor* sc = init_of(proj + ".MatMulNBits.scales"); + // `proj` arrives from a plan source string (`graph:model.layers.N.attn.q_proj`, no trailing dot) as well as + // from a role name (`...q_proj.`, with one), so normalise before appending a suffix -- otherwise the + // three-input spelling is looked up as `...q_projMatMul.weight_Q4` and never found, which is how a pack could + // plan all 252 projections, write only its norms, and fail on the device with `MLP dequant : data_size 0` + // (engine#1503). + std::string pre = proj; + if (!pre.empty() && pre.back() != '.') pre += '.'; + const OnnxTensor* qw = init_of(pre + "MatMulNBits.qweight"); + const OnnxTensor* sc = init_of(pre + "MatMulNBits.scales"); if (!qw || !sc) { - qw = init_of(proj + "MatMul.weight_Q4"); - sc = init_of(proj + "MatMul.weight_scales"); + qw = init_of(pre + "MatMul.weight_Q4"); + sc = init_of(pre + "MatMul.weight_scales"); } + if (!qw || !sc) return false; // named, so the caller's refusal says which if (!qw || !sc || qw->dims.size() != 3 || sc->dims.empty()) return false; const int N = int(qw->dims[0]); const int block = int(qw->dims[2]) * 2; // 64 B holds 128 four-bit codes From f87179e2af5244dbcd659e3412e88f777266a54a Mon Sep 17 00:00:00 2001 From: pi-npufeat Date: Mon, 9 Nov 2026 23:47:21 -0400 Subject: [PATCH 4/5] npu-pack: apply the implicit zero point to the PROJECTIONS too -- and the 4B pack answers "Paris" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The head path and the projection path share `decode_matmulnbits`, but only the head path applied `implicit_zp_zero()` when a node carries no qzeros. So every projection of a three-input pack was decoded with zero point 0 while the head was decoded with 2^(bits-1) -- a container that imported completely (399 tensors, 491,000 tiles), served on the device, and answered gibberish on **both** paths, which is what made it look like a kernel or rotary fault: device, before: "West什么呢ưởng中国联通nioUnnamed violateIOD" host, before: "chu cele隐ntlinskyDupturnstilefarewandMspacad在市场上" After applying the same rule in `decode_rows()`: host : " Paris. The capital of Spain is Madrid. The capital of" (12 tokens, 74.6 s = 0.16 tok/s) device : " Paris. The capital of Spain is Madrid" (8 tokens, 5.10 s = 1.57 tok/s, prompt 1.47 s) Both paths now answer the known-answer prompt, and the device is ~10x the host-GEMM rate on the same container -- which also shows the first gibberish was never the kernels. The lesson worth keeping: a decode convention applied at one call site and not its twin produces a *plausible* container with one consistent self-check (head-vs-embedding still read 0.993981, because both suffered the same defect) and garbage text everywhere. The control that would have caught it earlier is the host-GEMM run: gibberish there too means the container or the forward, never the device. --- npu/pack_import.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 8eb3886fb..0001d20e3 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -3260,7 +3260,12 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons const uint8_t* zp = zt ? bytes_of(*zt, z_need) : nullptr; uint64_t qw_need = 0; if (!mul_ok(groups_n, uint64_t(block) / 2, qw_need)) return false; - const Q4 q = decode_matmulnbits(bytes_of(*qw, qw_need), scales.data(), zp, N, K, block); + Q4 q = decode_matmulnbits(bytes_of(*qw, qw_need), scales.data(), zp, N, K, block); + // No qzeros tensor means the implicit zero point 2^(bits-1) -- the same rule the head path applies, and the + // one ONNX Runtime's MatMulNBits uses. Missing it here decoded every PROJECTION with zero point 0 while the + // head was decoded with 8, which is why the pack imported completely, served on the device, and answered + // gibberish on both the device and the host-GEMM path (engine#1503: "West什么呢ưởng中国联通nio…"). + if (!zt) implicit_zp_zero(q, 4); // the row slice, re-tiled exactly as the whole tensor would be Q4 part; part.N = rows; part.K = K; From 55b25ac72c180ad62fca4017c4b6d728e41d1f01 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 01:01:30 -0400 Subject: [PATCH 5/5] npu-pack: the external-data container carries a nested mRoPE declaration My own review of #1794. Reading the theta from `text_config.rope_parameters` fixes the number, but the container is what the forward reads, and the forward's `value_for_top()` (`q4nx_semantics.c:121`) returns only depth-1 keys. `import_pack()` starts its config from `load_geometry()`, which the lift fills -- so that writer already carries `mrope_interleaved`/`mrope_section`. `import_external()` (the 1.9 external-data genre, Qwen3-VL-4B's) starts from an EMPTY `json c` and writes a fixed key list, so the declaration was dropped and the pack would be served as plain RoPE: right theta, wrong axis layout, bit-identical for a text-only prompt and ~0.29 rad off once image tokens are in it (the failure `q4nx_semantics.c` documents for engine#1311). So this copies those two keys from `load_geometry()` in the external-data writer, only when declared. No arm: the crafted external-data fixture in `npu_pack_import_test` REFUSES by design ("this importer has no writer for that layout yet", engine#1503), so `import_external()` is not reachable from it -- I tried an arm, hit the refusal, and removed it rather than leaving a test that cannot fail. The path is exercised only by the real 1.9 pack leg, so the assertion belongs there or in a future external-genre writer test; saying that is better than an arm that proves nothing. Verified: full build, `npu_pack_import_test` -> `all npu-pack checks passed`. Refs #1794, #1311, #1503 --- npu/pack_import.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 0001d20e3..68076ab9f 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -2924,6 +2924,17 @@ Report import_external(const std::string& pack_dir, const std::string& out_dir, ++aux; } json c; + // engine#1794 review: the container's OWN config is what the forward reads (`q4nx_semantics.c:121` value_for_top, + // depth 1 only), and this writer started from an empty json -- so a mRoPE declaration that lives only at + // `text_config.rope_parameters` was dropped, and the pack would be served as plain RoPE with the right theta and the + // wrong axis layout (engine#1311: exact for a text-only prompt, ~0.29 rad off once image tokens are in it). The + // import_pack() writer starts from load_geometry() and already carries it; this one has to copy it. + { + std::string geom_src; + const json geom = load_geometry(pack_dir, &geom_src); + if (geom.value("mrope_interleaved", false)) c["mrope_interleaved"] = true; + if (geom.contains("mrope_section")) c["mrope_section"] = geom["mrope_section"]; + } c["model_type"] = p.dims.model_type; c["hidden_size"] = p.dims.hidden; c["intermediate_size"] = p.dims.intermediate;