From bea86be42e3db6b8f17fff54ecfd8e3c57ed041d Mon Sep 17 00:00:00 2001 From: pi-npufeat Date: Tue, 10 Nov 2026 06:36:32 -0400 Subject: [PATCH] npu-pack: the implicit zero point's bit width comes from the spelling, not a literal (engine#1841) The rule `2^(bits-1)` existed twice with two different derivations of the same parameter: the projection path passed a hardcoded `4`, the head path passed the node's own `bits`. Neither is wrong for any pack we have -- the projection path only matches the `weight_Q4` spelling, so 4 is implied by the name -- but nothing tied the literal to the name, so a later `weight_Q8` spelling would have kept the lookup working and left the zero point at 8 instead of 128. That is a silent wrong weight, not a crash. `zp_bits_for_spelling()` reads the width from the suffix that matched (`weight_Q4` -> 4, `weight_Q8` -> 8) and falls back to the node's own value for a spelling with no `_Q`, so both call sites now agree by construction. Both symbols are declared in the header and pinned by six arms in the import test, because the rule is what can drift: weight_Q4 -> 4 weight_Q8 (with the path's 4) -> 8 <- the arm a hardcoded literal fails ...MatMulNBits.qweight (no _Q) with node bits 3 -> 3 <- the documented fallback weight_Q4 with node bits 8 -> 4 <- the suffix wins implicit_zp_zero(q, 4) on scale 0.5 -> zero -4.0 implicit_zp_zero(q, 8) on scale 0.5 -> zero -64.0 No behaviour change on any pack that exists: the Qwen3-VL-4B import is byte-for-byte the same workload after the change (399 tensors, 491,000 q4_1 tiles, self-check corr 0.993981), and the whole npu-pack suite passes (0 FAIL). Closes the second point of engine#1841. The first (the trailing dot) and third (a lookup miss counted as a gap) both landed on main in aa673e3e and 486c8cbc; I verified the miss path now returns a named gap instead of a quiet skip. --- npu/pack_import.cpp | 18 ++++++++++++++++-- npu/pack_import.h | 7 +++++++ tests/npu_pack_import_test.cpp | 26 ++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 400e4fe29..d86171557 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -1525,6 +1525,20 @@ void implicit_zp_zero(Q4& q, int bits) { for (size_t i = 0; i < q.zero.size(); ++i) q.zero[i] -= zp * q.scale[i]; } +// The bit width, taken from the SPELLING that matched (engine#1841, second point). The rule above used to be called with a +// literal `4` on the projection path and with the node's own `bits` on the head path; nothing tied the literal to the name, +// so a later spelling like `weight_Q8` would have kept the lookup working while leaving the zero point at 2^3 instead of 2^7 +// -- a wrong weight with no error anywhere. Deriving it from the suffix makes the two agree by construction, and an +// unrecognised suffix falls back to the node's value (documented here) rather than to a silent 4. +int zp_bits_for_spelling(const std::string& spelling, int node_bits) { + const auto pos = spelling.rfind("_Q"); + if (pos != std::string::npos && pos + 2 < spelling.size()) { + const char d = spelling[pos + 2]; + if (d >= '2' && d <= '8') return d - '0'; + } + return node_bits; +} + // ── MatMulNBits → (codes, scale, zero) per 32-column group ─────────────────── Q4 decode_matmulnbits(const uint8_t* qweight, const float* scales, const uint8_t* zeros, int N, int K, int block, bool swap_nibbles, bool drop_zero) { @@ -3729,7 +3743,7 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons // one ONNX Runtime's MatMulNBits uses. Missing it here decoded every PROJECTION with zero point 0 while the // head was decoded with 8, which is why the pack imported completely, served on the device, and answered // gibberish on both the device and the host-GEMM path (engine#1503: "West什么呢ưởng中国联通nio…"). - if (!zt) implicit_zp_zero(q, 4); + if (!zt) implicit_zp_zero(q, zp_bits_for_spelling(qw->name, 4)); // the row slice, re-tiled exactly as the whole tensor would be Q4 part; part.N = rows; part.K = K; @@ -3855,7 +3869,7 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons if (!mul_ok(groups_n, uint64_t(mm.block) / 2, qw_need)) throw std::runtime_error("lm_head: qweight length overflows"); Q4 q = decode_matmulnbits(bytes_of(wt, qw_need), scales.data(), zp, int(mm.N), int(mm.K), mm.block); // no qzeros tensor -> the implicit zero point (engine#1503: Qwen3-VL-4B's spelling, three-input nodes) - if (!have_zp) implicit_zp_zero(q, mm.bits); + if (!have_zp) implicit_zp_zero(q, zp_bits_for_spelling(mm.weight, mm.bits)); std::vector tiled; long long pinexact = 0; diff --git a/npu/pack_import.h b/npu/pack_import.h index 838513fbb..21c8fa219 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -378,6 +378,13 @@ Q4 decode_external_tile(const uint8_t* data, size_t bytes, int N, int K, std::st Q4 decode_matmulnbits(const uint8_t* qweight, const float* scales, const uint8_t* zeros, int N, int K, int block, bool swap_nibbles = false, bool drop_zero = false); +// The implicit zero point 2^(bits-1) for a MatMulNBits with no zeros tensor, and the bit width its SPELLING implies +// (`weight_Q4` -> 4, `weight_Q8` -> 8, anything else -> the node's own value). Exposed because the pair is what can drift: +// the projection path used to pass a hardcoded 4 while the head path passed the node's bits, so a future `weight_Q8` +// spelling would have kept the lookup working and left the zero point at 8 instead of 128 (engine#1841). +void implicit_zp_zero(Q4& q, int bits); +int zp_bits_for_spelling(const std::string& spelling, int node_bits); + // One 64-row tile of the .fconst attention payload, as the pack stores it. Established byte-exactly // (amd-oss/npu-re docs/FACTS-oga-pack-weight-layout.md §7.13, §7.15): the tile is 2*(K/128)+2 subvolumes // of 2,176 B = one pair per (column half, K block of 128) plus the pair holding the writer's bias input. diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 019347612..331eaaa64 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -631,6 +631,31 @@ void test_canonical_copy_reusable() { // engine#1770: the canonical copy is scratch that used to only ever accumulate -- reuse writes a new file when the source // changes, so 699 files / 8.7 GiB of tmpfs (RAM) had built up on one box. The eviction rule is "same pack and geometry, // different source identity", and what can go wrong is deleting one of the neighbours that must survive, so each is an arm. +// engine#1841 (second point): the implicit zero point's bit width must follow the SPELLING that matched, not a literal at +// one call site and the node's attribute at the other. The arms below are the drift the issue described: `weight_Q4` must +// give 4 bits (zero point 8) and a hypothetical `weight_Q8` must give 8 (zero point 128) -- the number that would have +// stayed at 8, silently, if the literal were left in place. +void test_implicit_zp_bits_follow_the_spelling() { + check(pi::zp_bits_for_spelling("model.layers.0.attn.q_proj.MatMul.weight_Q4", 4) == 4, + "zp bits: weight_Q4 declares 4 bits"); + check(pi::zp_bits_for_spelling("model.layers.0.attn.q_proj.MatMul.weight_Q8", 4) == 8, + "zp bits: weight_Q8 overrides the path's 4 -- this is the arm a hardcoded literal fails"); + check(pi::zp_bits_for_spelling("model.layers.0.mlp.down_proj.MatMulNBits.qweight", 3) == 3, + "zp bits: a spelling with no _Q suffix falls back to the node's own value"); + check(pi::zp_bits_for_spelling("weight_Q4", 8) == 4, + "zp bits: the suffix wins over a node value for a spelled tensor"); + // and the effect on the tensor itself: the zero term moves by 2^(bits-1) * scale. + pi::Q4 q; + q.N = 1; q.K = 32; q.block = 128; + q.codes.assign(32, 0); q.scale.assign(1, 0.5f); q.zero.assign(1, 0.0f); + pi::implicit_zp_zero(q, 4); + check(std::fabs(q.zero[0] + 8.0f * 0.5f) < 1e-6f, "zp: 4 bits subtracts 8 * scale = 4.0"); + q.zero.assign(1, 0.0f); + pi::implicit_zp_zero(q, 8); + check(std::fabs(q.zero[0] + 128.0f * 0.5f) < 1e-6f, "zp: 8 bits subtracts 128 * scale = 64.0"); + std::printf(" implicit zero point: 6 arms (spelling -> bits x4, shift x2)\n"); +} + void test_evict_stale_canonical_copies() { const fs::path dir = fs::temp_directory_path() / ("1bit-evict-" + std::to_string(::getpid())); std::error_code ec; @@ -3390,6 +3415,7 @@ int main(int argc, char** argv) { test_external_container_names(); test_external_packed_genre(golden_dir.empty() ? "." : golden_dir); test_canonical_copy_reusable(); test_evict_stale_canonical_copies(); + test_implicit_zp_bits_follow_the_spelling(); test_crafted_refusals(golden_dir.empty() ? "." : golden_dir); test_modelbin_genre_refusal(golden_dir.empty() ? "." : golden_dir); test_1671_plan_head_arm(golden_dir.empty() ? "." : golden_dir);