Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 16 additions & 2 deletions npu/pack_import.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1545,6 +1545,20 @@ void implicit_zp_zero(Q4& q, int bits) {
for (size_t i = 0; i < q.zero.size(); ++i) q.zero[i] -= zp * q.scale[i];
}

// The bit width, taken from the SPELLING that matched (engine#1841, second point). The rule above used to be called with a
// literal `4` on the projection path and with the node's own `bits` on the head path; nothing tied the literal to the name,
// so a later spelling like `weight_Q8` would have kept the lookup working while leaving the zero point at 2^3 instead of 2^7
// -- a wrong weight with no error anywhere. Deriving it from the suffix makes the two agree by construction, and an
// unrecognised suffix falls back to the node's value (documented here) rather than to a silent 4.
int zp_bits_for_spelling(const std::string& spelling, int node_bits) {
const auto pos = spelling.rfind("_Q");
if (pos != std::string::npos && pos + 2 < spelling.size()) {
const char d = spelling[pos + 2];
if (d >= '2' && d <= '8') return d - '0';
}
return node_bits;
}

// ── MatMulNBits → (codes, scale, zero) per 32-column group ───────────────────
Q4 decode_matmulnbits(const uint8_t* qweight, const float* scales, const uint8_t* zeros,
int N, int K, int block, bool swap_nibbles, bool drop_zero) {
Expand Down Expand Up @@ -3775,7 +3789,7 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons
// one ONNX Runtime's MatMulNBits uses. Missing it here decoded every PROJECTION with zero point 0 while the
// head was decoded with 8, which is why the pack imported completely, served on the device, and answered
// gibberish on both the device and the host-GEMM path (engine#1503: "West什么呢ưởng中国联通nio…").
if (!zt) implicit_zp_zero(q, 4);
if (!zt) implicit_zp_zero(q, zp_bits_for_spelling(qw->name, 4));
// the row slice, re-tiled exactly as the whole tensor would be
Q4 part;
part.N = rows; part.K = K;
Expand Down Expand Up @@ -3901,7 +3915,7 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons
if (!mul_ok(groups_n, uint64_t(mm.block) / 2, qw_need)) throw std::runtime_error("lm_head: qweight length overflows");
Q4 q = decode_matmulnbits(bytes_of(wt, qw_need), scales.data(), zp, int(mm.N), int(mm.K), mm.block);
// no qzeros tensor -> the implicit zero point (engine#1503: Qwen3-VL-4B's spelling, three-input nodes)
if (!have_zp) implicit_zp_zero(q, mm.bits);
if (!have_zp) implicit_zp_zero(q, zp_bits_for_spelling(mm.weight, mm.bits));

std::vector<uint8_t> tiled;
long long pinexact = 0;
Expand Down
7 changes: 7 additions & 0 deletions npu/pack_import.h
Original file line number Diff line number Diff line change
Expand Up @@ -397,6 +397,13 @@ Q4 decode_external_tile(const uint8_t* data, size_t bytes, int N, int K, std::st
Q4 decode_matmulnbits(const uint8_t* qweight, const float* scales, const uint8_t* zeros,
int N, int K, int block, bool swap_nibbles = false, bool drop_zero = false);

// The implicit zero point 2^(bits-1) for a MatMulNBits with no zeros tensor, and the bit width its SPELLING implies
// (`weight_Q4` -> 4, `weight_Q8` -> 8, anything else -> the node's own value). Exposed because the pair is what can drift:
// the projection path used to pass a hardcoded 4 while the head path passed the node's bits, so a future `weight_Q8`
// spelling would have kept the lookup working and left the zero point at 8 instead of 128 (engine#1841).
void implicit_zp_zero(Q4& q, int bits);
int zp_bits_for_spelling(const std::string& spelling, int node_bits);

// One 64-row tile of the .fconst attention payload, as the pack stores it. Established byte-exactly
// (amd-oss/npu-re docs/FACTS-oga-pack-weight-layout.md §7.13, §7.15): the tile is 2*(K/128)+2 subvolumes
// of 2,176 B = one pair per (column half, K block of 128) plus the pair holding the writer's bias input.
Expand Down
26 changes: 26 additions & 0 deletions tests/npu_pack_import_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -631,6 +631,31 @@ void test_canonical_copy_reusable() {
// engine#1770: the canonical copy is scratch that used to only ever accumulate -- reuse writes a new file when the source
// changes, so 699 files / 8.7 GiB of tmpfs (RAM) had built up on one box. The eviction rule is "same pack and geometry,
// different source identity", and what can go wrong is deleting one of the neighbours that must survive, so each is an arm.
// engine#1841 (second point): the implicit zero point's bit width must follow the SPELLING that matched, not a literal at
// one call site and the node's attribute at the other. The arms below are the drift the issue described: `weight_Q4` must
// give 4 bits (zero point 8) and a hypothetical `weight_Q8` must give 8 (zero point 128) -- the number that would have
// stayed at 8, silently, if the literal were left in place.
void test_implicit_zp_bits_follow_the_spelling() {
check(pi::zp_bits_for_spelling("model.layers.0.attn.q_proj.MatMul.weight_Q4", 4) == 4,
"zp bits: weight_Q4 declares 4 bits");
check(pi::zp_bits_for_spelling("model.layers.0.attn.q_proj.MatMul.weight_Q8", 4) == 8,
"zp bits: weight_Q8 overrides the path's 4 -- this is the arm a hardcoded literal fails");
check(pi::zp_bits_for_spelling("model.layers.0.mlp.down_proj.MatMulNBits.qweight", 3) == 3,
"zp bits: a spelling with no _Q<d> suffix falls back to the node's own value");
check(pi::zp_bits_for_spelling("weight_Q4", 8) == 4,
"zp bits: the suffix wins over a node value for a spelled tensor");
// and the effect on the tensor itself: the zero term moves by 2^(bits-1) * scale.
pi::Q4 q;
q.N = 1; q.K = 32; q.block = 128;
q.codes.assign(32, 0); q.scale.assign(1, 0.5f); q.zero.assign(1, 0.0f);
pi::implicit_zp_zero(q, 4);
check(std::fabs(q.zero[0] + 8.0f * 0.5f) < 1e-6f, "zp: 4 bits subtracts 8 * scale = 4.0");
q.zero.assign(1, 0.0f);
pi::implicit_zp_zero(q, 8);
check(std::fabs(q.zero[0] + 128.0f * 0.5f) < 1e-6f, "zp: 8 bits subtracts 128 * scale = 64.0");
std::printf(" implicit zero point: 6 arms (spelling -> bits x4, shift x2)\n");
}

void test_evict_stale_canonical_copies() {
const fs::path dir = fs::temp_directory_path() / ("1bit-evict-" + std::to_string(::getpid()));
std::error_code ec;
Expand Down Expand Up @@ -3490,6 +3515,7 @@ int main(int argc, char** argv) {
test_external_container_names();
test_external_packed_genre(golden_dir.empty() ? "." : golden_dir);
test_canonical_copy_reusable(); test_evict_stale_canonical_copies();
test_implicit_zp_bits_follow_the_spelling();
test_crafted_refusals(golden_dir.empty() ? "." : golden_dir);
test_modelbin_genre_refusal(golden_dir.empty() ? "." : golden_dir);
test_1671_plan_head_arm(golden_dir.empty() ? "." : golden_dir);
Expand Down
Loading