From 89c7f4835ef92bc7f2c8f2dafbf17198e582ade7 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 03:42:13 -0400 Subject: [PATCH] npu-pack: a genre-A projection the decoder cannot read is refused by name, not skipped (engine#1841) plan() counts a graph-initializer projection present on its codes alone (`.MatMul.weight_Q4`, or any `.MatMulNBits.` tensor), while decode_rows() needs the scales beside them. A pack missing only the scales passed plan(), decode_rows() returned false, and every caller ignored it: the import reported no refusals and wrote a container without that projection. Every early exit of decode_rows() (no pair, bad shape, rows out of range, size overflow) is now a named entry in r.refused; without --allow-incomplete nothing is written. The qzeros lookup uses the same normalised prefix as the codes and scales. New arm test_genre_a_projection_gap: a crafted 1-layer three-input pack imports whole with no refusal, and with down_proj's scales misnamed is refused by name (no container) or listed under --allow-incomplete. With the pack_import.cpp change reverted, the arm fails 3 checks (0 refused, the container written). docs/npu.md says so under "Refusals are named". Co-Authored-By: Claude Opus 5.5 --- docs/npu.md | 4 ++ npu/pack_import.cpp | 34 ++++++++--- tests/npu_pack_import_test.cpp | 105 +++++++++++++++++++++++++++++++++ 3 files changed, 135 insertions(+), 8 deletions(-) diff --git a/docs/npu.md b/docs/npu.md index aedaaae7c..98d287a95 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1071,6 +1071,10 @@ what it can and records the rest in the container's `__metadata__` (`incomplete` `config.json` (`q4nx_source.incomplete`). The shipped `Qwen2.5-0.5B-Instruct_rai_1.8.0_npu_4K` now imports whole — **291 tensors, 67,088 tiles, no refusals**: embedding, head, the 48 layer norms, the final norm, 96 attention projections, 72 attention biases and 72 MLP projections. +That holds for a pack whose layer projections are graph initializers (no `.fconst`) too: a +projection the decoder cannot read, for example codes present but no scales tensor beside them, +is refused by name like any other. It used to be skipped, so the import reported no refusals over +a container missing that projection (engine#1841). ### The final norm, and the tail envelope diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 4ffb990d8..400e4fe29 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -3657,6 +3657,7 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons // this is a loop, not a second reader. The one projection that needs arithmetic is the fused qkv: one // [q|k|v] tensor that has to be split by head geometry, since only it is fused. if (graph_weights_a) { + std::vector proj_gaps; auto init_of = [&](const std::string& n) -> const OnnxTensor* { const auto it = onnx.index.find(n); return it == onnx.index.end() ? nullptr : &onnx.tensors[it->second]; @@ -3674,14 +3675,24 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons // (engine#1503). std::string pre = proj; if (!pre.empty() && pre.back() != '.') pre += '.'; + // Every way out of here that writes nothing is a named gap (engine#1841), never a quiet skip. + auto gap = [&](const std::string& why) { + proj_gaps.push_back(out_name + ": `" + pre + "` " + why); + return false; + }; const OnnxTensor* qw = init_of(pre + "MatMulNBits.qweight"); const OnnxTensor* sc = init_of(pre + "MatMulNBits.scales"); if (!qw || !sc) { qw = init_of(pre + "MatMul.weight_Q4"); sc = init_of(pre + "MatMul.weight_scales"); } - if (!qw || !sc) return false; // named, so the caller's refusal says which - if (!qw || !sc || qw->dims.size() != 3 || sc->dims.empty()) return false; + // plan() counts a projection present on its codes alone, so a pack missing only the scales passed it; returning + // quietly here then let the container report itself complete without the projection. + if (!qw || !sc) + return gap("has no quantised projection the decoder can pair (tried MatMulNBits.qweight + .scales and " + "MatMul.weight_Q4 + _scales)"); + if (qw->dims.size() != 3 || sc->dims.empty()) + return gap("qweight is not [N, K/block, block/2], or its scales have no shape"); const int N = int(qw->dims[0]); const int block = int(qw->dims[2]) * 2; // 64 B holds 128 four-bit codes const int K = k_override > 0 ? k_override : int(qw->dims[1]) * block; @@ -3690,26 +3701,29 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons // block the pack used, which is why a block-32 SmolLM-class pack tiles identically (pi-inferzo // verified the recipe on amd/SmolLM2-135M-Instruct_rai_1.7.1_hybrid, corr +0.9957 against the // pack's own tied embedding). - if (block <= 0 || N <= 0 || K <= 0 || K % block || row0 < 0 || row0 + rows > N) return false; + if (block <= 0 || N <= 0 || K <= 0 || K % block || row0 < 0 || row0 + rows > N) + return gap("has geometry N=" + std::to_string(N) + " K=" + std::to_string(K) + " block=" + + std::to_string(block) + " that does not hold rows [" + std::to_string(row0) + ", " + + std::to_string(row0 + rows) + ")"); const uint64_t groups = uint64_t(K) / uint64_t(block), nrows = uint64_t(N); uint64_t groups_n = 0; - if (!mul_ok(nrows, groups, groups_n)) return false; + if (!mul_ok(nrows, groups, groups_n)) return gap("declares a size that overflows"); std::vector scales(static_cast::size_type>(groups_n)); const uint64_t sw = sc->type == kF32 ? 4 : 2; uint64_t s_need = 0; - if (!mul_ok(groups_n, sw, s_need)) return false; + if (!mul_ok(groups_n, sw, s_need)) return gap("declares a size that overflows"); if (sc->type == kF32) std::memcpy(scales.data(), bytes_of(*sc, s_need), scales.size() * 4); else { const auto b = as_bf16(bytes_of(*sc, s_need), sc->type, scales.size()); const auto* d = reinterpret_cast(b.data()); for (size_t i = 0; i < scales.size(); ++i) scales[i] = bf16_to_f32(d[i]); } - const OnnxTensor* zt = init_of(proj + ".MatMulNBits.qzeros"); + const OnnxTensor* zt = init_of(pre + "MatMulNBits.qzeros"); uint64_t z_need = 0; - if (!mul_ok(nrows, (groups + 1) / 2, z_need)) return false; + if (!mul_ok(nrows, (groups + 1) / 2, z_need)) return gap("declares a size that overflows"); const uint8_t* zp = zt ? bytes_of(*zt, z_need) : nullptr; uint64_t qw_need = 0; - if (!mul_ok(groups_n, uint64_t(block) / 2, qw_need)) return false; + if (!mul_ok(groups_n, uint64_t(block) / 2, qw_need)) return gap("declares a size that overflows"); Q4 q = decode_matmulnbits(bytes_of(*qw, qw_need), scales.data(), zp, N, K, block); // No qzeros tensor means the implicit zero point 2^(bits-1) -- the same rule the head path applies, and the // one ONNX Runtime's MatMulNBits uses. Missing it here decoded every PROJECTION with zero point 0 while the @@ -3766,6 +3780,10 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons decode_rows(a + "v_proj", 0, kv_rows, base + ".self_attn.v_proj.weight", 0); } } + if (!proj_gaps.empty()) { + for (auto& s : proj_gaps) r.refused.push_back(std::move(s)); + if (!opt.allow_incomplete) return r; // refuse by name; the container is not written + } } for (const auto& mm : onnx.matmuls) { diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 25051340b..d93a71a27 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -2335,6 +2335,110 @@ void test_three_input_matmulnbits(const std::string& root) { fs::remove_all(dir); } +// engine#1841: genre A (projections are graph initializers, no .fconst) decodes each layer projection by name, and a +// lookup that missed used to return quietly -- the container then reported itself complete with no layer projections +// (the #1790 CHANGES: `...o_projMatMul.weight_Q4`, a missing dot). The dot is normalised on main; this pins the other +// half: a projection the decoder cannot read is a NAMED refusal, not a skip. plan() refuses a projection whose codes are +// absent, but it counts one present on `.MatMul.weight_Q4` (or any `.MatMulNBits.` tensor) alone, while the +// decoder needs the scales beside it -- so a pack missing only the scales passed plan() and lost the projection without a +// word. One crafted 1-layer pack in Qwen3-VL-4B's three-input spelling, imported whole (every projection lands, nothing +// refused), then with down_proj's scales misnamed (refused by name; nothing written without --allow-incomplete). +void test_genre_a_projection_gap(const std::string& root) { + const fs::path dir = fs::path(root) / "genre-a-gap"; + const int64_t H = 64, I = 64, V = 64, hd = 32, heads = 2, kv = 1, block = 32, groups = H / block; + auto build = [&](bool spell_down_wrong) { + fs::remove_all(dir); + fs::create_directories(dir); + std::string data; + std::vector inits, nodes; + auto append = [&](const void* q, size_t n) { + data.append(static_cast(q), n); + return uint64_t(data.size() - n); + }; + std::vector embed(size_t(V * H)); + for (size_t i = 0; i < embed.size(); ++i) embed[i] = 0.01f * float(i % 89) - 0.4f; + const uint64_t eo = append(embed.data(), embed.size() * 4); + inits.push_back(pb::tensor("model.embed_tokens.weight", {V, H}, "model.onnx.data", eo, uint64_t(V * H) * 4)); + std::vector ones(size_t(H), 0x3C00); // f16 1.0 + for (const char* nm : {"model.layers.0.input_layernorm.weight", "model.layers.0.post_attention_layernorm.weight", + "model.norm.weight"}) { + const uint64_t o = append(ones.data(), ones.size() * 2); + inits.push_back(pb::tensor(nm, {H}, "model.onnx.data", o, uint64_t(H) * 2, 10)); + } + auto proj = [&](const std::string& pre, int64_t N, int64_t K, bool wrong) { + const int64_t g = K / block; + std::vector qw(size_t(N * g * (block / 2))); + for (size_t i = 0; i < qw.size(); ++i) qw[i] = uint8_t((i * 7 + 3) & 0xFF); + std::vector sc(size_t(N * g)); + for (size_t i = 0; i < sc.size(); ++i) sc[i] = 0.01f * float(1 + i % 5); + // `wrong`: the codes keep their name and the scales sibling does not -- plan() counts a projection present + // on its weight_Q4 alone, so only the decoder can see that the pair is incomplete. + const std::string wq = pre + ".MatMul.weight_Q4"; + const std::string ws = pre + (wrong ? ".MatMul.weight_scale" : ".MatMul.weight_scales"); + const uint64_t qo = append(qw.data(), qw.size()), so = append(sc.data(), sc.size() * 4); + inits.push_back(pb::tensor(wq, {N, g, block / 2}, "model.onnx.data", qo, qw.size(), 2)); + inits.push_back(pb::tensor(ws, {N, g}, "model.onnx.data", so, sc.size() * 4, 1)); + nodes.push_back(pb::matmulnbits(pre, {"x", wq, ws}, K, N, 4, block)); + }; + const std::string a = "model.layers.0.attn.", m = "model.layers.0.mlp."; + proj(a + "q_proj", heads * hd, H, false); + proj(a + "k_proj", kv * hd, H, false); + proj(a + "v_proj", kv * hd, H, false); + proj(a + "o_proj", H, heads * hd, false); + proj(m + "gate_proj", I, H, false); + proj(m + "up_proj", I, H, false); + proj(m + "down_proj", H, I, spell_down_wrong); + proj("lm_head", V, H, false); + put_file((dir / "model.onnx.data").string(), data); + put_file((dir / "model.onnx").string(), pb::model(inits, nodes)); + put_file((dir / "config.json").string(), + R"({"model_type":"qwen2","hidden_size":64,"intermediate_size":64,"num_hidden_layers":1,)" + R"("num_attention_heads":2,"num_key_value_heads":1,"head_dim":32,"vocab_size":64,)" + R"("rope_theta":10000.0,"tie_word_embeddings":false,"rms_norm_eps":1e-06})"); + (void)groups; + }; + const std::string out = (fs::temp_directory_path() / ("oga_pack_genre_a." + std::to_string(::getpid()))).string(); + auto names_down = [](const pi::Report& r) { + for (const auto& s : r.refused) + if (s.find("mlp.down_proj.weight") != std::string::npos && s.find("no quantised projection the decoder can pair") != std::string::npos) + return true; + return false; + }; + + build(false); + fs::remove_all(out); + pi::Options allow; + allow.allow_incomplete = true; + pi::Report whole; + try { + whole = pi::import_pack(dir.string(), out, allow); + } catch (const std::exception& e) { + check(false, std::string("genre-A gap: the whole crafted pack imports (threw: ") + e.what() + ")"); + fs::remove_all(dir); + return; + } + for (const auto& s : whole.refused) std::fprintf(stderr, " genre-A whole: refused %s\n", s.c_str()); + bool proj_refused = false; + for (const auto& s : whole.refused) proj_refused |= s.find("no quantised projection the decoder can pair") != std::string::npos; + check(!proj_refused, "genre-A gap: with every projection present, none is refused as missing"); + check(whole.tensors >= 7, "genre-A gap: the seven layer projections land in the container (" + + std::to_string(whole.tensors) + " tensors)"); + + build(true); + fs::remove_all(out); + const pi::Report gap = pi::import_pack(dir.string(), out, pi::Options{}); + for (const auto& s : gap.refused) std::fprintf(stderr, " genre-A gap: refused %s\n", s.c_str()); + check(names_down(gap), "genre-A gap: a projection plan() accepted but the decoder cannot pair is refused BY NAME (down_proj), not skipped"); + check(!fs::exists(out + "/model.q4nx"), "genre-A gap: and without --allow-incomplete no container is written"); + const pi::Report gap2 = pi::import_pack(dir.string(), out, allow); + check(names_down(gap2) && fs::exists(out + "/model.q4nx"), + "genre-A gap: --allow-incomplete writes the rest and still lists down_proj"); + std::printf("genre-A gap: whole pack %zu refused / %d tensors; down_proj scales misnamed -> %zu refused, named %s\n", + whole.refused.size(), whole.tensors, gap.refused.size(), names_down(gap) ? "yes" : "NO"); + fs::remove_all(out); + fs::remove_all(dir); +} + void test_embedding_component(const std::string& root) { const fs::path dir = fs::path(root) / "embedcomp"; fs::remove_all(dir); @@ -3274,6 +3378,7 @@ int main(int argc, char** argv) { test_hybrid_pack_import(golden_dir.empty() ? "." : golden_dir); test_rope_parameters_theta(golden_dir.empty() ? "." : golden_dir); test_three_input_matmulnbits(golden_dir.empty() ? "." : golden_dir); + test_genre_a_projection_gap(golden_dir.empty() ? "." : golden_dir); test_embedding_component(golden_dir.empty() ? "." : golden_dir); test_fconst19_rule_table(); test_fconst_scratch_order();