Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions docs/npu.md
Original file line number Diff line number Diff line change
Expand Up @@ -1071,6 +1071,10 @@ what it can and records the rest in the container's `__metadata__` (`incomplete`
`config.json` (`q4nx_source.incomplete`). The shipped `Qwen2.5-0.5B-Instruct_rai_1.8.0_npu_4K`
now imports whole — **291 tensors, 67,088 tiles, no refusals**: embedding, head, the 48 layer
norms, the final norm, 96 attention projections, 72 attention biases and 72 MLP projections.
That holds for a pack whose layer projections are graph initializers (no `.fconst`) too: a
projection the decoder cannot read, for example codes present but no scales tensor beside them,
is refused by name like any other. It used to be skipped, so the import reported no refusals over
a container missing that projection (engine#1841).

### The final norm, and the tail envelope

Expand Down
34 changes: 26 additions & 8 deletions npu/pack_import.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3657,6 +3657,7 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons
// this is a loop, not a second reader. The one projection that needs arithmetic is the fused qkv: one
// [q|k|v] tensor that has to be split by head geometry, since only it is fused.
if (graph_weights_a) {
std::vector<std::string> proj_gaps;
auto init_of = [&](const std::string& n) -> const OnnxTensor* {
const auto it = onnx.index.find(n);
return it == onnx.index.end() ? nullptr : &onnx.tensors[it->second];
Expand All @@ -3674,14 +3675,24 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons
// (engine#1503).
std::string pre = proj;
if (!pre.empty() && pre.back() != '.') pre += '.';
// Every way out of here that writes nothing is a named gap (engine#1841), never a quiet skip.
auto gap = [&](const std::string& why) {
proj_gaps.push_back(out_name + ": `" + pre + "` " + why);
return false;
};
const OnnxTensor* qw = init_of(pre + "MatMulNBits.qweight");
const OnnxTensor* sc = init_of(pre + "MatMulNBits.scales");
if (!qw || !sc) {
qw = init_of(pre + "MatMul.weight_Q4");
sc = init_of(pre + "MatMul.weight_scales");
}
if (!qw || !sc) return false; // named, so the caller's refusal says which
if (!qw || !sc || qw->dims.size() != 3 || sc->dims.empty()) return false;
// plan() counts a projection present on its codes alone, so a pack missing only the scales passed it; returning
// quietly here then let the container report itself complete without the projection.
if (!qw || !sc)
return gap("has no quantised projection the decoder can pair (tried MatMulNBits.qweight + .scales and "
"MatMul.weight_Q4 + _scales)");
if (qw->dims.size() != 3 || sc->dims.empty())
return gap("qweight is not [N, K/block, block/2], or its scales have no shape");
const int N = int(qw->dims[0]);
const int block = int(qw->dims[2]) * 2; // 64 B holds 128 four-bit codes
const int K = k_override > 0 ? k_override : int(qw->dims[1]) * block;
Expand All @@ -3690,26 +3701,29 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons
// block the pack used, which is why a block-32 SmolLM-class pack tiles identically (pi-inferzo
// verified the recipe on amd/SmolLM2-135M-Instruct_rai_1.7.1_hybrid, corr +0.9957 against the
// pack's own tied embedding).
if (block <= 0 || N <= 0 || K <= 0 || K % block || row0 < 0 || row0 + rows > N) return false;
if (block <= 0 || N <= 0 || K <= 0 || K % block || row0 < 0 || row0 + rows > N)
return gap("has geometry N=" + std::to_string(N) + " K=" + std::to_string(K) + " block=" +
std::to_string(block) + " that does not hold rows [" + std::to_string(row0) + ", " +
std::to_string(row0 + rows) + ")");
const uint64_t groups = uint64_t(K) / uint64_t(block), nrows = uint64_t(N);
uint64_t groups_n = 0;
if (!mul_ok(nrows, groups, groups_n)) return false;
if (!mul_ok(nrows, groups, groups_n)) return gap("declares a size that overflows");
std::vector<float> scales(static_cast<std::vector<float>::size_type>(groups_n));
const uint64_t sw = sc->type == kF32 ? 4 : 2;
uint64_t s_need = 0;
if (!mul_ok(groups_n, sw, s_need)) return false;
if (!mul_ok(groups_n, sw, s_need)) return gap("declares a size that overflows");
if (sc->type == kF32) std::memcpy(scales.data(), bytes_of(*sc, s_need), scales.size() * 4);
else {
const auto b = as_bf16(bytes_of(*sc, s_need), sc->type, scales.size());
const auto* d = reinterpret_cast<const uint16_t*>(b.data());
for (size_t i = 0; i < scales.size(); ++i) scales[i] = bf16_to_f32(d[i]);
}
const OnnxTensor* zt = init_of(proj + ".MatMulNBits.qzeros");
const OnnxTensor* zt = init_of(pre + "MatMulNBits.qzeros");
uint64_t z_need = 0;
if (!mul_ok(nrows, (groups + 1) / 2, z_need)) return false;
if (!mul_ok(nrows, (groups + 1) / 2, z_need)) return gap("declares a size that overflows");
const uint8_t* zp = zt ? bytes_of(*zt, z_need) : nullptr;
uint64_t qw_need = 0;
if (!mul_ok(groups_n, uint64_t(block) / 2, qw_need)) return false;
if (!mul_ok(groups_n, uint64_t(block) / 2, qw_need)) return gap("declares a size that overflows");
Q4 q = decode_matmulnbits(bytes_of(*qw, qw_need), scales.data(), zp, N, K, block);
// No qzeros tensor means the implicit zero point 2^(bits-1) -- the same rule the head path applies, and the
// one ONNX Runtime's MatMulNBits uses. Missing it here decoded every PROJECTION with zero point 0 while the
Expand Down Expand Up @@ -3766,6 +3780,10 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons
decode_rows(a + "v_proj", 0, kv_rows, base + ".self_attn.v_proj.weight", 0);
}
}
if (!proj_gaps.empty()) {
for (auto& s : proj_gaps) r.refused.push_back(std::move(s));
if (!opt.allow_incomplete) return r; // refuse by name; the container is not written
}
}

for (const auto& mm : onnx.matmuls) {
Expand Down
105 changes: 105 additions & 0 deletions tests/npu_pack_import_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2335,6 +2335,110 @@ void test_three_input_matmulnbits(const std::string& root) {
fs::remove_all(dir);
}

// engine#1841: genre A (projections are graph initializers, no .fconst) decodes each layer projection by name, and a
// lookup that missed used to return quietly -- the container then reported itself complete with no layer projections
// (the #1790 CHANGES: `...o_projMatMul.weight_Q4`, a missing dot). The dot is normalised on main; this pins the other
// half: a projection the decoder cannot read is a NAMED refusal, not a skip. plan() refuses a projection whose codes are
// absent, but it counts one present on `<proj>.MatMul.weight_Q4` (or any `.MatMulNBits.` tensor) alone, while the
// decoder needs the scales beside it -- so a pack missing only the scales passed plan() and lost the projection without a
// word. One crafted 1-layer pack in Qwen3-VL-4B's three-input spelling, imported whole (every projection lands, nothing
// refused), then with down_proj's scales misnamed (refused by name; nothing written without --allow-incomplete).
void test_genre_a_projection_gap(const std::string& root) {
const fs::path dir = fs::path(root) / "genre-a-gap";
const int64_t H = 64, I = 64, V = 64, hd = 32, heads = 2, kv = 1, block = 32, groups = H / block;
auto build = [&](bool spell_down_wrong) {
fs::remove_all(dir);
fs::create_directories(dir);
std::string data;
std::vector<std::string> inits, nodes;
auto append = [&](const void* q, size_t n) {
data.append(static_cast<const char*>(q), n);
return uint64_t(data.size() - n);
};
std::vector<float> embed(size_t(V * H));
for (size_t i = 0; i < embed.size(); ++i) embed[i] = 0.01f * float(i % 89) - 0.4f;
const uint64_t eo = append(embed.data(), embed.size() * 4);
inits.push_back(pb::tensor("model.embed_tokens.weight", {V, H}, "model.onnx.data", eo, uint64_t(V * H) * 4));
std::vector<uint16_t> ones(size_t(H), 0x3C00); // f16 1.0
for (const char* nm : {"model.layers.0.input_layernorm.weight", "model.layers.0.post_attention_layernorm.weight",
"model.norm.weight"}) {
const uint64_t o = append(ones.data(), ones.size() * 2);
inits.push_back(pb::tensor(nm, {H}, "model.onnx.data", o, uint64_t(H) * 2, 10));
}
auto proj = [&](const std::string& pre, int64_t N, int64_t K, bool wrong) {
const int64_t g = K / block;
std::vector<uint8_t> qw(size_t(N * g * (block / 2)));
for (size_t i = 0; i < qw.size(); ++i) qw[i] = uint8_t((i * 7 + 3) & 0xFF);
std::vector<float> sc(size_t(N * g));
for (size_t i = 0; i < sc.size(); ++i) sc[i] = 0.01f * float(1 + i % 5);
// `wrong`: the codes keep their name and the scales sibling does not -- plan() counts a projection present
// on its weight_Q4 alone, so only the decoder can see that the pair is incomplete.
const std::string wq = pre + ".MatMul.weight_Q4";
const std::string ws = pre + (wrong ? ".MatMul.weight_scale" : ".MatMul.weight_scales");
const uint64_t qo = append(qw.data(), qw.size()), so = append(sc.data(), sc.size() * 4);
inits.push_back(pb::tensor(wq, {N, g, block / 2}, "model.onnx.data", qo, qw.size(), 2));
inits.push_back(pb::tensor(ws, {N, g}, "model.onnx.data", so, sc.size() * 4, 1));
nodes.push_back(pb::matmulnbits(pre, {"x", wq, ws}, K, N, 4, block));
};
const std::string a = "model.layers.0.attn.", m = "model.layers.0.mlp.";
proj(a + "q_proj", heads * hd, H, false);
proj(a + "k_proj", kv * hd, H, false);
proj(a + "v_proj", kv * hd, H, false);
proj(a + "o_proj", H, heads * hd, false);
proj(m + "gate_proj", I, H, false);
proj(m + "up_proj", I, H, false);
proj(m + "down_proj", H, I, spell_down_wrong);
proj("lm_head", V, H, false);
put_file((dir / "model.onnx.data").string(), data);
put_file((dir / "model.onnx").string(), pb::model(inits, nodes));
put_file((dir / "config.json").string(),
R"({"model_type":"qwen2","hidden_size":64,"intermediate_size":64,"num_hidden_layers":1,)"
R"("num_attention_heads":2,"num_key_value_heads":1,"head_dim":32,"vocab_size":64,)"
R"("rope_theta":10000.0,"tie_word_embeddings":false,"rms_norm_eps":1e-06})");
(void)groups;
};
const std::string out = (fs::temp_directory_path() / ("oga_pack_genre_a." + std::to_string(::getpid()))).string();
auto names_down = [](const pi::Report& r) {
for (const auto& s : r.refused)
if (s.find("mlp.down_proj.weight") != std::string::npos && s.find("no quantised projection the decoder can pair") != std::string::npos)
return true;
return false;
};

build(false);
fs::remove_all(out);
pi::Options allow;
allow.allow_incomplete = true;
pi::Report whole;
try {
whole = pi::import_pack(dir.string(), out, allow);
} catch (const std::exception& e) {
check(false, std::string("genre-A gap: the whole crafted pack imports (threw: ") + e.what() + ")");
fs::remove_all(dir);
return;
}
for (const auto& s : whole.refused) std::fprintf(stderr, " genre-A whole: refused %s\n", s.c_str());
bool proj_refused = false;
for (const auto& s : whole.refused) proj_refused |= s.find("no quantised projection the decoder can pair") != std::string::npos;
check(!proj_refused, "genre-A gap: with every projection present, none is refused as missing");
check(whole.tensors >= 7, "genre-A gap: the seven layer projections land in the container (" +
std::to_string(whole.tensors) + " tensors)");

build(true);
fs::remove_all(out);
const pi::Report gap = pi::import_pack(dir.string(), out, pi::Options{});
for (const auto& s : gap.refused) std::fprintf(stderr, " genre-A gap: refused %s\n", s.c_str());
check(names_down(gap), "genre-A gap: a projection plan() accepted but the decoder cannot pair is refused BY NAME (down_proj), not skipped");
check(!fs::exists(out + "/model.q4nx"), "genre-A gap: and without --allow-incomplete no container is written");
const pi::Report gap2 = pi::import_pack(dir.string(), out, allow);
check(names_down(gap2) && fs::exists(out + "/model.q4nx"),
"genre-A gap: --allow-incomplete writes the rest and still lists down_proj");
std::printf("genre-A gap: whole pack %zu refused / %d tensors; down_proj scales misnamed -> %zu refused, named %s\n",
whole.refused.size(), whole.tensors, gap.refused.size(), names_down(gap) ? "yes" : "NO");
fs::remove_all(out);
fs::remove_all(dir);
}

void test_embedding_component(const std::string& root) {
const fs::path dir = fs::path(root) / "embedcomp";
fs::remove_all(dir);
Expand Down Expand Up @@ -3274,6 +3378,7 @@ int main(int argc, char** argv) {
test_hybrid_pack_import(golden_dir.empty() ? "." : golden_dir);
test_rope_parameters_theta(golden_dir.empty() ? "." : golden_dir);
test_three_input_matmulnbits(golden_dir.empty() ? "." : golden_dir);
test_genre_a_projection_gap(golden_dir.empty() ? "." : golden_dir);
test_embedding_component(golden_dir.empty() ? "." : golden_dir);
test_fconst19_rule_table();
test_fconst_scratch_order();
Expand Down
Loading