Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 45 additions & 5 deletions npu/pack_import.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1168,6 +1168,22 @@ json load_geometry(const std::string& pack_dir, std::string* source) {
++from_text_config;
}
}
// `rope_theta` (and the rotary block's other keys) can be ONE LEVEL DEEPER still: Qwen3-VL-4B declares
// `text_config.rope_parameters = {mrope_interleaved: true, mrope_section: [24,20,20], rope_theta: 5000000,
// rope_type: default}` with no `rope_theta` anywhere else, so this reader fell back to its 10000 default and the
// importer warned that the pack's cos/sin caches "no plain theta explains (best 0.5918 at 3.2e+06, declared 10000)".
// Measured against the pack's own cos_cache (262144 x 64, external): **theta 5e6 reproduces row 1 to max|d|
// 0.000000**, 10000 is off by 0.088 and 3.2e6 by 0.005 -- so the caches are not unexplained at all, the theta was
// simply not read (engine#1503). Gaps only, never overriding, same rule as the lift above.
int from_rope_params = 0;
if (cfg.contains("rope_parameters") && cfg["rope_parameters"].is_object()) {
for (auto it = cfg["rope_parameters"].begin(); it != cfg["rope_parameters"].end(); ++it) {
if (cfg.contains(it.key()) || it.value().is_null()) continue;
cfg[it.key()] = it.value();
++from_rope_params;
}
}
if (from_rope_params > 0 && source) *source += " + config.json rope_parameters";
if (from_text_config > 0 && source) *source += " + config.json text_config";
if (fill_from_model_config_impl(pack_dir, cfg) > 0 && source) *source += " + model_config.json (text_config)";
return cfg;
Expand Down Expand Up @@ -3345,6 +3361,17 @@ Report import_external(const std::string& pack_dir, const std::string& out_dir,
if (const std::string why = weight_layout_refusal(opt.weight_layout); !why.empty())
throw std::runtime_error("--weight-layout: " + why);
json c;
// engine#1794 review: the container's OWN config is what the forward reads (`q4nx_semantics.c:121` value_for_top,
// depth 1 only), and this writer started from an empty json -- so a mRoPE declaration that lives only at
// `text_config.rope_parameters` was dropped, and the pack would be served as plain RoPE with the right theta and the
// wrong axis layout (engine#1311: exact for a text-only prompt, ~0.29 rad off once image tokens are in it). The
// import_pack() writer starts from load_geometry() and already carries it; this one has to copy it.
{
std::string geom_src;
const json geom = load_geometry(pack_dir, &geom_src);
if (geom.value("mrope_interleaved", false)) c["mrope_interleaved"] = true;
if (geom.contains("mrope_section")) c["mrope_section"] = geom["mrope_section"];
}
c["model_type"] = p.dims.model_type;
c["hidden_size"] = p.dims.hidden;
c["intermediate_size"] = p.dims.intermediate;
Expand Down Expand Up @@ -3640,12 +3667,20 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons
// The two spellings of a quantised projection (engine#1503): the 1.8/phi-4 form carries a qweight and a
// scales sibling; Qwen3-VL-4B names them `<proj>.MatMul.weight_Q4` / `<proj>.MatMul.weight_scales` on a
// three-input node with no qzeros (implicit zero point, applied below).
const OnnxTensor* qw = init_of(proj + ".MatMulNBits.qweight");
const OnnxTensor* sc = init_of(proj + ".MatMulNBits.scales");
// `proj` arrives from a plan source string (`graph:model.layers.N.attn.q_proj`, no trailing dot) as well as
// from a role name (`...q_proj.`, with one), so normalise before appending a suffix -- otherwise the
// three-input spelling is looked up as `...q_projMatMul.weight_Q4` and never found, which is how a pack could
// plan all 252 projections, write only its norms, and fail on the device with `MLP dequant : data_size 0`
// (engine#1503).
std::string pre = proj;
if (!pre.empty() && pre.back() != '.') pre += '.';
const OnnxTensor* qw = init_of(pre + "MatMulNBits.qweight");
const OnnxTensor* sc = init_of(pre + "MatMulNBits.scales");
if (!qw || !sc) {
qw = init_of(proj + "MatMul.weight_Q4");
sc = init_of(proj + "MatMul.weight_scales");
qw = init_of(pre + "MatMul.weight_Q4");
sc = init_of(pre + "MatMul.weight_scales");
}
if (!qw || !sc) return false; // named, so the caller's refusal says which
if (!qw || !sc || qw->dims.size() != 3 || sc->dims.empty()) return false;
const int N = int(qw->dims[0]);
const int block = int(qw->dims[2]) * 2; // 64 B holds 128 four-bit codes
Expand Down Expand Up @@ -3675,7 +3710,12 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons
const uint8_t* zp = zt ? bytes_of(*zt, z_need) : nullptr;
uint64_t qw_need = 0;
if (!mul_ok(groups_n, uint64_t(block) / 2, qw_need)) return false;
const Q4 q = decode_matmulnbits(bytes_of(*qw, qw_need), scales.data(), zp, N, K, block);
Q4 q = decode_matmulnbits(bytes_of(*qw, qw_need), scales.data(), zp, N, K, block);
// No qzeros tensor means the implicit zero point 2^(bits-1) -- the same rule the head path applies, and the
// one ONNX Runtime's MatMulNBits uses. Missing it here decoded every PROJECTION with zero point 0 while the
// head was decoded with 8, which is why the pack imported completely, served on the device, and answered
// gibberish on both the device and the host-GEMM path (engine#1503: "West什么呢ưởng中国联通nio…").
if (!zt) implicit_zp_zero(q, 4);
// the row slice, re-tiled exactly as the whole tensor would be
Q4 part;
part.N = rows; part.K = K;
Expand Down
32 changes: 32 additions & 0 deletions tests/npu_pack_import_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2276,6 +2276,37 @@ void test_text_config_geometry(const std::string& root) {
// pack: `info` printed matmulnbits_ops 0 and the import wrote the embedding only ("imported 1 tensors, 0 q4_1 tiles").
// On the real pack the fix yields 147 tensors / 47,480 tiles and the pack's own self-check reads corr 0.993981 against its
// embedding, with the no-zero-point control at 0.3333 -- this arm pins the mechanism without the 3 GB pack.
// engine#1503: `rope_theta` nested one level deeper still — `text_config.rope_parameters.rope_theta` — which is how
// Qwen3-VL-4B declares 5,000,000 with no `rope_theta` anywhere else. The reader's 10000 default made the importer warn that
// the pack's cos/sin caches "no plain theta explains"; measured against the pack's own cos_cache (262144 x 64), theta 5e6
// reproduces row 1 to max|d| 0.000000 while 10000 is off by 0.088. This pins the lift and the never-override rule.
void test_rope_parameters_theta(const std::string& root) {
const fs::path dir = fs::path(root) / "ropeparams";
fs::remove_all(dir);
fs::create_directories(dir);
put_file((dir / "config.json").string(),
R"({"model_type":"qwen3_vl","text_config":{"hidden_size":64,"num_hidden_layers":1,)"
R"("num_attention_heads":2,"num_key_value_heads":1,"head_dim":32,"intermediate_size":128,)"
R"("vocab_size":32,"rope_parameters":{"mrope_interleaved":true,"mrope_section":[24,20,20],)"
R"("rope_theta":5000000,"rope_type":"default"}}})");
std::string src;
const nlohmann::json g = pi::load_geometry_exported(dir.string(), &src);
const pi::Dims d = pi::read_dims_exported(g, src);
check(d.rope_theta == 5000000.0f,
"rope_parameters: rope_theta comes from the nested block (" + std::to_string(d.rope_theta) + ")");
check(src.find("rope_parameters") != std::string::npos, "rope_parameters: and the source says so (" + src + ")");
// a top-level rope_theta still wins
put_file((dir / "config.json").string(),
R"({"model_type":"qwen3_vl","rope_theta":10000.0,"text_config":{"hidden_size":64,)"
R"("num_hidden_layers":1,"num_attention_heads":2,"head_dim":32,"intermediate_size":128,"vocab_size":32,)"
R"("rope_parameters":{"rope_theta":5000000}}})");
const nlohmann::json g2 = pi::load_geometry_exported(dir.string(), &src);
check(pi::read_dims_exported(g2, src).rope_theta == 10000.0f,
"rope_parameters: a top-level rope_theta is NOT overridden by the nested one");
std::printf(" rope_parameters: theta %.0f nested, top-level still wins\n", (double)d.rope_theta);
fs::remove_all(dir);
}

void test_three_input_matmulnbits(const std::string& root) {
const fs::path dir = fs::path(root) / "mm3";
fs::remove_all(dir);
Expand Down Expand Up @@ -3241,6 +3272,7 @@ int main(int argc, char** argv) {
test_attn_payload_off();
test_text_config_geometry(golden_dir.empty() ? "." : golden_dir);
test_hybrid_pack_import(golden_dir.empty() ? "." : golden_dir);
test_rope_parameters_theta(golden_dir.empty() ? "." : golden_dir);
test_three_input_matmulnbits(golden_dir.empty() ? "." : golden_dir);
test_embedding_component(golden_dir.empty() ? "." : golden_dir);
test_fconst19_rule_table();
Expand Down
Loading