diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 7243c0898..4ffb990d8 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -1168,6 +1168,22 @@ json load_geometry(const std::string& pack_dir, std::string* source) { ++from_text_config; } } + // `rope_theta` (and the rotary block's other keys) can be ONE LEVEL DEEPER still: Qwen3-VL-4B declares + // `text_config.rope_parameters = {mrope_interleaved: true, mrope_section: [24,20,20], rope_theta: 5000000, + // rope_type: default}` with no `rope_theta` anywhere else, so this reader fell back to its 10000 default and the + // importer warned that the pack's cos/sin caches "no plain theta explains (best 0.5918 at 3.2e+06, declared 10000)". + // Measured against the pack's own cos_cache (262144 x 64, external): **theta 5e6 reproduces row 1 to max|d| + // 0.000000**, 10000 is off by 0.088 and 3.2e6 by 0.005 -- so the caches are not unexplained at all, the theta was + // simply not read (engine#1503). Gaps only, never overriding, same rule as the lift above. + int from_rope_params = 0; + if (cfg.contains("rope_parameters") && cfg["rope_parameters"].is_object()) { + for (auto it = cfg["rope_parameters"].begin(); it != cfg["rope_parameters"].end(); ++it) { + if (cfg.contains(it.key()) || it.value().is_null()) continue; + cfg[it.key()] = it.value(); + ++from_rope_params; + } + } + if (from_rope_params > 0 && source) *source += " + config.json rope_parameters"; if (from_text_config > 0 && source) *source += " + config.json text_config"; if (fill_from_model_config_impl(pack_dir, cfg) > 0 && source) *source += " + model_config.json (text_config)"; return cfg; @@ -3345,6 +3361,17 @@ Report import_external(const std::string& pack_dir, const std::string& out_dir, if (const std::string why = weight_layout_refusal(opt.weight_layout); !why.empty()) throw std::runtime_error("--weight-layout: " + why); json c; + // engine#1794 review: the container's OWN config is what the forward reads (`q4nx_semantics.c:121` value_for_top, + // depth 1 only), and this writer started from an empty json -- so a mRoPE declaration that lives only at + // `text_config.rope_parameters` was dropped, and the pack would be served as plain RoPE with the right theta and the + // wrong axis layout (engine#1311: exact for a text-only prompt, ~0.29 rad off once image tokens are in it). The + // import_pack() writer starts from load_geometry() and already carries it; this one has to copy it. + { + std::string geom_src; + const json geom = load_geometry(pack_dir, &geom_src); + if (geom.value("mrope_interleaved", false)) c["mrope_interleaved"] = true; + if (geom.contains("mrope_section")) c["mrope_section"] = geom["mrope_section"]; + } c["model_type"] = p.dims.model_type; c["hidden_size"] = p.dims.hidden; c["intermediate_size"] = p.dims.intermediate; @@ -3640,12 +3667,20 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons // The two spellings of a quantised projection (engine#1503): the 1.8/phi-4 form carries a qweight and a // scales sibling; Qwen3-VL-4B names them `.MatMul.weight_Q4` / `.MatMul.weight_scales` on a // three-input node with no qzeros (implicit zero point, applied below). - const OnnxTensor* qw = init_of(proj + ".MatMulNBits.qweight"); - const OnnxTensor* sc = init_of(proj + ".MatMulNBits.scales"); + // `proj` arrives from a plan source string (`graph:model.layers.N.attn.q_proj`, no trailing dot) as well as + // from a role name (`...q_proj.`, with one), so normalise before appending a suffix -- otherwise the + // three-input spelling is looked up as `...q_projMatMul.weight_Q4` and never found, which is how a pack could + // plan all 252 projections, write only its norms, and fail on the device with `MLP dequant : data_size 0` + // (engine#1503). + std::string pre = proj; + if (!pre.empty() && pre.back() != '.') pre += '.'; + const OnnxTensor* qw = init_of(pre + "MatMulNBits.qweight"); + const OnnxTensor* sc = init_of(pre + "MatMulNBits.scales"); if (!qw || !sc) { - qw = init_of(proj + "MatMul.weight_Q4"); - sc = init_of(proj + "MatMul.weight_scales"); + qw = init_of(pre + "MatMul.weight_Q4"); + sc = init_of(pre + "MatMul.weight_scales"); } + if (!qw || !sc) return false; // named, so the caller's refusal says which if (!qw || !sc || qw->dims.size() != 3 || sc->dims.empty()) return false; const int N = int(qw->dims[0]); const int block = int(qw->dims[2]) * 2; // 64 B holds 128 four-bit codes @@ -3675,7 +3710,12 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons const uint8_t* zp = zt ? bytes_of(*zt, z_need) : nullptr; uint64_t qw_need = 0; if (!mul_ok(groups_n, uint64_t(block) / 2, qw_need)) return false; - const Q4 q = decode_matmulnbits(bytes_of(*qw, qw_need), scales.data(), zp, N, K, block); + Q4 q = decode_matmulnbits(bytes_of(*qw, qw_need), scales.data(), zp, N, K, block); + // No qzeros tensor means the implicit zero point 2^(bits-1) -- the same rule the head path applies, and the + // one ONNX Runtime's MatMulNBits uses. Missing it here decoded every PROJECTION with zero point 0 while the + // head was decoded with 8, which is why the pack imported completely, served on the device, and answered + // gibberish on both the device and the host-GEMM path (engine#1503: "West什么呢ưởng中国联通nio…"). + if (!zt) implicit_zp_zero(q, 4); // the row slice, re-tiled exactly as the whole tensor would be Q4 part; part.N = rows; part.K = K; diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 8afb5cd79..25051340b 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -2276,6 +2276,37 @@ void test_text_config_geometry(const std::string& root) { // pack: `info` printed matmulnbits_ops 0 and the import wrote the embedding only ("imported 1 tensors, 0 q4_1 tiles"). // On the real pack the fix yields 147 tensors / 47,480 tiles and the pack's own self-check reads corr 0.993981 against its // embedding, with the no-zero-point control at 0.3333 -- this arm pins the mechanism without the 3 GB pack. +// engine#1503: `rope_theta` nested one level deeper still — `text_config.rope_parameters.rope_theta` — which is how +// Qwen3-VL-4B declares 5,000,000 with no `rope_theta` anywhere else. The reader's 10000 default made the importer warn that +// the pack's cos/sin caches "no plain theta explains"; measured against the pack's own cos_cache (262144 x 64), theta 5e6 +// reproduces row 1 to max|d| 0.000000 while 10000 is off by 0.088. This pins the lift and the never-override rule. +void test_rope_parameters_theta(const std::string& root) { + const fs::path dir = fs::path(root) / "ropeparams"; + fs::remove_all(dir); + fs::create_directories(dir); + put_file((dir / "config.json").string(), + R"({"model_type":"qwen3_vl","text_config":{"hidden_size":64,"num_hidden_layers":1,)" + R"("num_attention_heads":2,"num_key_value_heads":1,"head_dim":32,"intermediate_size":128,)" + R"("vocab_size":32,"rope_parameters":{"mrope_interleaved":true,"mrope_section":[24,20,20],)" + R"("rope_theta":5000000,"rope_type":"default"}}})"); + std::string src; + const nlohmann::json g = pi::load_geometry_exported(dir.string(), &src); + const pi::Dims d = pi::read_dims_exported(g, src); + check(d.rope_theta == 5000000.0f, + "rope_parameters: rope_theta comes from the nested block (" + std::to_string(d.rope_theta) + ")"); + check(src.find("rope_parameters") != std::string::npos, "rope_parameters: and the source says so (" + src + ")"); + // a top-level rope_theta still wins + put_file((dir / "config.json").string(), + R"({"model_type":"qwen3_vl","rope_theta":10000.0,"text_config":{"hidden_size":64,)" + R"("num_hidden_layers":1,"num_attention_heads":2,"head_dim":32,"intermediate_size":128,"vocab_size":32,)" + R"("rope_parameters":{"rope_theta":5000000}}})"); + const nlohmann::json g2 = pi::load_geometry_exported(dir.string(), &src); + check(pi::read_dims_exported(g2, src).rope_theta == 10000.0f, + "rope_parameters: a top-level rope_theta is NOT overridden by the nested one"); + std::printf(" rope_parameters: theta %.0f nested, top-level still wins\n", (double)d.rope_theta); + fs::remove_all(dir); +} + void test_three_input_matmulnbits(const std::string& root) { const fs::path dir = fs::path(root) / "mm3"; fs::remove_all(dir); @@ -3241,6 +3272,7 @@ int main(int argc, char** argv) { test_attn_payload_off(); test_text_config_geometry(golden_dir.empty() ? "." : golden_dir); test_hybrid_pack_import(golden_dir.empty() ? "." : golden_dir); + test_rope_parameters_theta(golden_dir.empty() ? "." : golden_dir); test_three_input_matmulnbits(golden_dir.empty() ? "." : golden_dir); test_embedding_component(golden_dir.empty() ? "." : golden_dir); test_fconst19_rule_table();