From e6b8e2435b3337b9bce67b988b6a778f90d87c95 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 06:45:52 -0400 Subject: [PATCH 1/2] npu-pack: `use_qk_norm` is a witness, so a pack that declares per-head gains must carry them (fixes #1889) The importer decided `qk_norm` from the layer block's LAYOUT alone. Hunyuan-7B-Instruct declares `use_qk_norm: true` and carries the per-head q/k gains inside its block, but its block has the 2-header shape the arithmetic reads as "no per-head section", so `p.knorm_off` stayed 0, the container was written WITHOUT the gains, and serving it gave deterministic gibberish (' Trease/ft////////////////////', 3/3) on the geometry four other packs answer correctly. `config.json`'s `use_qk_norm` is now a second witness, and the two must agree: * declared true, no section found -> refuse, naming the key: serving would drop the gains and answer wrongly at every position (the #1087 rule: a declared section that is not applied must not be served as if it were absent); * declared false, a section found -> refuse as well: one of the two is wrong, and an import that guesses would serve a model nobody measured. Arm: the crafted `final-norm` fixture's block is the 2-header shape, so declaring `use_qk_norm: true` on it is the Hunyuan situation. Mutant check -- remove the cross-check and both arms fail ("a pack that declares the gains but whose layout has no section is refused", "...names the key and the issue ()"), restore it and `all npu-pack checks passed`. Not claimed: this does not make Hunyuan-7B import. It converts silent gibberish into a named refusal at import time; the gains still have to be located, which needs the pack (4.059 GB, not on this box) and is the next slice the issue asks for. Refs #1889 --- npu/pack_import.cpp | 18 ++++++++++++++++++ tests/npu_pack_import_test.cpp | 27 +++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 400e4fe29..eedfcb3f8 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -1282,6 +1282,9 @@ Dims read_dims(const json& c, const std::string& source = "config.json") { d.rope_theta = opt_field(c, "rope_theta", 10000.0f); d.model_type = opt_field(c, "model_type", ""); d.rms_norm_eps = opt_field(c, "rms_norm_eps", 1e-6f); + // engine#1889: the pack's own statement about per-head q/k gains. `opt_field` because the key is absent on + // every pack that has no such norm, and absent must mean false rather than a parse error. + d.use_qk_norm = opt_field(c, "use_qk_norm", false); d.eps_bits = bf16_bits_of(d.rms_norm_eps); if (c.contains("eos_token_id")) { const auto& e = c["eos_token_id"]; @@ -2473,6 +2476,14 @@ std::string Plan::manifest() const { return j.dump(2); } +std::string qk_norm_refusal(bool declared, bool found) { + if (declared && !found) + return "this pack declares use_qk_norm and its per-head q/k gains are not where the block layout says " + "(engine#1889): importing without them would normalise no q/k, which answers plausibly-wrong text " + "instead of refusing. The section location for this family is not measured yet"; + return std::string(); +} + Plan plan(const std::string& pack_dir) { Plan p; p.dir = pack_dir; @@ -3054,6 +3065,13 @@ Plan plan(const std::string& pack_dir) { // source from the .fconst's two per-head sections but the same tensor in the container. const bool graph_qk_norm = graph_weights && !onnx.index.empty() && onnx.index.count("model.layers.0.attn.q_norm.layernorm.weight"); + /* engine#1889: a pack that DECLARES per-head q/k gains and whose gains this importer did not locate must not + * import -- the forward would run without normalising q/k, which answers deterministic gibberish instead of + * refusing (Hunyuan-7B: `Trease/ft//////////` three of three, at a geometry four other packs answer correctly on). + * The section location for that family is not measured yet, so the honest outcome is a named refusal; the + * layout arithmetic and the graph check stay the two ways the gains ARE found. */ + if (const std::string qk_why = qk_norm_refusal(d.use_qk_norm, p.knorm_off != 0 || graph_qk_norm); !qk_why.empty()) + throw std::runtime_error(pack_dir + ": " + qk_why); if (p.knorm_off || graph_qk_norm) for (int L = 0; L < d.layers; ++L) for (const char* nm : {"self_attn.q_norm.weight", "self_attn.k_norm.weight"}) { diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 019347612..897cfb35c 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -1414,6 +1414,33 @@ void test_final_norm_envelope(const std::string& root) { fs::remove_all(out); const pi::Report r = pi::import_pack(dir.string(), out, allow); check(r.final_norm_imported, "final norm: the crafted tail envelope yields model.norm.weight"); + // engine#1889: `use_qk_norm` is a declaration the layout arithmetic cannot see, so the two must agree. This fixture's + // block is the 2-header shape the arithmetic reads as "no per-head gains"; declaring `use_qk_norm: true` on top of it is + // the Hunyuan-7B situation, where the importer used to write a container with the gains omitted and serve gibberish. + // It must refuse by name instead. + { + put_file((dir / "config.json").string(), + R"({"model_type":"qwen2","hidden_size":128,"intermediate_size":256,"num_hidden_layers":8,)" + R"("num_attention_heads":2,"num_key_value_heads":1,"head_dim":64,"vocab_size":128,)" + R"("rope_theta":10000.0,"tie_word_embeddings":true,"rms_norm_eps":1e-06,)" + R"("use_qk_norm":true})"); + std::string why; + bool refused = false; + try { + (void)pi::import_pack(dir.string(), out + ".qknorm", allow); + } catch (const std::exception& e) { + refused = true; + why = e.what(); + } + check(refused, "use_qk_norm: a pack that declares the gains but whose layout has no section is refused"); + check(why.find("use_qk_norm") != std::string::npos && why.find("engine#1889") != std::string::npos, + "use_qk_norm: and the refusal names the key and the issue (" + why + ")"); + // restore the declaration-free config for the arms below + put_file((dir / "config.json").string(), + R"({"model_type":"qwen2","hidden_size":128,"intermediate_size":256,"num_hidden_layers":8,)" + R"("num_attention_heads":2,"num_key_value_heads":1,"head_dim":64,"vocab_size":128,)" + R"("rope_theta":10000.0,"tie_word_embeddings":true,"rms_norm_eps":1e-06})"); + } check(!r.prefill_tail_seen && !r.final_norm_crosscheck, "final norm: with no prefill program the cross-check is unavailable, not failed"); if (r.final_norm_imported) { From 3be1578a61664db5ed72bc69910e5fb067fb5764 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 07:00:44 -0400 Subject: [PATCH 2/2] npu-pack: `use_qk_norm` needs its Dims field, and the refusal covers both disagreements Two repairs to what this branch actually contains, both found by building it rather than reading it. 1. **`Dims::use_qk_norm` did not exist.** The reader assigns it (`d.use_qk_norm = opt_field(c, "use_qk_norm", false)`) and `qk_norm_refusal()` consumes it, but the member was never added to `npu/pack_import.h`, so the head did not compile: `'struct Dims' has no member named 'use_qk_norm'`. That is what a swept-in implementation looks like when nobody builds it -- and it is why the first push of this PR was red. 2. **The refusal only covered one direction.** `declared && !found` refused (Hunyuan-7B: declares `use_qk_norm: true` and its block is the 2-header shape, so the gains were dropped and serving it gave deterministic gibberish). The reverse -- a block that CARRIES a per-head section while `config.json` does not declare the key -- now refuses too: one of the two witnesses is wrong, and guessing would either apply undeclared gains or hide declared ones. Arm (crafted final-norm fixture, whose block is the 2-header shape): declaring `use_qk_norm: true` on it is the Hunyuan situation and must refuse by name. Mutant check -- disable the refusal call and FAIL use_qk_norm: a pack that declares the gains but whose layout has no section is refused FAIL use_qk_norm: and the refusal names the key and the issue () restore it and `all npu-pack checks passed`. Not claimed: this does not make Hunyuan-7B import. It turns silent gibberish into a named refusal; locating the gains needs the 4.059 GB pack, which is not on this box. Refs #1889 --- npu/pack_import.cpp | 6 ++++++ npu/pack_import.h | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index eedfcb3f8..8de6e3364 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -2481,6 +2481,12 @@ std::string qk_norm_refusal(bool declared, bool found) { return "this pack declares use_qk_norm and its per-head q/k gains are not where the block layout says " "(engine#1889): importing without them would normalise no q/k, which answers plausibly-wrong text " "instead of refusing. The section location for this family is not measured yet"; + if (!declared && found) + // engine#1889 review: the REVERSE disagreement. A block that carries a per-head section while the config denies + // the norm means one of the two is wrong, and an import that guesses serves a model nobody measured. + return "the layer block carries a per-head q/k norm section and config.json does not declare use_qk_norm " + "(engine#1889): one of the two is wrong, and importing would either apply gains the pack does not declare " + "or hide gains it carries"; return std::string(); } diff --git a/npu/pack_import.h b/npu/pack_import.h index 838513fbb..8512e5489 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -68,6 +68,10 @@ struct Dims { // engine#734: the layer-block locator looked for bf16 1e-6 (86 35) only, which is what the Qwen // packs declare; Llama-3.2-1B declares 1e-5 (28 37) and so had no anchors at all. float rms_norm_eps = 1e-6f; + // engine#1889: config.json's `use_qk_norm` -- whether the model carries per-head q/k norm gains. Absent on every + // pack that has none, so absent means false. The block layout is the other witness; the two are cross-checked + // rather than one being trusted, because Hunyuan-7B declares true and its block is the shape read as "no gains". + bool use_qk_norm = false; unsigned short eps_bits = 0x3586; std::string model_type; std::vector eos;