Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions npu/pack_import.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1282,6 +1282,9 @@ Dims read_dims(const json& c, const std::string& source = "config.json") {
d.rope_theta = opt_field<float>(c, "rope_theta", 10000.0f);
d.model_type = opt_field<std::string>(c, "model_type", "");
d.rms_norm_eps = opt_field<float>(c, "rms_norm_eps", 1e-6f);
// engine#1889: the pack's own statement about per-head q/k gains. `opt_field<bool>` because the key is absent on
// every pack that has no such norm, and absent must mean false rather than a parse error.
d.use_qk_norm = opt_field<bool>(c, "use_qk_norm", false);
d.eps_bits = bf16_bits_of(d.rms_norm_eps);
if (c.contains("eos_token_id")) {
const auto& e = c["eos_token_id"];
Expand Down Expand Up @@ -2473,6 +2476,20 @@ std::string Plan::manifest() const {
return j.dump(2);
}

std::string qk_norm_refusal(bool declared, bool found) {
if (declared && !found)
return "this pack declares use_qk_norm and its per-head q/k gains are not where the block layout says "
"(engine#1889): importing without them would normalise no q/k, which answers plausibly-wrong text "
"instead of refusing. The section location for this family is not measured yet";
if (!declared && found)
// engine#1889 review: the REVERSE disagreement. A block that carries a per-head section while the config denies
// the norm means one of the two is wrong, and an import that guesses serves a model nobody measured.
return "the layer block carries a per-head q/k norm section and config.json does not declare use_qk_norm "
"(engine#1889): one of the two is wrong, and importing would either apply gains the pack does not declare "
"or hide gains it carries";
return std::string();
}

Plan plan(const std::string& pack_dir) {
Plan p;
p.dir = pack_dir;
Expand Down Expand Up @@ -3054,6 +3071,13 @@ Plan plan(const std::string& pack_dir) {
// source from the .fconst's two per-head sections but the same tensor in the container.
const bool graph_qk_norm = graph_weights && !onnx.index.empty() &&
onnx.index.count("model.layers.0.attn.q_norm.layernorm.weight");
/* engine#1889: a pack that DECLARES per-head q/k gains and whose gains this importer did not locate must not
* import -- the forward would run without normalising q/k, which answers deterministic gibberish instead of
* refusing (Hunyuan-7B: `Trease/ft//////////` three of three, at a geometry four other packs answer correctly on).
* The section location for that family is not measured yet, so the honest outcome is a named refusal; the
* layout arithmetic and the graph check stay the two ways the gains ARE found. */
if (const std::string qk_why = qk_norm_refusal(d.use_qk_norm, p.knorm_off != 0 || graph_qk_norm); !qk_why.empty())
throw std::runtime_error(pack_dir + ": " + qk_why);
if (p.knorm_off || graph_qk_norm)
for (int L = 0; L < d.layers; ++L)
for (const char* nm : {"self_attn.q_norm.weight", "self_attn.k_norm.weight"}) {
Expand Down
4 changes: 4 additions & 0 deletions npu/pack_import.h
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,10 @@ struct Dims {
// engine#734: the layer-block locator looked for bf16 1e-6 (86 35) only, which is what the Qwen
// packs declare; Llama-3.2-1B declares 1e-5 (28 37) and so had no anchors at all.
float rms_norm_eps = 1e-6f;
// engine#1889: config.json's `use_qk_norm` -- whether the model carries per-head q/k norm gains. Absent on every
// pack that has none, so absent means false. The block layout is the other witness; the two are cross-checked
// rather than one being trusted, because Hunyuan-7B declares true and its block is the shape read as "no gains".
bool use_qk_norm = false;
unsigned short eps_bits = 0x3586;
std::string model_type;
std::vector<int> eos;
Expand Down
27 changes: 27 additions & 0 deletions tests/npu_pack_import_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1414,6 +1414,33 @@ void test_final_norm_envelope(const std::string& root) {
fs::remove_all(out);
const pi::Report r = pi::import_pack(dir.string(), out, allow);
check(r.final_norm_imported, "final norm: the crafted tail envelope yields model.norm.weight");
// engine#1889: `use_qk_norm` is a declaration the layout arithmetic cannot see, so the two must agree. This fixture's
// block is the 2-header shape the arithmetic reads as "no per-head gains"; declaring `use_qk_norm: true` on top of it is
// the Hunyuan-7B situation, where the importer used to write a container with the gains omitted and serve gibberish.
// It must refuse by name instead.
{
put_file((dir / "config.json").string(),
R"({"model_type":"qwen2","hidden_size":128,"intermediate_size":256,"num_hidden_layers":8,)"
R"("num_attention_heads":2,"num_key_value_heads":1,"head_dim":64,"vocab_size":128,)"
R"("rope_theta":10000.0,"tie_word_embeddings":true,"rms_norm_eps":1e-06,)"
R"("use_qk_norm":true})");
std::string why;
bool refused = false;
try {
(void)pi::import_pack(dir.string(), out + ".qknorm", allow);
} catch (const std::exception& e) {
refused = true;
why = e.what();
}
check(refused, "use_qk_norm: a pack that declares the gains but whose layout has no section is refused");
check(why.find("use_qk_norm") != std::string::npos && why.find("engine#1889") != std::string::npos,
"use_qk_norm: and the refusal names the key and the issue (" + why + ")");
// restore the declaration-free config for the arms below
put_file((dir / "config.json").string(),
R"({"model_type":"qwen2","hidden_size":128,"intermediate_size":256,"num_hidden_layers":8,)"
R"("num_attention_heads":2,"num_key_value_heads":1,"head_dim":64,"vocab_size":128,)"
R"("rope_theta":10000.0,"tie_word_embeddings":true,"rms_norm_eps":1e-06})");
}
check(!r.prefill_tail_seen && !r.final_norm_crosscheck,
"final norm: with no prefill program the cross-check is unavailable, not failed");
if (r.final_norm_imported) {
Expand Down
Loading