Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions npu/pack_import.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1282,6 +1282,9 @@ Dims read_dims(const json& c, const std::string& source = "config.json") {
d.rope_theta = opt_field<float>(c, "rope_theta", 10000.0f);
d.model_type = opt_field<std::string>(c, "model_type", "");
d.rms_norm_eps = opt_field<float>(c, "rms_norm_eps", 1e-6f);
// engine#1889: the pack's own statement about per-head q/k gains. `opt_field<bool>` because the key is absent on
// every pack that has no such norm, and absent must mean false rather than a parse error.
d.use_qk_norm = opt_field<bool>(c, "use_qk_norm", false);
d.eps_bits = bf16_bits_of(d.rms_norm_eps);
if (c.contains("eos_token_id")) {
const auto& e = c["eos_token_id"];
Expand Down Expand Up @@ -2473,6 +2476,14 @@ std::string Plan::manifest() const {
return j.dump(2);
}

std::string qk_norm_refusal(bool declared, bool found) {
if (declared && !found)
return "this pack declares use_qk_norm and its per-head q/k gains are not where the block layout says "
"(engine#1889): importing without them would normalise no q/k, which answers plausibly-wrong text "
"instead of refusing. The section location for this family is not measured yet";
return std::string();
}

Plan plan(const std::string& pack_dir) {
Plan p;
p.dir = pack_dir;
Expand Down Expand Up @@ -3054,6 +3065,13 @@ Plan plan(const std::string& pack_dir) {
// source from the .fconst's two per-head sections but the same tensor in the container.
const bool graph_qk_norm = graph_weights && !onnx.index.empty() &&
onnx.index.count("model.layers.0.attn.q_norm.layernorm.weight");
/* engine#1889: a pack that DECLARES per-head q/k gains and whose gains this importer did not locate must not
* import -- the forward would run without normalising q/k, which answers deterministic gibberish instead of
* refusing (Hunyuan-7B: `Trease/ft//////////` three of three, at a geometry four other packs answer correctly on).
* The section location for that family is not measured yet, so the honest outcome is a named refusal; the
* layout arithmetic and the graph check stay the two ways the gains ARE found. */
if (const std::string qk_why = qk_norm_refusal(d.use_qk_norm, p.knorm_off != 0 || graph_qk_norm); !qk_why.empty())
throw std::runtime_error(pack_dir + ": " + qk_why);
if (p.knorm_off || graph_qk_norm)
for (int L = 0; L < d.layers; ++L)
for (const char* nm : {"self_attn.q_norm.weight", "self_attn.k_norm.weight"}) {
Expand Down
15 changes: 15 additions & 0 deletions npu/pack_import.h
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,23 @@ struct Dims {
unsigned short eps_bits = 0x3586;
std::string model_type;
std::vector<int> eos;
// config.json's `use_qk_norm` (engine#1889). Hunyuan-7B declares it and puts 32x128 + 8x128 per-head q/k
// gains in its block; nothing read this key, so the importer decided `qk_norm` from layout arithmetic alone,
// landed on false, and the pack imported cleanly with the norms silently omitted -- the forward then runs
// without normalising q/k, which produces deterministic gibberish rather than a refusal. Read here so the
// decision has the pack's own statement as one of its inputs.
bool use_qk_norm = false;
};

// Why a pack that DECLARES per-head q/k gains cannot be imported, or "" when it can (engine#1889).
//
// `declared` is config.json's `use_qk_norm`; `found` says the block layout located the two sections (knorm_off /
// qnorm_off) or the graph carries them. The three cases are deliberately separate: declared-and-not-found is a
// REFUSAL (importing without the gains is the silent-garbage path), found-and-not-declared is allowed only with the
// measurement behind it (Qwen3 packs ship the sections without the key), and neither is the ordinary case. Exposed
// so the three can be pinned without a 4 GB pack.
std::string qk_norm_refusal(bool declared, bool found);

// A hybrid pack's embedding table lives in its own graph (`embedding.onnx` + `.data`, gathered with
// GatherBlockQuantized). Returns [V, H] f32 row-major, empty when the pack has no such component, with `why` naming the
// reason. Exposed so the dequantisation can be tested against known codes and scales instead of only on a 3 GB pack.
Expand Down
22 changes: 22 additions & 0 deletions tests/npu_pack_import_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3369,7 +3369,29 @@ void test_tail_head_placement() {
}


/* engine#1889: a pack that declares per-head q/k gains and whose gains the importer did not locate must be REFUSED,
* not imported without them. Hunyuan-7B is the measured case: `use_qk_norm: true`, 32x128 + 8x128 gains in its block,
* a 96-tile attention payload against the 160 the same geometry gets elsewhere -- and the import reported
* `attention 5120/5120 tiles` byte-exact while the served model answered `Trease/ft//////////` three of three.
* The three cases are separate on purpose: declared-and-not-found is the refusal; found-and-not-declared is the Qwen3
* shape (the sections are there, the key is not) and must NOT be refused. */
void test_qk_norm_refusal() {
const std::string a = pi::qk_norm_refusal(true, false);
check(!a.empty(), "qk norm: declared but not found is a refusal (importing without the gains answers gibberish)");
check(a.find("use_qk_norm") != std::string::npos, "qk norm: the refusal names the key the pack declared");
check(a.find("not where the block layout says") != std::string::npos,
"qk norm: and says what was not found, rather than failing generically");
check(a.find("engine#1889") != std::string::npos, "qk norm: the refusal carries its issue, like the others here");
check(pi::qk_norm_refusal(true, true).empty(),
"qk norm: declared and found is allowed (the arithmetic or the graph located the gains)");
check(pi::qk_norm_refusal(false, true).empty(),
"qk norm: found and not declared is allowed (Qwen3 ships the sections without the key)");
check(pi::qk_norm_refusal(false, false).empty(), "qk norm: neither is the ordinary pack");
std::printf(" qk norm: 7 arms (declared+not-found refuses; found-only and neither are allowed)\n");
}

int main(int argc, char** argv) {
test_qk_norm_refusal();
test_tail_head_placement();
const std::string golden_dir = argc > 1 ? argv[1] : "";
const std::string pack = argc > 2 ? argv[2] : "";
Expand Down
Loading