Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions npu/pack_import.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2493,6 +2493,19 @@ std::string Plan::manifest() const {
return j.dump(2);
}

// engine#1889: where the per-head QK gains sit when the attention section carries them as the 640 B between the last
// qkv tile and o_proj. Split out and exposed because this is the arithmetic a wrong answer silently mis-rotates every
// Hunyuan pack with: a right offset reads a gain (~1.0-2.0), a wrong one reads tile bytes as bf16 and the gains come out
// wild, and the test drives both shapes without a pack.
QkGainOffsets qk_gain_offsets(int64_t gap_start, int64_t head_dim, int64_t gap_bytes) {
const int64_t per_head = head_dim * 2;
if (gap_bytes != 2 * (kNormOff + per_head)) return QkGainOffsets{};
QkGainOffsets o;
o.k_norm = gap_start + kNormOff; // block 0 is k, as the separate-section family
o.q_norm = gap_start + (kNormOff + per_head) + kNormOff; // block 1 is q
return o;
}

std::string qk_norm_refusal(bool declared, bool found) {
if (declared && !found)
return "this pack declares use_qk_norm and its per-head q/k gains are not where the block layout says "
Expand Down Expand Up @@ -2813,6 +2826,17 @@ Plan plan(const std::string& pack_dir) {
p.attn_tiles = n_kv * 2 + n_q;
p.o_off = attn_off + p.attn_tiles * tile_bytes + rem;
p.o_k = int(H);
// engine#1889: that gap IS the per-head QK norms -- the note below has said so since it was
// measured, and its shape is the separate-section family's, twice: [64 B header][bf16
// [head_dim] gain], i.e. 2 * (kNormOff + head_dim*2) = 640 B at head_dim 128, which is
// exactly the `rem` Hunyuan-7B reports. Naming it is what turns #1895's refusal ("declares
// use_qk_norm, gains not located") into an import; the values are the check (a wrong offset
// reads tile bytes as bf16 and the gains come out wild, a right one gives the 1.0-2.0 a norm
// gain has).
const QkGainOffsets qkg =
qk_gain_offsets(attn_off + p.attn_tiles * tile_bytes, int64_t(d.head_dim), rem);
p.knorm_off = qkg.k_norm;
p.qnorm_off = qkg.q_norm;
static bool said2 = false;
if (!said2) {
said2 = true;
Expand Down
4 changes: 4 additions & 0 deletions npu/pack_import.h
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,10 @@ struct Dims {
// REFUSAL (importing without the gains is the silent-garbage path), found-and-not-declared is allowed only with the
// measurement behind it (Qwen3 packs ship the sections without the key), and neither is the ordinary case. Exposed
// so the three can be pinned without a 4 GB pack.
struct QkGainOffsets {
int64_t k_norm = 0, q_norm = 0; // 0,0 = the gap is not the gains' shape
};
QkGainOffsets qk_gain_offsets(int64_t gap_start, int64_t head_dim, int64_t gap_bytes);
std::string qk_norm_refusal(bool declared, bool found);

// A hybrid pack's embedding table lives in its own graph (`embedding.onnx` + `.data`, gathered with
Expand Down
19 changes: 19 additions & 0 deletions tests/npu_pack_import_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3468,8 +3468,27 @@ void test_qk_norm_refusal() {
std::printf(" qk norm: 7 arms (declared+not-found refuses; found-only and neither are allowed)\n");
}

// engine#1889: the arithmetic that claims the per-head QK gains out of the 640 B Hunyuan-7B carries between its last qkv
// tile and o_proj. A wrong offset there reads tile bytes as bf16 and mis-rotates every pack of the family, so both the
// right shape (2 * (64 + head_dim*2) = 640 at head_dim 128) and the neighbours that must NOT be taken for it are driven.
void test_qk_gain_offsets() {
const pi::QkGainOffsets o = pi::qk_gain_offsets(1000, 128, 640);
check(o.k_norm == 1064, "qk gains: block 0 starts one kNormOff (64 B) after the gap");
check(o.q_norm == 1384, "qk gains: block 1 starts after block 0 (64 B header + a 256 B bf16 [128] gain)");
check(o.q_norm - o.k_norm == 320, "qk gains: the two blocks are 320 B apart, the same stride as the separate family");
const pi::QkGainOffsets n1 = pi::qk_gain_offsets(1000, 128, 639);
const pi::QkGainOffsets n2 = pi::qk_gain_offsets(1000, 128, 641);
const pi::QkGainOffsets n3 = pi::qk_gain_offsets(1000, 128, 512); // just the two gains, no headers
const pi::QkGainOffsets n4 = pi::qk_gain_offsets(1000, 256, 640); // the right byte count at the wrong head_dim
check(n1.k_norm == 0 && n1.q_norm == 0, "qk gains: a gap one byte short is not the gains' shape");
check(n2.k_norm == 0 && n2.q_norm == 0, "qk gains: a gap one byte long is not either");
check(n3.k_norm == 0 && n3.q_norm == 0, "qk gains: two bare gains with no headers are not this shape");
check(n4.k_norm == 0 && n4.q_norm == 0, "qk gains: the byte count alone is not enough, head_dim has to agree");
}

int main(int argc, char** argv) {
test_qk_norm_refusal();
test_qk_gain_offsets();
test_tail_head_placement();
const std::string golden_dir = argc > 1 ? argv[1] : "";
const std::string pack = argc > 2 ? argv[2] : "";
Expand Down
Loading