From 41bf606e173211d0c1797cd5574427d7a2c8b006 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 13:17:48 -0400 Subject: [PATCH 01/10] npu: Hunyuan answers on the forward route: use_qk_norm is read, RoPE base and norm order follow the model, tiles are q,k,v (Fixes #1889) Four defects stood between a Hunyuan-7B pack and an answer. Each is measured on Hunyuan-7B-Instruct_rai_1.9.0_npu with the device-free forward. forward: - use_qk_norm in config.json decides whether q and k are normalised when the config states it. The model_type table (qwen3*, gemma*) has no Hunyuan row, so the imported gains were loaded and never applied. - rope_scaling {type: dynamic, alpha: A} is a fixed base theta * A^(dim/(dim-2)), as the pack's cos/sin cache and the reference implementation have it. It was read as plain theta. - Hunyuan applies RoPE first and the per-head RMSNorm second (the pack graph: q_proj -> RotaryEmbedding -> q_norm). Qwen3 and Gemma keep norm first. - a config that says q/k are normalised with no q_norm weight in the container is refused at load instead of zeroing every head's q. importer: - the Hunyuan attention block stores its tiles q, k, v. Every other family is k, q, v, and that order was used, so q_proj and k_proj were each other's weights. Top-1 next-token accuracy on a fixed 78-token text, teacher-forced: no q/k norm 33/77, q/k norm 62/77, all forward changes 59/77, all forward changes with the old k,q,v tiles 26/77. Co-Authored-By: Claude Fable 5.2 --- CMakeLists.txt | 8 ++ docs/npu.md | 19 +++++ npu/forward/include/q4nx_rope.h | 16 ++++ npu/forward/include/q4nx_semantics.h | 10 ++- npu/forward/src/q4nx_forward.cpp | 36 ++++++-- npu/forward/src/q4nx_semantics.c | 17 +++- npu/pack_import.cpp | 24 +++++- npu/pack_import.h | 13 +++ tests/npu_pack_import_test.cpp | 16 ++++ tests/npu_rope_llama3_test.c | 29 +++++++ tests/q4nx_qk_norm_test.c | 119 +++++++++++++++++++++++++++ 11 files changed, 295 insertions(+), 12 deletions(-) create mode 100644 tests/q4nx_qk_norm_test.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 7b6821e3e..c924ef1c1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1455,6 +1455,14 @@ target_include_directories(q4nx_mrope_test PRIVATE npu/forward/include) target_link_libraries(q4nx_mrope_test PRIVATE m) add_test(NAME q4nx_mrope COMMAND q4nx_mrope_test) add_dependencies(npu_sampling_test q4nx_mrope_test) +# Which models get the per-head q/k RMSNorm: config.json's use_qk_norm when the config states it, else the +# model_type table (engine#1889: Hunyuan declares it under a model_type the table has no row for, so its gains were +# loaded and never applied). Plain C, links the semantics TU directly, no model and no device. +add_executable(q4nx_qk_norm_test tests/q4nx_qk_norm_test.c npu/forward/src/q4nx_semantics.c) +target_include_directories(q4nx_qk_norm_test PRIVATE npu/forward/include) +target_link_libraries(q4nx_qk_norm_test PRIVATE m) +add_test(NAME q4nx_qk_norm COMMAND q4nx_qk_norm_test) +add_dependencies(npu_sampling_test q4nx_qk_norm_test) add_executable(q4nx_layer_kinds_test tests/q4nx_layer_kinds_test.c npu/forward/src/q4nx_semantics.c) target_include_directories(q4nx_layer_kinds_test PRIVATE npu/forward/include) target_link_libraries(q4nx_layer_kinds_test PRIVATE m) diff --git a/docs/npu.md b/docs/npu.md index ff48d28d6..e61da4a69 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -788,6 +788,25 @@ forward route, which reads the block from `config.json`: `rope_type: "llama3"` declared-but-unusable `llama3` block, and every other declared type, at `1bit serve` / `forward-serve` load time rather than at request time. +**Hunyuan (`hunyuan_v1_dense`) differs from the other q/k-norm families in four places: three in +the forward, which reads them from the model rather than from a family list where it can, and one in +the importer** +([engine#1889](https://github.com/1bit-MONSTER/engine/issues/1889)). Each was measured against +`Hunyuan-7B-Instruct`'s own pack: + +| what | rule | where it comes from | +|---|---|---| +| whether q and k are normalised | `use_qk_norm` in `config.json` decides when the config states it; the `qwen3*`/`gemma*` table answers only for configs that do not | the config key. The table has no Hunyuan row, so its gains were loaded and never applied | +| the RoPE base | `rope_scaling: {type: "dynamic", alpha: A}` is a **fixed** base `theta * A^(dim/(dim-2))` at every position (11,158,840 for theta 10000, alpha 1000, dim 128). Without `alpha` the block is the position-dependent NTK rule | the graph's `cos_cache`/`sin_cache` in `model.onnx.data` match that base to 0.00024; plain theta 10000 is off by up to 1.99 | +| the order of norm and RoPE | RoPE first, then the per-head RMSNorm (Qwen3 and Gemma normalise first) | the graph: `q_proj -> RotaryEmbedding -> q_norm -> GroupQueryAttention` in every layer | +| the order of the attention tiles in the pack (importer) | q tiles first, then k, then v. Every other family's block is k, q, v. `1bit npu-pack import` picks the order with the block shape that carries the packed q/k gains | measured: with k, q, v the imported model predicts 26 of 77 next tokens of a fixed text, with q, k, v 59 (the two projections were being read as each other) | + +A model whose config says q and k are normalised and whose container has no +`self_attn.q_norm.weight` is refused at load: the gain would read as zeros and zero every head's q. +That is the shape of a Hunyuan container imported before the importer carried the gains, so the +answer is to re-import it. A Hunyuan container imported before the tile order was right has +the gains and still answers filler words; re-import that one too. + | GGUF type | To q4_1 | Error | |---|---|---| | Q4_0 | scale `d`, zero `-8d`, codes unchanged | bf16 rounding of scale and zero | diff --git a/npu/forward/include/q4nx_rope.h b/npu/forward/include/q4nx_rope.h index 946e41670..aee089a16 100644 --- a/npu/forward/include/q4nx_rope.h +++ b/npu/forward/include/q4nx_rope.h @@ -69,10 +69,26 @@ typedef struct { int on; /* config declared rope_type "dynamic" with usable numbers */ double factor; /* 1.0 on Hunyuan-0.5B -- a factor of 1 is a no-op until pos > original */ double original_max_position; /* 262144 on Hunyuan-0.5B */ + /* engine#1889: Hunyuan's own form of "dynamic". Its rope_scaling block also carries `alpha` (1000.0 on every + * Hunyuan/HY-MT 1.9 pack), and when alpha is set the model's rotary embedding does not move the base with the + * position at all: it uses the FIXED base + * + * base = theta * alpha ^ (dim / (dim - 2)) + * + * (HunYuanDenseV1RotaryEmbedding, "DynamicNTKAlphaRotary"). Measured against the pack rather than taken from the + * reference: Hunyuan-7B's model.onnx.data holds the graph's cos_cache/sin_cache (FP16 [32768, 64]), and they match + * base 10000 * 1000^(128/126) = 11,158,840 to 0.00024 at positions 1, 2, 7, 50 and 300, where plain theta 10000 is + * off by up to 1.99. Reading the block as factor-only made it a no-op, i.e. every position rotated by the wrong + * angle. 0 = the block has no alpha, and the factor rule above applies. */ + double alpha; } Q4nxRopeDynamic; static inline double q4nx_rope_dynamic_base(double theta, const Q4nxRopeDynamic* d, double pos, int dim) { if (!d || !d->on) return theta; + if (d->alpha > 0.0) { /* the alpha form: one fixed base, every position */ + if (dim <= 2) return theta; + return theta * pow(d->alpha, (double)dim / (double)(dim - 2)); + } if (!(d->factor > 0.0) || !(d->original_max_position > 0.0) || dim <= 2) return theta; if (!(pos >= d->original_max_position)) return theta; /* inside the window: plain rope, bit for bit */ const double scaled = d->factor * pos / d->original_max_position - (d->factor - 1.0); diff --git a/npu/forward/include/q4nx_semantics.h b/npu/forward/include/q4nx_semantics.h index 77393f1b1..abd6201fa 100644 --- a/npu/forward/include/q4nx_semantics.h +++ b/npu/forward/include/q4nx_semantics.h @@ -62,8 +62,16 @@ typedef struct { * Source: model_type/architectures == qwen3_5*, resolved against the weights. */ int rmsnorm_offset_probe; /* Per-head RMSNorm applied to q and k before RoPE. - * Source: presence of self_attn.q_norm.weight / k_norm.weight. */ + * Source: config.json's `use_qk_norm` when the config states it (engine#1889: Hunyuan), else the + * model_type/architectures table (qwen3*, gemma*). The forward refuses a model whose flag is set and whose + * layers carry no self_attn.q_norm.weight, since normalising with an absent gain zeroes q. */ int qk_norm; + /* The q/k RMSNorm runs AFTER RoPE instead of before it (engine#1889). Qwen3 and Gemma normalise, then rotate; + * Hunyuan rotates, then normalises. With a per-dimension gain the two orders differ wherever a rotary pair's two + * gains differ, so the order is part of the model. Source: model_type/architectures (hunyuan*) -- no config key + * states it; the evidence is the pack's own graph, where every layer is + * q_proj -> RotaryEmbedding -> q_norm (SimplifiedLayerNormalization) -> GroupQueryAttention. */ + int qk_norm_after_rope; /* RoPE base. Source: rope_theta. */ float rope_theta; /* Channels actually rotated; 0 means the full head_dim. Partial rotary diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 18deb4d7b..4954bb265 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -197,9 +197,10 @@ static int dequant_any(const void* data, size_t nbytes, int rows, int cols, floa // transformers' _compute_llama3_parameters applies. `on == 0` unless the config names the type, so // every other architecture builds exactly the table it built before. Keys are searched with their // quotes, which is what keeps "factor" from matching inside "high_freq_factor". -// Dynamic NTK: `rope_scaling: {type: "dynamic"|"dynamic_ntk", factor: F}` and `max_position_embeddings`. Hunyuan's -// 1.9 packs declare factor 1.0 with a 262,144 window, i.e. a no-op until a prompt passes that window -- which is why -// accepting it is safe rather than a guess: below the window the base IS theta, so the served bytes are plain rope. +// Dynamic NTK: `rope_scaling: {type: "dynamic"|"dynamic_ntk", factor: F}` and `max_position_embeddings`: below the +// window the base IS theta, so a block with factor 1.0 serves plain rope. Hunyuan's 1.9 packs declare that AND +// `alpha: 1000.0`, which is a different rule (a fixed base theta * alpha^(dim/(dim-2)), engine#1889): reading only +// the factor made their block a no-op and rotated every position by the wrong angle. Q4nxRopeDynamic read_rope_dynamic(const std::string& cfg_path) { Q4nxRopeDynamic d{}; std::ifstream f(cfg_path); @@ -216,7 +217,10 @@ Q4nxRopeDynamic read_rope_dynamic(const std::string& cfg_path) { }; d.factor = number("factor"); d.original_max_position = number("max_position_embeddings"); - d.on = (d.factor > 0.0 && d.original_max_position > 0.0) ? 1 : 0; + // engine#1889: `alpha` selects Hunyuan's fixed-base form (q4nx_rope.h). The key is searched with its quotes and its + // colon, so "lora_alpha" or an "alpha" inside a string value cannot be taken for it. + d.alpha = number("alpha"); + d.on = ((d.factor > 0.0 && d.original_max_position > 0.0) || d.alpha > 0.0) ? 1 : 0; return d; } @@ -1626,6 +1630,17 @@ bool Q4nxNpuForward::load_norms() { layer_attn_dims(l, hd_l, nkv_l); if (!read_bf16(&mw_->layers[l].input_layernorm_weight, in_norm_[l].data(), H)) return false; if (!read_bf16(&mw_->layers[l].post_attention_layernorm_weight, post_norm_[l].data(), H)) return false; + // engine#1889: the config says q and k are normalised and this layer has no gain to normalise with. read_bf16 + // would hand back zeros for the absent tensor and rmsnorm(q, 0) is 0 for every head, so the layer would attend + // uniformly and answer fluent nonsense. A Hunyuan container imported before the importer claimed the gains is + // exactly this shape, so it is refused by name here instead. + if (sem_.qk_norm && absent(&mw_->layers[l].q_norm_weight)) { + err_ = "config.json says q and k are normalised per head (" + + std::string(q4nx_semantics_field(Q4NX_SEM_QK_NORM)) + ") and layer " + std::to_string(l) + + " has no self_attn.q_norm.weight: normalising with an absent gain zeroes q, so this container " + "would answer wrong text (engine#1889); re-import the pack with a build that carries the gains"; + return false; + } if (!read_bf16(&mw_->layers[l].q_norm_weight, q_norm_[l].data(), hd_l)) return false; // Qwen2.5-style attention biases (q/k/v projection biases); absent on the // bias-free families, so a miss is not an error. Loaded BEFORE the k_norm @@ -4555,7 +4570,11 @@ bool Q4nxNpuForward::attention(const std::vector& qkv, int pos, const int // per-head RMSNorm on q and k, only for architectures that have it. A KV-shared // layer (Gemma4) computed no k at all, so its k is never normalized or used. - if (sem_.qk_norm) { + // engine#1889: WHERE it runs is per family. Qwen3 and Gemma normalise and then rotate; Hunyuan's graph rotates + // and then normalises (q_proj -> RotaryEmbedding -> q_norm -> attention), so for it the same code runs after the + // RoPE block below. The RMS itself is rotation-invariant; the per-dimension gain is not, so the order matters + // wherever a rotary pair's two gains differ. + auto qk_normalise = [&]() { std::vector t(HD); for (int h = 0; h < NH; h++) { rmsnorm(&q[(size_t)h * HD], q_norm_[cur_layer_].data(), HD, t.data(), "q_norm"); @@ -4567,7 +4586,8 @@ bool Q4nxNpuForward::attention(const std::vector& qkv, int pos, const int memcpy(&k[(size_t)h * HD], t.data(), sizeof(float) * HD); } } - } + }; + if (sem_.qk_norm && !sem_.qk_norm_after_rope) qk_normalise(); // Gemma4 normalises v with a SCALE-FREE RMSNorm (its v_norm has no weight). if (sem_.v_norm && append_kv) { std::vector tv(HD); @@ -4587,7 +4607,8 @@ bool Q4nxNpuForward::attention(const std::vector& qkv, int pos, const int std::vector cos_(half), sin_(half); // Dynamic NTK: the base moves with the position, so it is one pow per position rather than one per dim pair. // Below the model's original window this returns theta unchanged, which is what makes a short prompt - // bit-identical to the same model served without the scaling. + // bit-identical to the same model served without the scaling. A block with `alpha` (Hunyuan) is the fixed + // base theta * alpha^(dim/(dim-2)) at every position instead (engine#1889). // engine#1792 review: the exponent is HF's `dim / (dim - 2)` over the ROTARY width (`dim = int(head_dim * // partial_rotary_factor)`), which is `rot` here -- passing `HD` was wrong for any model that declares dynamic // scaling together with partial_rotary_factor < 1. Unreachable in today's catalog, so this is a correctness @@ -4634,6 +4655,7 @@ bool Q4nxNpuForward::attention(const std::vector& qkv, int pos, const int for (int h = 0; h < NH; h++) rot_half(&q[(size_t)h * HD]); for (int h = 0; h < NKV; h++) rot_half(&k[(size_t)h * HD]); } + if (sem_.qk_norm && sem_.qk_norm_after_rope) qk_normalise(); // append to cache (a KV-shared layer reuses its owner's, which the owner has // already appended earlier in this same step). diff --git a/npu/forward/src/q4nx_semantics.c b/npu/forward/src/q4nx_semantics.c index c623e660d..775994d7b 100644 --- a/npu/forward/src/q4nx_semantics.c +++ b/npu/forward/src/q4nx_semantics.c @@ -26,7 +26,7 @@ static const char* kFields[Q4NX_SEM_COUNT] = { /* RMSNORM_OFFSET */ "model_type/architectures (gemma* uses the 1+w form)", - /* QK_NORM */ "model_type/architectures (q_norm/k_norm tensors)", + /* QK_NORM */ "use_qk_norm when stated, else model_type/architectures", /* ROPE_THETA */ "rope_theta", /* ROTARY_DIM */ "partial_rotary_factor * head_dim", /* MLP_ACT */ "hidden_act (falls back to model_type)", @@ -221,6 +221,18 @@ int q4nx_semantics_from_config(const char* json_path, const char* model_dir, /* per-head q/k RMSNorm: qwen3* and gemma* have q_norm/k_norm; llama/phi do not */ s->qk_norm = (is_qwen3 || is_gemma) ? 1 : 0; + /* engine#1889: a config that STATES `use_qk_norm` decides it, whatever the name table says. The table is a list + * of the families measured so far, so a family it has never heard of got 0 silently: Hunyuan-7B is + * `hunyuan_v1_dense` / `HunYuanDenseV1ForCausalLM` with `use_qk_norm: true`, its container carries + * self_attn.{q,k}_norm.weight for all 32 layers, and the forward loaded them and never applied them -- the same + * text with the gains, without them and with the key set to false, because nothing read the key. Families that + * do not state the key (qwen3, gemma) keep the table's answer. */ + { + double uq = 0; + if (get_num(json, "use_qk_norm", &uq)) s->qk_norm = uq != 0 ? 1 : 0; + } + /* Hunyuan rotates q and k first and normalises them after (see the header); everyone else normalises first. */ + s->qk_norm_after_rope = (starts_with(model_type, "hunyuan") || starts_with(arch, "HunYuan")) ? 1 : 0; /* RoPE base. Qwen3.5 nests it under rope_parameters (scalar); Gemma4 nests * it PER LAYER TYPE under rope_parameters.{full,sliding}_attention with @@ -541,6 +553,9 @@ void q4nx_semantics_dump(const Q4nxSemantics* s) { q4nx_semantics_field(Q4NX_SEM_RMSNORM_OFFSET)); fprintf(stderr, " [semantics] qk_norm=%d <- %s\n", s->qk_norm, q4nx_semantics_field(Q4NX_SEM_QK_NORM)); + if (s->qk_norm) + fprintf(stderr, " [semantics] qk_norm order: %s <- model_type/architectures (hunyuan* normalises after RoPE)\n", + s->qk_norm_after_rope ? "RoPE, then norm" : "norm, then RoPE"); fprintf(stderr, " [semantics] rope_theta=%.6g <- %s\n", s->rope_theta, q4nx_semantics_field(Q4NX_SEM_ROPE_THETA)); fprintf(stderr, " [semantics] rotary_dim=%d <- %s\n", s->rotary_dim, diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 492a7d494..cf89bd0d8 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -3313,6 +3313,14 @@ std::string header_witness_refusal(int vote_hdrs, bool declared_qk) { "neither is being preferred"; } +AttnFirstTiles attn_first_tiles(int n_kv, int n_q, bool q_first) { + AttnFirstTiles f; + f.k = q_first ? n_q : 0; + f.q = q_first ? 0 : n_kv; + f.v = n_kv + n_q; + return f; +} + std::string qk_norm_refusal(bool declared, bool found) { if (declared && !found) return "this pack declares use_qk_norm and its per-head q/k gains are not where the block layout says " @@ -3713,6 +3721,14 @@ Plan plan(const std::string& pack_dir) { (long long)rem); } handled = true; + // engine#1889: this block's projections are q, k, v -- not the k, q, v every other family + // writes. It cannot be read off the tiles (no biases here to identify the runs, and the + // round trip is byte-exact either way), so it was settled by the answer: Hunyuan-7B on the + // host forward, teacher-forced on "The capital of France is Paris. The capital of Germany is + // Berlin. The capital of Italy is", predicts ' Paris' / ' Berlin' / ' Rome' at the three + // "is" rows with q,k,v and ' the' / ' the' / ' the' with k,q,v. The same family puts q + // before k in its packed gains (engine#1934), which is the same writer's order. + p.attn_q_first = true; } } const int64_t o_tiles_h = H / kTileRows; @@ -5473,9 +5489,11 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons // o_proj's K is q_proj's output width). struct Slot { const char* proj; int rows; int64_t base; int k; int first; }; const int64_t attn_off = attn_payload_off(H); - const Slot slots[] = {{"k_proj", kv_out, attn_off, int(H), 0}, - {"q_proj", q_out, attn_off, int(H), n_kv}, - {"v_proj", kv_out, attn_off, int(H), n_kv + n_q}, + // engine#1889: k, q, v on every family but the Hunyuan block, which is q, k, v (Plan::attn_q_first). + const AttnFirstTiles ft = attn_first_tiles(n_kv, n_q, p.attn_q_first); + const Slot slots[] = {{"k_proj", kv_out, attn_off, int(H), ft.k}, + {"q_proj", q_out, attn_off, int(H), ft.q}, + {"v_proj", kv_out, attn_off, int(H), ft.v}, {"o_proj", int(H), p.o_off, p.o_k, 0}}; for (int L = 0; L < p.dims.layers; ++L) { const int64_t block = int64_t(L) * g.stride; diff --git a/npu/pack_import.h b/npu/pack_import.h index dc62f27ff..5541f612f 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -109,6 +109,14 @@ std::string qk_norm_refusal(bool declared, bool found); // pinned without a 2.6 GB pack. std::string header_witness_refusal(int vote_hdrs, bool declared_qk); +// The first tile of k_proj, q_proj and v_proj inside the attention payload, for n_kv k/v tiles and n_q q tiles +// (64 rows each). `q_first` false is k, q, v; true is q, k, v (engine#1889, the Hunyuan block). v is last either way. +// Split out so the order is pinned by a test instead of living in an initializer list. +struct AttnFirstTiles { + int k = 0, q = 0, v = 0; +}; +AttnFirstTiles attn_first_tiles(int n_kv, int n_q, bool q_first); + // A hybrid pack's embedding table lives in its own graph (`embedding.onnx` + `.data`, gathered with // GatherBlockQuantized). Returns [V, H] f32 row-major, empty when the pack has no such component, with `why` naming the // reason. Exposed so the dequantisation can be tested against known codes and scales instead of only on a 3 GB pack. @@ -209,6 +217,11 @@ struct Plan { // none. Identified and verified byte-exact against the public Qwen3-1.7B/4B checkpoints (0 of 128 // values differ on either pack, and the swap differs on 127-128 of 128). int64_t knorm_off = 0, qnorm_off = 0; + // The order of the three projections in the block's first payload. Every family measured before Hunyuan writes + // k, q, v (identified on Qwen2.5-0.5B by the tiles' bias subvolumes). The Hunyuan block -- q/k/v at K=hidden, the + // packed per-head gains, o_proj -- writes q, k, v (engine#1889): with k,q,v the container's k_proj was q_proj's + // first rows, and the model answered filler words; with q,k,v it answers. See attn_first_tiles(). + bool attn_q_first = false; int64_t mlp_off = 0; // the MLP section starts just past the block's post-attention norm // What the pack's *graph* carries, reported so `npu-pack info` can answer a pack's genre without a // third-party parser: the rai_1.3/1.6/1.7.1 line ships its 4-bit weights as ordinary ONNX initializers, while diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 9087bb95e..3508946b4 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -5432,6 +5432,21 @@ void test_header_witness_refusal() { std::printf(" header witness: 8 arms (one-header vote vs declaration refuses; 3/1-without-declaration/0 allowed)\n"); } +// engine#1889: which tile each projection starts at inside the attention payload. Every family before Hunyuan is +// k, q, v; the Hunyuan block is q, k, v, and reading it k, q, v handed the forward q_proj's first rows as k_proj +// (byte-exact round trip, filler-word answers). Hunyuan-7B's own counts: 16 k/v tiles, 64 q tiles. +void test_attn_first_tiles() { + const pi::AttnFirstTiles kqv = pi::attn_first_tiles(16, 64, false); + check(kqv.k == 0 && kqv.q == 16 && kqv.v == 80, "attn order: k,q,v puts k at tile 0, q at 16, v at 80"); + const pi::AttnFirstTiles qkv = pi::attn_first_tiles(16, 64, true); + check(qkv.q == 0 && qkv.k == 64 && qkv.v == 80, "attn order: q,k,v puts q at tile 0, k at 64, v at 80 (engine#1889)"); + check(qkv.k + 16 == qkv.v && kqv.q + 64 == kqv.v, "attn order: v follows the second run in both orders"); + const pi::AttnFirstTiles small = pi::attn_first_tiles(2, 14, false); // Qwen2.5-0.5B: k 2, q 14, v 2 + check(small.k == 0 && small.q == 2 && small.v == 16, "attn order: the Qwen2.5-0.5B run (2/14/2) is unchanged"); + pi::Plan p; + check(!p.attn_q_first, "attn order: a plan is k,q,v unless the block measured otherwise"); +} + int main(int argc, char** argv) { // engine#1907: every crafted 1.9 import writes a canonical copy into the scratch cache. Without this the suite left 0-byte // copies in the user's real ~/.cache/1bit/fconst19 and in /tmp on every run. Give the run its own directory, removed at exit. @@ -5444,6 +5459,7 @@ int main(int argc, char** argv) { test_header_witness_refusal(); test_encode_tile_code_range(); test_q4_writer_code_range(); + test_attn_first_tiles(); test_qk_norm_refusal(); test_qk_gain_offsets(); test_tail_head_placement(); diff --git a/tests/npu_rope_llama3_test.c b/tests/npu_rope_llama3_test.c index dd261d2df..63f310dcb 100644 --- a/tests/npu_rope_llama3_test.c +++ b/tests/npu_rope_llama3_test.c @@ -257,6 +257,35 @@ static void test_declared_type(void) { check(fabs(got64 - got128) > 1e3, "and the two widths differ by more than 1e3, so this arm cannot pass with the widths confused"); } + /* engine#1889: Hunyuan's block is `{"alpha": 1000.0, "factor": 1.0, "type": "dynamic"}`, and with alpha set its + * rotary embedding uses ONE fixed base, theta * alpha^(dim/(dim-2)), at every position. Read as factor-only the + * block was a no-op (base 10000), which rotated every position by the wrong angle. The expected numbers are the + * pack's, not the formula's: Hunyuan-7B's model.onnx.data carries the graph's cos_cache/sin_cache (FP16 + * [32768, 64]); at position 1 the angle of pair i is its inverse frequency, and the table reads + * pair 1: 0.77608 pair 2: 0.60209 pair 5: 0.28159 + * (FP16, so good to about 5e-4), against 0.86596 / 0.74989 / 0.48697 for plain theta 10000. */ + { + Q4nxRopeDynamic hy = {1, 1.0, 262144.0, 1000.0}; /* Hunyuan-7B's own numbers */ + const double base = q4nx_rope_dynamic_base(1e4, &hy, 1.0, 128); + check(fabs(base - 11158839.9) < 1.0, "alpha 1000 at theta 10000, dim 128: the base is 11,158,840"); + check(q4nx_rope_dynamic_base(1e4, &hy, 0.0, 128) == base && q4nx_rope_dynamic_base(1e4, &hy, 300000.0, 128) == base, + "... at every position, inside the window and past it (the alpha form does not move with pos)"); + check(fabs(pow(base, -1.0 / 64.0) - 0.77608) < 5e-4, "pair 1's inverse frequency is the pack table's 0.77608"); + check(fabs(pow(base, -2.0 / 64.0) - 0.60209) < 5e-4, "pair 2's is the pack table's 0.60209"); + check(fabs(pow(base, -5.0 / 64.0) - 0.28159) < 5e-4, "pair 5's is the pack table's 0.28159"); + check(fabs(pow(1e4, -1.0 / 64.0) - 0.77608) > 0.05, + "and plain theta 10000 misses pair 1 by more than 0.05, so these arms cannot pass on the no-op"); + Q4nxRopeDynamic hy64 = {1, 1.0, 262144.0, 1000.0}; + check(fabs(q4nx_rope_dynamic_base(1e4, &hy64, 1.0, 64) - 1e4 * pow(1000.0, 64.0 / 62.0)) < 1e-3, + "the exponent is dim/(dim-2) over the rotary width here too (dim 64 -> 64/62)"); + Q4nxRopeDynamic no_alpha = {1, 1.0, 262144.0, 0.0}; + check(q4nx_rope_dynamic_base(1e4, &no_alpha, 5.0, 128) == 1e4, + "alpha 0 is the factor rule: inside the window the base is theta, as before"); + Q4nxRopeDynamic off_alpha = {0, 1.0, 262144.0, 1000.0}; + check(q4nx_rope_dynamic_base(1e4, &off_alpha, 5.0, 128) == 1e4, "and an alpha on a block that is off does nothing"); + Q4nxRopeDynamic tiny = {1, 1.0, 262144.0, 1000.0}; + check(q4nx_rope_dynamic_base(1e4, &tiny, 5.0, 2) == 1e4, "a rotary width of 2 has no exponent, so the base stays theta"); + } } } diff --git a/tests/q4nx_qk_norm_test.c b/tests/q4nx_qk_norm_test.c new file mode 100644 index 000000000..1ea678b3e --- /dev/null +++ b/tests/q4nx_qk_norm_test.c @@ -0,0 +1,119 @@ +/* Copyright 2026 bong-water-water-bong + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* q4nx_qk_norm_test -- which models get the per-head q/k RMSNorm (engine#1889). + * + * The forward applies it only when the semantics flag is set, and the flag used to come from a model_type / + * architectures table alone (qwen3*, gemma*). Hunyuan-7B is `hunyuan_v1_dense` with `use_qk_norm: true`: the table had + * no row for it, so its container's self_attn.{q,k}_norm.weight were loaded and never applied. The config key now + * decides when the config states it; the table stays the answer for the families that do not state it. + * + * The strings below are the real keys of the configs on this box (Hunyuan-7B-Instruct's config.json, a Qwen3 and a + * Llama container's), cut down to the fields the rule reads. No model, no device. */ +#include "q4nx_semantics.h" + +#include +#include +#include +#include +#include + +static int failures = 0; + +static void check(int ok, const char* what) { + printf("%s %s\n", ok ? "ok " : "FAIL", what); + if (!ok) failures++; +} + +/* /config.json holding exactly `json`, 0600 (mkdtemp gives 0700; the file's own mode is what CodeQL reads). */ +static int after_rope = -1; /* the last config's qk_norm_after_rope */ + +static int qk_norm_of(const char* dir, const char* json) { + char path[512]; + snprintf(path, sizeof path, "%s/config.json", dir); + int fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0600); + FILE* f = fd >= 0 ? fdopen(fd, "w") : NULL; + if (!f) { printf("FAIL cannot write %s\n", path); failures++; return -1; } + fputs(json, f); + fclose(f); + Q4nxSemantics s; + if (q4nx_semantics_from_config(path, dir, &s) != 0) { printf("FAIL cannot read %s\n", path); failures++; return -1; } + after_rope = s.qk_norm_after_rope; + return s.qk_norm; +} + +int main(void) { + char dir[] = "/tmp/q4nx-qk-norm-XXXXXX"; + if (!mkdtemp(dir)) { printf("FAIL mkdtemp\n"); return 1; } + + /* The measured case: a family the name table has never heard of, declaring the norm. */ + check(qk_norm_of(dir, "{\"architectures\": [\"HunYuanDenseV1ForCausalLM\"], \"model_type\": \"hunyuan_v1_dense\"," + " \"num_attention_heads\": 32, \"num_key_value_heads\": 8, \"head_dim\": 128," + " \"use_qk_norm\": true}") == 1, + "hunyuan_v1_dense with use_qk_norm: true normalises q and k (engine#1889)"); + /* The same family saying it has none: the key is read, not assumed from the family. */ + check(qk_norm_of(dir, "{\"architectures\": [\"HunYuanDenseV1ForCausalLM\"], \"model_type\": \"hunyuan_v1_dense\"," + " \"use_qk_norm\": false}") == 0, + "hunyuan_v1_dense with use_qk_norm: false does not"); + /* A family with no row and no key keeps the neutral convention. */ + check(qk_norm_of(dir, "{\"architectures\": [\"HunYuanDenseV1ForCausalLM\"], \"model_type\": \"hunyuan_v1_dense\"}") == 0, + "an unlisted family that does not state the key keeps the plain convention"); + + /* The table is unchanged for the families that never state the key. */ + check(qk_norm_of(dir, "{\"architectures\": [\"Qwen3ForCausalLM\"], \"model_type\": \"qwen3\"}") == 1, + "qwen3 without the key still normalises (the name table)"); + check(qk_norm_of(dir, "{\"architectures\": [\"Gemma3ForCausalLM\"], \"model_type\": \"gemma3_text\"}") == 1, + "gemma without the key still normalises (the name table)"); + check(qk_norm_of(dir, "{\"architectures\": [\"LlamaForCausalLM\"], \"model_type\": \"llama\"}") == 0, + "llama without the key still does not"); + check(qk_norm_of(dir, "{\"architectures\": [\"Phi3ForCausalLM\"], \"model_type\": \"phi3\"}") == 0, + "phi without the key still does not"); + + /* A stated key beats the table in both directions, and a null is not a statement. */ + check(qk_norm_of(dir, "{\"model_type\": \"llama\", \"use_qk_norm\": true}") == 1, + "a llama-named config that states use_qk_norm: true normalises"); + check(qk_norm_of(dir, "{\"model_type\": \"qwen3\", \"use_qk_norm\": false}") == 0, + "a qwen3-named config that states use_qk_norm: false does not"); + check(qk_norm_of(dir, "{\"model_type\": \"qwen3\", \"use_qk_norm\": null}") == 1, + "use_qk_norm: null states nothing, so the table answers"); + /* Only the top-level key counts: a nested one belongs to another component's config. */ + check(qk_norm_of(dir, "{\"model_type\": \"llama\", \"vision_config\": {\"use_qk_norm\": true}}") == 0, + "a use_qk_norm nested under another component's config is not this model's"); + + /* The ORDER (engine#1889). Hunyuan's graph is q_proj -> RotaryEmbedding -> q_norm -> attention in every layer, so + * its norm runs after RoPE; Qwen3 and Gemma keep norm-then-RoPE. */ + (void)qk_norm_of(dir, "{\"architectures\": [\"HunYuanDenseV1ForCausalLM\"], \"model_type\": \"hunyuan_v1_dense\"," + " \"use_qk_norm\": true}"); + check(after_rope == 1, "hunyuan_v1_dense normalises q and k AFTER RoPE, as its graph does"); + (void)qk_norm_of(dir, "{\"architectures\": [\"HunYuanDenseV1ForCausalLM\"]}"); + check(after_rope == 1, "... and the architectures entry alone is enough to say so"); + (void)qk_norm_of(dir, "{\"architectures\": [\"Qwen3ForCausalLM\"], \"model_type\": \"qwen3\"}"); + check(after_rope == 0, "qwen3 keeps norm, then RoPE"); + (void)qk_norm_of(dir, "{\"architectures\": [\"Gemma3ForCausalLM\"], \"model_type\": \"gemma3_text\"}"); + check(after_rope == 0, "gemma keeps norm, then RoPE"); + (void)qk_norm_of(dir, "{\"model_type\": \"llama\", \"use_qk_norm\": true}"); + check(after_rope == 0, "a non-Hunyuan config that states use_qk_norm gets the default order"); + + check(strstr(q4nx_semantics_field(Q4NX_SEM_QK_NORM), "use_qk_norm") != NULL, + "the startup line names use_qk_norm as the flag's source"); + + char path[512]; + snprintf(path, sizeof path, "%s/config.json", dir); + unlink(path); + rmdir(dir); + printf(failures ? "q4nx_qk_norm: %d FAILED\n" : "q4nx_qk_norm: ok\n", failures); + return failures ? 1 : 0; +} From e2800c2047cbdea9b61b807d9c22e016c6a2a652 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 13:23:51 -0400 Subject: [PATCH 02/10] npu-pack: the RoPE cache check tries the rope_scaling alpha base first (engine#1889) Hunyuan-7B's import warned that no plain theta explains the pack's cos/sin cache (best 0.7222 at 1e7). The cache is theta * alpha^(dim/(dim-2)), which the forward now computes, so the check tries that base first and reports it verified. rope_theta stays as declared: the forward applies alpha itself. Co-Authored-By: Claude Fable 5.2 --- docs/npu.md | 2 +- npu/pack_import.cpp | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/docs/npu.md b/docs/npu.md index e61da4a69..0eeb5cc32 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -797,7 +797,7 @@ the importer** | what | rule | where it comes from | |---|---|---| | whether q and k are normalised | `use_qk_norm` in `config.json` decides when the config states it; the `qwen3*`/`gemma*` table answers only for configs that do not | the config key. The table has no Hunyuan row, so its gains were loaded and never applied | -| the RoPE base | `rope_scaling: {type: "dynamic", alpha: A}` is a **fixed** base `theta * A^(dim/(dim-2))` at every position (11,158,840 for theta 10000, alpha 1000, dim 128). Without `alpha` the block is the position-dependent NTK rule | the graph's `cos_cache`/`sin_cache` in `model.onnx.data` match that base to 0.00024; plain theta 10000 is off by up to 1.99 | +| the RoPE base | `rope_scaling: {type: "dynamic", alpha: A}` is a **fixed** base `theta * A^(dim/(dim-2))` at every position (11,158,840 for theta 10000, alpha 1000, dim 128). Without `alpha` the block is the position-dependent NTK rule | the graph's `cos_cache`/`sin_cache` in `model.onnx.data` match that base to 0.00024; plain theta 10000 is off by up to 1.99. `1bit npu-pack import` tests this base against the pack's cache first and prints `note: RoPE base ... verified`; before, it warned that no plain theta explained the cache | | the order of norm and RoPE | RoPE first, then the per-head RMSNorm (Qwen3 and Gemma normalise first) | the graph: `q_proj -> RotaryEmbedding -> q_norm -> GroupQueryAttention` in every layer | | the order of the attention tiles in the pack (importer) | q tiles first, then k, then v. Every other family's block is k, q, v. `1bit npu-pack import` picks the order with the block shape that carries the packed q/k gains | measured: with k, q, v the imported model predicts 26 of 77 next tokens of a fixed text, with q, k, v 59 (the two projections were being read as each other) | diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index cf89bd0d8..5cbc755c8 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -5796,6 +5796,23 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons } // candidates: what the metadata says first, then the values these families actually use std::vector cands{p.dims.rope_theta, 1e4, 5e4, 1e5, 5e5, 1e6, 1e7, 1.6e6, 3.2e6}; + // engine#1889: a `rope_scaling: {type: "dynamic", alpha: A}` block (Hunyuan) is a FIXED base + // theta * A^(dim/(dim-2)), and the forward computes it from the block (q4nx_rope_dynamic_base). + // That base is tried first: on Hunyuan-7B it is 11,158,840 and no plain candidate comes near + // (best 0.7222 at 1e7), so the import warned "no plain theta explains" about a cache the + // model's own config explains. rope_theta is left as declared when this one wins -- writing the + // scaled base there would have the forward apply alpha twice. + double alpha_base = 0.0, alpha = 0.0; + if (c.contains("rope_scaling") && c["rope_scaling"].is_object() && + c["rope_scaling"].contains("type") && c["rope_scaling"]["type"].is_string() && + c["rope_scaling"]["type"].get() == "dynamic" && + c["rope_scaling"].contains("alpha") && c["rope_scaling"]["alpha"].is_number()) { + alpha = c["rope_scaling"]["alpha"].get(); + const double hd = double(half) * 2.0; + if (alpha > 0.0 && hd > 2.0 && p.dims.rope_theta > 0) + alpha_base = double(p.dims.rope_theta) * std::pow(alpha, hd / (hd - 2.0)); + } + if (alpha_base > 0.0) cands.insert(cands.begin(), alpha_base); double best_corr = -2.0, best_theta = p.dims.rope_theta; const int HD2 = int(half) * 2; // head_dim implied by the cache const int64_t pmax = std::min(rows, 4096); @@ -5826,7 +5843,21 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons if (corr == corr && corr > best_corr) { best_corr = corr; best_theta = th; } } } - if (best_corr > 0.99 && std::fabs(best_theta - p.dims.rope_theta) > 1e-9) { + if (best_corr > 0.99 && alpha_base > 0.0 && best_theta == alpha_base) { + std::fprintf(stderr, + "note: RoPE base %.8g (rope_theta %.6g, rope_scaling alpha %.6g) verified against " + "the pack's cos_cache (corr %.4f)\n", + alpha_base, (double)p.dims.rope_theta, alpha, best_corr); + c.erase("q4nx_rope_note"); + } else if (best_corr > 0.99 && alpha_base > 0.0) { + // A plain theta fits and the alpha base does not, while the config still carries alpha: the + // forward would scale whatever is written to rope_theta, so nothing is adopted. + std::fprintf(stderr, + "WARNING: this pack's RoPE caches fit plain theta %.6g (corr %.4f), not the base " + "%.8g its rope_scaling alpha %.6g implies; this forward applies alpha, so " + "attention may be wrong from the first token\n", + best_theta, best_corr, alpha_base, alpha); + } else if (best_corr > 0.99 && std::fabs(best_theta - p.dims.rope_theta) > 1e-9) { std::fprintf(stderr, "note: rope_theta %.6g declared, %.6g adopted (correlation %.4f against the " "pack's own cos_cache)\n", (double)p.dims.rope_theta, best_theta, best_corr); From c948ef4316e5f37f736ef7638584e6d8e43274f5 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 19:18:29 -0400 Subject: [PATCH 03/10] npu-pack: Hunyuan-0.5B imports with its q/k gains (the wide-q attention section names them) (engine#1889) A pack whose q_proj is wider than hidden (Hunyuan-0.5B-Instruct: hidden 1024, q_out 2048) has a one-header attention section [q,k,v tiles at K=hidden][640 B][o_proj tiles at K=q_out]. The reader placed o_proj after the gap and never named the gains, so a pack declaring use_qk_norm was refused with "gains are not where the block layout says". The division is now wide_q_attn_section(), which also sets the gain offsets through qk_gain_offsets() when the gap has their shape. Measured on the pack: its model.onnx q_norm/k_norm initializers, rounded to bf16, are in the .fconst at block offset 2,508,928 and 2,509,248 in 24 of 24 layers; the imported container's 48 gain tensors are bit equal to them, and none equals the other gain. Co-Authored-By: Claude Fable 5.2 --- docs/npu.md | 6 ++++++ npu/pack_import.cpp | 32 ++++++++++++++++++++++++++------ npu/pack_import.h | 14 ++++++++++++++ tests/npu_pack_import_test.cpp | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 78 insertions(+), 6 deletions(-) diff --git a/docs/npu.md b/docs/npu.md index 2962883a6..4c6539869 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1264,6 +1264,12 @@ graph: `model.onnx` carries `attn.q_norm.layernorm.weight` and `attn.k_norm.laye FP16 `[128]` initializers, and on Hunyuan-7B-Instruct their bf16 rounding is the first and the second block bit for bit in 32 of 32 layers. `qk_gain_offsets()` holds that arithmetic. +The same gap exists when q_proj is wider than hidden. Hunyuan-0.5B-Instruct (hidden 1024, 16 heads +of 128, so q_proj has 2,048 rows) has `[64 q/k/v tiles at K=1024][640 B][16 o_proj tiles at +K=2048]`, and its graph's gains are in that gap in the same order in 24 of 24 layers (q_norm at +block offset 2,508,928, k_norm 320 B later). `wide_q_attn_section()` holds that division and names +the gains; before it did, this pack was refused as declaring gains the layout did not hold. + **Every tile read is its own check.** Before anything is written, each tile is decoded and re-encoded; the encoder has to reproduce the 34,816 bytes it came from, bias slots included. That one comparison holds every field, index and nibble order at once, and it is why a tile whose diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 492a7d494..15ae22d5c 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -3306,6 +3306,24 @@ QkGainOffsets qk_gain_offsets(int64_t gap_start, int64_t head_dim, int64_t gap_b return o; } +WideQAttnSection wide_q_attn_section(int64_t attn_off, int64_t payload, int64_t qkv_tiles, int64_t tile_bytes, + int64_t o_tiles, int64_t o_tile_bytes, int64_t head_dim) { + WideQAttnSection s; + if (payload <= 0 || qkv_tiles <= 0 || tile_bytes <= 0 || o_tiles <= 0 || o_tile_bytes <= 0) return s; + const int64_t qkv_bytes = qkv_tiles * tile_bytes; + s.gap = payload - qkv_bytes - o_tiles * o_tile_bytes; + if (s.gap < 0 || s.gap > int64_t(2) * head_dim * 2 + 256) return s; + s.ok = true; + s.o_off = attn_off + qkv_bytes + s.gap; + // Measured on Hunyuan-0.5B-Instruct_rai_1.9.0_npu against the q_norm/k_norm FP16 initializers of its own model.onnx: + // rounded to bf16, q_norm is at block offset 2,508,928 and k_norm 320 B later in 24 of 24 layers, each behind 64 + // zero bytes -- the same [header][q][header][k] gap as Hunyuan-7B's, at attn_off + 64 tiles. + const QkGainOffsets g = qk_gain_offsets(attn_off + qkv_bytes, head_dim, s.gap); + s.q_norm = g.q_norm; + s.k_norm = g.k_norm; + return s; +} + std::string header_witness_refusal(int vote_hdrs, bool declared_qk) { if (!declared_qk || vote_hdrs != 1) return std::string(); return "the epsilon-anchor vote recorded ONE in-block header, and this pack declares use_qk_norm, whose per-head q/k " @@ -3716,15 +3734,17 @@ Plan plan(const std::string& pack_dir) { } } const int64_t o_tiles_h = H / kTileRows; - const int64_t qkv_bytes_c = int64_t(n_kv * 2 + n_q) * tile_bytes; - const int64_t o_bytes_c = o_tiles_h * tile_o; - const int64_t gap_c = payload - qkv_bytes_c - o_bytes_c; - const int64_t norm_ceiling = int64_t(2) * int64_t(d.head_dim) * 2 + 256; - if (!handled && payload > 0 && q_out != H && o_tiles_h > 0 && gap_c >= 0 && gap_c <= norm_ceiling) { + const WideQAttnSection wq = wide_q_attn_section(attn_off, payload, int64_t(n_kv * 2 + n_q), tile_bytes, + o_tiles_h, tile_o, int64_t(d.head_dim)); + const int64_t gap_c = wq.gap; + if (!handled && q_out != H && wq.ok) { handled = true; p.attn_tiles = n_kv * 2 + n_q; - p.o_off = attn_off + qkv_bytes_c + gap_c; + p.o_off = wq.o_off; p.o_k = int(q_out); + // engine#1889: the gap IS the per-head gains when it has their shape; 0,0 otherwise. + p.qnorm_off = wq.q_norm; + p.knorm_off = wq.k_norm; // NOT p.attn_note: that field is the refusal channel -- a non-empty value there refuses the // attention biases by name (it means "why this section could not be read"). This is the // opposite: the section WAS read this way. Reported on stderr once instead. diff --git a/npu/pack_import.h b/npu/pack_import.h index dc62f27ff..f9e67e979 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -96,6 +96,20 @@ struct QkGainOffsets { QkGainOffsets qk_gain_offsets(int64_t gap_start, int64_t head_dim, int64_t gap_bytes); std::string qk_norm_refusal(bool declared, bool found); +// The one-header attention section of a pack whose q_proj is WIDER than hidden (Hunyuan-0.5B: hidden 1024, 16 heads x +// head_dim 128, so q_out = 2048): [q,k,v tiles at K=hidden][gap][o_proj tiles at K=q_out]. `payload` is the bytes from +// `attn_off` to the block's header. `ok` is false when the remainder is not a norms-sized gap (negative, or larger than +// two gains plus 256 B), which is a layout this reader has not measured. When the gap has the gains' shape +// (qk_gain_offsets()), `q_norm`/`k_norm` are their offsets; otherwise both are 0 and the section is still read +// (engine#1889: this branch placed o_proj and never named the gains, so a pack that declares use_qk_norm was refused +// with "gains are not where the block layout says" although they sit in the gap it had just computed). +struct WideQAttnSection { + bool ok = false; + int64_t gap = 0, o_off = 0, q_norm = 0, k_norm = 0; +}; +WideQAttnSection wide_q_attn_section(int64_t attn_off, int64_t payload, int64_t qkv_tiles, int64_t tile_bytes, + int64_t o_tiles, int64_t o_tile_bytes, int64_t head_dim); + // Why a pack's two layout witnesses cannot be reconciled, or "" when they can (engine#1985). // // `vote_hdrs` is the number of in-block headers the epsilon-anchor vote recorded (0 when it could not win); `declared_qk` diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index e494e8110..fa968122f 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -5417,6 +5417,37 @@ void test_qk_gain_offsets() { check(n4.k_norm == 0 && n4.q_norm == 0, "qk gains: the byte count alone is not enough, head_dim has to agree"); } +// engine#1889, the pack whose q_proj is wider than hidden (Hunyuan-0.5B-Instruct: hidden 1024, q_out 2048). Its +// one-header attention section is [64 q/k/v tiles at K=1024][640 B][16 o_proj tiles at K=2048]; the reader placed +// o_proj and left the gains unnamed, so the pack was refused as "gains are not where the block layout says". The +// numbers are the pack's own: payload 3,691,136 B from attn_off 2,112, tiles of 39,168 and 73,984 B, and the gains' +// bytes found in the .fconst from the graph's initializers at 2,508,928 (q) and 2,509,248 (k) in every block. +void test_wide_q_attn_section() { + const pi::WideQAttnSection s = pi::wide_q_attn_section(2112, 3691136, 64, 39168, 16, 73984, 128); + check(s.ok && s.gap == 640, "wide-q section: Hunyuan-0.5B's remainder is the 640 B gap"); + check(s.o_off == 2112 + 64 * 39168 + 640, "wide-q section: o_proj starts after the gap"); + check(s.q_norm == 2508928, "wide-q section: q_norm is named, where the graph's bytes are (engine#1889)"); + check(s.k_norm == 2509248, "wide-q section: k_norm is named, 320 B after q_norm"); + check(pi::qk_norm_refusal(true, s.k_norm != 0).empty(), + "wide-q section: a pack declaring use_qk_norm is therefore not refused for unlocated gains"); + // a pack of the same shape with no gap (no per-head norms): read, and no gain is invented + const pi::WideQAttnSection z = pi::wide_q_attn_section(2112, 3691136 - 640, 64, 39168, 16, 73984, 128); + check(z.ok && z.gap == 0 && z.q_norm == 0 && z.k_norm == 0 && z.o_off == 2112 + 64 * 39168, + "wide-q section: no gap is read as no gains, o_proj directly after q/k/v"); + // a norms-sized gap that is not the gains' shape: the section is read, the gains stay unnamed (so a pack that + // declares them is still refused rather than handed tile bytes) + const pi::WideQAttnSection odd = pi::wide_q_attn_section(2112, 3691136 - 128, 64, 39168, 16, 73984, 128); + check(odd.ok && odd.gap == 512 && odd.q_norm == 0 && odd.k_norm == 0, + "wide-q section: a 512 B gap is not the [header][gain] pair and names nothing"); + check(!pi::qk_norm_refusal(true, odd.k_norm != 0).empty(), "wide-q section: and such a pack is still refused"); + // not this layout at all: a tile too many, or a payload shorter than the tiles + check(!pi::wide_q_attn_section(2112, 3691136 + 39168, 64, 39168, 16, 73984, 128).ok, + "wide-q section: a remainder of a whole tile is not a norms gap"); + check(!pi::wide_q_attn_section(2112, 3691136 - 641, 64, 39168, 16, 73984, 128).ok, + "wide-q section: a payload shorter than the tiles is refused"); + check(!pi::wide_q_attn_section(2112, 0, 64, 39168, 16, 73984, 128).ok, "wide-q section: an empty payload is refused"); +} + /* engine#1985: a REGRESSION, not a new bug -- `Qwen3-1.7B` imported with 0 gaps on yesterday's main and refused with 280 * on today's, because #1838's early return trusted an epsilon-anchor vote that had recorded ONE in-block header on a block * that has three. The tell in the report is `qk_norm: true -> false`: a pack that declares per-head q/k gains keeps them in @@ -5454,6 +5485,7 @@ int main(int argc, char** argv) { test_q4_writer_code_range(); test_qk_norm_refusal(); test_qk_gain_offsets(); + test_wide_q_attn_section(); test_tail_head_placement(); const std::string golden_dir = argc > 1 ? argv[1] : ""; const std::string pack = argc > 2 ? argv[2] : ""; From 4821a5ea506de2e9da5529f70fb5f7150e475fa9 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 19:31:27 -0400 Subject: [PATCH 04/10] npu-pack: say the wide-q gap ceiling is a derived bound; docs: where the q/k gains are, by layout (review of #2063) Co-Authored-By: Claude Fable 5.2 --- docs/npu.md | 5 +++++ npu/pack_import.cpp | 2 ++ 2 files changed, 7 insertions(+) diff --git a/docs/npu.md b/docs/npu.md index 4c6539869..58c6a8585 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1270,6 +1270,11 @@ K=2048]`, and its graph's gains are in that gap in the same order in 24 of 24 la block offset 2,508,928, k_norm 320 B later). `wide_q_attn_section()` holds that division and names the gains; before it did, this pack was refused as declaring gains the layout did not hold. +Where the gains are, by layout, in one place: a 4-header block keeps them in their own two sections, +k first (Qwen3, above); a 2-header block keeps them in the 640 B gap between q/k/v and o_proj, q +first, whether o_proj's K is hidden (Hunyuan-7B, Hy-MT2-1.8B) or q_proj's width (Hunyuan-0.5B). A +pack that declares `use_qk_norm` and has them in none of these is refused (`qk_norm_refusal()`). + **Every tile read is its own check.** Before anything is written, each tile is decoded and re-encoded; the encoder has to reproduce the 34,816 bytes it came from, bias slots included. That one comparison holds every field, index and nibble order at once, and it is why a tile whose diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 15ae22d5c..6f5e5185a 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -3312,6 +3312,8 @@ WideQAttnSection wide_q_attn_section(int64_t attn_off, int64_t payload, int64_t if (payload <= 0 || qkv_tiles <= 0 || tile_bytes <= 0 || o_tiles <= 0 || o_tile_bytes <= 0) return s; const int64_t qkv_bytes = qkv_tiles * tile_bytes; s.gap = payload - qkv_bytes - o_tiles * o_tile_bytes; + // The ceiling is a derived bound, not a measurement: two bf16 [head_dim] gains plus 256 B of headers and padding + // (768 at head_dim 128, 512 at 64). Only the offsets below are measured. if (s.gap < 0 || s.gap > int64_t(2) * head_dim * 2 + 256) return s; s.ok = true; s.o_off = attn_off + qkv_bytes + s.gap; From 016ff0a43c73ec45e09e260debe3b87c026bdca2 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 19:55:25 -0400 Subject: [PATCH 05/10] npu-pack: check the q/k gap's form in every layer before writing a gain (review of #2063) The gap's offsets are block arithmetic, so one number serves every layer. qk_gap_block_why() is the per-layer witness: both 64 B headers zero, both gains finite and not all zero. A layer that differs is refused by name. Applies to both 2-header gap layouts (o_proj at K=hidden and at K=q_out). Test arms: a 960 B remainder is not the wide-q layout; the offsets follow the arguments at another head_dim. Co-Authored-By: Claude Fable 5.2 --- docs/npu.md | 3 +++ npu/pack_import.cpp | 37 ++++++++++++++++++++++++++++-- npu/pack_import.h | 9 ++++++++ tests/npu_pack_import_test.cpp | 41 ++++++++++++++++++++++++++++++++++ 4 files changed, 88 insertions(+), 2 deletions(-) diff --git a/docs/npu.md b/docs/npu.md index 58c6a8585..7433746fe 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1274,6 +1274,9 @@ Where the gains are, by layout, in one place: a 4-header block keeps them in the k first (Qwen3, above); a 2-header block keeps them in the 640 B gap between q/k/v and o_proj, q first, whether o_proj's K is hidden (Hunyuan-7B, Hy-MT2-1.8B) or q_proj's width (Hunyuan-0.5B). A pack that declares `use_qk_norm` and has them in none of these is refused (`qk_norm_refusal()`). +The gap's offsets are block arithmetic, one number for every layer, so the import checks each +layer's bytes to be `[64 B zeros][finite gain][64 B zeros][finite gain]` (`qk_gap_block_why()`) +and refuses a layer that is not, by name, instead of writing whatever is there. **Every tile read is its own check.** Before anything is written, each tile is decoded and re-encoded; the encoder has to reproduce the 34,816 bytes it came from, bias slots included. That diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 6f5e5185a..b73149a2f 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -3306,6 +3306,28 @@ QkGainOffsets qk_gain_offsets(int64_t gap_start, int64_t head_dim, int64_t gap_b return o; } +std::string qk_gap_block_why(const uint8_t* gap, int64_t head_dim) { + const int64_t per_head = head_dim * 2; + for (int b = 0; b < 2; ++b) { + const uint8_t* blk = gap + int64_t(b) * (kNormOff + per_head); + const char* nm = b == 0 ? "q_norm" : "k_norm"; + for (int64_t i = 0; i < kNormOff; ++i) + if (blk[i]) + return std::string("the 64 B before ") + nm + " are not zero (byte " + std::to_string(i) + " is " + + std::to_string(int(blk[i])) + "), so this is not the [header][gain] pair"; + bool any = false; + for (int64_t i = 0; i < head_dim; ++i) { + const uint16_t v = uint16_t(blk[kNormOff + 2 * i]) | uint16_t(uint16_t(blk[kNormOff + 2 * i + 1]) << 8); + if ((v & 0x7F80) == 0x7F80) + return std::string(nm) + " element " + std::to_string(i) + " is not finite (bf16 bits " + + std::to_string(int(v)) + "), so these bytes are not a gain"; + any = any || (v & 0x7FFF) != 0; + } + if (!any) return std::string(nm) + " is all zero, so these bytes are not a gain"; + } + return std::string(); +} + WideQAttnSection wide_q_attn_section(int64_t attn_off, int64_t payload, int64_t qkv_tiles, int64_t tile_bytes, int64_t o_tiles, int64_t o_tile_bytes, int64_t head_dim) { WideQAttnSection s; @@ -3681,7 +3703,7 @@ Plan plan(const std::string& pack_dir) { const int64_t n_kv = kv_out / 64, n_q = q_out / 64; auto refuse = [&](const std::string& s) { p.attn_note = s; - p.knorm_off = p.qnorm_off = p.o_off = 0; // an unread section is not a half-read one + p.knorm_off = p.qnorm_off = p.o_off = p.qk_gap_off = 0; // an unread section is not a half-read one p.o_k = 0; }; if (g.hdr.size() == 1) { @@ -3724,6 +3746,7 @@ Plan plan(const std::string& pack_dir) { qk_gain_offsets(attn_off + p.attn_tiles * tile_bytes, int64_t(d.head_dim), rem); p.knorm_off = qkg.k_norm; p.qnorm_off = qkg.q_norm; + if (qkg.k_norm) p.qk_gap_off = attn_off + p.attn_tiles * tile_bytes; static bool said2 = false; if (!said2) { said2 = true; @@ -3747,6 +3770,7 @@ Plan plan(const std::string& pack_dir) { // engine#1889: the gap IS the per-head gains when it has their shape; 0,0 otherwise. p.qnorm_off = wq.q_norm; p.knorm_off = wq.k_norm; + if (wq.k_norm) p.qk_gap_off = wq.o_off - wq.gap; // NOT p.attn_note: that field is the refusal channel -- a non-empty value there refuses the // attention biases by name (it means "why this section could not be read"). This is the // opposite: the section WAS read this way. Reported on stderr once instead. @@ -5556,7 +5580,16 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons } // per-head QK norms (Qwen3 and later): plain bf16 [head_dim] gains that the // container already carries for families that have them. - if (p.knorm_off) + // The gap families: the offsets are one number for every layer, so each layer's bytes are checked + // to be the [header][gain] pair before a gain is written; a layer that differs is refused by name. + std::string gap_why; + if (p.qk_gap_off) + gap_why = qk_gap_block_why( + at(fc, block + p.qk_gap_off, 2 * (kNormOff + int64_t(p.dims.head_dim) * 2)), + int64_t(p.dims.head_dim)); + if (!gap_why.empty()) + late.push_back("model.layers." + std::to_string(L) + ".self_attn.q_norm.weight: " + gap_why); + else if (p.knorm_off) for (const auto& [off, nm] : {std::pair{p.knorm_off, "k_norm"}, std::pair{p.qnorm_off, "q_norm"}}) { const int64_t n = p.dims.head_dim; diff --git a/npu/pack_import.h b/npu/pack_import.h index f9e67e979..c5616c25a 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -107,6 +107,12 @@ struct WideQAttnSection { bool ok = false; int64_t gap = 0, o_off = 0, q_norm = 0, k_norm = 0; }; +// Why one block's gap is NOT `[64 B zeros][bf16 [head_dim] gain][64 B zeros][bf16 [head_dim] gain]`, or "" when it is. +// `gap` points at 2 * (64 + head_dim*2) bytes. The offsets come from the block arithmetic, which is the same number +// for every layer; this is the per-layer witness that the bytes there are gains: both headers all zero, and each gain +// finite and not all zero. A gain window displaced by a section (320 B at head_dim 128) lands on a header or on tile +// bytes and fails one of these. The import refuses the layer by name on a non-empty answer (review of engine#2063). +std::string qk_gap_block_why(const uint8_t* gap, int64_t head_dim); WideQAttnSection wide_q_attn_section(int64_t attn_off, int64_t payload, int64_t qkv_tiles, int64_t tile_bytes, int64_t o_tiles, int64_t o_tile_bytes, int64_t head_dim); @@ -223,6 +229,9 @@ struct Plan { // none. Identified and verified byte-exact against the public Qwen3-1.7B/4B checkpoints (0 of 128 // values differ on either pack, and the swap differs on 127-128 of 128). int64_t knorm_off = 0, qnorm_off = 0; + // Where the 2-header families' gap `[64 B zeros][q_norm][64 B zeros][k_norm]` starts in the block, 0 when the gains + // are not in that form. The import checks the form in EVERY layer (qk_gap_block_why()) before it writes a gain. + int64_t qk_gap_off = 0; int64_t mlp_off = 0; // the MLP section starts just past the block's post-attention norm // What the pack's *graph* carries, reported so `npu-pack info` can answer a pack's genre without a // third-party parser: the rai_1.3/1.6/1.7.1 line ships its 4-bit weights as ordinary ONNX initializers, while diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index fa968122f..1b64abd6c 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -5446,6 +5446,46 @@ void test_wide_q_attn_section() { check(!pi::wide_q_attn_section(2112, 3691136 - 641, 64, 39168, 16, 73984, 128).ok, "wide-q section: a payload shorter than the tiles is refused"); check(!pi::wide_q_attn_section(2112, 0, 64, 39168, 16, 73984, 128).ok, "wide-q section: an empty payload is refused"); + // the same layout with one more [header][gain] section (the gains displaced by 320 B): not a norms gap, so the + // caller falls through to the refusals it had before + check(!pi::wide_q_attn_section(2112, 3691136 + 320, 64, 39168, 16, 73984, 128).ok, + "wide-q section: a 960 B remainder is not this layout"); + // the offsets are arithmetic on the arguments, not this pack's constants: another head_dim and tile count + const pi::WideQAttnSection h64 = pi::wide_q_attn_section(1000, 10 * 500 + 384 + 4 * 900, 10, 500, 4, 900, 64); + check(h64.ok && h64.gap == 384 && h64.q_norm == 1000 + 5000 + 64 && h64.k_norm == h64.q_norm + 128 + 64 && + h64.o_off == 1000 + 5000 + 384, + "wide-q section: at head_dim 64 the gap is 384 B and every offset follows the arguments"); +} + +// Review of engine#2063: the gap's offsets are one number for every layer, so the import checks each layer's bytes to be +// the [64 B zeros][gain][64 B zeros][gain] pair before writing a gain, and refuses the layer otherwise. +void test_qk_gap_block_why() { + std::vector gap(640, 0); + auto put = [&](size_t off, uint16_t v) { gap[off] = uint8_t(v & 0xFF); gap[off + 1] = uint8_t(v >> 8); }; + for (int i = 0; i < 128; ++i) { + put(64 + 2 * size_t(i), 0x3F80); // q_norm = 1.0 + put(384 + 2 * size_t(i), 0x3FC0); // k_norm = 1.5 + } + check(pi::qk_gap_block_why(gap.data(), 128).empty(), "qk gap: two zero headers and two finite gains are the pair"); + // the windows displaced by one section (320 B): q's window is then k's header + gain start, as a reader that + // placed the gains one section late would see it + std::vector shifted(640, 0); + std::copy(gap.begin() + 320, gap.end(), shifted.begin()); // [hdr][k] then zeros where k should be + const std::string a = pi::qk_gap_block_why(shifted.data(), 128); + check(!a.empty() && a.find("k_norm is all zero") != std::string::npos, + "qk gap: a gain displaced by 320 B is refused, naming the gain that is not there"); + std::vector hdr = gap; + hdr[320 + 7] = 1; + const std::string b = pi::qk_gap_block_why(hdr.data(), 128); + check(b.find("before k_norm are not zero") != std::string::npos && b.find("byte 7") != std::string::npos, + "qk gap: a non-zero header byte is refused with its index"); + std::vector inf = gap; + inf[64 + 2 * 5] = 0x80; inf[64 + 2 * 5 + 1] = 0x7F; // +inf in element 5 of q_norm + const std::string c = pi::qk_gap_block_why(inf.data(), 128); + check(c.find("q_norm element 5 is not finite") != std::string::npos, "qk gap: a non-finite gain is refused by element"); + std::vector tile(640); + for (size_t i = 0; i < tile.size(); ++i) tile[i] = uint8_t(17 * i + 3); // tile-like bytes where the gap should be + check(!pi::qk_gap_block_why(tile.data(), 128).empty(), "qk gap: tile bytes are not the pair"); } /* engine#1985: a REGRESSION, not a new bug -- `Qwen3-1.7B` imported with 0 gaps on yesterday's main and refused with 280 @@ -5486,6 +5526,7 @@ int main(int argc, char** argv) { test_qk_norm_refusal(); test_qk_gain_offsets(); test_wide_q_attn_section(); + test_qk_gap_block_why(); test_tail_head_placement(); const std::string golden_dir = argc > 1 ? argv[1] : ""; const std::string pack = argc > 2 ? argv[2] : ""; From edbf39f061baf920fbb48a911fcc4e74e3b464d5 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 20:37:56 -0400 Subject: [PATCH 06/10] npu-pack: the wide-q Hunyuan block is q, k, v too (Hunyuan-0.5B, on #1976's attn_q_first) With the gains named the pack imported, and its q_proj/k_proj were wrong: read as k, q, v the container's k_proj is q's first 1,024 rows and q_proj is q's last 1,024 rows then k (correlation 0.01 with tencent/Hunyuan-0.5B-Instruct in layers 0/11/23). Read as q, k, v all projections correlate 0.95-0.99 and the container picks the checkpoint's next token at 39 of 42 chat-format positions (mean KL 0.030 nats, host numpy forward that reproduces the official implementation's logits). Co-Authored-By: Claude Fable 5.2 --- docs/npu.md | 6 ++++++ npu/pack_import.cpp | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/docs/npu.md b/docs/npu.md index a20f4bc75..a4821ca74 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1297,6 +1297,12 @@ The gap's offsets are block arithmetic, one number for every layer, so the impor layer's bytes to be `[64 B zeros][finite gain][64 B zeros][finite gain]` (`qk_gap_block_why()`) and refuses a layer that is not, by name, instead of writing whatever is there. +Hunyuan-0.5B's first payload is in the order q, k, v, like Hunyuan-7B's (`Plan::attn_q_first`). +Checked against `tencent/Hunyuan-0.5B-Instruct` on the host: read as k, q, v, q_proj and k_proj +correlate 0.01 with the checkpoint in layers 0, 11 and 23; read as q, k, v they correlate 0.96 to +0.99, and in chat format the container picks the checkpoint's next token at 39 of 42 positions +(mean KL 0.030 nats). + **Every tile read is its own check.** Before anything is written, each tile is decoded and re-encoded; the encoder has to reproduce the 34,816 bytes it came from, bias slots included. That one comparison holds every field, index and nibble order at once, and it is why a tile whose diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 64543c5bd..7d17f4dd6 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -3787,6 +3787,11 @@ Plan plan(const std::string& pack_dir) { p.qnorm_off = wq.q_norm; p.knorm_off = wq.k_norm; if (wq.k_norm) p.qk_gap_off = wq.o_off - wq.gap; + // The Hunyuan block's first payload is q, k, v here too (see attn_q_first). Measured on + // Hunyuan-0.5B-Instruct against the public checkpoint: read as k, q, v, the container's k_proj + // is q_proj's first 1,024 rows and its q_proj is q's last 1,024 rows followed by k (per-tensor + // correlation 0.01 for both in three layers); read as q, k, v all three correlate above 0.95. + if (wq.k_norm) p.attn_q_first = true; // NOT p.attn_note: that field is the refusal channel -- a non-empty value there refuses the // attention biases by name (it means "why this section could not be read"). This is the // opposite: the section WAS read this way. Reported on stderr once instead. From f065efb0cb9362ca82b9fc561ce20dabe264271c Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 13:17:48 -0400 Subject: [PATCH 07/10] npu: Hunyuan answers on the forward route: use_qk_norm is read, RoPE base and norm order follow the model, tiles are q,k,v (Fixes #1889) Four defects stood between a Hunyuan-7B pack and an answer. Each is measured on Hunyuan-7B-Instruct_rai_1.9.0_npu with the device-free forward. forward: - use_qk_norm in config.json decides whether q and k are normalised when the config states it. The model_type table (qwen3*, gemma*) has no Hunyuan row, so the imported gains were loaded and never applied. - rope_scaling {type: dynamic, alpha: A} is a fixed base theta * A^(dim/(dim-2)), as the pack's cos/sin cache and the reference implementation have it. It was read as plain theta. - Hunyuan applies RoPE first and the per-head RMSNorm second (the pack graph: q_proj -> RotaryEmbedding -> q_norm). Qwen3 and Gemma keep norm first. - a config that says q/k are normalised with no q_norm weight in the container is refused at load instead of zeroing every head's q. importer: - the Hunyuan attention block stores its tiles q, k, v. Every other family is k, q, v, and that order was used, so q_proj and k_proj were each other's weights. Top-1 next-token accuracy on a fixed 78-token text, teacher-forced: no q/k norm 33/77, q/k norm 62/77, all forward changes 59/77, all forward changes with the old k,q,v tiles 26/77. Co-Authored-By: Claude Fable 5.2 --- CMakeLists.txt | 8 ++ docs/npu.md | 19 +++++ npu/forward/include/q4nx_rope.h | 16 ++++ npu/forward/include/q4nx_semantics.h | 10 ++- npu/forward/src/q4nx_forward.cpp | 36 ++++++-- npu/forward/src/q4nx_semantics.c | 17 +++- npu/pack_import.cpp | 24 +++++- npu/pack_import.h | 13 +++ tests/npu_pack_import_test.cpp | 16 ++++ tests/npu_rope_llama3_test.c | 29 +++++++ tests/q4nx_qk_norm_test.c | 119 +++++++++++++++++++++++++++ 11 files changed, 295 insertions(+), 12 deletions(-) create mode 100644 tests/q4nx_qk_norm_test.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 3f60faae3..92c82c50e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1525,6 +1525,14 @@ target_include_directories(q4nx_mrope_test PRIVATE npu/forward/include) target_link_libraries(q4nx_mrope_test PRIVATE m) add_test(NAME q4nx_mrope COMMAND q4nx_mrope_test) add_dependencies(npu_sampling_test q4nx_mrope_test) +# Which models get the per-head q/k RMSNorm: config.json's use_qk_norm when the config states it, else the +# model_type table (engine#1889: Hunyuan declares it under a model_type the table has no row for, so its gains were +# loaded and never applied). Plain C, links the semantics TU directly, no model and no device. +add_executable(q4nx_qk_norm_test tests/q4nx_qk_norm_test.c npu/forward/src/q4nx_semantics.c) +target_include_directories(q4nx_qk_norm_test PRIVATE npu/forward/include) +target_link_libraries(q4nx_qk_norm_test PRIVATE m) +add_test(NAME q4nx_qk_norm COMMAND q4nx_qk_norm_test) +add_dependencies(npu_sampling_test q4nx_qk_norm_test) add_executable(q4nx_layer_kinds_test tests/q4nx_layer_kinds_test.c npu/forward/src/q4nx_semantics.c) target_include_directories(q4nx_layer_kinds_test PRIVATE npu/forward/include) target_link_libraries(q4nx_layer_kinds_test PRIVATE m) diff --git a/docs/npu.md b/docs/npu.md index 71ac34ff1..4e36a39f1 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -823,6 +823,25 @@ forward route, which reads the block from `config.json`: `rope_type: "llama3"` declared-but-unusable `llama3` block, and every other declared type, at `1bit serve` / `forward-serve` load time rather than at request time. +**Hunyuan (`hunyuan_v1_dense`) differs from the other q/k-norm families in four places: three in +the forward, which reads them from the model rather than from a family list where it can, and one in +the importer** +([engine#1889](https://github.com/1bit-MONSTER/engine/issues/1889)). Each was measured against +`Hunyuan-7B-Instruct`'s own pack: + +| what | rule | where it comes from | +|---|---|---| +| whether q and k are normalised | `use_qk_norm` in `config.json` decides when the config states it; the `qwen3*`/`gemma*` table answers only for configs that do not | the config key. The table has no Hunyuan row, so its gains were loaded and never applied | +| the RoPE base | `rope_scaling: {type: "dynamic", alpha: A}` is a **fixed** base `theta * A^(dim/(dim-2))` at every position (11,158,840 for theta 10000, alpha 1000, dim 128). Without `alpha` the block is the position-dependent NTK rule | the graph's `cos_cache`/`sin_cache` in `model.onnx.data` match that base to 0.00024; plain theta 10000 is off by up to 1.99 | +| the order of norm and RoPE | RoPE first, then the per-head RMSNorm (Qwen3 and Gemma normalise first) | the graph: `q_proj -> RotaryEmbedding -> q_norm -> GroupQueryAttention` in every layer | +| the order of the attention tiles in the pack (importer) | q tiles first, then k, then v. Every other family's block is k, q, v. `1bit npu-pack import` picks the order with the block shape that carries the packed q/k gains | measured: with k, q, v the imported model predicts 26 of 77 next tokens of a fixed text, with q, k, v 59 (the two projections were being read as each other) | + +A model whose config says q and k are normalised and whose container has no +`self_attn.q_norm.weight` is refused at load: the gain would read as zeros and zero every head's q. +That is the shape of a Hunyuan container imported before the importer carried the gains, so the +answer is to re-import it. A Hunyuan container imported before the tile order was right has +the gains and still answers filler words; re-import that one too. + | GGUF type | To q4_1 | Error | |---|---|---| | Q4_0 | scale `d`, zero `-8d`, codes unchanged | bf16 rounding of scale and zero | diff --git a/npu/forward/include/q4nx_rope.h b/npu/forward/include/q4nx_rope.h index 946e41670..aee089a16 100644 --- a/npu/forward/include/q4nx_rope.h +++ b/npu/forward/include/q4nx_rope.h @@ -69,10 +69,26 @@ typedef struct { int on; /* config declared rope_type "dynamic" with usable numbers */ double factor; /* 1.0 on Hunyuan-0.5B -- a factor of 1 is a no-op until pos > original */ double original_max_position; /* 262144 on Hunyuan-0.5B */ + /* engine#1889: Hunyuan's own form of "dynamic". Its rope_scaling block also carries `alpha` (1000.0 on every + * Hunyuan/HY-MT 1.9 pack), and when alpha is set the model's rotary embedding does not move the base with the + * position at all: it uses the FIXED base + * + * base = theta * alpha ^ (dim / (dim - 2)) + * + * (HunYuanDenseV1RotaryEmbedding, "DynamicNTKAlphaRotary"). Measured against the pack rather than taken from the + * reference: Hunyuan-7B's model.onnx.data holds the graph's cos_cache/sin_cache (FP16 [32768, 64]), and they match + * base 10000 * 1000^(128/126) = 11,158,840 to 0.00024 at positions 1, 2, 7, 50 and 300, where plain theta 10000 is + * off by up to 1.99. Reading the block as factor-only made it a no-op, i.e. every position rotated by the wrong + * angle. 0 = the block has no alpha, and the factor rule above applies. */ + double alpha; } Q4nxRopeDynamic; static inline double q4nx_rope_dynamic_base(double theta, const Q4nxRopeDynamic* d, double pos, int dim) { if (!d || !d->on) return theta; + if (d->alpha > 0.0) { /* the alpha form: one fixed base, every position */ + if (dim <= 2) return theta; + return theta * pow(d->alpha, (double)dim / (double)(dim - 2)); + } if (!(d->factor > 0.0) || !(d->original_max_position > 0.0) || dim <= 2) return theta; if (!(pos >= d->original_max_position)) return theta; /* inside the window: plain rope, bit for bit */ const double scaled = d->factor * pos / d->original_max_position - (d->factor - 1.0); diff --git a/npu/forward/include/q4nx_semantics.h b/npu/forward/include/q4nx_semantics.h index 77393f1b1..abd6201fa 100644 --- a/npu/forward/include/q4nx_semantics.h +++ b/npu/forward/include/q4nx_semantics.h @@ -62,8 +62,16 @@ typedef struct { * Source: model_type/architectures == qwen3_5*, resolved against the weights. */ int rmsnorm_offset_probe; /* Per-head RMSNorm applied to q and k before RoPE. - * Source: presence of self_attn.q_norm.weight / k_norm.weight. */ + * Source: config.json's `use_qk_norm` when the config states it (engine#1889: Hunyuan), else the + * model_type/architectures table (qwen3*, gemma*). The forward refuses a model whose flag is set and whose + * layers carry no self_attn.q_norm.weight, since normalising with an absent gain zeroes q. */ int qk_norm; + /* The q/k RMSNorm runs AFTER RoPE instead of before it (engine#1889). Qwen3 and Gemma normalise, then rotate; + * Hunyuan rotates, then normalises. With a per-dimension gain the two orders differ wherever a rotary pair's two + * gains differ, so the order is part of the model. Source: model_type/architectures (hunyuan*) -- no config key + * states it; the evidence is the pack's own graph, where every layer is + * q_proj -> RotaryEmbedding -> q_norm (SimplifiedLayerNormalization) -> GroupQueryAttention. */ + int qk_norm_after_rope; /* RoPE base. Source: rope_theta. */ float rope_theta; /* Channels actually rotated; 0 means the full head_dim. Partial rotary diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 61480b018..38aab38a9 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -198,9 +198,10 @@ static int dequant_any(const void* data, size_t nbytes, int rows, int cols, floa // transformers' _compute_llama3_parameters applies. `on == 0` unless the config names the type, so // every other architecture builds exactly the table it built before. Keys are searched with their // quotes, which is what keeps "factor" from matching inside "high_freq_factor". -// Dynamic NTK: `rope_scaling: {type: "dynamic"|"dynamic_ntk", factor: F}` and `max_position_embeddings`. Hunyuan's -// 1.9 packs declare factor 1.0 with a 262,144 window, i.e. a no-op until a prompt passes that window -- which is why -// accepting it is safe rather than a guess: below the window the base IS theta, so the served bytes are plain rope. +// Dynamic NTK: `rope_scaling: {type: "dynamic"|"dynamic_ntk", factor: F}` and `max_position_embeddings`: below the +// window the base IS theta, so a block with factor 1.0 serves plain rope. Hunyuan's 1.9 packs declare that AND +// `alpha: 1000.0`, which is a different rule (a fixed base theta * alpha^(dim/(dim-2)), engine#1889): reading only +// the factor made their block a no-op and rotated every position by the wrong angle. Q4nxRopeDynamic read_rope_dynamic(const std::string& cfg_path) { Q4nxRopeDynamic d{}; std::ifstream f(cfg_path); @@ -217,7 +218,10 @@ Q4nxRopeDynamic read_rope_dynamic(const std::string& cfg_path) { }; d.factor = number("factor"); d.original_max_position = number("max_position_embeddings"); - d.on = (d.factor > 0.0 && d.original_max_position > 0.0) ? 1 : 0; + // engine#1889: `alpha` selects Hunyuan's fixed-base form (q4nx_rope.h). The key is searched with its quotes and its + // colon, so "lora_alpha" or an "alpha" inside a string value cannot be taken for it. + d.alpha = number("alpha"); + d.on = ((d.factor > 0.0 && d.original_max_position > 0.0) || d.alpha > 0.0) ? 1 : 0; return d; } @@ -1698,6 +1702,17 @@ bool Q4nxNpuForward::load_norms() { layer_attn_dims(l, hd_l, nkv_l); if (!read_bf16(&mw_->layers[l].input_layernorm_weight, in_norm_[l].data(), H)) return false; if (!read_bf16(&mw_->layers[l].post_attention_layernorm_weight, post_norm_[l].data(), H)) return false; + // engine#1889: the config says q and k are normalised and this layer has no gain to normalise with. read_bf16 + // would hand back zeros for the absent tensor and rmsnorm(q, 0) is 0 for every head, so the layer would attend + // uniformly and answer fluent nonsense. A Hunyuan container imported before the importer claimed the gains is + // exactly this shape, so it is refused by name here instead. + if (sem_.qk_norm && absent(&mw_->layers[l].q_norm_weight)) { + err_ = "config.json says q and k are normalised per head (" + + std::string(q4nx_semantics_field(Q4NX_SEM_QK_NORM)) + ") and layer " + std::to_string(l) + + " has no self_attn.q_norm.weight: normalising with an absent gain zeroes q, so this container " + "would answer wrong text (engine#1889); re-import the pack with a build that carries the gains"; + return false; + } if (!read_bf16(&mw_->layers[l].q_norm_weight, q_norm_[l].data(), hd_l)) return false; // Qwen2.5-style attention biases (q/k/v projection biases); absent on the // bias-free families, so a miss is not an error. Loaded BEFORE the k_norm @@ -4627,7 +4642,11 @@ bool Q4nxNpuForward::attention(const std::vector& qkv, int pos, const int // per-head RMSNorm on q and k, only for architectures that have it. A KV-shared // layer (Gemma4) computed no k at all, so its k is never normalized or used. - if (sem_.qk_norm) { + // engine#1889: WHERE it runs is per family. Qwen3 and Gemma normalise and then rotate; Hunyuan's graph rotates + // and then normalises (q_proj -> RotaryEmbedding -> q_norm -> attention), so for it the same code runs after the + // RoPE block below. The RMS itself is rotation-invariant; the per-dimension gain is not, so the order matters + // wherever a rotary pair's two gains differ. + auto qk_normalise = [&]() { std::vector t(HD); for (int h = 0; h < NH; h++) { rmsnorm(&q[(size_t)h * HD], q_norm_[cur_layer_].data(), HD, t.data(), "q_norm"); @@ -4639,7 +4658,8 @@ bool Q4nxNpuForward::attention(const std::vector& qkv, int pos, const int memcpy(&k[(size_t)h * HD], t.data(), sizeof(float) * HD); } } - } + }; + if (sem_.qk_norm && !sem_.qk_norm_after_rope) qk_normalise(); // Gemma4 normalises v with a SCALE-FREE RMSNorm (its v_norm has no weight). if (sem_.v_norm && append_kv) { std::vector tv(HD); @@ -4659,7 +4679,8 @@ bool Q4nxNpuForward::attention(const std::vector& qkv, int pos, const int std::vector cos_(half), sin_(half); // Dynamic NTK: the base moves with the position, so it is one pow per position rather than one per dim pair. // Below the model's original window this returns theta unchanged, which is what makes a short prompt - // bit-identical to the same model served without the scaling. + // bit-identical to the same model served without the scaling. A block with `alpha` (Hunyuan) is the fixed + // base theta * alpha^(dim/(dim-2)) at every position instead (engine#1889). // engine#1792 review: the exponent is HF's `dim / (dim - 2)` over the ROTARY width (`dim = int(head_dim * // partial_rotary_factor)`), which is `rot` here -- passing `HD` was wrong for any model that declares dynamic // scaling together with partial_rotary_factor < 1. Unreachable in today's catalog, so this is a correctness @@ -4706,6 +4727,7 @@ bool Q4nxNpuForward::attention(const std::vector& qkv, int pos, const int for (int h = 0; h < NH; h++) rot_half(&q[(size_t)h * HD]); for (int h = 0; h < NKV; h++) rot_half(&k[(size_t)h * HD]); } + if (sem_.qk_norm && sem_.qk_norm_after_rope) qk_normalise(); // append to cache (a KV-shared layer reuses its owner's, which the owner has // already appended earlier in this same step). diff --git a/npu/forward/src/q4nx_semantics.c b/npu/forward/src/q4nx_semantics.c index c623e660d..775994d7b 100644 --- a/npu/forward/src/q4nx_semantics.c +++ b/npu/forward/src/q4nx_semantics.c @@ -26,7 +26,7 @@ static const char* kFields[Q4NX_SEM_COUNT] = { /* RMSNORM_OFFSET */ "model_type/architectures (gemma* uses the 1+w form)", - /* QK_NORM */ "model_type/architectures (q_norm/k_norm tensors)", + /* QK_NORM */ "use_qk_norm when stated, else model_type/architectures", /* ROPE_THETA */ "rope_theta", /* ROTARY_DIM */ "partial_rotary_factor * head_dim", /* MLP_ACT */ "hidden_act (falls back to model_type)", @@ -221,6 +221,18 @@ int q4nx_semantics_from_config(const char* json_path, const char* model_dir, /* per-head q/k RMSNorm: qwen3* and gemma* have q_norm/k_norm; llama/phi do not */ s->qk_norm = (is_qwen3 || is_gemma) ? 1 : 0; + /* engine#1889: a config that STATES `use_qk_norm` decides it, whatever the name table says. The table is a list + * of the families measured so far, so a family it has never heard of got 0 silently: Hunyuan-7B is + * `hunyuan_v1_dense` / `HunYuanDenseV1ForCausalLM` with `use_qk_norm: true`, its container carries + * self_attn.{q,k}_norm.weight for all 32 layers, and the forward loaded them and never applied them -- the same + * text with the gains, without them and with the key set to false, because nothing read the key. Families that + * do not state the key (qwen3, gemma) keep the table's answer. */ + { + double uq = 0; + if (get_num(json, "use_qk_norm", &uq)) s->qk_norm = uq != 0 ? 1 : 0; + } + /* Hunyuan rotates q and k first and normalises them after (see the header); everyone else normalises first. */ + s->qk_norm_after_rope = (starts_with(model_type, "hunyuan") || starts_with(arch, "HunYuan")) ? 1 : 0; /* RoPE base. Qwen3.5 nests it under rope_parameters (scalar); Gemma4 nests * it PER LAYER TYPE under rope_parameters.{full,sliding}_attention with @@ -541,6 +553,9 @@ void q4nx_semantics_dump(const Q4nxSemantics* s) { q4nx_semantics_field(Q4NX_SEM_RMSNORM_OFFSET)); fprintf(stderr, " [semantics] qk_norm=%d <- %s\n", s->qk_norm, q4nx_semantics_field(Q4NX_SEM_QK_NORM)); + if (s->qk_norm) + fprintf(stderr, " [semantics] qk_norm order: %s <- model_type/architectures (hunyuan* normalises after RoPE)\n", + s->qk_norm_after_rope ? "RoPE, then norm" : "norm, then RoPE"); fprintf(stderr, " [semantics] rope_theta=%.6g <- %s\n", s->rope_theta, q4nx_semantics_field(Q4NX_SEM_ROPE_THETA)); fprintf(stderr, " [semantics] rotary_dim=%d <- %s\n", s->rotary_dim, diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index d1e844bc7..f0941a20c 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -3363,6 +3363,14 @@ ContainerGeometry container_geometry(const json& meta, const json& config) { return g; } +AttnFirstTiles attn_first_tiles(int n_kv, int n_q, bool q_first) { + AttnFirstTiles f; + f.k = q_first ? n_q : 0; + f.q = q_first ? 0 : n_kv; + f.v = n_kv + n_q; + return f; +} + std::string qk_norm_refusal(bool declared, bool found) { if (declared && !found) return "this pack declares use_qk_norm and its per-head q/k gains are not where the block layout says " @@ -3785,6 +3793,14 @@ Plan plan(const std::string& pack_dir) { (long long)rem); } handled = true; + // engine#1889: this block's projections are q, k, v -- not the k, q, v every other family + // writes. It cannot be read off the tiles (no biases here to identify the runs, and the + // round trip is byte-exact either way), so it was settled by the answer: Hunyuan-7B on the + // host forward, teacher-forced on "The capital of France is Paris. The capital of Germany is + // Berlin. The capital of Italy is", predicts ' Paris' / ' Berlin' / ' Rome' at the three + // "is" rows with q,k,v and ' the' / ' the' / ' the' with k,q,v. The same family puts q + // before k in its packed gains (engine#1934), which is the same writer's order. + p.attn_q_first = true; } } const int64_t o_tiles_h = H / kTileRows; @@ -5545,9 +5561,11 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons // o_proj's K is q_proj's output width). struct Slot { const char* proj; int rows; int64_t base; int k; int first; }; const int64_t attn_off = attn_payload_off(H); - const Slot slots[] = {{"k_proj", kv_out, attn_off, int(H), 0}, - {"q_proj", q_out, attn_off, int(H), n_kv}, - {"v_proj", kv_out, attn_off, int(H), n_kv + n_q}, + // engine#1889: k, q, v on every family but the Hunyuan block, which is q, k, v (Plan::attn_q_first). + const AttnFirstTiles ft = attn_first_tiles(n_kv, n_q, p.attn_q_first); + const Slot slots[] = {{"k_proj", kv_out, attn_off, int(H), ft.k}, + {"q_proj", q_out, attn_off, int(H), ft.q}, + {"v_proj", kv_out, attn_off, int(H), ft.v}, {"o_proj", int(H), p.o_off, p.o_k, 0}}; for (int L = 0; L < p.dims.layers; ++L) { const int64_t block = int64_t(L) * g.stride; diff --git a/npu/pack_import.h b/npu/pack_import.h index 513025dbf..970182529 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -109,6 +109,14 @@ std::string qk_norm_refusal(bool declared, bool found); // pinned without a 2.6 GB pack. std::string header_witness_refusal(int vote_hdrs, bool declared_qk); +// The first tile of k_proj, q_proj and v_proj inside the attention payload, for n_kv k/v tiles and n_q q tiles +// (64 rows each). `q_first` false is k, q, v; true is q, k, v (engine#1889, the Hunyuan block). v is last either way. +// Split out so the order is pinned by a test instead of living in an initializer list. +struct AttnFirstTiles { + int k = 0, q = 0, v = 0; +}; +AttnFirstTiles attn_first_tiles(int n_kv, int n_q, bool q_first); + // A hybrid pack's embedding table lives in its own graph (`embedding.onnx` + `.data`, gathered with // GatherBlockQuantized). Returns [V, H] f32 row-major, empty when the pack has no such component, with `why` naming the // reason. Exposed so the dequantisation can be tested against known codes and scales instead of only on a 3 GB pack. @@ -209,6 +217,11 @@ struct Plan { // none. Identified and verified byte-exact against the public Qwen3-1.7B/4B checkpoints (0 of 128 // values differ on either pack, and the swap differs on 127-128 of 128). int64_t knorm_off = 0, qnorm_off = 0; + // The order of the three projections in the block's first payload. Every family measured before Hunyuan writes + // k, q, v (identified on Qwen2.5-0.5B by the tiles' bias subvolumes). The Hunyuan block -- q/k/v at K=hidden, the + // packed per-head gains, o_proj -- writes q, k, v (engine#1889): with k,q,v the container's k_proj was q_proj's + // first rows, and the model answered filler words; with q,k,v it answers. See attn_first_tiles(). + bool attn_q_first = false; int64_t mlp_off = 0; // the MLP section starts just past the block's post-attention norm // What the pack's *graph* carries, reported so `npu-pack info` can answer a pack's genre without a // third-party parser: the rai_1.3/1.6/1.7.1 line ships its 4-bit weights as ordinary ONNX initializers, while diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 57e3e8862..6e38cd1fd 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -5665,6 +5665,21 @@ void test_header_witness_refusal() { std::printf(" header witness: 8 arms (one-header vote vs declaration refuses; 3/1-without-declaration/0 allowed)\n"); } +// engine#1889: which tile each projection starts at inside the attention payload. Every family before Hunyuan is +// k, q, v; the Hunyuan block is q, k, v, and reading it k, q, v handed the forward q_proj's first rows as k_proj +// (byte-exact round trip, filler-word answers). Hunyuan-7B's own counts: 16 k/v tiles, 64 q tiles. +void test_attn_first_tiles() { + const pi::AttnFirstTiles kqv = pi::attn_first_tiles(16, 64, false); + check(kqv.k == 0 && kqv.q == 16 && kqv.v == 80, "attn order: k,q,v puts k at tile 0, q at 16, v at 80"); + const pi::AttnFirstTiles qkv = pi::attn_first_tiles(16, 64, true); + check(qkv.q == 0 && qkv.k == 64 && qkv.v == 80, "attn order: q,k,v puts q at tile 0, k at 64, v at 80 (engine#1889)"); + check(qkv.k + 16 == qkv.v && kqv.q + 64 == kqv.v, "attn order: v follows the second run in both orders"); + const pi::AttnFirstTiles small = pi::attn_first_tiles(2, 14, false); // Qwen2.5-0.5B: k 2, q 14, v 2 + check(small.k == 0 && small.q == 2 && small.v == 16, "attn order: the Qwen2.5-0.5B run (2/14/2) is unchanged"); + pi::Plan p; + check(!p.attn_q_first, "attn order: a plan is k,q,v unless the block measured otherwise"); +} + int main(int argc, char** argv) { // engine#1907: every crafted 1.9 import writes a canonical copy into the scratch cache. Without this the suite left 0-byte // copies in the user's real ~/.cache/1bit/fconst19 and in /tmp on every run. Give the run its own directory, removed at exit. @@ -5677,6 +5692,7 @@ int main(int argc, char** argv) { test_header_witness_refusal(); test_encode_tile_code_range(); test_q4_writer_code_range(); + test_attn_first_tiles(); test_qk_norm_refusal(); test_qk_gain_offsets(); test_tail_head_placement(); diff --git a/tests/npu_rope_llama3_test.c b/tests/npu_rope_llama3_test.c index dd261d2df..63f310dcb 100644 --- a/tests/npu_rope_llama3_test.c +++ b/tests/npu_rope_llama3_test.c @@ -257,6 +257,35 @@ static void test_declared_type(void) { check(fabs(got64 - got128) > 1e3, "and the two widths differ by more than 1e3, so this arm cannot pass with the widths confused"); } + /* engine#1889: Hunyuan's block is `{"alpha": 1000.0, "factor": 1.0, "type": "dynamic"}`, and with alpha set its + * rotary embedding uses ONE fixed base, theta * alpha^(dim/(dim-2)), at every position. Read as factor-only the + * block was a no-op (base 10000), which rotated every position by the wrong angle. The expected numbers are the + * pack's, not the formula's: Hunyuan-7B's model.onnx.data carries the graph's cos_cache/sin_cache (FP16 + * [32768, 64]); at position 1 the angle of pair i is its inverse frequency, and the table reads + * pair 1: 0.77608 pair 2: 0.60209 pair 5: 0.28159 + * (FP16, so good to about 5e-4), against 0.86596 / 0.74989 / 0.48697 for plain theta 10000. */ + { + Q4nxRopeDynamic hy = {1, 1.0, 262144.0, 1000.0}; /* Hunyuan-7B's own numbers */ + const double base = q4nx_rope_dynamic_base(1e4, &hy, 1.0, 128); + check(fabs(base - 11158839.9) < 1.0, "alpha 1000 at theta 10000, dim 128: the base is 11,158,840"); + check(q4nx_rope_dynamic_base(1e4, &hy, 0.0, 128) == base && q4nx_rope_dynamic_base(1e4, &hy, 300000.0, 128) == base, + "... at every position, inside the window and past it (the alpha form does not move with pos)"); + check(fabs(pow(base, -1.0 / 64.0) - 0.77608) < 5e-4, "pair 1's inverse frequency is the pack table's 0.77608"); + check(fabs(pow(base, -2.0 / 64.0) - 0.60209) < 5e-4, "pair 2's is the pack table's 0.60209"); + check(fabs(pow(base, -5.0 / 64.0) - 0.28159) < 5e-4, "pair 5's is the pack table's 0.28159"); + check(fabs(pow(1e4, -1.0 / 64.0) - 0.77608) > 0.05, + "and plain theta 10000 misses pair 1 by more than 0.05, so these arms cannot pass on the no-op"); + Q4nxRopeDynamic hy64 = {1, 1.0, 262144.0, 1000.0}; + check(fabs(q4nx_rope_dynamic_base(1e4, &hy64, 1.0, 64) - 1e4 * pow(1000.0, 64.0 / 62.0)) < 1e-3, + "the exponent is dim/(dim-2) over the rotary width here too (dim 64 -> 64/62)"); + Q4nxRopeDynamic no_alpha = {1, 1.0, 262144.0, 0.0}; + check(q4nx_rope_dynamic_base(1e4, &no_alpha, 5.0, 128) == 1e4, + "alpha 0 is the factor rule: inside the window the base is theta, as before"); + Q4nxRopeDynamic off_alpha = {0, 1.0, 262144.0, 1000.0}; + check(q4nx_rope_dynamic_base(1e4, &off_alpha, 5.0, 128) == 1e4, "and an alpha on a block that is off does nothing"); + Q4nxRopeDynamic tiny = {1, 1.0, 262144.0, 1000.0}; + check(q4nx_rope_dynamic_base(1e4, &tiny, 5.0, 2) == 1e4, "a rotary width of 2 has no exponent, so the base stays theta"); + } } } diff --git a/tests/q4nx_qk_norm_test.c b/tests/q4nx_qk_norm_test.c new file mode 100644 index 000000000..1ea678b3e --- /dev/null +++ b/tests/q4nx_qk_norm_test.c @@ -0,0 +1,119 @@ +/* Copyright 2026 bong-water-water-bong + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* q4nx_qk_norm_test -- which models get the per-head q/k RMSNorm (engine#1889). + * + * The forward applies it only when the semantics flag is set, and the flag used to come from a model_type / + * architectures table alone (qwen3*, gemma*). Hunyuan-7B is `hunyuan_v1_dense` with `use_qk_norm: true`: the table had + * no row for it, so its container's self_attn.{q,k}_norm.weight were loaded and never applied. The config key now + * decides when the config states it; the table stays the answer for the families that do not state it. + * + * The strings below are the real keys of the configs on this box (Hunyuan-7B-Instruct's config.json, a Qwen3 and a + * Llama container's), cut down to the fields the rule reads. No model, no device. */ +#include "q4nx_semantics.h" + +#include +#include +#include +#include +#include + +static int failures = 0; + +static void check(int ok, const char* what) { + printf("%s %s\n", ok ? "ok " : "FAIL", what); + if (!ok) failures++; +} + +/* /config.json holding exactly `json`, 0600 (mkdtemp gives 0700; the file's own mode is what CodeQL reads). */ +static int after_rope = -1; /* the last config's qk_norm_after_rope */ + +static int qk_norm_of(const char* dir, const char* json) { + char path[512]; + snprintf(path, sizeof path, "%s/config.json", dir); + int fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0600); + FILE* f = fd >= 0 ? fdopen(fd, "w") : NULL; + if (!f) { printf("FAIL cannot write %s\n", path); failures++; return -1; } + fputs(json, f); + fclose(f); + Q4nxSemantics s; + if (q4nx_semantics_from_config(path, dir, &s) != 0) { printf("FAIL cannot read %s\n", path); failures++; return -1; } + after_rope = s.qk_norm_after_rope; + return s.qk_norm; +} + +int main(void) { + char dir[] = "/tmp/q4nx-qk-norm-XXXXXX"; + if (!mkdtemp(dir)) { printf("FAIL mkdtemp\n"); return 1; } + + /* The measured case: a family the name table has never heard of, declaring the norm. */ + check(qk_norm_of(dir, "{\"architectures\": [\"HunYuanDenseV1ForCausalLM\"], \"model_type\": \"hunyuan_v1_dense\"," + " \"num_attention_heads\": 32, \"num_key_value_heads\": 8, \"head_dim\": 128," + " \"use_qk_norm\": true}") == 1, + "hunyuan_v1_dense with use_qk_norm: true normalises q and k (engine#1889)"); + /* The same family saying it has none: the key is read, not assumed from the family. */ + check(qk_norm_of(dir, "{\"architectures\": [\"HunYuanDenseV1ForCausalLM\"], \"model_type\": \"hunyuan_v1_dense\"," + " \"use_qk_norm\": false}") == 0, + "hunyuan_v1_dense with use_qk_norm: false does not"); + /* A family with no row and no key keeps the neutral convention. */ + check(qk_norm_of(dir, "{\"architectures\": [\"HunYuanDenseV1ForCausalLM\"], \"model_type\": \"hunyuan_v1_dense\"}") == 0, + "an unlisted family that does not state the key keeps the plain convention"); + + /* The table is unchanged for the families that never state the key. */ + check(qk_norm_of(dir, "{\"architectures\": [\"Qwen3ForCausalLM\"], \"model_type\": \"qwen3\"}") == 1, + "qwen3 without the key still normalises (the name table)"); + check(qk_norm_of(dir, "{\"architectures\": [\"Gemma3ForCausalLM\"], \"model_type\": \"gemma3_text\"}") == 1, + "gemma without the key still normalises (the name table)"); + check(qk_norm_of(dir, "{\"architectures\": [\"LlamaForCausalLM\"], \"model_type\": \"llama\"}") == 0, + "llama without the key still does not"); + check(qk_norm_of(dir, "{\"architectures\": [\"Phi3ForCausalLM\"], \"model_type\": \"phi3\"}") == 0, + "phi without the key still does not"); + + /* A stated key beats the table in both directions, and a null is not a statement. */ + check(qk_norm_of(dir, "{\"model_type\": \"llama\", \"use_qk_norm\": true}") == 1, + "a llama-named config that states use_qk_norm: true normalises"); + check(qk_norm_of(dir, "{\"model_type\": \"qwen3\", \"use_qk_norm\": false}") == 0, + "a qwen3-named config that states use_qk_norm: false does not"); + check(qk_norm_of(dir, "{\"model_type\": \"qwen3\", \"use_qk_norm\": null}") == 1, + "use_qk_norm: null states nothing, so the table answers"); + /* Only the top-level key counts: a nested one belongs to another component's config. */ + check(qk_norm_of(dir, "{\"model_type\": \"llama\", \"vision_config\": {\"use_qk_norm\": true}}") == 0, + "a use_qk_norm nested under another component's config is not this model's"); + + /* The ORDER (engine#1889). Hunyuan's graph is q_proj -> RotaryEmbedding -> q_norm -> attention in every layer, so + * its norm runs after RoPE; Qwen3 and Gemma keep norm-then-RoPE. */ + (void)qk_norm_of(dir, "{\"architectures\": [\"HunYuanDenseV1ForCausalLM\"], \"model_type\": \"hunyuan_v1_dense\"," + " \"use_qk_norm\": true}"); + check(after_rope == 1, "hunyuan_v1_dense normalises q and k AFTER RoPE, as its graph does"); + (void)qk_norm_of(dir, "{\"architectures\": [\"HunYuanDenseV1ForCausalLM\"]}"); + check(after_rope == 1, "... and the architectures entry alone is enough to say so"); + (void)qk_norm_of(dir, "{\"architectures\": [\"Qwen3ForCausalLM\"], \"model_type\": \"qwen3\"}"); + check(after_rope == 0, "qwen3 keeps norm, then RoPE"); + (void)qk_norm_of(dir, "{\"architectures\": [\"Gemma3ForCausalLM\"], \"model_type\": \"gemma3_text\"}"); + check(after_rope == 0, "gemma keeps norm, then RoPE"); + (void)qk_norm_of(dir, "{\"model_type\": \"llama\", \"use_qk_norm\": true}"); + check(after_rope == 0, "a non-Hunyuan config that states use_qk_norm gets the default order"); + + check(strstr(q4nx_semantics_field(Q4NX_SEM_QK_NORM), "use_qk_norm") != NULL, + "the startup line names use_qk_norm as the flag's source"); + + char path[512]; + snprintf(path, sizeof path, "%s/config.json", dir); + unlink(path); + rmdir(dir); + printf(failures ? "q4nx_qk_norm: %d FAILED\n" : "q4nx_qk_norm: ok\n", failures); + return failures ? 1 : 0; +} From 4dd11395d323e9ed2cdf58c6e48f65bd4b292ee2 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 13:23:51 -0400 Subject: [PATCH 08/10] npu-pack: the RoPE cache check tries the rope_scaling alpha base first (engine#1889) Hunyuan-7B's import warned that no plain theta explains the pack's cos/sin cache (best 0.7222 at 1e7). The cache is theta * alpha^(dim/(dim-2)), which the forward now computes, so the check tries that base first and reports it verified. rope_theta stays as declared: the forward applies alpha itself. Co-Authored-By: Claude Fable 5.2 --- docs/npu.md | 2 +- npu/pack_import.cpp | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/docs/npu.md b/docs/npu.md index 4e36a39f1..ec3236f8e 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -832,7 +832,7 @@ the importer** | what | rule | where it comes from | |---|---|---| | whether q and k are normalised | `use_qk_norm` in `config.json` decides when the config states it; the `qwen3*`/`gemma*` table answers only for configs that do not | the config key. The table has no Hunyuan row, so its gains were loaded and never applied | -| the RoPE base | `rope_scaling: {type: "dynamic", alpha: A}` is a **fixed** base `theta * A^(dim/(dim-2))` at every position (11,158,840 for theta 10000, alpha 1000, dim 128). Without `alpha` the block is the position-dependent NTK rule | the graph's `cos_cache`/`sin_cache` in `model.onnx.data` match that base to 0.00024; plain theta 10000 is off by up to 1.99 | +| the RoPE base | `rope_scaling: {type: "dynamic", alpha: A}` is a **fixed** base `theta * A^(dim/(dim-2))` at every position (11,158,840 for theta 10000, alpha 1000, dim 128). Without `alpha` the block is the position-dependent NTK rule | the graph's `cos_cache`/`sin_cache` in `model.onnx.data` match that base to 0.00024; plain theta 10000 is off by up to 1.99. `1bit npu-pack import` tests this base against the pack's cache first and prints `note: RoPE base ... verified`; before, it warned that no plain theta explained the cache | | the order of norm and RoPE | RoPE first, then the per-head RMSNorm (Qwen3 and Gemma normalise first) | the graph: `q_proj -> RotaryEmbedding -> q_norm -> GroupQueryAttention` in every layer | | the order of the attention tiles in the pack (importer) | q tiles first, then k, then v. Every other family's block is k, q, v. `1bit npu-pack import` picks the order with the block shape that carries the packed q/k gains | measured: with k, q, v the imported model predicts 26 of 77 next tokens of a fixed text, with q, k, v 59 (the two projections were being read as each other) | diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index f0941a20c..65bf147d0 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -5868,6 +5868,23 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons } // candidates: what the metadata says first, then the values these families actually use std::vector cands{p.dims.rope_theta, 1e4, 5e4, 1e5, 5e5, 1e6, 1e7, 1.6e6, 3.2e6}; + // engine#1889: a `rope_scaling: {type: "dynamic", alpha: A}` block (Hunyuan) is a FIXED base + // theta * A^(dim/(dim-2)), and the forward computes it from the block (q4nx_rope_dynamic_base). + // That base is tried first: on Hunyuan-7B it is 11,158,840 and no plain candidate comes near + // (best 0.7222 at 1e7), so the import warned "no plain theta explains" about a cache the + // model's own config explains. rope_theta is left as declared when this one wins -- writing the + // scaled base there would have the forward apply alpha twice. + double alpha_base = 0.0, alpha = 0.0; + if (c.contains("rope_scaling") && c["rope_scaling"].is_object() && + c["rope_scaling"].contains("type") && c["rope_scaling"]["type"].is_string() && + c["rope_scaling"]["type"].get() == "dynamic" && + c["rope_scaling"].contains("alpha") && c["rope_scaling"]["alpha"].is_number()) { + alpha = c["rope_scaling"]["alpha"].get(); + const double hd = double(half) * 2.0; + if (alpha > 0.0 && hd > 2.0 && p.dims.rope_theta > 0) + alpha_base = double(p.dims.rope_theta) * std::pow(alpha, hd / (hd - 2.0)); + } + if (alpha_base > 0.0) cands.insert(cands.begin(), alpha_base); double best_corr = -2.0, best_theta = p.dims.rope_theta; const int HD2 = int(half) * 2; // head_dim implied by the cache const int64_t pmax = std::min(rows, 4096); @@ -5898,7 +5915,21 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons if (corr == corr && corr > best_corr) { best_corr = corr; best_theta = th; } } } - if (best_corr > 0.99 && std::fabs(best_theta - p.dims.rope_theta) > 1e-9) { + if (best_corr > 0.99 && alpha_base > 0.0 && best_theta == alpha_base) { + std::fprintf(stderr, + "note: RoPE base %.8g (rope_theta %.6g, rope_scaling alpha %.6g) verified against " + "the pack's cos_cache (corr %.4f)\n", + alpha_base, (double)p.dims.rope_theta, alpha, best_corr); + c.erase("q4nx_rope_note"); + } else if (best_corr > 0.99 && alpha_base > 0.0) { + // A plain theta fits and the alpha base does not, while the config still carries alpha: the + // forward would scale whatever is written to rope_theta, so nothing is adopted. + std::fprintf(stderr, + "WARNING: this pack's RoPE caches fit plain theta %.6g (corr %.4f), not the base " + "%.8g its rope_scaling alpha %.6g implies; this forward applies alpha, so " + "attention may be wrong from the first token\n", + best_theta, best_corr, alpha_base, alpha); + } else if (best_corr > 0.99 && std::fabs(best_theta - p.dims.rope_theta) > 1e-9) { std::fprintf(stderr, "note: rope_theta %.6g declared, %.6g adopted (correlation %.4f against the " "pack's own cos_cache)\n", (double)p.dims.rope_theta, best_theta, best_corr); From f910f9198d24009da0f6545acc5035c7e1616da1 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 22:29:26 -0400 Subject: [PATCH 09/10] npu-pack: one attn_first_tiles() after merging #1976's rebased head, placed where #1976 has it Co-Authored-By: Claude Fable 5.2 --- npu/pack_import.cpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 70c80d1ae..8d4dbe8d9 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -3354,14 +3354,6 @@ std::string header_witness_refusal(int vote_hdrs, bool declared_qk) { "neither is being preferred"; } -AttnFirstTiles attn_first_tiles(int n_kv, int n_q, bool q_first) { - AttnFirstTiles f; - f.k = q_first ? n_q : 0; - f.q = q_first ? 0 : n_kv; - f.v = n_kv + n_q; - return f; -} - // engine#1786 (item F1): the container genre's geometry, from the container's own `dims` block and then config.json. // The order is authority, not preference: `dims` is the pack's own statement about itself, config.json is a file beside // it that a vendor converter wrote. When the two disagree the container's statement wins and `source` says so -- the From 770797f81669cdd482d420ca72aafcfe84797143 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Wed, 11 Nov 2026 16:07:09 -0400 Subject: [PATCH 10/10] npu-pack: the wide-q branch sets q,k,v through apply_wide_q_section(), and a test arm pins it (review of #2063) The CHANGES at af80a93: the line that makes Hunyuan-0.5B's container right is the tile order (`if (wq.k_norm) p.attn_q_first = true;`, q_proj correlation -0.008 -> 0.964), and removing it left npu_pack_import_test at rc 0. plan()'s wide-q branch is now one function, apply_wide_q_section(Plan&, section, n_kv, n_q, q_out, hidden), which plan() calls, so its effect on the plan can be tested without a fabricated 1.9 pack. New arms in test_attn_first_tiles(), using Hunyuan-0.5B's own section (hidden 1024, q_out 2048): - gains in the gap: attn_q_first is set, and attn_first_tiles gives q 0, k 32, v 48 (the rows the checkpoint matches); 64 q/k/v tiles, o_proj at K=q_out after the gap, both gains named; - the same shape with no gap: the section is read, and attn_q_first stays unset; - q_out == hidden, or a section that was not read: not applied, and the plan is untouched. Mutant (that line removed, rebuilt): rc 1, 2 FAIL (the q,k,v arm and the offsets arm). Restored: rc 0 with the Hunyuan-0.5B pack leg. The re-import gives container sha256 0c047c02f9d0381a, the same as before the refactor. Co-Authored-By: Claude Opus 5.5 --- npu/pack_import.cpp | 31 ++++++++++++++++++------------- npu/pack_import.h | 6 ++++++ tests/npu_pack_import_test.cpp | 24 ++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 13 deletions(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 0e9fa0551..33b4470a7 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -3382,6 +3382,23 @@ WideQAttnSection wide_q_attn_section(int64_t attn_off, int64_t payload, int64_t return s; } +bool apply_wide_q_section(Plan& p, const WideQAttnSection& wq, int64_t n_kv, int64_t n_q, int q_out, int64_t hidden) { + if (int64_t(q_out) == hidden || !wq.ok) return false; + p.attn_tiles = n_kv * 2 + n_q; + p.o_off = wq.o_off; + p.o_k = q_out; + // engine#1889: the gap IS the per-head gains when it has their shape; 0,0 otherwise. + p.qnorm_off = wq.q_norm; + p.knorm_off = wq.k_norm; + if (wq.k_norm) p.qk_gap_off = wq.o_off - wq.gap; + // The Hunyuan block's first payload is q, k, v here too (see attn_q_first). Measured on Hunyuan-0.5B-Instruct + // against the public checkpoint: read as k, q, v, the container's k_proj is q_proj's first 1,024 rows and its q_proj + // is q's last 1,024 rows followed by k (per-tensor correlation 0.01 for both in three layers); read as q, k, v all + // three correlate above 0.95. + if (wq.k_norm) p.attn_q_first = true; + return true; +} + std::string header_witness_refusal(int vote_hdrs, bool declared_qk) { if (!declared_qk || vote_hdrs != 1) return std::string(); return "the epsilon-anchor vote recorded ONE in-block header, and this pack declares use_qk_norm, whose per-head q/k " @@ -3895,20 +3912,8 @@ Plan plan(const std::string& pack_dir) { const WideQAttnSection wq = wide_q_attn_section(attn_off, payload, int64_t(n_kv * 2 + n_q), tile_bytes, o_tiles_h, tile_o, int64_t(d.head_dim)); const int64_t gap_c = wq.gap; - if (!handled && q_out != H && wq.ok) { + if (!handled && apply_wide_q_section(p, wq, n_kv, n_q, q_out, H)) { handled = true; - p.attn_tiles = n_kv * 2 + n_q; - p.o_off = wq.o_off; - p.o_k = int(q_out); - // engine#1889: the gap IS the per-head gains when it has their shape; 0,0 otherwise. - p.qnorm_off = wq.q_norm; - p.knorm_off = wq.k_norm; - if (wq.k_norm) p.qk_gap_off = wq.o_off - wq.gap; - // The Hunyuan block's first payload is q, k, v here too (see attn_q_first). Measured on - // Hunyuan-0.5B-Instruct against the public checkpoint: read as k, q, v, the container's k_proj - // is q_proj's first 1,024 rows and its q_proj is q's last 1,024 rows followed by k (per-tensor - // correlation 0.01 for both in three layers); read as q, k, v all three correlate above 0.95. - if (wq.k_norm) p.attn_q_first = true; // NOT p.attn_note: that field is the refusal channel -- a non-empty value there refuses the // attention biases by name (it means "why this section could not be read"). This is the // opposite: the section WAS read this way. Reported on stderr once instead. diff --git a/npu/pack_import.h b/npu/pack_import.h index 477ae6f75..730cf866a 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -365,6 +365,12 @@ ContainerGeometry container_geometry(const nlohmann::json& meta, const nlohmann: // Reads the pack (model.onnx, its external data, both .fconst headers, config.json). // Throws only when the pack is structurally unreadable. +// engine#1889 (review of #2063): plan()'s wide-q branch (q_out != hidden), as one function so its effect on the plan is +// testable without a fabricated 1.9 pack. When the section was read (`wq.ok`) and q is wider than hidden, it places the +// q/k/v run, o_proj and the gains, and sets attn_q_first when the gap held the gains: that block is the Hunyuan writer's, +// whose first payload is q, k, v (read as k, q, v, the container's k_proj was q_proj's first rows, correlation 0.01). +// Returns false and leaves `p` untouched otherwise. +bool apply_wide_q_section(Plan& p, const WideQAttnSection& wq, int64_t n_kv, int64_t n_q, int q_out, int64_t hidden); Plan plan(const std::string& pack_dir); // Whether the canonical Token .fconst already on the scratch path holds exactly the bytes this run would write, diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 1899afad7..c1a31e7d2 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -5849,6 +5849,30 @@ void test_attn_first_tiles() { check(small.k == 0 && small.q == 2 && small.v == 16, "attn order: the Qwen2.5-0.5B run (2/14/2) is unchanged"); pi::Plan p; check(!p.attn_q_first, "attn order: a plan is k,q,v unless the block measured otherwise"); + + // Review of engine#2063: the line that makes the wide-q container right is the tile ORDER, and nothing guarded it. + // Hunyuan-0.5B's own section (hidden 1024, q_out 2048: 32 q, 16 k and 16 v tiles of 64 rows), with its gains in the gap. + const pi::WideQAttnSection wq = pi::wide_q_attn_section(2112, 3691136, 64, 39168, 16, 73984, 128); + pi::Plan w; + check(pi::apply_wide_q_section(w, wq, 16, 32, 2048, 1024), "wide-q plan: Hunyuan-0.5B's section is taken"); + check(w.attn_q_first, "wide-q plan: a wide-q block with its gains in the gap is q,k,v (engine#1889, review of #2063)"); + const pi::AttnFirstTiles wt = pi::attn_first_tiles(int(16), int(32), w.attn_q_first); + check(wt.q == 0 && wt.k == 32 && wt.v == 48, + "wide-q plan: q at tile 0, k at 32, v at 48 -- the rows the checkpoint matches (q 0-31, k 32-47, v 48-63)"); + check(w.attn_tiles == 64 && w.o_k == 2048 && w.o_off == wq.o_off && w.qnorm_off == 2508928 && w.knorm_off == 2509248, + "wide-q plan: 64 q/k/v tiles, o_proj at K=q_out after the gap, both gains named"); + // the same shape without gains in the gap: the section is still read, the order is not claimed + const pi::WideQAttnSection wz = pi::wide_q_attn_section(2112, 3691136 - 640, 64, 39168, 16, 73984, 128); + pi::Plan z; + check(pi::apply_wide_q_section(z, wz, 16, 32, 2048, 1024) && !z.attn_q_first && z.knorm_off == 0, + "wide-q plan: no gains in the gap, so no q,k,v claim (k,q,v stays the default)"); + // q no wider than hidden is not this branch: the plan is left as it was + pi::Plan n; + check(!pi::apply_wide_q_section(n, wq, 16, 32, 1024, 1024) && !n.attn_q_first && n.attn_tiles == 0, + "wide-q plan: q_out == hidden is not the wide-q branch, and the plan is untouched"); + pi::Plan bad; + check(!pi::apply_wide_q_section(bad, pi::WideQAttnSection{}, 16, 32, 2048, 1024) && !bad.attn_q_first, + "wide-q plan: a section that was not read is not applied"); } // engine#1503: the tied-head rule as ONE predicate -- correlated AND not mis-scaled. Two packs on this box correlate