From e8842ab1ba8e4871bfb0a70d97db7cce7a4b143b Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 13:17:48 -0400 Subject: [PATCH 1/8] npu: Hunyuan answers on the forward route: use_qk_norm is read, RoPE base and norm order follow the model, tiles are q,k,v (Fixes #1889) Four defects stood between a Hunyuan-7B pack and an answer. Each is measured on Hunyuan-7B-Instruct_rai_1.9.0_npu with the device-free forward. forward: - use_qk_norm in config.json decides whether q and k are normalised when the config states it. The model_type table (qwen3*, gemma*) has no Hunyuan row, so the imported gains were loaded and never applied. - rope_scaling {type: dynamic, alpha: A} is a fixed base theta * A^(dim/(dim-2)), as the pack's cos/sin cache and the reference implementation have it. It was read as plain theta. - Hunyuan applies RoPE first and the per-head RMSNorm second (the pack graph: q_proj -> RotaryEmbedding -> q_norm). Qwen3 and Gemma keep norm first. - a config that says q/k are normalised with no q_norm weight in the container is refused at load instead of zeroing every head's q. importer: - the Hunyuan attention block stores its tiles q, k, v. Every other family is k, q, v, and that order was used, so q_proj and k_proj were each other's weights. Top-1 next-token accuracy on a fixed 78-token text, teacher-forced: no q/k norm 33/77, q/k norm 62/77, all forward changes 59/77, all forward changes with the old k,q,v tiles 26/77. Co-Authored-By: Claude Fable 5.2 --- CMakeLists.txt | 8 ++ docs/npu.md | 19 +++++ npu/forward/include/q4nx_rope.h | 16 ++++ npu/forward/include/q4nx_semantics.h | 10 ++- npu/forward/src/q4nx_forward.cpp | 36 ++++++-- npu/forward/src/q4nx_semantics.c | 17 +++- npu/pack_import.cpp | 24 +++++- npu/pack_import.h | 13 +++ tests/npu_pack_import_test.cpp | 16 ++++ tests/npu_rope_llama3_test.c | 29 +++++++ tests/q4nx_qk_norm_test.c | 119 +++++++++++++++++++++++++++ 11 files changed, 295 insertions(+), 12 deletions(-) create mode 100644 tests/q4nx_qk_norm_test.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 610142905..d2361d2a5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1535,6 +1535,14 @@ target_include_directories(q4nx_mrope_test PRIVATE npu/forward/include) target_link_libraries(q4nx_mrope_test PRIVATE m) add_test(NAME q4nx_mrope COMMAND q4nx_mrope_test) add_dependencies(npu_sampling_test q4nx_mrope_test) +# Which models get the per-head q/k RMSNorm: config.json's use_qk_norm when the config states it, else the +# model_type table (engine#1889: Hunyuan declares it under a model_type the table has no row for, so its gains were +# loaded and never applied). Plain C, links the semantics TU directly, no model and no device. +add_executable(q4nx_qk_norm_test tests/q4nx_qk_norm_test.c npu/forward/src/q4nx_semantics.c) +target_include_directories(q4nx_qk_norm_test PRIVATE npu/forward/include) +target_link_libraries(q4nx_qk_norm_test PRIVATE m) +add_test(NAME q4nx_qk_norm COMMAND q4nx_qk_norm_test) +add_dependencies(npu_sampling_test q4nx_qk_norm_test) add_executable(q4nx_layer_kinds_test tests/q4nx_layer_kinds_test.c npu/forward/src/q4nx_semantics.c) target_include_directories(q4nx_layer_kinds_test PRIVATE npu/forward/include) target_link_libraries(q4nx_layer_kinds_test PRIVATE m) diff --git a/docs/npu.md b/docs/npu.md index 85857e81f..c112ae570 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -849,6 +849,25 @@ forward route, which reads the block from `config.json`: `rope_type: "llama3"` declared-but-unusable `llama3` block, and every other declared type, at `1bit serve` / `forward-serve` load time rather than at request time. +**Hunyuan (`hunyuan_v1_dense`) differs from the other q/k-norm families in four places: three in +the forward, which reads them from the model rather than from a family list where it can, and one in +the importer** +([engine#1889](https://github.com/1bit-MONSTER/engine/issues/1889)). Each was measured against +`Hunyuan-7B-Instruct`'s own pack: + +| what | rule | where it comes from | +|---|---|---| +| whether q and k are normalised | `use_qk_norm` in `config.json` decides when the config states it; the `qwen3*`/`gemma*` table answers only for configs that do not | the config key. The table has no Hunyuan row, so its gains were loaded and never applied | +| the RoPE base | `rope_scaling: {type: "dynamic", alpha: A}` is a **fixed** base `theta * A^(dim/(dim-2))` at every position (11,158,840 for theta 10000, alpha 1000, dim 128). Without `alpha` the block is the position-dependent NTK rule | the graph's `cos_cache`/`sin_cache` in `model.onnx.data` match that base to 0.00024; plain theta 10000 is off by up to 1.99 | +| the order of norm and RoPE | RoPE first, then the per-head RMSNorm (Qwen3 and Gemma normalise first) | the graph: `q_proj -> RotaryEmbedding -> q_norm -> GroupQueryAttention` in every layer | +| the order of the attention tiles in the pack (importer) | q tiles first, then k, then v. Every other family's block is k, q, v. `1bit npu-pack import` picks the order with the block shape that carries the packed q/k gains | measured: with k, q, v the imported model predicts 26 of 77 next tokens of a fixed text, with q, k, v 59 (the two projections were being read as each other) | + +A model whose config says q and k are normalised and whose container has no +`self_attn.q_norm.weight` is refused at load: the gain would read as zeros and zero every head's q. +That is the shape of a Hunyuan container imported before the importer carried the gains, so the +answer is to re-import it. A Hunyuan container imported before the tile order was right has +the gains and still answers filler words; re-import that one too. + | GGUF type | To q4_1 | Error | |---|---|---| | Q4_0 | scale `d`, zero `-8d`, codes unchanged | bf16 rounding of scale and zero | diff --git a/npu/forward/include/q4nx_rope.h b/npu/forward/include/q4nx_rope.h index 946e41670..aee089a16 100644 --- a/npu/forward/include/q4nx_rope.h +++ b/npu/forward/include/q4nx_rope.h @@ -69,10 +69,26 @@ typedef struct { int on; /* config declared rope_type "dynamic" with usable numbers */ double factor; /* 1.0 on Hunyuan-0.5B -- a factor of 1 is a no-op until pos > original */ double original_max_position; /* 262144 on Hunyuan-0.5B */ + /* engine#1889: Hunyuan's own form of "dynamic". Its rope_scaling block also carries `alpha` (1000.0 on every + * Hunyuan/HY-MT 1.9 pack), and when alpha is set the model's rotary embedding does not move the base with the + * position at all: it uses the FIXED base + * + * base = theta * alpha ^ (dim / (dim - 2)) + * + * (HunYuanDenseV1RotaryEmbedding, "DynamicNTKAlphaRotary"). Measured against the pack rather than taken from the + * reference: Hunyuan-7B's model.onnx.data holds the graph's cos_cache/sin_cache (FP16 [32768, 64]), and they match + * base 10000 * 1000^(128/126) = 11,158,840 to 0.00024 at positions 1, 2, 7, 50 and 300, where plain theta 10000 is + * off by up to 1.99. Reading the block as factor-only made it a no-op, i.e. every position rotated by the wrong + * angle. 0 = the block has no alpha, and the factor rule above applies. */ + double alpha; } Q4nxRopeDynamic; static inline double q4nx_rope_dynamic_base(double theta, const Q4nxRopeDynamic* d, double pos, int dim) { if (!d || !d->on) return theta; + if (d->alpha > 0.0) { /* the alpha form: one fixed base, every position */ + if (dim <= 2) return theta; + return theta * pow(d->alpha, (double)dim / (double)(dim - 2)); + } if (!(d->factor > 0.0) || !(d->original_max_position > 0.0) || dim <= 2) return theta; if (!(pos >= d->original_max_position)) return theta; /* inside the window: plain rope, bit for bit */ const double scaled = d->factor * pos / d->original_max_position - (d->factor - 1.0); diff --git a/npu/forward/include/q4nx_semantics.h b/npu/forward/include/q4nx_semantics.h index 77393f1b1..abd6201fa 100644 --- a/npu/forward/include/q4nx_semantics.h +++ b/npu/forward/include/q4nx_semantics.h @@ -62,8 +62,16 @@ typedef struct { * Source: model_type/architectures == qwen3_5*, resolved against the weights. */ int rmsnorm_offset_probe; /* Per-head RMSNorm applied to q and k before RoPE. - * Source: presence of self_attn.q_norm.weight / k_norm.weight. */ + * Source: config.json's `use_qk_norm` when the config states it (engine#1889: Hunyuan), else the + * model_type/architectures table (qwen3*, gemma*). The forward refuses a model whose flag is set and whose + * layers carry no self_attn.q_norm.weight, since normalising with an absent gain zeroes q. */ int qk_norm; + /* The q/k RMSNorm runs AFTER RoPE instead of before it (engine#1889). Qwen3 and Gemma normalise, then rotate; + * Hunyuan rotates, then normalises. With a per-dimension gain the two orders differ wherever a rotary pair's two + * gains differ, so the order is part of the model. Source: model_type/architectures (hunyuan*) -- no config key + * states it; the evidence is the pack's own graph, where every layer is + * q_proj -> RotaryEmbedding -> q_norm (SimplifiedLayerNormalization) -> GroupQueryAttention. */ + int qk_norm_after_rope; /* RoPE base. Source: rope_theta. */ float rope_theta; /* Channels actually rotated; 0 means the full head_dim. Partial rotary diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index e3bebd1b2..b5c7639cd 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -198,9 +198,10 @@ static int dequant_any(const void* data, size_t nbytes, int rows, int cols, floa // transformers' _compute_llama3_parameters applies. `on == 0` unless the config names the type, so // every other architecture builds exactly the table it built before. Keys are searched with their // quotes, which is what keeps "factor" from matching inside "high_freq_factor". -// Dynamic NTK: `rope_scaling: {type: "dynamic"|"dynamic_ntk", factor: F}` and `max_position_embeddings`. Hunyuan's -// 1.9 packs declare factor 1.0 with a 262,144 window, i.e. a no-op until a prompt passes that window -- which is why -// accepting it is safe rather than a guess: below the window the base IS theta, so the served bytes are plain rope. +// Dynamic NTK: `rope_scaling: {type: "dynamic"|"dynamic_ntk", factor: F}` and `max_position_embeddings`: below the +// window the base IS theta, so a block with factor 1.0 serves plain rope. Hunyuan's 1.9 packs declare that AND +// `alpha: 1000.0`, which is a different rule (a fixed base theta * alpha^(dim/(dim-2)), engine#1889): reading only +// the factor made their block a no-op and rotated every position by the wrong angle. Q4nxRopeDynamic read_rope_dynamic(const std::string& cfg_path) { Q4nxRopeDynamic d{}; std::ifstream f(cfg_path); @@ -217,7 +218,10 @@ Q4nxRopeDynamic read_rope_dynamic(const std::string& cfg_path) { }; d.factor = number("factor"); d.original_max_position = number("max_position_embeddings"); - d.on = (d.factor > 0.0 && d.original_max_position > 0.0) ? 1 : 0; + // engine#1889: `alpha` selects Hunyuan's fixed-base form (q4nx_rope.h). The key is searched with its quotes and its + // colon, so "lora_alpha" or an "alpha" inside a string value cannot be taken for it. + d.alpha = number("alpha"); + d.on = ((d.factor > 0.0 && d.original_max_position > 0.0) || d.alpha > 0.0) ? 1 : 0; return d; } @@ -1722,6 +1726,17 @@ bool Q4nxNpuForward::load_norms() { layer_attn_dims(l, hd_l, nkv_l); if (!read_bf16(&mw_->layers[l].input_layernorm_weight, in_norm_[l].data(), H)) return false; if (!read_bf16(&mw_->layers[l].post_attention_layernorm_weight, post_norm_[l].data(), H)) return false; + // engine#1889: the config says q and k are normalised and this layer has no gain to normalise with. read_bf16 + // would hand back zeros for the absent tensor and rmsnorm(q, 0) is 0 for every head, so the layer would attend + // uniformly and answer fluent nonsense. A Hunyuan container imported before the importer claimed the gains is + // exactly this shape, so it is refused by name here instead. + if (sem_.qk_norm && absent(&mw_->layers[l].q_norm_weight)) { + err_ = "config.json says q and k are normalised per head (" + + std::string(q4nx_semantics_field(Q4NX_SEM_QK_NORM)) + ") and layer " + std::to_string(l) + + " has no self_attn.q_norm.weight: normalising with an absent gain zeroes q, so this container " + "would answer wrong text (engine#1889); re-import the pack with a build that carries the gains"; + return false; + } if (!read_bf16(&mw_->layers[l].q_norm_weight, q_norm_[l].data(), hd_l)) return false; // Qwen2.5-style attention biases (q/k/v projection biases); absent on the // bias-free families, so a miss is not an error. Loaded BEFORE the k_norm @@ -4651,7 +4666,11 @@ bool Q4nxNpuForward::attention(const std::vector& qkv, int pos, const int // per-head RMSNorm on q and k, only for architectures that have it. A KV-shared // layer (Gemma4) computed no k at all, so its k is never normalized or used. - if (sem_.qk_norm) { + // engine#1889: WHERE it runs is per family. Qwen3 and Gemma normalise and then rotate; Hunyuan's graph rotates + // and then normalises (q_proj -> RotaryEmbedding -> q_norm -> attention), so for it the same code runs after the + // RoPE block below. The RMS itself is rotation-invariant; the per-dimension gain is not, so the order matters + // wherever a rotary pair's two gains differ. + auto qk_normalise = [&]() { std::vector t(HD); for (int h = 0; h < NH; h++) { rmsnorm(&q[(size_t)h * HD], q_norm_[cur_layer_].data(), HD, t.data(), "q_norm"); @@ -4663,7 +4682,8 @@ bool Q4nxNpuForward::attention(const std::vector& qkv, int pos, const int memcpy(&k[(size_t)h * HD], t.data(), sizeof(float) * HD); } } - } + }; + if (sem_.qk_norm && !sem_.qk_norm_after_rope) qk_normalise(); // Gemma4 normalises v with a SCALE-FREE RMSNorm (its v_norm has no weight). if (sem_.v_norm && append_kv) { std::vector tv(HD); @@ -4683,7 +4703,8 @@ bool Q4nxNpuForward::attention(const std::vector& qkv, int pos, const int std::vector cos_(half), sin_(half); // Dynamic NTK: the base moves with the position, so it is one pow per position rather than one per dim pair. // Below the model's original window this returns theta unchanged, which is what makes a short prompt - // bit-identical to the same model served without the scaling. + // bit-identical to the same model served without the scaling. A block with `alpha` (Hunyuan) is the fixed + // base theta * alpha^(dim/(dim-2)) at every position instead (engine#1889). // engine#1792 review: the exponent is HF's `dim / (dim - 2)` over the ROTARY width (`dim = int(head_dim * // partial_rotary_factor)`), which is `rot` here -- passing `HD` was wrong for any model that declares dynamic // scaling together with partial_rotary_factor < 1. Unreachable in today's catalog, so this is a correctness @@ -4730,6 +4751,7 @@ bool Q4nxNpuForward::attention(const std::vector& qkv, int pos, const int for (int h = 0; h < NH; h++) rot_half(&q[(size_t)h * HD]); for (int h = 0; h < NKV; h++) rot_half(&k[(size_t)h * HD]); } + if (sem_.qk_norm && sem_.qk_norm_after_rope) qk_normalise(); // append to cache (a KV-shared layer reuses its owner's, which the owner has // already appended earlier in this same step). diff --git a/npu/forward/src/q4nx_semantics.c b/npu/forward/src/q4nx_semantics.c index c623e660d..775994d7b 100644 --- a/npu/forward/src/q4nx_semantics.c +++ b/npu/forward/src/q4nx_semantics.c @@ -26,7 +26,7 @@ static const char* kFields[Q4NX_SEM_COUNT] = { /* RMSNORM_OFFSET */ "model_type/architectures (gemma* uses the 1+w form)", - /* QK_NORM */ "model_type/architectures (q_norm/k_norm tensors)", + /* QK_NORM */ "use_qk_norm when stated, else model_type/architectures", /* ROPE_THETA */ "rope_theta", /* ROTARY_DIM */ "partial_rotary_factor * head_dim", /* MLP_ACT */ "hidden_act (falls back to model_type)", @@ -221,6 +221,18 @@ int q4nx_semantics_from_config(const char* json_path, const char* model_dir, /* per-head q/k RMSNorm: qwen3* and gemma* have q_norm/k_norm; llama/phi do not */ s->qk_norm = (is_qwen3 || is_gemma) ? 1 : 0; + /* engine#1889: a config that STATES `use_qk_norm` decides it, whatever the name table says. The table is a list + * of the families measured so far, so a family it has never heard of got 0 silently: Hunyuan-7B is + * `hunyuan_v1_dense` / `HunYuanDenseV1ForCausalLM` with `use_qk_norm: true`, its container carries + * self_attn.{q,k}_norm.weight for all 32 layers, and the forward loaded them and never applied them -- the same + * text with the gains, without them and with the key set to false, because nothing read the key. Families that + * do not state the key (qwen3, gemma) keep the table's answer. */ + { + double uq = 0; + if (get_num(json, "use_qk_norm", &uq)) s->qk_norm = uq != 0 ? 1 : 0; + } + /* Hunyuan rotates q and k first and normalises them after (see the header); everyone else normalises first. */ + s->qk_norm_after_rope = (starts_with(model_type, "hunyuan") || starts_with(arch, "HunYuan")) ? 1 : 0; /* RoPE base. Qwen3.5 nests it under rope_parameters (scalar); Gemma4 nests * it PER LAYER TYPE under rope_parameters.{full,sliding}_attention with @@ -541,6 +553,9 @@ void q4nx_semantics_dump(const Q4nxSemantics* s) { q4nx_semantics_field(Q4NX_SEM_RMSNORM_OFFSET)); fprintf(stderr, " [semantics] qk_norm=%d <- %s\n", s->qk_norm, q4nx_semantics_field(Q4NX_SEM_QK_NORM)); + if (s->qk_norm) + fprintf(stderr, " [semantics] qk_norm order: %s <- model_type/architectures (hunyuan* normalises after RoPE)\n", + s->qk_norm_after_rope ? "RoPE, then norm" : "norm, then RoPE"); fprintf(stderr, " [semantics] rope_theta=%.6g <- %s\n", s->rope_theta, q4nx_semantics_field(Q4NX_SEM_ROPE_THETA)); fprintf(stderr, " [semantics] rotary_dim=%d <- %s\n", s->rotary_dim, diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index f2ad9c895..e6cdc9bef 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -3305,6 +3305,14 @@ QkGainOffsets qk_gain_offsets(int64_t gap_start, int64_t head_dim, int64_t gap_b return o; } +AttnFirstTiles attn_first_tiles(int n_kv, int n_q, bool q_first) { + AttnFirstTiles f; + f.k = q_first ? n_q : 0; + f.q = q_first ? 0 : n_kv; + f.v = n_kv + n_q; + return f; +} + std::string header_witness_refusal(int vote_hdrs, bool declared_qk) { if (!declared_qk || vote_hdrs != 1) return std::string(); return "the epsilon-anchor vote recorded ONE in-block header, and this pack declares use_qk_norm, whose per-head q/k " @@ -3790,6 +3798,14 @@ Plan plan(const std::string& pack_dir) { (long long)rem); } handled = true; + // engine#1889: this block's projections are q, k, v -- not the k, q, v every other family + // writes. It cannot be read off the tiles (no biases here to identify the runs, and the + // round trip is byte-exact either way), so it was settled by the answer: Hunyuan-7B on the + // host forward, teacher-forced on "The capital of France is Paris. The capital of Germany is + // Berlin. The capital of Italy is", predicts ' Paris' / ' Berlin' / ' Rome' at the three + // "is" rows with q,k,v and ' the' / ' the' / ' the' with k,q,v. The same family puts q + // before k in its packed gains (engine#1934), which is the same writer's order. + p.attn_q_first = true; } } const int64_t o_tiles_h = H / kTileRows; @@ -5550,9 +5566,11 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons // o_proj's K is q_proj's output width). struct Slot { const char* proj; int rows; int64_t base; int k; int first; }; const int64_t attn_off = attn_payload_off(H); - const Slot slots[] = {{"k_proj", kv_out, attn_off, int(H), 0}, - {"q_proj", q_out, attn_off, int(H), n_kv}, - {"v_proj", kv_out, attn_off, int(H), n_kv + n_q}, + // engine#1889: k, q, v on every family but the Hunyuan block, which is q, k, v (Plan::attn_q_first). + const AttnFirstTiles ft = attn_first_tiles(n_kv, n_q, p.attn_q_first); + const Slot slots[] = {{"k_proj", kv_out, attn_off, int(H), ft.k}, + {"q_proj", q_out, attn_off, int(H), ft.q}, + {"v_proj", kv_out, attn_off, int(H), ft.v}, {"o_proj", int(H), p.o_off, p.o_k, 0}}; for (int L = 0; L < p.dims.layers; ++L) { const int64_t block = int64_t(L) * g.stride; diff --git a/npu/pack_import.h b/npu/pack_import.h index 5fc37dc35..d8fa1a233 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -115,6 +115,14 @@ int64_t head_stored_rows(int64_t vocab, bool from_fconst_tail); // pinned without a 2.6 GB pack. std::string header_witness_refusal(int vote_hdrs, bool declared_qk); +// The first tile of k_proj, q_proj and v_proj inside the attention payload, for n_kv k/v tiles and n_q q tiles +// (64 rows each). `q_first` false is k, q, v; true is q, k, v (engine#1889, the Hunyuan block). v is last either way. +// Split out so the order is pinned by a test instead of living in an initializer list. +struct AttnFirstTiles { + int k = 0, q = 0, v = 0; +}; +AttnFirstTiles attn_first_tiles(int n_kv, int n_q, bool q_first); + // A hybrid pack's embedding table lives in its own graph (`embedding.onnx` + `.data`, gathered with // GatherBlockQuantized). Returns [V, H] f32 row-major, empty when the pack has no such component, with `why` naming the // reason. Exposed so the dequantisation can be tested against known codes and scales instead of only on a 3 GB pack. @@ -215,6 +223,11 @@ struct Plan { // none. Identified and verified byte-exact against the public Qwen3-1.7B/4B checkpoints (0 of 128 // values differ on either pack, and the swap differs on 127-128 of 128). int64_t knorm_off = 0, qnorm_off = 0; + // The order of the three projections in the block's first payload. Every family measured before Hunyuan writes + // k, q, v (identified on Qwen2.5-0.5B by the tiles' bias subvolumes). The Hunyuan block -- q/k/v at K=hidden, the + // packed per-head gains, o_proj -- writes q, k, v (engine#1889): with k,q,v the container's k_proj was q_proj's + // first rows, and the model answered filler words; with q,k,v it answers. See attn_first_tiles(). + bool attn_q_first = false; int64_t mlp_off = 0; // the MLP section starts just past the block's post-attention norm // What the pack's *graph* carries, reported so `npu-pack info` can answer a pack's genre without a // third-party parser: the rai_1.3/1.6/1.7.1 line ships its 4-bit weights as ordinary ONNX initializers, while diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 0dcad1ff0..a883fd303 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -5683,6 +5683,21 @@ void test_header_witness_refusal() { std::printf(" header witness: 8 arms (one-header vote vs declaration refuses; 3/1-without-declaration/0 allowed)\n"); } +// engine#1889: which tile each projection starts at inside the attention payload. Every family before Hunyuan is +// k, q, v; the Hunyuan block is q, k, v, and reading it k, q, v handed the forward q_proj's first rows as k_proj +// (byte-exact round trip, filler-word answers). Hunyuan-7B's own counts: 16 k/v tiles, 64 q tiles. +void test_attn_first_tiles() { + const pi::AttnFirstTiles kqv = pi::attn_first_tiles(16, 64, false); + check(kqv.k == 0 && kqv.q == 16 && kqv.v == 80, "attn order: k,q,v puts k at tile 0, q at 16, v at 80"); + const pi::AttnFirstTiles qkv = pi::attn_first_tiles(16, 64, true); + check(qkv.q == 0 && qkv.k == 64 && qkv.v == 80, "attn order: q,k,v puts q at tile 0, k at 64, v at 80 (engine#1889)"); + check(qkv.k + 16 == qkv.v && kqv.q + 64 == kqv.v, "attn order: v follows the second run in both orders"); + const pi::AttnFirstTiles small = pi::attn_first_tiles(2, 14, false); // Qwen2.5-0.5B: k 2, q 14, v 2 + check(small.k == 0 && small.q == 2 && small.v == 16, "attn order: the Qwen2.5-0.5B run (2/14/2) is unchanged"); + pi::Plan p; + check(!p.attn_q_first, "attn order: a plan is k,q,v unless the block measured otherwise"); +} + int main(int argc, char** argv) { // engine#1907: every crafted 1.9 import writes a canonical copy into the scratch cache. Without this the suite left 0-byte // copies in the user's real ~/.cache/1bit/fconst19 and in /tmp on every run. Give the run its own directory, removed at exit. @@ -5695,6 +5710,7 @@ int main(int argc, char** argv) { test_header_witness_refusal(); test_encode_tile_code_range(); test_q4_writer_code_range(); + test_attn_first_tiles(); test_qk_norm_refusal(); test_head_stored_rows(); test_qk_gain_offsets(); diff --git a/tests/npu_rope_llama3_test.c b/tests/npu_rope_llama3_test.c index dd261d2df..63f310dcb 100644 --- a/tests/npu_rope_llama3_test.c +++ b/tests/npu_rope_llama3_test.c @@ -257,6 +257,35 @@ static void test_declared_type(void) { check(fabs(got64 - got128) > 1e3, "and the two widths differ by more than 1e3, so this arm cannot pass with the widths confused"); } + /* engine#1889: Hunyuan's block is `{"alpha": 1000.0, "factor": 1.0, "type": "dynamic"}`, and with alpha set its + * rotary embedding uses ONE fixed base, theta * alpha^(dim/(dim-2)), at every position. Read as factor-only the + * block was a no-op (base 10000), which rotated every position by the wrong angle. The expected numbers are the + * pack's, not the formula's: Hunyuan-7B's model.onnx.data carries the graph's cos_cache/sin_cache (FP16 + * [32768, 64]); at position 1 the angle of pair i is its inverse frequency, and the table reads + * pair 1: 0.77608 pair 2: 0.60209 pair 5: 0.28159 + * (FP16, so good to about 5e-4), against 0.86596 / 0.74989 / 0.48697 for plain theta 10000. */ + { + Q4nxRopeDynamic hy = {1, 1.0, 262144.0, 1000.0}; /* Hunyuan-7B's own numbers */ + const double base = q4nx_rope_dynamic_base(1e4, &hy, 1.0, 128); + check(fabs(base - 11158839.9) < 1.0, "alpha 1000 at theta 10000, dim 128: the base is 11,158,840"); + check(q4nx_rope_dynamic_base(1e4, &hy, 0.0, 128) == base && q4nx_rope_dynamic_base(1e4, &hy, 300000.0, 128) == base, + "... at every position, inside the window and past it (the alpha form does not move with pos)"); + check(fabs(pow(base, -1.0 / 64.0) - 0.77608) < 5e-4, "pair 1's inverse frequency is the pack table's 0.77608"); + check(fabs(pow(base, -2.0 / 64.0) - 0.60209) < 5e-4, "pair 2's is the pack table's 0.60209"); + check(fabs(pow(base, -5.0 / 64.0) - 0.28159) < 5e-4, "pair 5's is the pack table's 0.28159"); + check(fabs(pow(1e4, -1.0 / 64.0) - 0.77608) > 0.05, + "and plain theta 10000 misses pair 1 by more than 0.05, so these arms cannot pass on the no-op"); + Q4nxRopeDynamic hy64 = {1, 1.0, 262144.0, 1000.0}; + check(fabs(q4nx_rope_dynamic_base(1e4, &hy64, 1.0, 64) - 1e4 * pow(1000.0, 64.0 / 62.0)) < 1e-3, + "the exponent is dim/(dim-2) over the rotary width here too (dim 64 -> 64/62)"); + Q4nxRopeDynamic no_alpha = {1, 1.0, 262144.0, 0.0}; + check(q4nx_rope_dynamic_base(1e4, &no_alpha, 5.0, 128) == 1e4, + "alpha 0 is the factor rule: inside the window the base is theta, as before"); + Q4nxRopeDynamic off_alpha = {0, 1.0, 262144.0, 1000.0}; + check(q4nx_rope_dynamic_base(1e4, &off_alpha, 5.0, 128) == 1e4, "and an alpha on a block that is off does nothing"); + Q4nxRopeDynamic tiny = {1, 1.0, 262144.0, 1000.0}; + check(q4nx_rope_dynamic_base(1e4, &tiny, 5.0, 2) == 1e4, "a rotary width of 2 has no exponent, so the base stays theta"); + } } } diff --git a/tests/q4nx_qk_norm_test.c b/tests/q4nx_qk_norm_test.c new file mode 100644 index 000000000..1ea678b3e --- /dev/null +++ b/tests/q4nx_qk_norm_test.c @@ -0,0 +1,119 @@ +/* Copyright 2026 bong-water-water-bong + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* q4nx_qk_norm_test -- which models get the per-head q/k RMSNorm (engine#1889). + * + * The forward applies it only when the semantics flag is set, and the flag used to come from a model_type / + * architectures table alone (qwen3*, gemma*). Hunyuan-7B is `hunyuan_v1_dense` with `use_qk_norm: true`: the table had + * no row for it, so its container's self_attn.{q,k}_norm.weight were loaded and never applied. The config key now + * decides when the config states it; the table stays the answer for the families that do not state it. + * + * The strings below are the real keys of the configs on this box (Hunyuan-7B-Instruct's config.json, a Qwen3 and a + * Llama container's), cut down to the fields the rule reads. No model, no device. */ +#include "q4nx_semantics.h" + +#include +#include +#include +#include +#include + +static int failures = 0; + +static void check(int ok, const char* what) { + printf("%s %s\n", ok ? "ok " : "FAIL", what); + if (!ok) failures++; +} + +/* /config.json holding exactly `json`, 0600 (mkdtemp gives 0700; the file's own mode is what CodeQL reads). */ +static int after_rope = -1; /* the last config's qk_norm_after_rope */ + +static int qk_norm_of(const char* dir, const char* json) { + char path[512]; + snprintf(path, sizeof path, "%s/config.json", dir); + int fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0600); + FILE* f = fd >= 0 ? fdopen(fd, "w") : NULL; + if (!f) { printf("FAIL cannot write %s\n", path); failures++; return -1; } + fputs(json, f); + fclose(f); + Q4nxSemantics s; + if (q4nx_semantics_from_config(path, dir, &s) != 0) { printf("FAIL cannot read %s\n", path); failures++; return -1; } + after_rope = s.qk_norm_after_rope; + return s.qk_norm; +} + +int main(void) { + char dir[] = "/tmp/q4nx-qk-norm-XXXXXX"; + if (!mkdtemp(dir)) { printf("FAIL mkdtemp\n"); return 1; } + + /* The measured case: a family the name table has never heard of, declaring the norm. */ + check(qk_norm_of(dir, "{\"architectures\": [\"HunYuanDenseV1ForCausalLM\"], \"model_type\": \"hunyuan_v1_dense\"," + " \"num_attention_heads\": 32, \"num_key_value_heads\": 8, \"head_dim\": 128," + " \"use_qk_norm\": true}") == 1, + "hunyuan_v1_dense with use_qk_norm: true normalises q and k (engine#1889)"); + /* The same family saying it has none: the key is read, not assumed from the family. */ + check(qk_norm_of(dir, "{\"architectures\": [\"HunYuanDenseV1ForCausalLM\"], \"model_type\": \"hunyuan_v1_dense\"," + " \"use_qk_norm\": false}") == 0, + "hunyuan_v1_dense with use_qk_norm: false does not"); + /* A family with no row and no key keeps the neutral convention. */ + check(qk_norm_of(dir, "{\"architectures\": [\"HunYuanDenseV1ForCausalLM\"], \"model_type\": \"hunyuan_v1_dense\"}") == 0, + "an unlisted family that does not state the key keeps the plain convention"); + + /* The table is unchanged for the families that never state the key. */ + check(qk_norm_of(dir, "{\"architectures\": [\"Qwen3ForCausalLM\"], \"model_type\": \"qwen3\"}") == 1, + "qwen3 without the key still normalises (the name table)"); + check(qk_norm_of(dir, "{\"architectures\": [\"Gemma3ForCausalLM\"], \"model_type\": \"gemma3_text\"}") == 1, + "gemma without the key still normalises (the name table)"); + check(qk_norm_of(dir, "{\"architectures\": [\"LlamaForCausalLM\"], \"model_type\": \"llama\"}") == 0, + "llama without the key still does not"); + check(qk_norm_of(dir, "{\"architectures\": [\"Phi3ForCausalLM\"], \"model_type\": \"phi3\"}") == 0, + "phi without the key still does not"); + + /* A stated key beats the table in both directions, and a null is not a statement. */ + check(qk_norm_of(dir, "{\"model_type\": \"llama\", \"use_qk_norm\": true}") == 1, + "a llama-named config that states use_qk_norm: true normalises"); + check(qk_norm_of(dir, "{\"model_type\": \"qwen3\", \"use_qk_norm\": false}") == 0, + "a qwen3-named config that states use_qk_norm: false does not"); + check(qk_norm_of(dir, "{\"model_type\": \"qwen3\", \"use_qk_norm\": null}") == 1, + "use_qk_norm: null states nothing, so the table answers"); + /* Only the top-level key counts: a nested one belongs to another component's config. */ + check(qk_norm_of(dir, "{\"model_type\": \"llama\", \"vision_config\": {\"use_qk_norm\": true}}") == 0, + "a use_qk_norm nested under another component's config is not this model's"); + + /* The ORDER (engine#1889). Hunyuan's graph is q_proj -> RotaryEmbedding -> q_norm -> attention in every layer, so + * its norm runs after RoPE; Qwen3 and Gemma keep norm-then-RoPE. */ + (void)qk_norm_of(dir, "{\"architectures\": [\"HunYuanDenseV1ForCausalLM\"], \"model_type\": \"hunyuan_v1_dense\"," + " \"use_qk_norm\": true}"); + check(after_rope == 1, "hunyuan_v1_dense normalises q and k AFTER RoPE, as its graph does"); + (void)qk_norm_of(dir, "{\"architectures\": [\"HunYuanDenseV1ForCausalLM\"]}"); + check(after_rope == 1, "... and the architectures entry alone is enough to say so"); + (void)qk_norm_of(dir, "{\"architectures\": [\"Qwen3ForCausalLM\"], \"model_type\": \"qwen3\"}"); + check(after_rope == 0, "qwen3 keeps norm, then RoPE"); + (void)qk_norm_of(dir, "{\"architectures\": [\"Gemma3ForCausalLM\"], \"model_type\": \"gemma3_text\"}"); + check(after_rope == 0, "gemma keeps norm, then RoPE"); + (void)qk_norm_of(dir, "{\"model_type\": \"llama\", \"use_qk_norm\": true}"); + check(after_rope == 0, "a non-Hunyuan config that states use_qk_norm gets the default order"); + + check(strstr(q4nx_semantics_field(Q4NX_SEM_QK_NORM), "use_qk_norm") != NULL, + "the startup line names use_qk_norm as the flag's source"); + + char path[512]; + snprintf(path, sizeof path, "%s/config.json", dir); + unlink(path); + rmdir(dir); + printf(failures ? "q4nx_qk_norm: %d FAILED\n" : "q4nx_qk_norm: ok\n", failures); + return failures ? 1 : 0; +} From a43aea359ea7b284ef57aaf12ccb381ff6cb1d9c Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 13:23:51 -0400 Subject: [PATCH 2/8] npu-pack: the RoPE cache check tries the rope_scaling alpha base first (engine#1889) Hunyuan-7B's import warned that no plain theta explains the pack's cos/sin cache (best 0.7222 at 1e7). The cache is theta * alpha^(dim/(dim-2)), which the forward now computes, so the check tries that base first and reports it verified. rope_theta stays as declared: the forward applies alpha itself. Co-Authored-By: Claude Fable 5.2 --- docs/npu.md | 2 +- npu/pack_import.cpp | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/docs/npu.md b/docs/npu.md index c112ae570..d412a5fad 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -858,7 +858,7 @@ the importer** | what | rule | where it comes from | |---|---|---| | whether q and k are normalised | `use_qk_norm` in `config.json` decides when the config states it; the `qwen3*`/`gemma*` table answers only for configs that do not | the config key. The table has no Hunyuan row, so its gains were loaded and never applied | -| the RoPE base | `rope_scaling: {type: "dynamic", alpha: A}` is a **fixed** base `theta * A^(dim/(dim-2))` at every position (11,158,840 for theta 10000, alpha 1000, dim 128). Without `alpha` the block is the position-dependent NTK rule | the graph's `cos_cache`/`sin_cache` in `model.onnx.data` match that base to 0.00024; plain theta 10000 is off by up to 1.99 | +| the RoPE base | `rope_scaling: {type: "dynamic", alpha: A}` is a **fixed** base `theta * A^(dim/(dim-2))` at every position (11,158,840 for theta 10000, alpha 1000, dim 128). Without `alpha` the block is the position-dependent NTK rule | the graph's `cos_cache`/`sin_cache` in `model.onnx.data` match that base to 0.00024; plain theta 10000 is off by up to 1.99. `1bit npu-pack import` tests this base against the pack's cache first and prints `note: RoPE base ... verified`; before, it warned that no plain theta explained the cache | | the order of norm and RoPE | RoPE first, then the per-head RMSNorm (Qwen3 and Gemma normalise first) | the graph: `q_proj -> RotaryEmbedding -> q_norm -> GroupQueryAttention` in every layer | | the order of the attention tiles in the pack (importer) | q tiles first, then k, then v. Every other family's block is k, q, v. `1bit npu-pack import` picks the order with the block shape that carries the packed q/k gains | measured: with k, q, v the imported model predicts 26 of 77 next tokens of a fixed text, with q, k, v 59 (the two projections were being read as each other) | diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index e6cdc9bef..b8e362361 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -5873,6 +5873,23 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons } // candidates: what the metadata says first, then the values these families actually use std::vector cands{p.dims.rope_theta, 1e4, 5e4, 1e5, 5e5, 1e6, 1e7, 1.6e6, 3.2e6}; + // engine#1889: a `rope_scaling: {type: "dynamic", alpha: A}` block (Hunyuan) is a FIXED base + // theta * A^(dim/(dim-2)), and the forward computes it from the block (q4nx_rope_dynamic_base). + // That base is tried first: on Hunyuan-7B it is 11,158,840 and no plain candidate comes near + // (best 0.7222 at 1e7), so the import warned "no plain theta explains" about a cache the + // model's own config explains. rope_theta is left as declared when this one wins -- writing the + // scaled base there would have the forward apply alpha twice. + double alpha_base = 0.0, alpha = 0.0; + if (c.contains("rope_scaling") && c["rope_scaling"].is_object() && + c["rope_scaling"].contains("type") && c["rope_scaling"]["type"].is_string() && + c["rope_scaling"]["type"].get() == "dynamic" && + c["rope_scaling"].contains("alpha") && c["rope_scaling"]["alpha"].is_number()) { + alpha = c["rope_scaling"]["alpha"].get(); + const double hd = double(half) * 2.0; + if (alpha > 0.0 && hd > 2.0 && p.dims.rope_theta > 0) + alpha_base = double(p.dims.rope_theta) * std::pow(alpha, hd / (hd - 2.0)); + } + if (alpha_base > 0.0) cands.insert(cands.begin(), alpha_base); double best_corr = -2.0, best_theta = p.dims.rope_theta; const int HD2 = int(half) * 2; // head_dim implied by the cache const int64_t pmax = std::min(rows, 4096); @@ -5903,7 +5920,21 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons if (corr == corr && corr > best_corr) { best_corr = corr; best_theta = th; } } } - if (best_corr > 0.99 && std::fabs(best_theta - p.dims.rope_theta) > 1e-9) { + if (best_corr > 0.99 && alpha_base > 0.0 && best_theta == alpha_base) { + std::fprintf(stderr, + "note: RoPE base %.8g (rope_theta %.6g, rope_scaling alpha %.6g) verified against " + "the pack's cos_cache (corr %.4f)\n", + alpha_base, (double)p.dims.rope_theta, alpha, best_corr); + c.erase("q4nx_rope_note"); + } else if (best_corr > 0.99 && alpha_base > 0.0) { + // A plain theta fits and the alpha base does not, while the config still carries alpha: the + // forward would scale whatever is written to rope_theta, so nothing is adopted. + std::fprintf(stderr, + "WARNING: this pack's RoPE caches fit plain theta %.6g (corr %.4f), not the base " + "%.8g its rope_scaling alpha %.6g implies; this forward applies alpha, so " + "attention may be wrong from the first token\n", + best_theta, best_corr, alpha_base, alpha); + } else if (best_corr > 0.99 && std::fabs(best_theta - p.dims.rope_theta) > 1e-9) { std::fprintf(stderr, "note: rope_theta %.6g declared, %.6g adopted (correlation %.4f against the " "pack's own cos_cache)\n", (double)p.dims.rope_theta, best_theta, best_corr); From 3d183af19652df4853b6b7738df119e8b513da5c Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Wed, 11 Nov 2026 10:04:30 -0400 Subject: [PATCH 3/8] npu: read the 4,608-byte symmetric chunk, so Hy-MT2-1.8B-NPU2 serves FastFlowLM's Hy-MT2 pack stores its projections in a tile no reader knew: 256 bf16 scales, then 4,096 code bytes in column order, value = two's complement nibble * scale. The size names the kind. Settled by the tie test against the pack's own bf16 embedding; the pack then answers on the per-op route with kernel shapes that already existed. Co-Authored-By: Claude Fable 5.2 --- docs/npu.md | 23 ++++++++++- npu/forward/include/pad_head_rows.h | 2 +- npu/forward/include/q4nx_dequant.h | 10 +++++ npu/forward/src/q4nx_dequant.c | 30 +++++++++++--- npu/forward/src/q4nx_forward.cpp | 2 +- npu/model.cpp | 2 +- tests/npu_dequant_bf16_test.c | 63 ++++++++++++++++++++++++++--- 7 files changed, 117 insertions(+), 15 deletions(-) diff --git a/docs/npu.md b/docs/npu.md index baedf4d8e..7100a4d4c 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1843,7 +1843,7 @@ engine#587 measured the route on it): | `qwen35` | `FastFlowLM/D3-*-NPU2` (Edge, Flash, Mini, Nano, Lite, Turbo, Ultra), `FastFlowLM/Qwen3.8-27B-NPU2`, `FastFlowLM/Qwen3.5-0.8B-NPU2` | hrx, npu | **served, and measured** (engine#587): the same `Qwen3.8-27B-NPU2` family runs on the NPU route — 0.2902 s/token (3.45 tok/s) and 19.55 tok/s of prefill on the sealed single-ELF ternary rung, 22.811 s to a first token with the weight load in it (`bench/npu-leaderboard/27b-t2s2-*-20261106.json`) — int8 Q4NX decodes at 63.1 s/token and takes 145.7 s to a first token (`bench/npu-leaderboard/27b-*.json`, 2026-10-27), a 5-token prefill chunk is 58.14 s, and the ternary sealed-ELF rung on the same container is 1.72 s/token with 3/3 text-identical output (engine#697, closed by #728). Quality on the same route: **GSM8K 97.5 % (39/40)** with greedy decoding, **6 of the 40 answers cut off by the 512-token budget and exactly 1 of those 6 also wrong** — so among answers that finished, 34/34 were right (`27b-gsm8k-20261107`); and the per-token cost at those answer lengths is 0.3857 s/token against 0.2902 on a 13-token prompt, i.e. it grows ~1.33x with context (`27b-decode-cot-20261107`). What a stock install still needs is a kernel set for the model's geometry — none ships in this repository — so the row read `not served` until the route itself was measured | | `dflash` | `FastFlowLM/Nemotron-3.5-Bolt-E4B-DFlash-NPU2` | hrx | not served: a draft head, not a decoder; "The draft loop, and whether to serve it" below covers what the HRX route does with it | | `gemma4` | `FastFlowLM/Nemotron-3.5-Bolt-E4B-NPU2`, `FastFlowLM/Gemma4-12B-IT-NPU2` | hrx | not served: no gemma4 model type or kernels in the route | -| `hunyuan-dense` | `FastFlowLM/Hy-MT2-1.8B-NPU2` | hrx | not served | +| `hunyuan-dense` | `FastFlowLM/Hy-MT2-1.8B-NPU2` | hrx, npu | **served and measured** on the per-op route ("FastFlowLM's own containers" below) | | `gemma3` | `FastFlowLM/medgemma-1.5-4b-it-NPU2` | hrx | not served | | `lfm2` | `FastFlowLM/LFM2-2.6B-NPU2`, `FastFlowLM/LFM2-1.2B-NPU2`, `FastFlowLM/LFM2.5-1.2B-NPU2` | hrx, npu | `LFM2.5-1.2B-NPU2` **served and measured** on the per-op route ("FastFlowLM's own containers" below); `LFM2-1.2B-NPU2` serves too (one paired ticket, same section); `LFM2-2.6B-NPU2` was not run | | `gpt-oss` | `FastFlowLM/GPT-OSS-20B-NPU2`, `FastFlowLM/GPT-OSS-Safeguard-20b-NPU2` | hrx | not served | @@ -1900,6 +1900,7 @@ does on a geometry that has a sealed set: `Qwen3-1.7B-NPU2` is 3.0 tok/s here an | `Llama-3.2-3B-NPU2` | `llama` | identical, `Paris.` | 1.7 to 1.8 tok/s | | `Phi4-mini-Instruct-NPU2` | `phi3` | identical, `Paris`, once the turn end is read from the template (below) | 1.4 to 1.5 tok/s | | `Qwen3-VL-4B-Instruct-NPU2`, text tower only | `qwen3` | identical, `Paris` | 1.4 to 1.5 tok/s | +| `Hy-MT2-1.8B-NPU2` | `hunyuan_v1_dense` | identical, `Paris`; five translations against FastFlowLM's runtime below | 2.0 to 2.8 tok/s | | `LFM2.5-1.2B-NPU2` | `lfm2`, hybrid: 10 conv layers of 16 | identical, `Paris`; against FastFlowLM's runtime below | 3.5 to 4.6 tok/s | | `LFM2-1.2B-NPU2` | `lfm2`, hybrid: 10 conv layers of 16 | `Paris`; one request per prompt, against FastFlowLM's runtime below | 3.7 tok/s | @@ -1964,6 +1965,26 @@ does not carry the three-identical-answers condition. reasoning, so there is no answer text to compare and no row; the reasoning differs from FastFlowLM's from its first words. +**`Hy-MT2-1.8B-NPU2`: a third tile.** This pack's projections are stored 4,608 bytes to a tile where every +other pack on the box uses 5,120, and no reader knew the size, so it was refused at the first dequant. The tile is +256 bf16 scales (index `g*32 + r`) followed by 4,096 code bytes in column order (`c*16 + r/2`, even row in the low +nibble), and a weight is the two's-complement nibble times its scale: the symmetric reading of the LFM2 packs with +the zero-point block left out and the codes in the Q4K chunk's order. It was settled the same way, against the +pack's own bf16 embedding (its `lm_head` is the quantised copy): mean 0.244 of a step, maximum 1.000, and the +eleven other combinations of scale order, code order and code sign are 3 to 8 steps off. Here the size names the +kind, so nothing is read off the data. The kernel directory is four shapes that were already built for other +packs (`QKV K2048 N3072`, `O K2048 N2048`, `GU K2048 N12288`, `D K6144 N2048`); q/k norm and the template +(`chat_template.jinja`) are read from the pack. + +Measured 2026-11-11, same `model.q4nx` (sha256 `a992683e04c2110e…`) for both runtimes, one ticket: five +translation requests (into German, French, Spanish, Chinese and English), all five answered with a correct +translation by both; one byte-identical (`Où se trouve la gare de train la plus proche ?`), four worded +differently (`Me gustaría una taza de café…` here, `Quisiera una taza de café…` there). The host float run of the +first request gives this route's wording, so the array agrees with float arithmetic on these weights as it does +for LFM2; no reference independent of both runtimes was available for this model, so the row does not say whose +wording is the model's. Three requests of the known-answer prompt returned `Paris` three times. Decode 2.0 to +2.8 tok/s (32 layers on the per-op route) against FastFlowLM's 50 to 53. + Not established by any row here: that a pack goes through the importer (`npu-pack` still reports these containers as recognised, not importable), anything about a vision or audio tower, and any pack whose geometry has no kernel set on the box. Of the 57 `*-NPU2` repos with a readable geometry on 2026-11-11, 19 had all four diff --git a/npu/forward/include/pad_head_rows.h b/npu/forward/include/pad_head_rows.h index 89e0f77f7..91bb7794d 100644 --- a/npu/forward/include/pad_head_rows.h +++ b/npu/forward/include/pad_head_rows.h @@ -45,7 +45,7 @@ // tiles gives 64,064, which fails the bound and falls through to the row spelling (review of #2026 at 63a47be). // Anything the three spellings cannot place returns NV, and the caller's own dequant then refuses it by name. inline bool head_chunk_bytes(int64_t b) { - return b == Q4NX_TILE_BYTES || b == 6144 /* q4_1f, the imported-container tile */ || b == Q4NX_CHUNK_Q4K || b == Q4NX_CHUNK_Q8; + return b == Q4NX_TILE_BYTES || b == 6144 /* q4_1f, the imported-container tile */ || b == Q4NX_CHUNK_Q4K || b == Q4NX_CHUNK_Q8 || b == Q4NX_CHUNK_Q4S; } inline int padded_head_rows(const int64_t* shape, int ndim, int kpad, int NV) { if (!shape || kpad <= 0 || kpad % Q4NX_TILE_COLS != 0 || NV <= 0) return NV; diff --git a/npu/forward/include/q4nx_dequant.h b/npu/forward/include/q4nx_dequant.h index adf2736c3..9476d9dbc 100644 --- a/npu/forward/include/q4nx_dequant.h +++ b/npu/forward/include/q4nx_dequant.h @@ -90,6 +90,16 @@ extern "C" { * the cross-validation against the checkpoint's own `.bf16` twin. */ #define Q4NX_CHUNK_Q4K 4736 #define Q4NX_CHUNK_Q8 8704 +/* The symmetric 4-bit chunk with no zero-point block (FastFlowLM lists the pack as "Q4_0"; Hy-MT2-1.8B-NPU2): + * scales [256] bf16 @[0,512) index g*32 + r (group-major, as in the 5120-byte tile) + * codes [256][16] @[512,4608) byte c*16 + r/2, even row = low nibble (column-major, as in the Q4K chunk) + * value[r][c] = (code >= 8 ? code - 16 : code) * scale[g][r] + * Settled by the tie test on that pack (2026-11-11): its lm_head is the quantised copy of its own bf16 + * embed_tokens, and over the first 1024 rows |dequant - embedding| / |scale| has mean 0.244 and maximum 1.000 with + * this reading; the eleven other combinations of scale order, code order and code sign are 3 to 8 steps off. + * The size names the kind (no tile of another kind is 4608 bytes), so nothing is inferred from the data. */ +#define Q4NX_CHUNK_Q4S 4608 +void q4nx_dequant_tile_q4s(const uint8_t* chunk, float* out, int ld); /* Q4_K chunk: 32 rows x 256 K, 4736 B. * scales[8][32] uint8 @[0,256) index g*32 + r diff --git a/npu/forward/src/q4nx_dequant.c b/npu/forward/src/q4nx_dequant.c index 25acac18b..dca7810ce 100644 --- a/npu/forward/src/q4nx_dequant.c +++ b/npu/forward/src/q4nx_dequant.c @@ -136,12 +136,26 @@ void q4nx_dequant_tile_q8(const uint8_t* t, float* out, int ld) { } } +void q4nx_dequant_tile_q4s(const uint8_t* t, float* out, int ld) { + const uint8_t* sc_b = t; /* [256] bf16, index g*32 + r */ + const uint8_t* qs = t + 512; /* [256][16], byte c*16 + r/2 */ + for (int r = 0; r < Q4NX_TILE_ROWS; r++) { + for (int c = 0; c < Q4NX_TILE_COLS; c++) { + const int g = c / Q4NX_GROUP_SIZE; + const uint8_t byte = qs[c * 16 + r / 2]; + int code = (r % 2 == 0) ? (byte & 0xF) : (byte >> 4); + if (code >= 8) code -= 16; /* two's complement nibble */ + out[(size_t)r * ld + c] = (float)code * bf16_to_f32(le16(sc_b + 2 * (g * Q4NX_TILE_ROWS + r))); + } + } +} + int q4nx_dequant_tensor_chunked(const uint8_t* data, size_t nbytes, int rows, int cols, int chunk_bytes, float* out) { if (!data || !out || rows <= 0 || cols <= 0) return -1; if (rows % Q4NX_TILE_ROWS || cols % Q4NX_TILE_COLS) return -1; if (chunk_bytes != Q4NX_TILE_BYTES && chunk_bytes != Q4NX_CHUNK_Q4K && - chunk_bytes != Q4NX_CHUNK_Q8) + chunk_bytes != Q4NX_CHUNK_Q8 && chunk_bytes != Q4NX_CHUNK_Q4S) return -1; const int ntr = rows / Q4NX_TILE_ROWS, ntc = cols / Q4NX_TILE_COLS; if (nbytes < (size_t)ntr * (size_t)ntc * (size_t)chunk_bytes) return -1; @@ -161,6 +175,8 @@ int q4nx_dequant_tensor_chunked(const uint8_t* data, size_t nbytes, int rows, q4nx_dequant_tile_q4k(c, tmp, Q4NX_TILE_COLS); else if (chunk_bytes == Q4NX_CHUNK_Q8) q4nx_dequant_tile_q8(c, tmp, Q4NX_TILE_COLS); + else if (chunk_bytes == Q4NX_CHUNK_Q4S) + q4nx_dequant_tile_q4s(c, tmp, Q4NX_TILE_COLS); else q4nx_dequant_tile_as(c, tmp, Q4NX_TILE_COLS, sym); for (int r = 0; r < Q4NX_TILE_ROWS; r++) @@ -181,7 +197,7 @@ int q4nx_dequant_tile_row(const uint8_t* data, size_t nbytes, int rows, int cols if (r < 0 || r >= rows) return -1; if (rows % Q4NX_TILE_ROWS || cols % Q4NX_TILE_COLS) return -1; if (chunk_bytes != Q4NX_TILE_BYTES && chunk_bytes != Q4NX_CHUNK_Q4K && - chunk_bytes != Q4NX_CHUNK_Q8) + chunk_bytes != Q4NX_CHUNK_Q8 && chunk_bytes != Q4NX_CHUNK_Q4S) return -1; const int ntr = rows / Q4NX_TILE_ROWS, ntc = cols / Q4NX_TILE_COLS; if (nbytes < (size_t)ntr * (size_t)ntc * (size_t)chunk_bytes) return -1; @@ -199,6 +215,8 @@ int q4nx_dequant_tile_row(const uint8_t* data, size_t nbytes, int rows, int cols q4nx_dequant_tile_q4k(c, tmp, Q4NX_TILE_COLS); else if (chunk_bytes == Q4NX_CHUNK_Q8) q4nx_dequant_tile_q8(c, tmp, Q4NX_TILE_COLS); + else if (chunk_bytes == Q4NX_CHUNK_Q4S) + q4nx_dequant_tile_q4s(c, tmp, Q4NX_TILE_COLS); else q4nx_dequant_tile_as(c, tmp, Q4NX_TILE_COLS, sym); memcpy(out + (size_t)tc * Q4NX_TILE_COLS, tmp + (size_t)rr * Q4NX_TILE_COLS, @@ -210,8 +228,8 @@ int q4nx_dequant_tile_row(const uint8_t* data, size_t nbytes, int rows, int cols int q4nx_tile_row_kind(size_t nbytes, int rows, int cols) { if (rows <= 0 || cols <= 0 || rows % Q4NX_TILE_ROWS || cols % Q4NX_TILE_COLS) return 0; const size_t tiles = (size_t)(rows / Q4NX_TILE_ROWS) * (size_t)(cols / Q4NX_TILE_COLS); - const int kinds[3] = {Q4NX_TILE_BYTES, Q4NX_CHUNK_Q4K, Q4NX_CHUNK_Q8}; - for (int i = 0; i < 3; i++) + const int kinds[4] = {Q4NX_TILE_BYTES, Q4NX_CHUNK_Q4K, Q4NX_CHUNK_Q8, Q4NX_CHUNK_Q4S}; + for (int i = 0; i < 4; i++) if (nbytes == tiles * (size_t)kinds[i]) return kinds[i]; return 0; } @@ -262,7 +280,7 @@ int q4nx_dequant_tensor_packed(const uint8_t* data, size_t nbytes, int rows, int if (rows_packed < rows || cols_packed < cols) return -1; if (rows_packed % Q4NX_TILE_ROWS || cols_packed % Q4NX_TILE_COLS) return -1; if (chunk_bytes != Q4NX_TILE_BYTES && chunk_bytes != Q4NX_CHUNK_Q4K && - chunk_bytes != Q4NX_CHUNK_Q8) + chunk_bytes != Q4NX_CHUNK_Q8 && chunk_bytes != Q4NX_CHUNK_Q4S) return -1; if (rows_packed == rows && cols_packed == cols) return q4nx_dequant_tensor_chunked(data, nbytes, rows, cols, chunk_bytes, out); @@ -285,6 +303,8 @@ int q4nx_dequant_tensor_packed(const uint8_t* data, size_t nbytes, int rows, int q4nx_dequant_tile_q4k(c, tmp, Q4NX_TILE_COLS); else if (chunk_bytes == Q4NX_CHUNK_Q8) q4nx_dequant_tile_q8(c, tmp, Q4NX_TILE_COLS); + else if (chunk_bytes == Q4NX_CHUNK_Q4S) + q4nx_dequant_tile_q4s(c, tmp, Q4NX_TILE_COLS); else q4nx_dequant_tile_as(c, tmp, Q4NX_TILE_COLS, sym); for (int r = 0; r < Q4NX_TILE_ROWS; r++) diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 9dca11690..a2b7f3733 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -187,7 +187,7 @@ static int dequant_any(const void* data, size_t nbytes, int rows, int cols, floa const size_t n = (size_t)(rows / Q4NX_TILE_ROWS) * (size_t)(cols / Q4NX_TILE_COLS); if (n == 0) return -1; const int cb = (int)(nbytes / n); - if (cb == Q4NX_TILE_BYTES || cb == Q4NX_CHUNK_Q4K || cb == Q4NX_CHUNK_Q8) + if (cb == Q4NX_TILE_BYTES || cb == Q4NX_CHUNK_Q4K || cb == Q4NX_CHUNK_Q8 || cb == Q4NX_CHUNK_Q4S) return q4nx_dequant_tensor_chunked((const uint8_t*)data, nbytes, rows, cols, cb, out); // The container may store this weight on a wider grid than the model's dimensions (engine#1097). // Read it on the container's grid and write only the model's [rows, cols], so buffers, row diff --git a/npu/model.cpp b/npu/model.cpp index fb8a5fed6..2b1beb5e2 100644 --- a/npu/model.cpp +++ b/npu/model.cpp @@ -67,7 +67,7 @@ bool table_holds_vocab(const Tensor& t, int64_t vocab, int64_t hidden, std::stri } if (t.dtype != "I8") return true; if (t.bytes == uint64_t(vocab) * uint64_t(hidden)) return true; // plain int8 + per-32-column scale (Gemma4) - const auto is_chunk = [](int64_t b) { return b == Q4NX_TILE_BYTES || b == Q4NX_CHUNK_Q4K || b == Q4NX_CHUNK_Q8; }; + const auto is_chunk = [](int64_t b) { return b == Q4NX_TILE_BYTES || b == Q4NX_CHUNK_Q4K || b == Q4NX_CHUNK_Q8 || b == Q4NX_CHUNK_Q4S; }; int64_t r = -1; if (t.shape.size() == 3 && is_chunk(t.shape[2]) && t.shape[1] == col_tiles) r = t.shape[0] * Q4NX_TILE_ROWS; else if (t.shape.size() == 2 && is_chunk(t.shape[1]) && t.shape[0] % col_tiles == 0) diff --git a/tests/npu_dequant_bf16_test.c b/tests/npu_dequant_bf16_test.c index 0840e21a9..86626d70c 100644 --- a/tests/npu_dequant_bf16_test.c +++ b/tests/npu_dequant_bf16_test.c @@ -121,11 +121,11 @@ static void embed_row_arms(void) { static void chunked_row_arms(void) { const int rows = 64, cols = 512; /* 2x2 tiles: several rows per tile, several tiles per row */ const int ntc = cols / Q4NX_TILE_COLS; - const int kinds[3] = {Q4NX_TILE_BYTES, Q4NX_CHUNK_Q4K, Q4NX_CHUNK_Q8}; + const int kinds[4] = {Q4NX_TILE_BYTES, Q4NX_CHUNK_Q4K, Q4NX_CHUNK_Q8, Q4NX_CHUNK_Q4S}; float* whole = malloc((size_t)rows * cols * sizeof(float)); float* row = malloc((size_t)cols * sizeof(float)); if (!whole || !row) { check(0, "chunked row: the arms allocate"); free(whole); free(row); return; } - for (int k = 0; k < 3; k++) { + for (int k = 0; k < 4; k++) { const int cb = kinds[k]; const size_t nbytes = (size_t)(rows / Q4NX_TILE_ROWS) * (size_t)ntc * (size_t)cb; uint8_t* q = malloc(nbytes); @@ -141,7 +141,8 @@ static void chunked_row_arms(void) { } check(bad == 0, cb == Q4NX_TILE_BYTES ? "chunked row: every row matches the whole-table decode (5,120 B tiles)" : cb == Q4NX_CHUNK_Q4K ? "chunked row: every row matches the whole-table decode (Q4K chunks)" - : "chunked row: every row matches the whole-table decode (Q8 chunks)"); + : cb == Q4NX_CHUNK_Q8 ? "chunked row: every row matches the whole-table decode (Q8 chunks)" + : "chunked row: every row matches the whole-table decode (Q4S chunks, 4,608 B)"); check(q4nx_dequant_tile_row(q, nbytes, rows, cols, cb, rows, row) == -1, "chunked row: row == rows is refused"); check(q4nx_dequant_tile_row(q, nbytes, rows, cols, cb, -1, row) == -1, "chunked row: row -1 is refused"); check(q4nx_dequant_tile_row(q, nbytes - 1, rows, cols, cb, 0, row) == -1, @@ -152,7 +153,7 @@ static void chunked_row_arms(void) { free(q); } free(whole); free(row); - printf(" chunked row: 3 kinds x 64 rows bit-exact + 5 refusal arms\n"); + printf(" chunked row: 4 kinds x 64 rows bit-exact + 5 refusal arms\n"); } /* engine#1842: the forward picks the per-row path for a chunked embedding with q4nx_tile_row_kind(). It used to pick it @@ -165,8 +166,8 @@ static int old_rule(size_t esz, int nv, int kpad) { return tiles > 0 && esz % tiles == 0; } static void tile_row_kind_arms(void) { - const int kinds[3] = {Q4NX_TILE_BYTES, Q4NX_CHUNK_Q4K, Q4NX_CHUNK_Q8}; - for (int k = 0; k < 3; k++) + const int kinds[4] = {Q4NX_TILE_BYTES, Q4NX_CHUNK_Q4K, Q4NX_CHUNK_Q8, Q4NX_CHUNK_Q4S}; + for (int k = 0; k < 4; k++) check(q4nx_tile_row_kind((size_t)4 * (size_t)kinds[k], 64, 512) == kinds[k], "tile-row kind: each supported chunk kind on a 2x2 grid is accepted as itself"); struct { size_t esz; int nv, kpad; const char* what; } bad[] = { @@ -329,7 +330,57 @@ static void tile_kind_is_per_tensor(void) { free(t); free(out); free(row); } +/* The 4,608-byte symmetric chunk (Hy-MT2-1.8B-NPU2): 256 bf16 scales at g*32 + r, then 4,096 code bytes at + * c*16 + r/2 (even row = low nibble), value = two's-complement nibble * scale. The expected values below index the + * bytes from that description, not through the decoder; and each of the three ways the layout was NOT (the 5,120 + * tile's code order, row-major scales, offset-binary codes) is shown to give a different table, so the fixture can + * tell them apart. */ +static void q4s_chunk(void) { + uint8_t* t = (uint8_t*)calloc(1, Q4NX_CHUNK_Q4S); + float* out = (float*)malloc(sizeof(float) * Q4NX_TILE_ROWS * Q4NX_TILE_COLS); + if (!t || !out) { check(0, "q4s: the buffers allocate"); free(t); free(out); return; } + for (int i = 0; i < 256; i++) put_bf16(t, (size_t)i, ((i % 5 == 0) ? -1.0f : 1.0f) * 0.125f * (float)(1 + i % 7)); + for (int i = 0; i < 4096; i++) t[512 + i] = (uint8_t)((i * 113 + 29) & 0xFF); + #define Q4S_U(r, c) (((r) % 2 == 0) ? (t[512 + (c) * 16 + (r) / 2] & 0xF) : (t[512 + (c) * 16 + (r) / 2] >> 4)) + #define Q4S_SC(i) ((((i) % 5 == 0) ? -1.0f : 1.0f) * 0.125f * (float)(1 + (i) % 7)) + q4nx_dequant_tile_q4s(t, out, Q4NX_TILE_COLS); + int bad = 0, as_5120_order = 0, as_row_major = 0, as_offset = 0, neg = 0; + for (int r = 0; r < Q4NX_TILE_ROWS; r++) + for (int c = 0; c < Q4NX_TILE_COLS; c++) { + const int g = c / 32, u = Q4S_U(r, c), sgn = u >= 8 ? u - 16 : u; + const float got = out[r * Q4NX_TILE_COLS + c]; + if (u >= 8) neg = 1; + if (got != (float)sgn * Q4S_SC(g * 32 + r)) bad++; + /* the layouts it is not */ + const uint8_t b5 = t[512 + ((r / 16) * 8 + g) * 256 + 8 * (c % 32) + (r % 16) / 2]; + const int u5 = ((r % 16) % 2 == 0) ? (b5 & 0xF) : (b5 >> 4); + if (got != (float)(u5 >= 8 ? u5 - 16 : u5) * Q4S_SC(g * 32 + r)) as_5120_order++; + if (got != (float)sgn * Q4S_SC(r * 8 + g)) as_row_major++; + if (got != (float)(u - 8) * Q4S_SC(g * 32 + r)) as_offset++; + } + check(neg, "q4s: the fixture has codes of 8 and above"); + check(bad == 0, "q4s: every element is (two's-complement nibble at c*16 + r/2) * (scale at g*32 + r)"); + check(as_5120_order > 1000, "q4s: the 5,120 tile's code order would give a different table"); + check(as_row_major > 1000, "q4s: row-major scales would give a different table"); + check(as_offset > 1000, "q4s: offset-binary codes would give a different table"); + float* whole = (float*)malloc(sizeof(float) * Q4NX_TILE_ROWS * Q4NX_TILE_COLS); + if (whole) { + check(q4nx_dequant_tensor_chunked(t, Q4NX_CHUNK_Q4S, Q4NX_TILE_ROWS, Q4NX_TILE_COLS, Q4NX_CHUNK_Q4S, whole) == 0 && + memcmp(whole, out, sizeof(float) * Q4NX_TILE_ROWS * Q4NX_TILE_COLS) == 0, + "q4s: the tensor reader takes the kind by its size and decodes the same values"); + check(q4nx_dequant_tensor_chunked(t, Q4NX_CHUNK_Q4S - 1, Q4NX_TILE_ROWS, Q4NX_TILE_COLS, Q4NX_CHUNK_Q4S, whole) == -1, + "q4s: one byte short is refused"); + } else check(0, "q4s: the reader buffer allocates"); + check(q4nx_tile_row_kind((size_t)Q4NX_CHUNK_Q4S, 32, 256) == Q4NX_CHUNK_Q4S && + q4nx_tile_row_kind((size_t)Q4NX_TILE_BYTES, 32, 256) == Q4NX_TILE_BYTES, + "q4s: 4,608 and 5,120 bytes per tile are told apart by size alone"); + #undef Q4S_U + #undef Q4S_SC + free(whole); free(t); free(out); +} + int main(void) { + q4s_chunk(); tile_kinds(); tile_kind_is_per_tensor(); tile_row_kind_arms(); From f94f8b02565a6e7fef3477eef054741976c8653f Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Wed, 11 Nov 2026 11:31:19 -0400 Subject: [PATCH 4/8] docs: Hy-MT2 on the tile reader alone is 1 of 5 against FastFlowLM, 4 of 5 with #1976's RoPE base Measured with #1976 merged locally. main runs hunyuan_v1_dense at RoPE base 10000 where alpha makes it about 1.116e7, so the row says the pack serves and is not yet the model. Co-Authored-By: Claude Fable 5.2 --- docs/npu.md | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/docs/npu.md b/docs/npu.md index 7100a4d4c..4ce3fe04d 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1843,7 +1843,7 @@ engine#587 measured the route on it): | `qwen35` | `FastFlowLM/D3-*-NPU2` (Edge, Flash, Mini, Nano, Lite, Turbo, Ultra), `FastFlowLM/Qwen3.8-27B-NPU2`, `FastFlowLM/Qwen3.5-0.8B-NPU2` | hrx, npu | **served, and measured** (engine#587): the same `Qwen3.8-27B-NPU2` family runs on the NPU route — 0.2902 s/token (3.45 tok/s) and 19.55 tok/s of prefill on the sealed single-ELF ternary rung, 22.811 s to a first token with the weight load in it (`bench/npu-leaderboard/27b-t2s2-*-20261106.json`) — int8 Q4NX decodes at 63.1 s/token and takes 145.7 s to a first token (`bench/npu-leaderboard/27b-*.json`, 2026-10-27), a 5-token prefill chunk is 58.14 s, and the ternary sealed-ELF rung on the same container is 1.72 s/token with 3/3 text-identical output (engine#697, closed by #728). Quality on the same route: **GSM8K 97.5 % (39/40)** with greedy decoding, **6 of the 40 answers cut off by the 512-token budget and exactly 1 of those 6 also wrong** — so among answers that finished, 34/34 were right (`27b-gsm8k-20261107`); and the per-token cost at those answer lengths is 0.3857 s/token against 0.2902 on a 13-token prompt, i.e. it grows ~1.33x with context (`27b-decode-cot-20261107`). What a stock install still needs is a kernel set for the model's geometry — none ships in this repository — so the row read `not served` until the route itself was measured | | `dflash` | `FastFlowLM/Nemotron-3.5-Bolt-E4B-DFlash-NPU2` | hrx | not served: a draft head, not a decoder; "The draft loop, and whether to serve it" below covers what the HRX route does with it | | `gemma4` | `FastFlowLM/Nemotron-3.5-Bolt-E4B-NPU2`, `FastFlowLM/Gemma4-12B-IT-NPU2` | hrx | not served: no gemma4 model type or kernels in the route | -| `hunyuan-dense` | `FastFlowLM/Hy-MT2-1.8B-NPU2` | hrx, npu | **served and measured** on the per-op route ("FastFlowLM's own containers" below) | +| `hunyuan-dense` | `FastFlowLM/Hy-MT2-1.8B-NPU2` | hrx, npu | loads and answers on the per-op route once its tile is read; its RoPE base and q/k-norm order are engine#1976 ("FastFlowLM's own containers" below) | | `gemma3` | `FastFlowLM/medgemma-1.5-4b-it-NPU2` | hrx | not served | | `lfm2` | `FastFlowLM/LFM2-2.6B-NPU2`, `FastFlowLM/LFM2-1.2B-NPU2`, `FastFlowLM/LFM2.5-1.2B-NPU2` | hrx, npu | `LFM2.5-1.2B-NPU2` **served and measured** on the per-op route ("FastFlowLM's own containers" below); `LFM2-1.2B-NPU2` serves too (one paired ticket, same section); `LFM2-2.6B-NPU2` was not run | | `gpt-oss` | `FastFlowLM/GPT-OSS-20B-NPU2`, `FastFlowLM/GPT-OSS-Safeguard-20b-NPU2` | hrx | not served | @@ -1900,7 +1900,7 @@ does on a geometry that has a sealed set: `Qwen3-1.7B-NPU2` is 3.0 tok/s here an | `Llama-3.2-3B-NPU2` | `llama` | identical, `Paris.` | 1.7 to 1.8 tok/s | | `Phi4-mini-Instruct-NPU2` | `phi3` | identical, `Paris`, once the turn end is read from the template (below) | 1.4 to 1.5 tok/s | | `Qwen3-VL-4B-Instruct-NPU2`, text tower only | `qwen3` | identical, `Paris` | 1.4 to 1.5 tok/s | -| `Hy-MT2-1.8B-NPU2` | `hunyuan_v1_dense` | identical, `Paris`; five translations against FastFlowLM's runtime below | 2.0 to 2.8 tok/s | +| `Hy-MT2-1.8B-NPU2` | `hunyuan_v1_dense` | identical, `Paris`; serves, and needs engine#1976 to be the model (below) | 2.0 to 2.8 tok/s | | `LFM2.5-1.2B-NPU2` | `lfm2`, hybrid: 10 conv layers of 16 | identical, `Paris`; against FastFlowLM's runtime below | 3.5 to 4.6 tok/s | | `LFM2-1.2B-NPU2` | `lfm2`, hybrid: 10 conv layers of 16 | `Paris`; one request per prompt, against FastFlowLM's runtime below | 3.7 tok/s | @@ -1976,14 +1976,24 @@ kind, so nothing is read off the data. The kernel directory is four shapes that packs (`QKV K2048 N3072`, `O K2048 N2048`, `GU K2048 N12288`, `D K6144 N2048`); q/k norm and the template (`chat_template.jinja`) are read from the pack. -Measured 2026-11-11, same `model.q4nx` (sha256 `a992683e04c2110e…`) for both runtimes, one ticket: five -translation requests (into German, French, Spanish, Chinese and English), all five answered with a correct -translation by both; one byte-identical (`Où se trouve la gare de train la plus proche ?`), four worded -differently (`Me gustaría una taza de café…` here, `Quisiera una taza de café…` there). The host float run of the -first request gives this route's wording, so the array agrees with float arithmetic on these weights as it does -for LFM2; no reference independent of both runtimes was available for this model, so the row does not say whose -wording is the model's. Three requests of the known-answer prompt returned `Paris` three times. Decode 2.0 to -2.8 tok/s (32 layers on the per-op route) against FastFlowLM's 50 to 53. +Measured 2026-11-11, same `model.q4nx` (sha256 `a992683e04c2110e…`) for both runtimes, five translation requests +(into German, French, Spanish, Chinese and English), FastFlowLM 1.0.12's runtime then this route in one ticket, +twice: + +| this route's code | byte-identical to FastFlowLM's answer | +|---|---| +| the tile reader alone (this change on `main`) | 1 of 5 | +| the tile reader plus the Hunyuan RoPE base and q/k-norm order of engine#1976 (merged locally for the measurement) | 4 of 5 | + +So reading the tiles makes the pack serve, and it is **not yet this model as its authors define it**: `main` runs +every `hunyuan_v1_dense` pack at RoPE base 10000 where the family's `rope_scaling.alpha` of 1000 makes it +10000 x 1000^(128/126), about 1.116e7, at every position, and normalises q/k before RoPE where the family does it +after (engine#1889). All five answers were correct translations either way, which is why fluency was no evidence: +`Das Wetter ist heute schön. Wir werden zum Spazierengehen gehen.` on the wrong base, +`Das Wetter ist heute schön, und wir gehen spazieren.` on the right one and from FastFlowLM. The one request still +different with both changes is `Me gustaría una taza de café…` here against `Quisiera una taza de café…` there. +Three requests of the known-answer prompt returned `Paris` three times, and decode was 2.0 to 2.8 tok/s (32 layers +on the per-op route) against FastFlowLM's 50 to 53; both of those were taken on the tile reader alone. Not established by any row here: that a pack goes through the importer (`npu-pack` still reports these containers as recognised, not importable), anything about a vision or audio tower, and any pack whose geometry From 03145fa6ed0d3298f7367fb59a0314ee10a329b8 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Wed, 11 Nov 2026 12:08:22 -0400 Subject: [PATCH 5/8] npu: the missing-q-gain refusal is about attention layers, so LFM2 still loads #1976 refuses a container whose layer has no q_norm when q/k norm is on. Applied to every layer it refused both LFM2 packs at load: their layer 0 is a short-convolution layer with no attention. The rule is now q4nx_qk_gain_missing(qk_norm, layer_has_attention, gain_present), tested. Co-Authored-By: Claude Fable 5.2 --- npu/forward/include/q4nx_semantics.h | 9 +++++++++ npu/forward/src/q4nx_forward.cpp | 3 ++- tests/q4nx_qk_norm_test.c | 7 +++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/npu/forward/include/q4nx_semantics.h b/npu/forward/include/q4nx_semantics.h index 4b981c002..fb794ce77 100644 --- a/npu/forward/include/q4nx_semantics.h +++ b/npu/forward/include/q4nx_semantics.h @@ -262,6 +262,15 @@ int q4nx_rmsnorm_offset_from_norm(const float* w, int n); * off: a family the table knows keeps its answer whatever the tensors are. Exported so the rule is testable. */ int q4nx_semantics_qk_norm_from_tensors(Q4nxSemantics* s, const unsigned char* has_q, const unsigned char* has_k, int layers); +/* engine#1889's refusal, as a rule: with q/k norm on, a layer whose q gain is absent would normalise q to zero, so the + * container is refused. That is a statement about ATTENTION layers. A layer with no attention at all carries no + * q_norm because it has no q: ten of LFM2's sixteen layers are gated short convolutions, layer 0 among them, and the + * rule applied to every layer refused both LFM2 packs at load with "layer 0 has no self_attn.q_norm.weight". + * Returns 1 when layer l must be refused. Exported so the rule is testable with no model present. */ +static inline int q4nx_qk_gain_missing(int qk_norm, int layer_has_attention, int q_gain_present) { + return qk_norm && layer_has_attention && !q_gain_present; +} + /* attention.mode for one container, from the two places that can state it (engine#796). * A d3 pack declares the mask it was trained with in its own `decision_config.json` diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index f400b78b9..4f582c804 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -1790,7 +1790,8 @@ bool Q4nxNpuForward::load_norms() { // would hand back zeros for the absent tensor and rmsnorm(q, 0) is 0 for every head, so the layer would attend // uniformly and answer fluent nonsense. A Hunyuan container imported before the importer claimed the gains is // exactly this shape, so it is refused by name here instead. - if (sem_.qk_norm && absent(&mw_->layers[l].q_norm_weight)) { + // A layer with no attention (LFM2's short-convolution layers) has no q to normalise and so no gain to miss. + if (q4nx_qk_gain_missing(sem_.qk_norm, !is_conv_layer(l), !absent(&mw_->layers[l].q_norm_weight))) { err_ = "config.json says q and k are normalised per head (" + std::string(q4nx_semantics_field(Q4NX_SEM_QK_NORM)) + ") and layer " + std::to_string(l) + " has no self_attn.q_norm.weight: normalising with an absent gain zeroes q, so this container " diff --git a/tests/q4nx_qk_norm_test.c b/tests/q4nx_qk_norm_test.c index 1ea678b3e..e3711c70f 100644 --- a/tests/q4nx_qk_norm_test.c +++ b/tests/q4nx_qk_norm_test.c @@ -110,6 +110,13 @@ int main(void) { check(strstr(q4nx_semantics_field(Q4NX_SEM_QK_NORM), "use_qk_norm") != NULL, "the startup line names use_qk_norm as the flag's source"); + /* The missing-gain refusal is about attention layers. LFM2: q/k norm on, layer 0 a short-convolution layer with no + * attention and so no q_norm. Applied to every layer, the rule refused both LFM2 packs at load. */ + check(q4nx_qk_gain_missing(1, 1, 0) == 1, "q/k norm on, an attention layer with no q gain: refused (the engine#1889 case)"); + check(q4nx_qk_gain_missing(1, 1, 1) == 0, "q/k norm on, an attention layer with its gain: fine"); + check(q4nx_qk_gain_missing(1, 0, 0) == 0, "q/k norm on, a layer with no attention and no gain: not a refusal (LFM2's conv layers)"); + check(q4nx_qk_gain_missing(0, 1, 0) == 0, "q/k norm off: no gain is expected anywhere"); + char path[512]; snprintf(path, sizeof path, "%s/config.json", dir); unlink(path); From c066ffc8b7881d30daff13433d6354e0e6aa28b5 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Wed, 11 Nov 2026 12:25:51 -0400 Subject: [PATCH 6/8] tests: batch_check_state on a fabricated LFM2 hybrid, so CI loads a container with conv layers The suite was green with a rule in it that refused every LFM2 pack at load. batch_check_state_ci.py now also writes a three-layer lfm2-shaped container (conv, attention with q/k norms, conv) and runs the same test on it: it must load, and a batched prefill must leave the logits and the state a token-by-token run leaves. Co-Authored-By: Claude Fable 5.2 --- CMakeLists.txt | 6 ++++++ tests/batch_check_state_ci.py | 37 +++++++++++++++++++++++++++++------ 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b3f6d826c..ec382ef6d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1958,6 +1958,12 @@ if(ONEBIT_NPU_HOST AND NOT ONEBIT_NPU AND NOT WIN32 AND Python3_Interpreter_FOUN COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/batch_check_state_ci.py $) set_tests_properties(batch_check_state PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 120) + # The same test on an LFM2-shaped hybrid the script fabricates (conv, attention, conv): the only container with + # short-convolution layers CI loads through the forward (engine#1786). + add_test(NAME batch_check_state_lfm2 + COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/batch_check_state_ci.py + $ lfm2) + set_tests_properties(batch_check_state_lfm2 PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 120) endif() # Private NPU routes (docs/npu.md, "Private routes"): closed-source add-ons in a checkout of diff --git a/tests/batch_check_state_ci.py b/tests/batch_check_state_ci.py index ddad781b7..7fe15a2ba 100755 --- a/tests/batch_check_state_ci.py +++ b/tests/batch_check_state_ci.py @@ -15,7 +15,7 @@ # limitations under the License. """The batch-check state pin (tests/batch_check_state_test.cpp, engine#1905/#1946) on a container made here. -usage: tests/batch_check_state_ci.py (needs numpy) +usage: tests/batch_check_state_ci.py [lfm2] (needs numpy) batch_check_state_test needs a Q4NX container, and CI has none, so on its own it skips (77) there and a regression of #1909 -- the NPU_BATCH_CHECK oracle resetting the prefix it replays from -- lands green. This writes a two-layer qwen3 @@ -25,6 +25,13 @@ Measured when it was written (engine#1946): passes on main; with #1909 reverted it fails 3 of 4 arms (the forward holds 2 and 1 positions instead of 5 and 6, last-chunk rel-L1 0.78) -- the same failure the real containers show. Weights are seeded, so the run is the same every time. Exits 77 (skip) without numpy. + +With `lfm2` the container is an LFM2-shaped hybrid instead (engine#1786): three layers, a gated short convolution, an +attention layer with q/k norms, another short convolution, `model_type: lfm2`, `norm_eps` and no `layer_types`. It is +the only place CI loads a container with conv layers through the forward. What it pins: the container loads at all +(on 2026-11-11 a rule added for Hunyuan, "q/k norm is on and this layer has no q_norm, refuse", refused every LFM2 +pack at layer 0, a conv layer with no attention, and the suite was green with that in it), and a batched prefill of +the conv layers leaves the logits and the state a token-by-token run leaves, which is what the test binary compares. """ import json import os @@ -56,8 +63,11 @@ def write_safetensors(path, arrays): f.write(b) -def make_model(d, np): +def make_model(d, np, family="qwen3"): rng = np.random.default_rng(1946) + lfm2 = family == "lfm2" + layers = 3 if lfm2 else LAYERS + conv_layers = (0, 2) if lfm2 else () def w(*shape, sc=0.05): return (rng.standard_normal(shape) * sc).astype(np.float32) @@ -67,8 +77,18 @@ def norm(n): weights = {"model.embed_tokens.weight": w(VOCAB, H, sc=0.5), "model.norm.weight": norm(H), "lm_head.weight": w(VOCAB, H, sc=0.2)} - for layer in range(LAYERS): + for layer in range(layers): p = f"model.layers.{layer}." + if layer in conv_layers: + # in_proj is H -> 3H ([B | C | X]), the taps are [H][L] and stay bf16 (3 columns are not a tile), out_proj + # is H -> H. No self_attn.* at all: that absence is what makes it a conv layer to the loader. + weights.update({ + p + "input_layernorm.weight": norm(H), p + "post_attention_layernorm.weight": norm(H), + p + "shortconv.in_proj.weight": w(3 * H, H), p + "shortconv.conv.weight": w(H, 3, sc=0.4), + p + "shortconv.out_proj.weight": w(H, H), + p + "mlp.gate_proj.weight": w(FFN, H), p + "mlp.up_proj.weight": w(FFN, H), + p + "mlp.down_proj.weight": w(H, FFN)}) + continue weights.update({ p + "input_layernorm.weight": norm(H), p + "post_attention_layernorm.weight": norm(H), p + "self_attn.q_proj.weight": w(HEADS * HEAD_DIM, H), @@ -85,6 +105,10 @@ def norm(n): "head_dim": HEAD_DIM, "intermediate_size": FFN, "vocab_size": VOCAB, "rms_norm_eps": 1e-6, "rope_theta": 1000000.0, "tie_word_embeddings": False, "max_position_embeddings": 64, "hidden_act": "silu", "torch_dtype": "float32"} + if lfm2: + del cfg["rms_norm_eps"] + cfg.update({"architectures": ["Lfm2ForCausalLM"], "model_type": "lfm2", "num_hidden_layers": layers, + "norm_eps": 1e-5, "conv_L_cache": 3, "full_attn_idxs": [1]}) with open(os.path.join(d, "config.json"), "w") as f: json.dump(cfg, f) with open(os.path.join(d, "tokenizer.json"), "w") as f: @@ -111,9 +135,10 @@ def tmp_root(): def main(): - if len(sys.argv) != 2: + if len(sys.argv) not in (2, 3) or (len(sys.argv) == 3 and sys.argv[2] != "lfm2"): print(__doc__.strip().splitlines()[2]) return 2 + family = sys.argv[2] if len(sys.argv) == 3 else "qwen3" try: import numpy as np except ImportError: @@ -121,7 +146,7 @@ def main(): return 77 with tempfile.TemporaryDirectory(prefix="batch_check_state_ci.", dir=tmp_root()) as tmp: src, out = os.path.join(tmp, "hf"), os.path.join(tmp, "q4nx") - make_model(src, np) + make_model(src, np, family) r = subprocess.run([sys.executable, TOOL, "pack", src, out], capture_output=True, text=True) if r.returncode != 0: print(f"FAIL packing the fabricated container: {r.stderr[-600:]}") @@ -130,7 +155,7 @@ def main(): env.pop("NPU_INFER_HOST_GEMM", None) # init_host is host-GEMM already; nothing else should steer it r = subprocess.run([sys.argv[1]], env=env, capture_output=True, text=True, timeout=110) for line in (r.stdout + r.stderr).splitlines(): - if line.startswith(("PASS", "FAIL", "SKIP", "ALL PASS", "FAILED", "[batchcheck]")): + if line.startswith(("PASS", "FAIL", "SKIP", "ALL PASS", "FAILED", "[batchcheck]")) or "short-convolution" in line: print(line) if r.returncode == 77: # The container is made here, so a skip would mean the test could not load it: that is a failure of this From adb81c4fac82e2d6f00bc5abef7203ccdf4d65a8 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Wed, 11 Nov 2026 12:48:25 -0400 Subject: [PATCH 7/8] =?UTF-8?q?docs:=20Hy-MT2=20measured=20at=20the=20comb?= =?UTF-8?q?ined=20code=20=E2=80=94=204=20of=205=20byte-identical=20to=20Fa?= =?UTF-8?q?stFlowLM,=20Paris=20x3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5.2 --- docs/npu.md | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/docs/npu.md b/docs/npu.md index 63b65c5dc..da16f4c18 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1862,7 +1862,7 @@ engine#587 measured the route on it): | `qwen35` | `FastFlowLM/D3-*-NPU2` (Edge, Flash, Mini, Nano, Lite, Turbo, Ultra), `FastFlowLM/Qwen3.8-27B-NPU2`, `FastFlowLM/Qwen3.5-0.8B-NPU2` | hrx, npu | **served, and measured** (engine#587): the same `Qwen3.8-27B-NPU2` family runs on the NPU route — 0.2902 s/token (3.45 tok/s) and 19.55 tok/s of prefill on the sealed single-ELF ternary rung, 22.811 s to a first token with the weight load in it (`bench/npu-leaderboard/27b-t2s2-*-20261106.json`) — int8 Q4NX decodes at 63.1 s/token and takes 145.7 s to a first token (`bench/npu-leaderboard/27b-*.json`, 2026-10-27), a 5-token prefill chunk is 58.14 s, and the ternary sealed-ELF rung on the same container is 1.72 s/token with 3/3 text-identical output (engine#697, closed by #728). Quality on the same route: **GSM8K 97.5 % (39/40)** with greedy decoding, **6 of the 40 answers cut off by the 512-token budget and exactly 1 of those 6 also wrong** — so among answers that finished, 34/34 were right (`27b-gsm8k-20261107`); and the per-token cost at those answer lengths is 0.3857 s/token against 0.2902 on a 13-token prompt, i.e. it grows ~1.33x with context (`27b-decode-cot-20261107`). What a stock install still needs is a kernel set for the model's geometry — none ships in this repository — so the row read `not served` until the route itself was measured | | `dflash` | `FastFlowLM/Nemotron-3.5-Bolt-E4B-DFlash-NPU2` | hrx | not served: a draft head, not a decoder; "The draft loop, and whether to serve it" below covers what the HRX route does with it | | `gemma4` | `FastFlowLM/Nemotron-3.5-Bolt-E4B-NPU2`, `FastFlowLM/Gemma4-12B-IT-NPU2` | hrx | not served: no gemma4 model type or kernels in the route | -| `hunyuan-dense` | `FastFlowLM/Hy-MT2-1.8B-NPU2` | hrx, npu | loads and answers on the per-op route once its tile is read; its RoPE base and q/k-norm order are engine#1976 ("FastFlowLM's own containers" below) | +| `hunyuan-dense` | `FastFlowLM/Hy-MT2-1.8B-NPU2` | hrx, npu | **served and measured** on the per-op route ("FastFlowLM's own containers" below) | | `gemma3` | `FastFlowLM/medgemma-1.5-4b-it-NPU2` | hrx | not served | | `lfm2` | `FastFlowLM/LFM2-2.6B-NPU2`, `FastFlowLM/LFM2-1.2B-NPU2`, `FastFlowLM/LFM2.5-1.2B-NPU2` | hrx, npu | `LFM2.5-1.2B-NPU2` **served and measured** on the per-op route ("FastFlowLM's own containers" below); `LFM2-1.2B-NPU2` serves too (one paired ticket, same section); `LFM2-2.6B-NPU2` was not run | | `gpt-oss` | `FastFlowLM/GPT-OSS-20B-NPU2`, `FastFlowLM/GPT-OSS-Safeguard-20b-NPU2` | hrx | not served | @@ -1919,7 +1919,7 @@ does on a geometry that has a sealed set: `Qwen3-1.7B-NPU2` is 3.0 tok/s here an | `Llama-3.2-3B-NPU2` | `llama` | identical, `Paris.` | 1.7 to 1.8 tok/s | | `Phi4-mini-Instruct-NPU2` | `phi3` | identical, `Paris`, once the turn end is read from the template (below) | 1.4 to 1.5 tok/s | | `Qwen3-VL-4B-Instruct-NPU2`, text tower only | `qwen3` | identical, `Paris` | 1.4 to 1.5 tok/s | -| `Hy-MT2-1.8B-NPU2` | `hunyuan_v1_dense` | identical, `Paris`; serves, and needs engine#1976 to be the model (below) | 2.0 to 2.8 tok/s | +| `Hy-MT2-1.8B-NPU2` | `hunyuan_v1_dense` | identical, `Paris`; 4 of 5 translations byte-identical to FastFlowLM's runtime (below) | 2.3 to 2.8 tok/s | | `LFM2.5-1.2B-NPU2` | `lfm2`, hybrid: 10 conv layers of 16 | identical, `Paris`; against FastFlowLM's runtime below | 3.5 to 4.6 tok/s | | `LFM2-1.2B-NPU2` | `lfm2`, hybrid: 10 conv layers of 16 | `Paris`; one request per prompt, against FastFlowLM's runtime below | 3.7 tok/s | @@ -1996,23 +1996,21 @@ packs (`QKV K2048 N3072`, `O K2048 N2048`, `GU K2048 N12288`, `D K6144 N2048`); (`chat_template.jinja`) are read from the pack. Measured 2026-11-11, same `model.q4nx` (sha256 `a992683e04c2110e…`) for both runtimes, five translation requests -(into German, French, Spanish, Chinese and English), FastFlowLM 1.0.12's runtime then this route in one ticket, -twice: +(into German, French, Spanish, Chinese and English), FastFlowLM 1.0.12's runtime then this route in one ticket: | this route's code | byte-identical to FastFlowLM's answer | |---|---| -| the tile reader alone (this change on `main`) | 1 of 5 | -| the tile reader plus the Hunyuan RoPE base and q/k-norm order of engine#1976 (merged locally for the measurement) | 4 of 5 | - -So reading the tiles makes the pack serve, and it is **not yet this model as its authors define it**: `main` runs -every `hunyuan_v1_dense` pack at RoPE base 10000 where the family's `rope_scaling.alpha` of 1000 makes it -10000 x 1000^(128/126), about 1.116e7, at every position, and normalises q/k before RoPE where the family does it -after (engine#1889). All five answers were correct translations either way, which is why fluency was no evidence: -`Das Wetter ist heute schön. Wir werden zum Spazierengehen gehen.` on the wrong base, -`Das Wetter ist heute schön, und wir gehen spazieren.` on the right one and from FastFlowLM. The one request still -different with both changes is `Me gustaría una taza de café…` here against `Quisiera una taza de café…` there. -Three requests of the known-answer prompt returned `Paris` three times, and decode was 2.0 to 2.8 tok/s (32 layers -on the per-op route) against FastFlowLM's 50 to 53; both of those were taken on the tile reader alone. +| the tile reader alone, on the semantics `main` had for this family | 1 of 5 | +| the tile reader with the family's RoPE base and q/k-norm order (engine#1889), which is this change | 4 of 5 | + +Reading the tiles made the pack serve; it did not make it this model. `hunyuan_v1_dense` was run at RoPE base 10000 +where the family's `rope_scaling.alpha` of 1000 makes it 10000 x 1000^(128/126), about 1.116e7, at every position, +and q/k were normalised before RoPE where the family does it after. All five answers were correct translations +either way, which is why fluency was no evidence: `Das Wetter ist heute schön. Wir werden zum Spazierengehen gehen.` +on the wrong base, `Das Wetter ist heute schön, und wir gehen spazieren.` on the right one and from FastFlowLM. The +one request still different is `Me gustaría una taza de café…` here against `Quisiera una taza de café…` there. At +the same code three requests of the known-answer prompt returned `Paris` three times, and decode was 2.3 to 2.8 +tok/s (32 layers on the per-op route) against FastFlowLM's 50 to 53. Not established by any row here: that a pack goes through the importer (`npu-pack` still reports these containers as recognised, not importable), anything about a vision or audio tower, and any pack whose geometry From a7bbf94a8b366bfe7b4fdabed8b83a5440434666 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Wed, 11 Nov 2026 13:13:46 -0400 Subject: [PATCH 8/8] npu: the missing-q-gain refusal is for softmax-attention layers, so qwen3_5 hybrids load too Exempting only conv layers still refused every qwen3_5 hybrid: a linear-attention (GDN) layer has no self_attn.q_norm either, and layer 0 of those packs is one. The rule now takes 'this layer has softmax attention', the test the step bodies use. batch_check_state_ci.py gains a fabricated qwen3_5 hybrid as a load-only row. Co-Authored-By: Claude Fable 5.2 --- CMakeLists.txt | 5 +++ npu/forward/include/q4nx_semantics.h | 11 +++--- npu/forward/src/q4nx_forward.cpp | 8 +++- tests/batch_check_state_ci.py | 55 +++++++++++++++++++++++++--- tests/batch_check_state_test.cpp | 7 ++++ tests/q4nx_qk_norm_test.c | 3 +- 6 files changed, 75 insertions(+), 14 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ec382ef6d..576d51b37 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1964,6 +1964,11 @@ if(ONEBIT_NPU_HOST AND NOT ONEBIT_NPU AND NOT WIN32 AND Python3_Interpreter_FOUN COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/batch_check_state_ci.py $ lfm2) set_tests_properties(batch_check_state_lfm2 PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 120) + # A fabricated qwen3_5 hybrid (a linear-attention layer, then attention): load only, see the script's header. + add_test(NAME batch_check_state_qwen35_load + COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/batch_check_state_ci.py + $ qwen3_5) + set_tests_properties(batch_check_state_qwen35_load PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 120) endif() # Private NPU routes (docs/npu.md, "Private routes"): closed-source add-ons in a checkout of diff --git a/npu/forward/include/q4nx_semantics.h b/npu/forward/include/q4nx_semantics.h index fb794ce77..1a5af0c9b 100644 --- a/npu/forward/include/q4nx_semantics.h +++ b/npu/forward/include/q4nx_semantics.h @@ -263,12 +263,13 @@ int q4nx_rmsnorm_offset_from_norm(const float* w, int n); int q4nx_semantics_qk_norm_from_tensors(Q4nxSemantics* s, const unsigned char* has_q, const unsigned char* has_k, int layers); /* engine#1889's refusal, as a rule: with q/k norm on, a layer whose q gain is absent would normalise q to zero, so the - * container is refused. That is a statement about ATTENTION layers. A layer with no attention at all carries no - * q_norm because it has no q: ten of LFM2's sixteen layers are gated short convolutions, layer 0 among them, and the - * rule applied to every layer refused both LFM2 packs at load with "layer 0 has no self_attn.q_norm.weight". + * container is refused. That is a statement about layers with SOFTMAX ATTENTION. A layer of another kind carries no + * q_norm because it has no q to normalise: ten of LFM2's sixteen layers are gated short convolutions, and 18 of + * Qwen3.5-2B's 24 are linear-attention (GDN) layers, layer 0 in both. Applied to every layer the rule refused every + * LFM2 pack and every qwen3_5 hybrid at load with "layer 0 has no self_attn.q_norm.weight". * Returns 1 when layer l must be refused. Exported so the rule is testable with no model present. */ -static inline int q4nx_qk_gain_missing(int qk_norm, int layer_has_attention, int q_gain_present) { - return qk_norm && layer_has_attention && !q_gain_present; +static inline int q4nx_qk_gain_missing(int qk_norm, int layer_has_softmax_attention, int q_gain_present) { + return qk_norm && layer_has_softmax_attention && !q_gain_present; } /* attention.mode for one container, from the two places that can state it (engine#796). diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index bcbb8c212..eb5babe50 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -1790,8 +1790,12 @@ bool Q4nxNpuForward::load_norms() { // would hand back zeros for the absent tensor and rmsnorm(q, 0) is 0 for every head, so the layer would attend // uniformly and answer fluent nonsense. A Hunyuan container imported before the importer claimed the gains is // exactly this shape, so it is refused by name here instead. - // A layer with no attention (LFM2's short-convolution layers) has no q to normalise and so no gain to miss. - if (q4nx_qk_gain_missing(sem_.qk_norm, !is_conv_layer(l), !absent(&mw_->layers[l].q_norm_weight))) { + // Only a layer with softmax attention has a q to normalise, so only such a layer can miss its gain. LFM2's + // short-convolution layers have no attention at all, and a qwen3_5 linear-attention (GDN) layer has no + // self_attn.q_norm either (18 of Qwen3.5-2B's 24 layers, layer 0 among them): the test is the one the step + // bodies use to send a layer to gdn_layer() or conv_layer() instead of the attention path. + const bool gdn_l = lin_kd_ > 0 && l < sem_.n_layer_types && sem_.layer_types[l] == Q4NX_LAYER_LINEAR; + if (q4nx_qk_gain_missing(sem_.qk_norm, !is_conv_layer(l) && !gdn_l, !absent(&mw_->layers[l].q_norm_weight))) { err_ = "config.json says q and k are normalised per head (" + std::string(q4nx_semantics_field(Q4NX_SEM_QK_NORM)) + ") and layer " + std::to_string(l) + " has no self_attn.q_norm.weight: normalising with an absent gain zeroes q, so this container " diff --git a/tests/batch_check_state_ci.py b/tests/batch_check_state_ci.py index 7fe15a2ba..9a10dc597 100755 --- a/tests/batch_check_state_ci.py +++ b/tests/batch_check_state_ci.py @@ -15,7 +15,7 @@ # limitations under the License. """The batch-check state pin (tests/batch_check_state_test.cpp, engine#1905/#1946) on a container made here. -usage: tests/batch_check_state_ci.py [lfm2] (needs numpy) +usage: tests/batch_check_state_ci.py [lfm2|qwen3_5] (needs numpy) batch_check_state_test needs a Q4NX container, and CI has none, so on its own it skips (77) there and a regression of #1909 -- the NPU_BATCH_CHECK oracle resetting the prefix it replays from -- lands green. This writes a two-layer qwen3 @@ -32,6 +32,18 @@ (on 2026-11-11 a rule added for Hunyuan, "q/k norm is on and this layer has no q_norm, refuse", refused every LFM2 pack at layer 0, a conv layer with no attention, and the suite was green with that in it), and a batched prefill of the conv layers leaves the logits and the state a token-by-token run leaves, which is what the test binary compares. + +With `qwen3_5` it is a qwen3_5 hybrid (engine#1889): a linear-attention (GDN) layer, then a full-attention layer with +q/k norms, written under `language_model.*` so the packer takes its qwen3_5 path (renames, the conv1d transpose, +F32 `ssm_a` / `ssm_dt.bias`). The same rule, scoped to "not a conv layer", still refused every qwen3_5 hybrid: a GDN +layer has no self_attn.q_norm either, and layer 0 of every such pack is one. The one test that loads a real +Qwen3.5 pack skips when the pack is not on disk, which in CI is always, so that too was green. +The qwen3_5 row is LOAD ONLY (ONEBIT_TEST_Q4NX_LOAD_ONLY=1), and that is a limit, not a choice of convenience: on this +fabricated model the full comparison fails its last check (the batched last chunk's logits against the sequential +reference: rel-L1 0.71, same argmax; 0.45 with two GDN layers and no attention layer; 0 with attention layers only), +while FastFlowLM's real Qwen3.5-2B passes the same binary with rel-L1 0. The GDN path's own NPU_GDN_CHECK reports +both of its internal comparisons identical on the fabricated model. Which of the fabrication and the batched path +is at fault is not established; until it is, this row claims the load and nothing about the arithmetic. """ import json import os @@ -66,8 +78,15 @@ def write_safetensors(path, arrays): def make_model(d, np, family="qwen3"): rng = np.random.default_rng(1946) lfm2 = family == "lfm2" + q35 = family == "qwen3_5" layers = 3 if lfm2 else LAYERS conv_layers = (0, 2) if lfm2 else () + gdn_layers = (0,) if q35 else () + # a qwen3_5 pack nests its language model under language_model.* and ties the head to the embedding + top = "language_model." if q35 else "model." + NKH = NVH = 2 + KHD = VHD = 128 # key_dim = value_dim = 256, so the fused q|k|v is 768 x H: whole tiles + CK = 4 def w(*shape, sc=0.05): return (rng.standard_normal(shape) * sc).astype(np.float32) @@ -75,10 +94,26 @@ def w(*shape, sc=0.05): def norm(n): return (1.0 + 0.1 * rng.standard_normal(n)).astype(np.float32) - weights = {"model.embed_tokens.weight": w(VOCAB, H, sc=0.5), "model.norm.weight": norm(H), - "lm_head.weight": w(VOCAB, H, sc=0.2)} + weights = {top + "embed_tokens.weight": w(VOCAB, H, sc=0.5), top + "norm.weight": norm(H)} + if not q35: + weights["lm_head.weight"] = w(VOCAB, H, sc=0.2) for layer in range(layers): - p = f"model.layers.{layer}." + p = f"{top}layers.{layer}." + if layer in gdn_layers: + # HF names; the packer renames them and transposes conv1d ([CD, 1, CK] -> [CK][CD]). No self_attn.q_norm: + # a GDN layer normalises inside its own recurrence (linear_attn.norm), not per attention head. + kd, vd = NKH * KHD, NVH * VHD + weights.update({ + p + "input_layernorm.weight": norm(H), p + "post_attention_layernorm.weight": norm(H), + p + "linear_attn.in_proj_qkv.weight": w(2 * kd + vd, H), p + "linear_attn.in_proj_z.weight": w(vd, H), + p + "linear_attn.in_proj_a.weight": w(NVH, H), p + "linear_attn.in_proj_b.weight": w(NVH, H), + p + "linear_attn.out_proj.weight": w(H, vd), + p + "linear_attn.conv1d.weight": w(2 * kd + vd, 1, CK, sc=0.3), + p + "linear_attn.norm.weight": norm(VHD), + p + "linear_attn.A_log": w(NVH, sc=0.3), p + "linear_attn.dt_bias": w(NVH, sc=0.3), + p + "mlp.gate_proj.weight": w(FFN, H), p + "mlp.up_proj.weight": w(FFN, H), + p + "mlp.down_proj.weight": w(H, FFN)}) + continue if layer in conv_layers: # in_proj is H -> 3H ([B | C | X]), the taps are [H][L] and stay bf16 (3 columns are not a tile), out_proj # is H -> H. No self_attn.* at all: that absence is what makes it a conv layer to the loader. @@ -105,6 +140,11 @@ def norm(n): "head_dim": HEAD_DIM, "intermediate_size": FFN, "vocab_size": VOCAB, "rms_norm_eps": 1e-6, "rope_theta": 1000000.0, "tie_word_embeddings": False, "max_position_embeddings": 64, "hidden_act": "silu", "torch_dtype": "float32"} + if q35: + cfg.update({"architectures": ["Qwen3_5ForCausalLM"], "model_type": "qwen3_5_text", "tie_word_embeddings": True, + "layer_types": ["linear_attention", "full_attention"], "linear_num_key_heads": NKH, + "linear_key_head_dim": KHD, "linear_num_value_heads": NVH, "linear_value_head_dim": VHD, + "linear_conv_kernel_dim": CK}) if lfm2: del cfg["rms_norm_eps"] cfg.update({"architectures": ["Lfm2ForCausalLM"], "model_type": "lfm2", "num_hidden_layers": layers, @@ -135,7 +175,7 @@ def tmp_root(): def main(): - if len(sys.argv) not in (2, 3) or (len(sys.argv) == 3 and sys.argv[2] != "lfm2"): + if len(sys.argv) not in (2, 3) or (len(sys.argv) == 3 and sys.argv[2] not in ("lfm2", "qwen3_5")): print(__doc__.strip().splitlines()[2]) return 2 family = sys.argv[2] if len(sys.argv) == 3 else "qwen3" @@ -153,9 +193,12 @@ def main(): return 1 env = dict(os.environ, ONEBIT_TEST_Q4NX=out) env.pop("NPU_INFER_HOST_GEMM", None) # init_host is host-GEMM already; nothing else should steer it + if family == "qwen3_5": + env["ONEBIT_TEST_Q4NX_LOAD_ONLY"] = "1" r = subprocess.run([sys.argv[1]], env=env, capture_output=True, text=True, timeout=110) for line in (r.stdout + r.stderr).splitlines(): - if line.startswith(("PASS", "FAIL", "SKIP", "ALL PASS", "FAILED", "[batchcheck]")) or "short-convolution" in line: + if (line.startswith(("PASS", "FAIL", "SKIP", "ALL PASS", "FAILED", "[batchcheck]")) or + "short-convolution" in line or "linear/GDN" in line or "linear_attention" in line): print(line) if r.returncode == 77: # The container is made here, so a skip would mean the test could not load it: that is a failure of this diff --git a/tests/batch_check_state_test.cpp b/tests/batch_check_state_test.cpp index 74fdf5308..9c7fb849a 100644 --- a/tests/batch_check_state_test.cpp +++ b/tests/batch_check_state_test.cpp @@ -90,6 +90,13 @@ int main() { std::printf("SKIP: init_host failed for %s (%s)\n", model.c_str(), fwd.last_error()); return 77; } + // ONEBIT_TEST_Q4NX_LOAD_ONLY=1: stop here. The row is "this container loads through the forward" and nothing more; + // tests/batch_check_state_ci.py uses it for a fabricated family whose only claim is that the load is not refused. + if (const char* lo = std::getenv("ONEBIT_TEST_Q4NX_LOAD_ONLY"); lo && lo[0] == '1') { + std::printf("PASS init_host accepted %s\n", model.c_str()); + std::printf("ALL PASS (load only)\n"); + return 0; + } // Values do not matter; the split does. Three chunks: [0,3), [3,5), [5,6) -- the last is the serve path's // split-off final token, which is a step_batch at start_pos > 0 on every request. diff --git a/tests/q4nx_qk_norm_test.c b/tests/q4nx_qk_norm_test.c index e3711c70f..0937c6fb1 100644 --- a/tests/q4nx_qk_norm_test.c +++ b/tests/q4nx_qk_norm_test.c @@ -114,7 +114,8 @@ int main(void) { * attention and so no q_norm. Applied to every layer, the rule refused both LFM2 packs at load. */ check(q4nx_qk_gain_missing(1, 1, 0) == 1, "q/k norm on, an attention layer with no q gain: refused (the engine#1889 case)"); check(q4nx_qk_gain_missing(1, 1, 1) == 0, "q/k norm on, an attention layer with its gain: fine"); - check(q4nx_qk_gain_missing(1, 0, 0) == 0, "q/k norm on, a layer with no attention and no gain: not a refusal (LFM2's conv layers)"); + check(q4nx_qk_gain_missing(1, 0, 0) == 0, "q/k norm on, a layer with no softmax attention and no gain: not a refusal " + "(LFM2's conv layers, qwen3_5's linear-attention layers)"); check(q4nx_qk_gain_missing(0, 1, 0) == 0, "q/k norm off: no gain is expected anywhere"); char path[512];