From 32dcc9782281c3acb8a8fae4f0fd6d31993f67dd Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 12 Nov 2026 03:01:43 -0400 Subject: [PATCH] npu: Qwen2's q/k/v biases are read from the container, so Qwen2.5-3B-Instruct-NPU2 answers (Refs #1786) The forward adds the q/k/v projection biases only when attention_bias is on, and took that from config.json alone. Qwen2 configs never state the key (our importers write it, FastFlowLM's packs do not), so on Qwen2.5-3B-Instruct-NPU2 the biases were loaded and never added: the known-answer prompt answered noise. Now, as for q/k norm: the key absent and a layer carrying a q, k or v bias turns the biases on (said on the startup line); a config stating false against a container with biases is refused. After: 'Paris<|im_end|>' on the host. batch_check_state_qwen2: a fabricated Qwen2 with biases and no key, against a numpy Qwen2 forward on the container's own weights: corr 1.0000000, max |d| 6.6e-06. The binary without the fix fails it (corr 0.444, argmax 200 vs 7) while passing batched == sequential. Four probe checks added to q4nx_qk_norm_test. Of the 24 q4nx containers on the box with biases, only this pack and flm-whisper lack the key; Whisper does not go through this forward. Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 5 ++ docs/npu.md | 12 +++++ npu/forward/include/q4nx_semantics.h | 8 +++ npu/forward/src/q4nx_forward.cpp | 18 +++++++ npu/forward/src/q4nx_semantics.c | 13 ++++- tests/batch_check_state_ci.py | 80 +++++++++++++++++++++++++--- tests/q4nx_qk_norm_test.c | 21 ++++++++ 7 files changed, 150 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 100c69d37..49c22f0a6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2149,6 +2149,11 @@ if(ONEBIT_NPU_HOST AND NOT ONEBIT_NPU AND NOT WIN32 AND Python3_Interpreter_FOUN COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/batch_check_state_ci.py $ lfm3) set_tests_properties(batch_check_state_lfm3 PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 120) + # Qwen2 (engine#1786): q/k/v biases with no attention_bias key in config.json, against a numpy Qwen2 reference. + add_test(NAME batch_check_state_qwen2 + COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/batch_check_state_ci.py + $ qwen2) + set_tests_properties(batch_check_state_qwen2 PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 120) # A fabricated qwen3_5 hybrid (a linear-attention layer, then attention), off the kernel's N grid on purpose. add_test(NAME batch_check_state_qwen35 COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/batch_check_state_ci.py diff --git a/docs/npu.md b/docs/npu.md index 99475c43e..5020928e8 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -2029,6 +2029,18 @@ off (the template's ``), gives `ParisParis<|im_end|>`. Th choices: `` after the first `Paris`, then `Paris` again. Asked why the sky is blue, it explains Rayleigh scattering in correct sentences (80 tokens, greedy). No kernel set for its shapes was built or run (the NPU is off). +**Qwen2's attention biases are read from the container (engine#1786).** The forward adds the q, k and v projection +biases only when `attention_bias` is on. A config that states the key decides it. Qwen2's `config.json` never states +it, though every Qwen2 layer has the three biases. With the key absent the forward loaded the biases and never added +them, so FastFlowLM's `Qwen2.5-3B-Instruct-NPU2` answered the known-answer prompt with noise +(`UNS �leon�leon�ucz …`). The rule is now the one q/k normalisation follows: with the key absent, a layer that carries +a q, k or v bias turns the biases on, and the startup line says so (`attention_bias=1 <- the container`). A config +that says `attention_bias: false` while the container carries biases is refused by name rather than guessed. +After: `Paris<|im_end|>` on the host. `batch_check_state_qwen2` fabricates a Qwen2 with biases and no key and +compares the forward's logits with a numpy Qwen2 forward on the container's own weights: corr 1.0000000, max |d| +6.6e-06 of 9.6. The binary without the fix fails it (corr 0.444, another argmax), and batched == sequential passed +on that binary all the same. + **Per-head q/k normalisation is read from the container (engine#1788).** Whether the forward normalises q and k per head used to be decided by `config.json`'s `model_type` alone: on for `qwen3*` and `gemma*`. A pack of the same architecture under another name got it wrong in silence. FastFlowLM's MiniCPM-V 4.7 diff --git a/npu/forward/include/q4nx_semantics.h b/npu/forward/include/q4nx_semantics.h index bc04f6602..72227e637 100644 --- a/npu/forward/include/q4nx_semantics.h +++ b/npu/forward/include/q4nx_semantics.h @@ -86,6 +86,10 @@ typedef struct { /* Linear biases on q/k/v/o and on the MLP projections. * Source: attention_bias / mlp_bias (presence of corresponding tensors). */ int attention_bias; + /* 1 when config.json states attention_bias (true or false); 0 when the key is absent, which is Qwen2's case: + * its configs never state it and every layer carries q/k/v biases. q4nx_semantics_attention_bias_from_tensors + * decides the absent case from the container. */ + int attention_bias_stated; int mlp_bias; /* Multiplier applied to token embeddings before layer 0. * Source: model_type (gemma* uses sqrt(hidden_size)); else 1.0. */ @@ -278,6 +282,10 @@ int q4nx_rmsnorm_offset_from_norm(const float* w, int n); * nothing, when qk_norm is already on, when no layer carries both, or on null/empty input. It never turns the norm * off: a family the table knows keeps its answer whatever the tensors are. Exported so the rule is testable. */ int q4nx_semantics_qk_norm_from_tensors(Q4nxSemantics* s, const unsigned char* has_q, const unsigned char* has_k, int layers); +/* engine#1786: config.json did not state attention_bias and a layer carries a q, k or v projection bias -> the + * biases are applied (attention_bias = 1); returns that layer, else -1. A config that states attention_bias=false + * while the container carries biases is a disagreement the caller refuses: returns -2. */ +int q4nx_semantics_attention_bias_from_tensors(Q4nxSemantics* s, const unsigned char* has_bias, int layers); /* engine#1889's refusal, as a rule: with q/k norm on, a layer whose q gain is absent would normalise q to zero, so the * container is refused. That is a statement about layers with SOFTMAX ATTENTION. A layer of another kind carries no diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index d06ee16bc..ba9451f7d 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -1099,6 +1099,24 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, fprintf(stderr, " [semantics] qk_norm=1 <- the container: layer %d carries self_attn.q_norm and k_norm " "(config.json's model_type is not a family the table knows)\n", by); } + // engine#1786: attention_bias the same way. Qwen2's config.json never states it and every layer carries q/k/v + // biases; with the key absent they were loaded and never added, so Qwen2.5-3B-Instruct-NPU2 answered noise. + if (mw_ && mw_->config.num_layers > 0) { + std::vector has_b(size_t(mw_->config.num_layers), 0); + auto present = [](const TensorDesc& t) { return t.name[0] && t.data_size > 0; }; + for (int l = 0; l < mw_->config.num_layers; l++) + has_b[size_t(l)] = present(mw_->layers[l].q_proj_bias) || present(mw_->layers[l].k_proj_bias) || + present(mw_->layers[l].v_proj_bias); + const int by = q4nx_semantics_attention_bias_from_tensors(&sem_, has_b.data(), mw_->config.num_layers); + if (by == -2) { + err_ = "config.json says attention_bias is false and model.q4nx carries q/k/v projection biases; " + "serving it either way would be a guess"; + return false; + } + if (by >= 0) + fprintf(stderr, " [semantics] attention_bias=1 <- the container: layer %d carries q/k/v projection " + "biases (config.json does not state attention_bias)\n", by); + } q4nx_semantics_dump(&sem_); // A layer kind nothing implements is refused BY NAME, here beside the other precise refusals (engine#1087's // shape): q4nx_semantics maps an unrecognised layer_types entry to Q4NX_LAYER_OTHER and the attention path diff --git a/npu/forward/src/q4nx_semantics.c b/npu/forward/src/q4nx_semantics.c index 01702acfb..2b933762f 100644 --- a/npu/forward/src/q4nx_semantics.c +++ b/npu/forward/src/q4nx_semantics.c @@ -478,7 +478,7 @@ int q4nx_semantics_from_config(const char* json_path, const char* model_dir, /* biases: absent field means "no bias" for llama-family configs, but Phi * and Gemma configs omit it as well, so default to 0 and let phase 2's * tensor-level probe (bias tensors present?) override it. */ - if (get_num(json, "attention_bias", &d)) s->attention_bias = d != 0; + if (get_num(json, "attention_bias", &d)) { s->attention_bias = d != 0; s->attention_bias_stated = 1; } if (get_num(json, "mlp_bias", &d)) s->mlp_bias = d != 0; /* embedding scale: gemma scales embeddings by sqrt(hidden_size) */ @@ -601,6 +601,17 @@ void q4nx_semantics_layer_mix(const Q4nxSemantics* s, char* out, size_t out_sz) snprintf(out, out_sz, "full=%d sliding=%d linear=%d other=%d", full, slid, lin, oth); } +int q4nx_semantics_attention_bias_from_tensors(Q4nxSemantics* s, const unsigned char* has_bias, int layers) { + if (!s || !has_bias || layers <= 0 || s->attention_bias) return -1; + for (int l = 0; l < layers; l++) + if (has_bias[l]) { + if (s->attention_bias_stated) return -2; + s->attention_bias = 1; + return l; + } + return -1; +} + int q4nx_semantics_qk_norm_from_tensors(Q4nxSemantics* s, const unsigned char* has_q, const unsigned char* has_k, int layers) { if (!s || !has_q || !has_k || layers <= 0 || s->qk_norm) return -1; for (int l = 0; l < layers; l++) diff --git a/tests/batch_check_state_ci.py b/tests/batch_check_state_ci.py index b52a96e59..74bf3b387 100755 --- a/tests/batch_check_state_ci.py +++ b/tests/batch_check_state_ci.py @@ -15,7 +15,7 @@ # limitations under the License. """The batch-check state pin (tests/batch_check_state_test.cpp, engine#1905/#1946) on a container made here. -usage: tests/batch_check_state_ci.py [lfm2|lfm3|qwen3_5] (needs numpy) +usage: tests/batch_check_state_ci.py [lfm2|lfm3|qwen2|qwen3_5] (needs numpy) batch_check_state_test needs a Q4NX container, and CI has none, so on its own it skips (77) there and a regression of #1909 -- the NPU_BATCH_CHECK oracle resetting the prefix it replays from -- lands green. This writes a two-layer qwen3 @@ -44,6 +44,12 @@ measured at engine#2254, corr 1.0000000 and max |d| 1.2e-05 of 7.63. A dead head gate (in both paths) gives corr 0.90 and another argmax, the usual partial RoPE 0.98, no sandwich 0.38; every one of them passes batched == sequential. +With `qwen2` it is a two-layer Qwen2 (engine#1786): q/k/v projection biases, no q/k norms, and a config.json with no +`attention_bias` key, which is how every Qwen2 config is written. With the key absent the forward loaded the biases +and never added them, so FastFlowLM's Qwen2.5-3B-Instruct-NPU2 answered noise. batched == sequential cannot see that +(both paths skip the biases alike), so the arm also compares the sequential logits with a numpy Qwen2 forward on the +container's own weights (qwen2_reference). Measured at the fix: see QWEN2_* below. + With `qwen3_5` it is a qwen3_5 hybrid (engine#1889): a linear-attention (GDN) layer, then a full-attention layer with q/k norms, written under `language_model.*` so the packer takes its qwen3_5 path (renames, the conv1d transpose, F32 `ssm_a` / `ssm_dt.bias`). The same rule, scoped to "not a conv layer", still refused every qwen3_5 hybrid: a GDN @@ -91,6 +97,7 @@ def make_model(d, np, family="qwen3"): rng = np.random.default_rng(1946) lfm2 = family == "lfm2" lfm3 = family == "lfm3" + qwen2 = family == "qwen2" q35 = family == "qwen3_5" layers = 3 if lfm2 else (4 if lfm3 else LAYERS) conv_layers = (0, 2) if (lfm2 or lfm3) else () @@ -165,6 +172,12 @@ def norm(n): p + "self_attn.q_norm.weight": norm(HEAD_DIM), p + "self_attn.k_norm.weight": norm(HEAD_DIM), p + "mlp.gate_proj.weight": w(FFN, H), p + "mlp.up_proj.weight": w(FFN, H), p + "mlp.down_proj.weight": w(H, FFN)}) + if qwen2: + # Qwen2: biases on q/k/v (large enough to matter), no per-head q/k norms + del weights[p + "self_attn.q_norm.weight"], weights[p + "self_attn.k_norm.weight"] + weights.update({p + "self_attn.q_proj.bias": w(HEADS * HEAD_DIM, sc=0.5), + p + "self_attn.k_proj.bias": w(KV_HEADS * HEAD_DIM, sc=0.5), + p + "self_attn.v_proj.bias": w(KV_HEADS * HEAD_DIM, sc=0.5)}) os.makedirs(d) write_safetensors(os.path.join(d, "model.safetensors"), weights) cfg = {"architectures": ["Qwen3ForCausalLM"], "model_type": "qwen3", "hidden_size": H, @@ -181,6 +194,8 @@ def norm(n): del cfg["rms_norm_eps"] cfg.update({"architectures": ["Lfm2ForCausalLM"], "model_type": "lfm2", "num_hidden_layers": layers, "norm_eps": 1e-5, "conv_L_cache": 3, "full_attn_idxs": [1]}) + if qwen2: + cfg.update({"architectures": ["Qwen2ForCausalLM"], "model_type": "qwen2"}) # and no attention_bias key if lfm3: del cfg["rms_norm_eps"], cfg["rope_theta"] cfg.update({"architectures": ["Lfm3ForCausalLM"], "model_type": "lfm3", "num_hidden_layers": layers, @@ -276,6 +291,59 @@ def rope(x, layer_type): return rms(x, W("model.embedding_norm.weight"))[-1] @ W("lm_head.weight").T +def qwen2_reference(q4nx_dir, np, ids, shapes): + """HF's Qwen2 forward (llama with q/k/v biases) on the container's own weights; the last position's logits.""" + sys.path.insert(0, os.path.join(HERE, "..", "tools")) + from q4nx_pack import untile + with open(os.path.join(q4nx_dir, "config.json")) as f: + cfg = json.load(f) + with open(os.path.join(q4nx_dir, "model.q4nx"), "rb") as f: + n = struct.unpack("hts", q, k) / np.sqrt(hd) + sc = np.where(np.tril(np.ones((T, T), bool))[None], sc, -np.inf) + sc = np.exp(sc - sc.max(-1, keepdims=True)) + ctx = np.einsum("hts,shd->thd", sc / sc.sum(-1, keepdims=True), v) + x = x + ctx.reshape(T, nh * hd) @ W(p + "self_attn.o_proj.weight").T + h = rms(x, W(p + "post_attention_layernorm.weight")) + a = h @ W(p + "mlp.gate_proj.weight").T + x = x + ((a / (1.0 + np.exp(-a))) * (h @ W(p + "mlp.up_proj.weight").T)) @ W(p + "mlp.down_proj.weight").T + return rms(x, W("model.norm.weight"))[-1] @ W("lm_head.weight").T + + def tmp_root(): """tests/tmp_root.h's rule (engine#2029): scratch goes on disk, not in tmpfs. ONEBIT_TEST_TMP if set, else ~/.cache/scratch-tests/tmp, else the system temp directory.""" @@ -294,7 +362,7 @@ def tmp_root(): def main(): - if len(sys.argv) not in (2, 3) or (len(sys.argv) == 3 and sys.argv[2] not in ("lfm2", "lfm3", "qwen3_5")): + if len(sys.argv) not in (2, 3) or (len(sys.argv) == 3 and sys.argv[2] not in ("lfm2", "lfm3", "qwen2", "qwen3_5")): print(__doc__.strip().splitlines()[2]) return 2 family = sys.argv[2] if len(sys.argv) == 3 else "qwen3" @@ -312,7 +380,7 @@ def main(): return 1 env = dict(os.environ, ONEBIT_TEST_Q4NX=out) logits_out = os.path.join(tmp, "sequential.f32") - if family == "lfm3": + if family in ("lfm3", "qwen2"): env["ONEBIT_TEST_LOGITS_OUT"] = logits_out env.pop("NPU_INFER_HOST_GEMM", None) # init_host is host-GEMM already; nothing else should steer it r = subprocess.run([sys.argv[1]], env=env, capture_output=True, text=True, timeout=110) @@ -325,13 +393,13 @@ def main(): # arm, not a reason to skip it. print("FAIL batch_check_state_test skipped on the fabricated container (init_host refused it)") return 1 - if family == "lfm3" and r.returncode == 0: + if family in ("lfm3", "qwen2") and r.returncode == 0: ours = np.fromfile(logits_out, dtype=np.float32)[:VOCAB] - ref = lfm3_reference(out, np, [1, 2, 3, 4, 5, 6], shapes) + ref = (lfm3_reference if family == "lfm3" else qwen2_reference)(out, np, [1, 2, 3, 4, 5, 6], shapes) corr = float(np.corrcoef(ours, ref)[0, 1]) ok = (corr >= LFM3_MIN_CORR and float(np.abs(ours - ref).max()) <= LFM3_MAX_DIFF * float(np.abs(ref).max()) and int(np.argmax(ours)) == int(np.argmax(ref))) - print(f"{'PASS' if ok else 'FAIL'} sequential logits vs the modeling_lfm3 reference on the container's " + print(f"{'PASS' if ok else 'FAIL'} sequential logits vs the {family} reference on the container's " f"weights: corr {corr:.7f} (needs {LFM3_MIN_CORR}), max |d| {float(np.abs(ours - ref).max()):.3g} of " f"{float(np.abs(ref).max()):.3g}, argmax {int(np.argmax(ours))} vs " f"{int(np.argmax(ref))}") diff --git a/tests/q4nx_qk_norm_test.c b/tests/q4nx_qk_norm_test.c index 0937c6fb1..4857823a6 100644 --- a/tests/q4nx_qk_norm_test.c +++ b/tests/q4nx_qk_norm_test.c @@ -118,6 +118,27 @@ int main(void) { "(LFM2's conv layers, qwen3_5's linear-attention layers)"); check(q4nx_qk_gain_missing(0, 1, 0) == 0, "q/k norm off: no gain is expected anywhere"); + /* engine#1786: attention_bias from the container, as qk_norm. Qwen2's config.json never states the key and every + * layer carries q/k/v biases; with the key absent they were loaded and never added (Qwen2.5-3B answered noise). */ + { + const unsigned char none[3] = {0, 0, 0}, l1[3] = {0, 1, 1}; + Q4nxSemantics b; + memset(&b, 0, sizeof b); + check(q4nx_semantics_attention_bias_from_tensors(&b, l1, 3) == 1 && b.attention_bias == 1, + "attention_bias absent from config.json, layer 1 carries biases: on, and layer 1 is named"); + memset(&b, 0, sizeof b); + check(q4nx_semantics_attention_bias_from_tensors(&b, none, 3) == -1 && b.attention_bias == 0, + "absent, and no layer carries a bias: stays off"); + memset(&b, 0, sizeof b); + b.attention_bias_stated = 1; + check(q4nx_semantics_attention_bias_from_tensors(&b, l1, 3) == -2 && b.attention_bias == 0, + "config.json says attention_bias=false and the container carries biases: a disagreement (-2, refused)"); + memset(&b, 0, sizeof b); + b.attention_bias = b.attention_bias_stated = 1; + check(q4nx_semantics_attention_bias_from_tensors(&b, l1, 3) == -1 && b.attention_bias == 1, + "stated true: left as stated"); + } + char path[512]; snprintf(path, sizeof path, "%s/config.json", dir); unlink(path);