From 48a088d0772a2800cc97f8d23031be10e1b61adc Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 12 Nov 2026 03:23:55 -0400 Subject: [PATCH] npu: Qwen2.5-VL's contiguous "mrope" is a position layout, so its text tower loads and answers (Refs #1786) Qwen2.5-VL-3B-Instruct-NPU2 was refused at load: rope_scaling {"type": "mrope", "mrope_section": [16, 24, 24]} was read as a frequency scaling this forward lacks. It is the Qwen2-VL position layout: plain frequencies, CONTIGUOUS sections (transformers apply_multimodal_rotary_pos_emb splits cos by mrope_section * 2), t 0-15, h 16-39, w 40-63 -- not Qwen3-VL's interleave. New mrope_sectioned and q4nx_mrope_axis_sectioned; a block whose sections do not sum to the rotary half-width is refused. With the attention-bias fix of the previous commit the text tower answers 'Paris<|im_end|>' on the host. q4nx_mrope_test: +6 checks (all 64 frequencies against the reference's split; Qwen3-VL unchanged). batch_check_state_qwen2vl: the fabricated Qwen2 declaring that block gives the plain-RoPE reference's logits (corr 1.0000000); with "mrope" refused again it fails. ctest 268/268. Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 5 ++++ docs/npu.md | 11 +++++++++ npu/forward/include/q4nx_forward.h | 6 ++++- npu/forward/include/q4nx_semantics.h | 14 +++++++++++ npu/forward/src/q4nx_forward.cpp | 20 ++++++++++++++- npu/forward/src/q4nx_semantics.c | 8 ++++++ tests/batch_check_state_ci.py | 16 +++++++----- tests/q4nx_mrope_test.c | 37 ++++++++++++++++++++++++++++ 8 files changed, 109 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 49c22f0a6..cdb9ada5b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2154,6 +2154,11 @@ if(ONEBIT_NPU_HOST AND NOT ONEBIT_NPU AND NOT WIN32 AND Python3_Interpreter_FOUN COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/batch_check_state_ci.py $ qwen2) set_tests_properties(batch_check_state_qwen2 PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 120) + # The same Qwen2 declaring Qwen2.5-VL's contiguous "mrope": accepted, and exact on a text-only prompt. + add_test(NAME batch_check_state_qwen2vl + COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/batch_check_state_ci.py + $ qwen2vl) + set_tests_properties(batch_check_state_qwen2vl PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 120) # A fabricated qwen3_5 hybrid (a linear-attention layer, then attention), off the kernel's N grid on purpose. add_test(NAME batch_check_state_qwen35 COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/batch_check_state_ci.py diff --git a/docs/npu.md b/docs/npu.md index 5020928e8..2b675926b 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -2041,6 +2041,17 @@ compares the forward's logits with a numpy Qwen2 forward on the container's own 6.6e-06 of 9.6. The binary without the fix fails it (corr 0.444, another argmax), and batched == sequential passed on that binary all the same. +Qwen2.5-VL's text tower is the same Qwen2, and it was refused one step earlier. Its config declares +`rope_scaling: {"type": "mrope", "mrope_section": [16, 24, 24]}`, and the forward read `"mrope"` as a frequency scaling +it does not implement. It is a position layout instead: plain RoPE frequencies, each taking its position from the t, +h or w plane. Qwen2-VL and Qwen2.5-VL lay the sections out CONTIGUOUSLY (transformers' `apply_multimodal_rotary_pos_emb` +splits cos by `mrope_section * 2`): frequencies 0-15 are t, 16-39 h and 40-63 w. Qwen3-VL interleaves them +instead. The forward now reads that form (`mrope_sectioned`, `q4nx_mrope_axis_sectioned`), and refuses a block whose +three sections do not sum to the rotary half-width. On a text-only prompt the three planes are equal, so it is plain +RoPE. `Qwen2.5-VL-3B-Instruct-NPU2`'s text tower now answers `Paris<|im_end|>` on the host; its vision tower is not +loaded here. `batch_check_state_qwen2vl` is the Qwen2 above declaring that block: the same logits as the plain-RoPE +reference (corr 1.0000000). With `"mrope"` refused again (main's behaviour) the arm fails. + **Per-head q/k normalisation is read from the container (engine#1788).** Whether the forward normalises q and k per head used to be decided by `config.json`'s `model_type` alone: on for `qwen3*` and `gemma*`. A pack of the same architecture under another name got it wrong in silence. FastFlowLM's MiniCPM-V 4.7 diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index f43e91fd9..bccdd9908 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -492,7 +492,10 @@ class Q4nxNpuForward { int deepstack_lo_ = -1, deepstack_hi_ = -1; // Which plane a rotary frequency belongs to under interleaved MRoPE. The rule lives in q4nx_semantics.h as // q4nx_mrope_axis() so the C unit test can pin it without building a forward pass. - int mrope_plane(int freq) const { return q4nx_mrope_axis(freq, sem_.n_mrope_section, sem_.mrope_section); } + int mrope_plane(int freq) const { + return sem_.mrope_sectioned ? q4nx_mrope_axis_sectioned(freq, sem_.n_mrope_section, sem_.mrope_section) + : q4nx_mrope_axis(freq, sem_.n_mrope_section, sem_.mrope_section); + } // Some models store the embedding QUANTIZED (Gemma4-E2B/E4B: dtype I8), and // reading it as bf16 yields ~zero - which invalidated the whole forward and // the tied LM head (all-zero logits). When the dtype is not BF16 the @@ -647,6 +650,7 @@ class Q4nxNpuForward { // Layout and maths: docs/qwen35-4b-weight-format.md. int lin_kd_ = 0, lin_vd_ = 0; // key_dim = nkh*khd, value_dim = nvh*vhd int attn_gate_ = 0; // qwen3_5 attn_output_gate + bool mrope_declared_ = false; // rope_scaling type "mrope" (Qwen2-VL form), validated after semantics // LFM3's per-head output gate, [NH][H] per attention layer (empty for conv layers and other models): // ctx of head h *= sigmoid(head_gate_[l][h] . xn), xn the operator-normed layer input. std::vector> head_gate_; diff --git a/npu/forward/include/q4nx_semantics.h b/npu/forward/include/q4nx_semantics.h index 72227e637..bfc7d3e32 100644 --- a/npu/forward/include/q4nx_semantics.h +++ b/npu/forward/include/q4nx_semantics.h @@ -116,6 +116,11 @@ typedef struct { * guess a layout for. */ int mrope_section[3]; int n_mrope_section; + /* Qwen2-VL / Qwen2.5-VL spell MRoPE as rope_scaling {"type": "mrope", "mrope_section": [16, 24, 24]}, and their + * sections are CONTIGUOUS runs of frequencies (transformers apply_multimodal_rotary_pos_emb: cos split by + * mrope_section * 2, chunk i taking plane i % 3), not Qwen3-VL's interleave. 1 for that form (mrope is 1 too). + * Source: rope_scaling.type / rope_type == "mrope". */ + int mrope_sectioned; /* DEEPSTACK: which decoder layers take one of the vision tower's extra feature sets. Read from * vision_config.deepstack_visual_indexes (Qwen3-VL: [5, 11, 17]); the consumer adds set s to the hidden state at * layer deepstack_layers[s]. Empty means the pack has none. */ @@ -256,6 +261,15 @@ static inline int q4nx_mrope_axis(int freq, int n_section, const int* section) { return 0; } +/* The same question for the CONTIGUOUS layout (mrope_sectioned, Qwen2-VL / Qwen2.5-VL): frequencies + * [0, s0) are t, [s0, s0 + s1) are h, and [s0 + s1, s0 + s1 + s2) are w. For [16, 24, 24]: 16 t, 24 h, 24 w. */ +static inline int q4nx_mrope_axis_sectioned(int freq, int n_section, const int* section) { + if (n_section < 3 || !section) return 0; + if (freq < section[0]) return 0; + if (freq < section[0] + section[1]) return 1; + return 2; +} + /* Human-readable one-line dump of every flag with its source field. */ void q4nx_semantics_dump(const Q4nxSemantics* s); diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index ba9451f7d..55ebf170a 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -847,7 +847,11 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, char declared[64]; q4nx_rope_declared_type(t.c_str(), declared, (int)sizeof declared); const bool linear_ok = !std::strcmp(declared, "linear") && rope_linear_factor_ > 0.0; - if (!linear_ok && + // Qwen2.5-VL's "mrope" is a position layout, not a frequency scaling: plain RoPE frequencies, each taking its + // position from the t, h or w plane. Accepted here and checked against its sections once the semantics are + // read (below), where a malformed block is refused. + mrope_declared_ = !std::strcmp(declared, "mrope"); + if (!linear_ok && !mrope_declared_ && !q4nx_rope_type_supported(declared, &rope_llama3_, rope_scale_.empty() ? 0 : 1, rope_dynamic_.on)) { if (!std::strcmp(declared, "llama3")) err_ = "config.json declares rope_type \"llama3\" but its numbers are not usable (factor > 0, " @@ -1117,6 +1121,20 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, fprintf(stderr, " [semantics] attention_bias=1 <- the container: layer %d carries q/k/v projection " "biases (config.json does not state attention_bias)\n", by); } + if (mrope_declared_) { + const int s3 = sem_.mrope_section[0] + sem_.mrope_section[1] + sem_.mrope_section[2]; + const int rot = sem_.rotary_dim > 0 ? sem_.rotary_dim : c_.HD; + if (!sem_.mrope_sectioned || sem_.n_mrope_section != 3 || s3 != rot / 2) { + err_ = "config.json declares rope scaling \"mrope\" and its mrope_section (" + + std::to_string(sem_.n_mrope_section) + " values, summing to " + std::to_string(s3) + + ") does not cover the " + std::to_string(rot / 2) + " rotary frequencies; the (t, h, w) " + "layout would be a guess"; + return false; + } + fprintf(stderr, " [semantics] mrope: contiguous sections t=%d h=%d w=%d (Qwen2-VL form) <- " + "rope_scaling.type \"mrope\"\n", + sem_.mrope_section[0], sem_.mrope_section[1], sem_.mrope_section[2]); + } q4nx_semantics_dump(&sem_); // A layer kind nothing implements is refused BY NAME, here beside the other precise refusals (engine#1087's // shape): q4nx_semantics maps an unrecognised layer_types entry to Q4NX_LAYER_OTHER and the attention path diff --git a/npu/forward/src/q4nx_semantics.c b/npu/forward/src/q4nx_semantics.c index 2b933762f..681fcc696 100644 --- a/npu/forward/src/q4nx_semantics.c +++ b/npu/forward/src/q4nx_semantics.c @@ -310,6 +310,14 @@ int q4nx_semantics_from_config(const char* json_path, const char* model_dir, const char* rs = value_for_top(json, "rope_scaling"); double mr = 0; if (rs && *rs == '{' && get_num(rs, "mrope_interleaved", &mr) && mr != 0) s->mrope = 1; + /* Qwen2-VL / Qwen2.5-VL: the contiguous form, named by the block's type */ + char rt[32] = {0}; + if (rs && *rs == '{' && !s->mrope && + ((get_str(rs, "type", rt, sizeof rt) && strcmp(rt, "mrope") == 0) || + (get_str(rs, "rope_type", rt, sizeof rt) && strcmp(rt, "mrope") == 0))) { + s->mrope = 1; + s->mrope_sectioned = 1; + } } const char* rope_blk = value_for_top(json, "rope_parameters"); if (rope_blk && *rope_blk == '{') { diff --git a/tests/batch_check_state_ci.py b/tests/batch_check_state_ci.py index 74bf3b387..eb6f7ff0f 100755 --- a/tests/batch_check_state_ci.py +++ b/tests/batch_check_state_ci.py @@ -15,7 +15,7 @@ # limitations under the License. """The batch-check state pin (tests/batch_check_state_test.cpp, engine#1905/#1946) on a container made here. -usage: tests/batch_check_state_ci.py [lfm2|lfm3|qwen2|qwen3_5] (needs numpy) +usage: tests/batch_check_state_ci.py [lfm2|lfm3|qwen2|qwen2vl|qwen3_5] (needs numpy) batch_check_state_test needs a Q4NX container, and CI has none, so on its own it skips (77) there and a regression of #1909 -- the NPU_BATCH_CHECK oracle resetting the prefix it replays from -- lands green. This writes a two-layer qwen3 @@ -48,7 +48,9 @@ `attention_bias` key, which is how every Qwen2 config is written. With the key absent the forward loaded the biases and never added them, so FastFlowLM's Qwen2.5-3B-Instruct-NPU2 answered noise. batched == sequential cannot see that (both paths skip the biases alike), so the arm also compares the sequential logits with a numpy Qwen2 forward on the -container's own weights (qwen2_reference). Measured at the fix: see QWEN2_* below. +container's own weights (qwen2_reference). `qwen2vl` is the same model declaring Qwen2.5-VL's rope_scaling +{"type": "mrope", "mrope_section": [16, 24, 24]}: the forward refused that at load, and on a text-only prompt (all +three position planes equal) it has to give exactly the plain-RoPE reference. With `qwen3_5` it is a qwen3_5 hybrid (engine#1889): a linear-attention (GDN) layer, then a full-attention layer with q/k norms, written under `language_model.*` so the packer takes its qwen3_5 path (renames, the conv1d transpose, @@ -97,7 +99,7 @@ def make_model(d, np, family="qwen3"): rng = np.random.default_rng(1946) lfm2 = family == "lfm2" lfm3 = family == "lfm3" - qwen2 = family == "qwen2" + qwen2 = family in ("qwen2", "qwen2vl") q35 = family == "qwen3_5" layers = 3 if lfm2 else (4 if lfm3 else LAYERS) conv_layers = (0, 2) if (lfm2 or lfm3) else () @@ -196,6 +198,8 @@ def norm(n): "norm_eps": 1e-5, "conv_L_cache": 3, "full_attn_idxs": [1]}) if qwen2: cfg.update({"architectures": ["Qwen2ForCausalLM"], "model_type": "qwen2"}) # and no attention_bias key + if family == "qwen2vl": + cfg["rope_scaling"] = {"type": "mrope", "mrope_section": [16, 24, 24]} # 64 = HEAD_DIM / 2 if lfm3: del cfg["rms_norm_eps"], cfg["rope_theta"] cfg.update({"architectures": ["Lfm3ForCausalLM"], "model_type": "lfm3", "num_hidden_layers": layers, @@ -362,7 +366,7 @@ def tmp_root(): def main(): - if len(sys.argv) not in (2, 3) or (len(sys.argv) == 3 and sys.argv[2] not in ("lfm2", "lfm3", "qwen2", "qwen3_5")): + if len(sys.argv) not in (2, 3) or (len(sys.argv) == 3 and sys.argv[2] not in ("lfm2", "lfm3", "qwen2", "qwen2vl", "qwen3_5")): print(__doc__.strip().splitlines()[2]) return 2 family = sys.argv[2] if len(sys.argv) == 3 else "qwen3" @@ -380,7 +384,7 @@ def main(): return 1 env = dict(os.environ, ONEBIT_TEST_Q4NX=out) logits_out = os.path.join(tmp, "sequential.f32") - if family in ("lfm3", "qwen2"): + if family in ("lfm3", "qwen2", "qwen2vl"): env["ONEBIT_TEST_LOGITS_OUT"] = logits_out env.pop("NPU_INFER_HOST_GEMM", None) # init_host is host-GEMM already; nothing else should steer it r = subprocess.run([sys.argv[1]], env=env, capture_output=True, text=True, timeout=110) @@ -393,7 +397,7 @@ def main(): # arm, not a reason to skip it. print("FAIL batch_check_state_test skipped on the fabricated container (init_host refused it)") return 1 - if family in ("lfm3", "qwen2") and r.returncode == 0: + if family in ("lfm3", "qwen2", "qwen2vl") and r.returncode == 0: ours = np.fromfile(logits_out, dtype=np.float32)[:VOCAB] ref = (lfm3_reference if family == "lfm3" else qwen2_reference)(out, np, [1, 2, 3, 4, 5, 6], shapes) corr = float(np.corrcoef(ours, ref)[0, 1]) diff --git a/tests/q4nx_mrope_test.c b/tests/q4nx_mrope_test.c index e8a42bfbf..63b4d9a3d 100644 --- a/tests/q4nx_mrope_test.c +++ b/tests/q4nx_mrope_test.c @@ -123,6 +123,43 @@ int main(void) { check(q4nx_mrope_axis(1, 0, sec) == 0, "no mrope_section -> plain (t) axis"); } + /* 6. Qwen2-VL / Qwen2.5-VL (engine#1786): rope_scaling {"type": "mrope", "mrope_section": [16, 24, 24]}, the + * CONTIGUOUS form. transformers' apply_multimodal_rotary_pos_emb splits cos by mrope_section * 2 and takes chunk + * k from plane k % 3, so frequency i is t below 16, h below 40 and w below 64. FastFlowLM's + * Qwen2.5-VL-3B-Instruct-NPU2 was refused at load ("mrope" was taken for a frequency scaling). */ + write_config(dir, + "{\"num_hidden_layers\": 2, \"hidden_size\": 2048, \"rope_theta\": 1000000.0," + " \"rope_scaling\": {\"type\": \"mrope\", \"mrope_section\": [16, 24, 24]}}", + cfg, sizeof cfg); + q4nx_semantics_from_config(cfg, dir, &s); + check(s.mrope == 1 && s.mrope_sectioned == 1, "qwen2.5_vl rope_scaling.type mrope -> mrope=1, sectioned"); + check(s.n_mrope_section == 3 && s.mrope_section[0] == 16 && s.mrope_section[1] == 24 && s.mrope_section[2] == 24, + "its sections are read: 16 / 24 / 24"); + { + const int sec[3] = {16, 24, 24}; + /* the reference's own construction, over the 64 frequencies of a 128-wide head */ + int ref[64], k = 0, agree = 0; + const int split[6] = {16, 24, 24, 16, 24, 24}; + for (int c = 0; c < 6 && k < 128; c++) + for (int j = 0; j < split[c]; j++, k++) + if (k < 64) ref[k] = c % 3; + for (int i = 0; i < 64; i++) agree += q4nx_mrope_axis_sectioned(i, 3, sec) == ref[i]; + check(agree == 64, "contiguous axis = the reference's split of cos by mrope_section * 2, all 64 frequencies"); + check(q4nx_mrope_axis_sectioned(15, 3, sec) == 0 && q4nx_mrope_axis_sectioned(16, 3, sec) == 1 && + q4nx_mrope_axis_sectioned(39, 3, sec) == 1 && q4nx_mrope_axis_sectioned(40, 3, sec) == 2, + "the run boundaries: 15 t, 16 h, 39 h, 40 w"); + int differ = 0; + for (int i = 0; i < 64; i++) differ += q4nx_mrope_axis_sectioned(i, 3, sec) != q4nx_mrope_axis(i, 3, sec); + check(differ > 0, "and it is not the interleaved layout (the two disagree on some frequencies)"); + } + /* the Qwen3-VL spelling is untouched by the new form */ + write_config(dir, + "{\"num_hidden_layers\": 2, \"rope_scaling\": {\"mrope_interleaved\": true," + " \"mrope_section\": [24, 20, 20], \"rope_type\": \"default\"}}", + cfg, sizeof cfg); + q4nx_semantics_from_config(cfg, dir, &s); + check(s.mrope == 1 && s.mrope_sectioned == 0, "qwen3_vl stays interleaved (mrope_sectioned 0)"); + if (failures) { printf("q4nx_mrope: %d failure(s)\n", failures); return 1; } printf("q4nx_mrope: ok\n"); return 0;