Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2154,6 +2154,11 @@ if(ONEBIT_NPU_HOST AND NOT ONEBIT_NPU AND NOT WIN32 AND Python3_Interpreter_FOUN
COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/batch_check_state_ci.py
$<TARGET_FILE:batch_check_state_test> qwen2)
set_tests_properties(batch_check_state_qwen2 PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 120)
# The same Qwen2 declaring Qwen2.5-VL's contiguous "mrope": accepted, and exact on a text-only prompt.
add_test(NAME batch_check_state_qwen2vl
COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/batch_check_state_ci.py
$<TARGET_FILE:batch_check_state_test> qwen2vl)
set_tests_properties(batch_check_state_qwen2vl PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 120)
# A fabricated qwen3_5 hybrid (a linear-attention layer, then attention), off the kernel's N grid on purpose.
add_test(NAME batch_check_state_qwen35
COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/batch_check_state_ci.py
Expand Down
11 changes: 11 additions & 0 deletions docs/npu.md
Original file line number Diff line number Diff line change
Expand Up @@ -2041,6 +2041,17 @@ compares the forward's logits with a numpy Qwen2 forward on the container's own
6.6e-06 of 9.6. The binary without the fix fails it (corr 0.444, another argmax), and batched == sequential passed
on that binary all the same.

Qwen2.5-VL's text tower is the same Qwen2, and it was refused one step earlier. Its config declares
`rope_scaling: {"type": "mrope", "mrope_section": [16, 24, 24]}`, and the forward read `"mrope"` as a frequency scaling
it does not implement. It is a position layout instead: plain RoPE frequencies, each taking its position from the t,
h or w plane. Qwen2-VL and Qwen2.5-VL lay the sections out CONTIGUOUSLY (transformers' `apply_multimodal_rotary_pos_emb`
splits cos by `mrope_section * 2`): frequencies 0-15 are t, 16-39 h and 40-63 w. Qwen3-VL interleaves them
instead. The forward now reads that form (`mrope_sectioned`, `q4nx_mrope_axis_sectioned`), and refuses a block whose
three sections do not sum to the rotary half-width. On a text-only prompt the three planes are equal, so it is plain
RoPE. `Qwen2.5-VL-3B-Instruct-NPU2`'s text tower now answers `Paris<|im_end|>` on the host; its vision tower is not
loaded here. `batch_check_state_qwen2vl` is the Qwen2 above declaring that block: the same logits as the plain-RoPE
reference (corr 1.0000000). With `"mrope"` refused again (main's behaviour) the arm fails.

**Per-head q/k normalisation is read from the container (engine#1788).** Whether the forward normalises q
and k per head used to be decided by `config.json`'s `model_type` alone: on for `qwen3*` and `gemma*`.
A pack of the same architecture under another name got it wrong in silence. FastFlowLM's MiniCPM-V 4.7
Expand Down
6 changes: 5 additions & 1 deletion npu/forward/include/q4nx_forward.h
Original file line number Diff line number Diff line change
Expand Up @@ -492,7 +492,10 @@ class Q4nxNpuForward {
int deepstack_lo_ = -1, deepstack_hi_ = -1;
// Which plane a rotary frequency belongs to under interleaved MRoPE. The rule lives in q4nx_semantics.h as
// q4nx_mrope_axis() so the C unit test can pin it without building a forward pass.
int mrope_plane(int freq) const { return q4nx_mrope_axis(freq, sem_.n_mrope_section, sem_.mrope_section); }
int mrope_plane(int freq) const {
return sem_.mrope_sectioned ? q4nx_mrope_axis_sectioned(freq, sem_.n_mrope_section, sem_.mrope_section)
: q4nx_mrope_axis(freq, sem_.n_mrope_section, sem_.mrope_section);
}
// Some models store the embedding QUANTIZED (Gemma4-E2B/E4B: dtype I8), and
// reading it as bf16 yields ~zero - which invalidated the whole forward and
// the tied LM head (all-zero logits). When the dtype is not BF16 the
Expand Down Expand Up @@ -647,6 +650,7 @@ class Q4nxNpuForward {
// Layout and maths: docs/qwen35-4b-weight-format.md.
int lin_kd_ = 0, lin_vd_ = 0; // key_dim = nkh*khd, value_dim = nvh*vhd
int attn_gate_ = 0; // qwen3_5 attn_output_gate
bool mrope_declared_ = false; // rope_scaling type "mrope" (Qwen2-VL form), validated after semantics
// LFM3's per-head output gate, [NH][H] per attention layer (empty for conv layers and other models):
// ctx of head h *= sigmoid(head_gate_[l][h] . xn), xn the operator-normed layer input.
std::vector<std::vector<float>> head_gate_;
Expand Down
14 changes: 14 additions & 0 deletions npu/forward/include/q4nx_semantics.h
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,11 @@ typedef struct {
* guess a layout for. */
int mrope_section[3];
int n_mrope_section;
/* Qwen2-VL / Qwen2.5-VL spell MRoPE as rope_scaling {"type": "mrope", "mrope_section": [16, 24, 24]}, and their
* sections are CONTIGUOUS runs of frequencies (transformers apply_multimodal_rotary_pos_emb: cos split by
* mrope_section * 2, chunk i taking plane i % 3), not Qwen3-VL's interleave. 1 for that form (mrope is 1 too).
* Source: rope_scaling.type / rope_type == "mrope". */
int mrope_sectioned;
/* DEEPSTACK: which decoder layers take one of the vision tower's extra feature sets. Read from
* vision_config.deepstack_visual_indexes (Qwen3-VL: [5, 11, 17]); the consumer adds set s to the hidden state at
* layer deepstack_layers[s]. Empty means the pack has none. */
Expand Down Expand Up @@ -256,6 +261,15 @@ static inline int q4nx_mrope_axis(int freq, int n_section, const int* section) {
return 0;
}

/* The same question for the CONTIGUOUS layout (mrope_sectioned, Qwen2-VL / Qwen2.5-VL): frequencies
* [0, s0) are t, [s0, s0 + s1) are h, and [s0 + s1, s0 + s1 + s2) are w. For [16, 24, 24]: 16 t, 24 h, 24 w. */
static inline int q4nx_mrope_axis_sectioned(int freq, int n_section, const int* section) {
if (n_section < 3 || !section) return 0;
if (freq < section[0]) return 0;
if (freq < section[0] + section[1]) return 1;
return 2;
}

/* Human-readable one-line dump of every flag with its source field. */
void q4nx_semantics_dump(const Q4nxSemantics* s);

Expand Down
20 changes: 19 additions & 1 deletion npu/forward/src/q4nx_forward.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -847,7 +847,11 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path,
char declared[64];
q4nx_rope_declared_type(t.c_str(), declared, (int)sizeof declared);
const bool linear_ok = !std::strcmp(declared, "linear") && rope_linear_factor_ > 0.0;
if (!linear_ok &&
// Qwen2.5-VL's "mrope" is a position layout, not a frequency scaling: plain RoPE frequencies, each taking its
// position from the t, h or w plane. Accepted here and checked against its sections once the semantics are
// read (below), where a malformed block is refused.
mrope_declared_ = !std::strcmp(declared, "mrope");
if (!linear_ok && !mrope_declared_ &&
!q4nx_rope_type_supported(declared, &rope_llama3_, rope_scale_.empty() ? 0 : 1, rope_dynamic_.on)) {
if (!std::strcmp(declared, "llama3"))
err_ = "config.json declares rope_type \"llama3\" but its numbers are not usable (factor > 0, "
Expand Down Expand Up @@ -1117,6 +1121,20 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path,
fprintf(stderr, " [semantics] attention_bias=1 <- the container: layer %d carries q/k/v projection "
"biases (config.json does not state attention_bias)\n", by);
}
if (mrope_declared_) {
const int s3 = sem_.mrope_section[0] + sem_.mrope_section[1] + sem_.mrope_section[2];
const int rot = sem_.rotary_dim > 0 ? sem_.rotary_dim : c_.HD;
if (!sem_.mrope_sectioned || sem_.n_mrope_section != 3 || s3 != rot / 2) {
err_ = "config.json declares rope scaling \"mrope\" and its mrope_section (" +
std::to_string(sem_.n_mrope_section) + " values, summing to " + std::to_string(s3) +
") does not cover the " + std::to_string(rot / 2) + " rotary frequencies; the (t, h, w) "
"layout would be a guess";
return false;
}
fprintf(stderr, " [semantics] mrope: contiguous sections t=%d h=%d w=%d (Qwen2-VL form) <- "
"rope_scaling.type \"mrope\"\n",
sem_.mrope_section[0], sem_.mrope_section[1], sem_.mrope_section[2]);
}
q4nx_semantics_dump(&sem_);
// A layer kind nothing implements is refused BY NAME, here beside the other precise refusals (engine#1087's
// shape): q4nx_semantics maps an unrecognised layer_types entry to Q4NX_LAYER_OTHER and the attention path
Expand Down
8 changes: 8 additions & 0 deletions npu/forward/src/q4nx_semantics.c
Original file line number Diff line number Diff line change
Expand Up @@ -310,6 +310,14 @@ int q4nx_semantics_from_config(const char* json_path, const char* model_dir,
const char* rs = value_for_top(json, "rope_scaling");
double mr = 0;
if (rs && *rs == '{' && get_num(rs, "mrope_interleaved", &mr) && mr != 0) s->mrope = 1;
/* Qwen2-VL / Qwen2.5-VL: the contiguous form, named by the block's type */
char rt[32] = {0};
if (rs && *rs == '{' && !s->mrope &&
((get_str(rs, "type", rt, sizeof rt) && strcmp(rt, "mrope") == 0) ||
(get_str(rs, "rope_type", rt, sizeof rt) && strcmp(rt, "mrope") == 0))) {
s->mrope = 1;
s->mrope_sectioned = 1;
}
}
const char* rope_blk = value_for_top(json, "rope_parameters");
if (rope_blk && *rope_blk == '{') {
Expand Down
16 changes: 10 additions & 6 deletions tests/batch_check_state_ci.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
# limitations under the License.
"""The batch-check state pin (tests/batch_check_state_test.cpp, engine#1905/#1946) on a container made here.

usage: tests/batch_check_state_ci.py <batch_check_state_test binary> [lfm2|lfm3|qwen2|qwen3_5] (needs numpy)
usage: tests/batch_check_state_ci.py <batch_check_state_test binary> [lfm2|lfm3|qwen2|qwen2vl|qwen3_5] (needs numpy)

batch_check_state_test needs a Q4NX container, and CI has none, so on its own it skips (77) there and a regression of
#1909 -- the NPU_BATCH_CHECK oracle resetting the prefix it replays from -- lands green. This writes a two-layer qwen3
Expand Down Expand Up @@ -48,7 +48,9 @@
`attention_bias` key, which is how every Qwen2 config is written. With the key absent the forward loaded the biases
and never added them, so FastFlowLM's Qwen2.5-3B-Instruct-NPU2 answered noise. batched == sequential cannot see that
(both paths skip the biases alike), so the arm also compares the sequential logits with a numpy Qwen2 forward on the
container's own weights (qwen2_reference). Measured at the fix: see QWEN2_* below.
container's own weights (qwen2_reference). `qwen2vl` is the same model declaring Qwen2.5-VL's rope_scaling
{"type": "mrope", "mrope_section": [16, 24, 24]}: the forward refused that at load, and on a text-only prompt (all
three position planes equal) it has to give exactly the plain-RoPE reference.

With `qwen3_5` it is a qwen3_5 hybrid (engine#1889): a linear-attention (GDN) layer, then a full-attention layer with
q/k norms, written under `language_model.*` so the packer takes its qwen3_5 path (renames, the conv1d transpose,
Expand Down Expand Up @@ -97,7 +99,7 @@ def make_model(d, np, family="qwen3"):
rng = np.random.default_rng(1946)
lfm2 = family == "lfm2"
lfm3 = family == "lfm3"
qwen2 = family == "qwen2"
qwen2 = family in ("qwen2", "qwen2vl")
q35 = family == "qwen3_5"
layers = 3 if lfm2 else (4 if lfm3 else LAYERS)
conv_layers = (0, 2) if (lfm2 or lfm3) else ()
Expand Down Expand Up @@ -196,6 +198,8 @@ def norm(n):
"norm_eps": 1e-5, "conv_L_cache": 3, "full_attn_idxs": [1]})
if qwen2:
cfg.update({"architectures": ["Qwen2ForCausalLM"], "model_type": "qwen2"}) # and no attention_bias key
if family == "qwen2vl":
cfg["rope_scaling"] = {"type": "mrope", "mrope_section": [16, 24, 24]} # 64 = HEAD_DIM / 2
if lfm3:
del cfg["rms_norm_eps"], cfg["rope_theta"]
cfg.update({"architectures": ["Lfm3ForCausalLM"], "model_type": "lfm3", "num_hidden_layers": layers,
Expand Down Expand Up @@ -362,7 +366,7 @@ def tmp_root():


def main():
if len(sys.argv) not in (2, 3) or (len(sys.argv) == 3 and sys.argv[2] not in ("lfm2", "lfm3", "qwen2", "qwen3_5")):
if len(sys.argv) not in (2, 3) or (len(sys.argv) == 3 and sys.argv[2] not in ("lfm2", "lfm3", "qwen2", "qwen2vl", "qwen3_5")):
print(__doc__.strip().splitlines()[2])
return 2
family = sys.argv[2] if len(sys.argv) == 3 else "qwen3"
Expand All @@ -380,7 +384,7 @@ def main():
return 1
env = dict(os.environ, ONEBIT_TEST_Q4NX=out)
logits_out = os.path.join(tmp, "sequential.f32")
if family in ("lfm3", "qwen2"):
if family in ("lfm3", "qwen2", "qwen2vl"):
env["ONEBIT_TEST_LOGITS_OUT"] = logits_out
env.pop("NPU_INFER_HOST_GEMM", None) # init_host is host-GEMM already; nothing else should steer it
r = subprocess.run([sys.argv[1]], env=env, capture_output=True, text=True, timeout=110)
Expand All @@ -393,7 +397,7 @@ def main():
# arm, not a reason to skip it.
print("FAIL batch_check_state_test skipped on the fabricated container (init_host refused it)")
return 1
if family in ("lfm3", "qwen2") and r.returncode == 0:
if family in ("lfm3", "qwen2", "qwen2vl") and r.returncode == 0:
ours = np.fromfile(logits_out, dtype=np.float32)[:VOCAB]
ref = (lfm3_reference if family == "lfm3" else qwen2_reference)(out, np, [1, 2, 3, 4, 5, 6], shapes)
corr = float(np.corrcoef(ours, ref)[0, 1])
Expand Down
37 changes: 37 additions & 0 deletions tests/q4nx_mrope_test.c
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,43 @@ int main(void) {
check(q4nx_mrope_axis(1, 0, sec) == 0, "no mrope_section -> plain (t) axis");
}

/* 6. Qwen2-VL / Qwen2.5-VL (engine#1786): rope_scaling {"type": "mrope", "mrope_section": [16, 24, 24]}, the
* CONTIGUOUS form. transformers' apply_multimodal_rotary_pos_emb splits cos by mrope_section * 2 and takes chunk
* k from plane k % 3, so frequency i is t below 16, h below 40 and w below 64. FastFlowLM's
* Qwen2.5-VL-3B-Instruct-NPU2 was refused at load ("mrope" was taken for a frequency scaling). */
write_config(dir,
"{\"num_hidden_layers\": 2, \"hidden_size\": 2048, \"rope_theta\": 1000000.0,"
" \"rope_scaling\": {\"type\": \"mrope\", \"mrope_section\": [16, 24, 24]}}",
cfg, sizeof cfg);
q4nx_semantics_from_config(cfg, dir, &s);
check(s.mrope == 1 && s.mrope_sectioned == 1, "qwen2.5_vl rope_scaling.type mrope -> mrope=1, sectioned");
check(s.n_mrope_section == 3 && s.mrope_section[0] == 16 && s.mrope_section[1] == 24 && s.mrope_section[2] == 24,
"its sections are read: 16 / 24 / 24");
{
const int sec[3] = {16, 24, 24};
/* the reference's own construction, over the 64 frequencies of a 128-wide head */
int ref[64], k = 0, agree = 0;
const int split[6] = {16, 24, 24, 16, 24, 24};
for (int c = 0; c < 6 && k < 128; c++)
for (int j = 0; j < split[c]; j++, k++)
if (k < 64) ref[k] = c % 3;
for (int i = 0; i < 64; i++) agree += q4nx_mrope_axis_sectioned(i, 3, sec) == ref[i];
check(agree == 64, "contiguous axis = the reference's split of cos by mrope_section * 2, all 64 frequencies");
check(q4nx_mrope_axis_sectioned(15, 3, sec) == 0 && q4nx_mrope_axis_sectioned(16, 3, sec) == 1 &&
q4nx_mrope_axis_sectioned(39, 3, sec) == 1 && q4nx_mrope_axis_sectioned(40, 3, sec) == 2,
"the run boundaries: 15 t, 16 h, 39 h, 40 w");
int differ = 0;
for (int i = 0; i < 64; i++) differ += q4nx_mrope_axis_sectioned(i, 3, sec) != q4nx_mrope_axis(i, 3, sec);
check(differ > 0, "and it is not the interleaved layout (the two disagree on some frequencies)");
}
/* the Qwen3-VL spelling is untouched by the new form */
write_config(dir,
"{\"num_hidden_layers\": 2, \"rope_scaling\": {\"mrope_interleaved\": true,"
" \"mrope_section\": [24, 20, 20], \"rope_type\": \"default\"}}",
cfg, sizeof cfg);
q4nx_semantics_from_config(cfg, dir, &s);
check(s.mrope == 1 && s.mrope_sectioned == 0, "qwen3_vl stays interleaved (mrope_sectioned 0)");

if (failures) { printf("q4nx_mrope: %d failure(s)\n", failures); return 1; }
printf("q4nx_mrope: ok\n");
return 0;
Expand Down
Loading