diff --git a/CMakeLists.txt b/CMakeLists.txt index 47413431c..c4578c4f1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2219,6 +2219,12 @@ if(ONEBIT_NPU_HOST AND NOT ONEBIT_NPU AND NOT WIN32 AND Python3_Interpreter_FOUN COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/batch_check_state_ci.py $ qwen2vl) set_tests_properties(batch_check_state_qwen2vl PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 120) + # The same Qwen2 with tie_word_embeddings removed from the container's config.json, as FastFlowLM's + # Qwen3.5-9B-NPU2 ships it: the separate lm_head must still be the head (engine#1786). + add_test(NAME batch_check_state_qwen2_untied + COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/batch_check_state_ci.py + $ qwen2_untied) + set_tests_properties(batch_check_state_qwen2_untied PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 120) # Hy-Memory (hy_v3, engine#1786): sigmoid routing with a selection bias, router [H][E], a leading dense layer, # [N][K] expert slices under FLM names, expert_hidden_dim, against a numpy hy_v3 reference. add_test(NAME batch_check_state_hy_v3 diff --git a/docs/npu.md b/docs/npu.md index 0477c7725..10efc60ca 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -2451,6 +2451,17 @@ one request still different is `Me gustaría una taza de café…` here against the same code three requests of the known-answer prompt returned `Paris` three times, and decode was 2.3 to 2.8 tok/s (32 layers on the per-op route) against FastFlowLM's 50 to 53. +**`Qwen3.5-9B-NPU2`: a head the config does not declare.** This pack's `config.json` has no +`tie_word_embeddings` key, and the forward read an absent key as "tied", the usual default. So it took the logits +off the embedding, although the pack ships a separate `lm_head.weight` and the model is untied (upstream states +`false`). The answer was noise on the host and on the NPU, from weights that all decode correctly. When the key is +absent and the container holds both tables, the forward now samples 16 rows of each and lets the weights decide. +A tied head is the embedding's quantised copy (correlation near 1), and an untied one is unrelated. The load says +which, for example `tie_word_embeddings not stated: lm_head.weight vs the embedding on 16 rows, corr 0.0142 -> +untied (the separate head is used)`. A stated key is still obeyed. `batch_check_state_qwen2_untied` pins this: the +Qwen2 container with the key removed must match the numpy reference that uses its `lm_head` (corr 1.0000000). With +the check disabled it reads corr 0.03 and argmax 6 against 7. + Not established by any row here: that a pack goes through the importer (`npu-pack` still reports these containers as recognised, not importable), anything about a vision or audio tower, and any pack whose geometry has no kernel set on the box. Of the 57 `*-NPU2` repos with a readable geometry on 2026-11-11, 19 had all four diff --git a/npu/forward/include/q4nx_semantics.h b/npu/forward/include/q4nx_semantics.h index 9dcfc27e2..a5906aa31 100644 --- a/npu/forward/include/q4nx_semantics.h +++ b/npu/forward/include/q4nx_semantics.h @@ -209,6 +209,10 @@ typedef struct { * logits from the embedding gives a completely different distribution * (corr 0.013 with the real head). Source: tie_word_embeddings. */ int tie_embeddings; + /* Whether config.json states tie_word_embeddings at all. FastFlowLM's Qwen3.5-9B-NPU2 omits it although the + * model is untied (its lm_head.weight is not the embedding), so an absent key is not evidence of a tie: the + * forward then decides from the container's own two tables (engine#1786). */ + int tie_stated; } Q4nxSemantics; /* Identifier for each flag, for the auditable field mapping. */ diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 6fc4e52a9..e0e8e0e6e 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -1757,6 +1757,65 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, // (packed I8); dequantize it once here. Nanbeige4.1-3B is such a model and // its head is unrelated to the embedding (corr 0.013), so decoding logits // from the embedding returns '的' instead of 'Paris'. + // engine#1786: a config that does not state tie_word_embeddings is not evidence of a tie. FastFlowLM's + // Qwen3.5-9B-NPU2 omits the key, ships a separate lm_head.weight and is untied (upstream says false), so reading + // the logits off the embedding answered noise on the host and on the NPU. When the key is absent and the + // container carries both tables, sample rows of each and let the weights say: a tied head is the embedding's + // quantised copy (corr ~1), an untied one is unrelated (measured 0.00x on the 9B). + // Every way this check can end prints one line: an un-taken measurement names the condition that stopped it and + // the default it keeps, so the assumption this replaces can never come back silently (review of #2309). + if (!sem_.tie_stated && sem_.tie_embeddings) { + TensorDesc* ltd = model_tensor_by_name(mw_, "lm_head.weight"); + TensorDesc* etd = &mw_->embed_tokens; + char why[160] = ""; + if (!ltd || !ltd->data_size) + snprintf(why, sizeof(why), "no separate lm_head.weight"); + else if (!etd->data_size || etd->name[0] == '\0' || !strcmp(ltd->name, etd->name)) + snprintf(why, sizeof(why), "lm_head.weight is the only table (it is also the embedding)"); + else if (kpad_ <= 0 || kpad_ % Q4NX_TILE_COLS != 0) + snprintf(why, sizeof(why), "lm_head.weight present but kpad %d is not a multiple of %d", kpad_, Q4NX_TILE_COLS); + const int samples = 16; + double corr = 0.0; + bool measured = false; + if (!why[0]) { + const int64_t head_shape[3] = {ltd->shape[0], ltd->ndim >= 2 ? ltd->shape[1] : 0, ltd->ndim >= 3 ? ltd->shape[2] : 0}; + const int head_rows = padded_head_rows(head_shape, ltd->ndim, kpad_, NV); + const int cb = q4nx_tile_row_kind((size_t)ltd->data_size, head_rows, kpad_); + const uint8_t* lp = (const uint8_t*)model_tensor_data(mw_, ltd); + if (cb == 0 || !lp) + snprintf(why, sizeof(why), "lm_head.weight (%llu bytes) has no row-readable chunk kind on a %d x %d grid", + (unsigned long long)ltd->data_size, head_rows, kpad_); + else { + std::vector row((size_t)kpad_); + double sx = 0, sy = 0, sxx = 0, syy = 0, sxy = 0, n = 0; + for (int i = 0; i < samples && !why[0]; i++) { + const int r = (int)((int64_t)i * (NV - 1) / (samples - 1)); + if (q4nx_dequant_tile_row(lp, (size_t)ltd->data_size, head_rows, kpad_, cb, r, row.data()) != 0) { + snprintf(why, sizeof(why), "lm_head.weight row %d did not decode", r); + break; + } + for (int k = 0; k < H; k++) { + const double x = row[(size_t)k], y = embed_at(r, k); + sx += x; sy += y; sxx += x * x; syy += y * y; sxy += x * y; n += 1; + } + } + if (!why[0]) { + const double cov = sxy - sx * sy / n, vx = sxx - sx * sx / n, vy = syy - sy * sy / n; + if (vx > 0 && vy > 0) { corr = cov / std::sqrt(vx * vy); measured = true; } + else snprintf(why, sizeof(why), "corr n/a (zero variance: %s)", vy > 0 ? "lm_head rows" : "embedding rows"); + } + } + } + if (measured) { + const bool untied = corr < 0.5; + std::fprintf(stderr, " [forward] tie_word_embeddings not stated: lm_head.weight vs the embedding on %d rows, " + "corr %.4f -> %s\n", samples, corr, + untied ? "untied (the separate head is used)" : "tied (the embedding is the head)"); + if (untied) sem_.tie_embeddings = 0; + } else { + std::fprintf(stderr, " [forward] tie_word_embeddings not stated: %s, keeping tied\n", why); + } + } if (!sem_.tie_embeddings) { TensorDesc* ltd = model_tensor_by_name(mw_, "lm_head.weight"); if (ltd) { diff --git a/npu/forward/src/q4nx_semantics.c b/npu/forward/src/q4nx_semantics.c index 1f02ea20d..acacd0a0e 100644 --- a/npu/forward/src/q4nx_semantics.c +++ b/npu/forward/src/q4nx_semantics.c @@ -431,7 +431,8 @@ int q4nx_semantics_from_config(const char* json_path, const char* model_dir, /* LM head: tied to the token embedding (the common case) or a SEPARATE * lm_head.weight (Nanbeige4.1-3B sets tie_word_embeddings = false). */ s->tie_embeddings = 1; - if (get_num(json, "tie_word_embeddings", &d)) s->tie_embeddings = d != 0; + s->tie_stated = 0; + if (get_num(json, "tie_word_embeddings", &d)) { s->tie_embeddings = d != 0; s->tie_stated = 1; } /* layer_types -> the per-layer attention kind the forward dispatches on */ { diff --git a/tests/batch_check_state_ci.py b/tests/batch_check_state_ci.py index 1edbd2146..9b2d59d44 100755 --- a/tests/batch_check_state_ci.py +++ b/tests/batch_check_state_ci.py @@ -15,7 +15,7 @@ # limitations under the License. """The batch-check state pin (tests/batch_check_state_test.cpp, engine#1905/#1946) on a container made here. -usage: tests/batch_check_state_ci.py [lfm2|lfm3|qwen2|qwen2vl|qwen3_5|hy_v3] (needs numpy) +usage: tests/batch_check_state_ci.py [lfm2|lfm3|qwen2|qwen2vl|qwen2_untied|qwen3_5|hy_v3] (needs numpy) batch_check_state_test needs a Q4NX container, and CI has none, so on its own it skips (77) there and a regression of #1909 -- the NPU_BATCH_CHECK oracle resetting the prefix it replays from -- lands green. This writes a two-layer qwen3 @@ -107,7 +107,7 @@ def make_model(d, np, family="qwen3"): rng = np.random.default_rng(1946) lfm2 = family == "lfm2" lfm3 = family == "lfm3" - qwen2 = family in ("qwen2", "qwen2vl") + qwen2 = family in ("qwen2", "qwen2vl", "qwen2_untied") q35 = family == "qwen3_5" layers = 3 if lfm2 else (4 if lfm3 else LAYERS) conv_layers = (0, 2) if (lfm2 or lfm3) else () @@ -501,7 +501,7 @@ def tmp_root(): def main(): - if len(sys.argv) not in (2, 3) or (len(sys.argv) == 3 and sys.argv[2] not in ("lfm2", "lfm3", "qwen2", "qwen2vl", "qwen3_5", "hy_v3")): + if len(sys.argv) not in (2, 3) or (len(sys.argv) == 3 and sys.argv[2] not in ("lfm2", "lfm3", "qwen2", "qwen2vl", "qwen2_untied", "qwen3_5", "hy_v3")): print(__doc__.strip().splitlines()[2]) return 2 family = sys.argv[2] if len(sys.argv) == 3 else "qwen3" @@ -526,9 +526,18 @@ def main(): redeclare_3d(out, {"model.layer.1.mlp.gate_exps_proj.weight": (c["E"] * c["EIM"] // 32, c["H"] // 256), "model.layer.1.mlp.up_exps_proj.weight": (c["E"] * c["EIM"] // 32, c["H"] // 256), "model.layer.1.mlp.down_exps_proj.weight": (c["E"] * c["H"] // 32, c["EIM"] // 256)}) + if family == "qwen2_untied": + # FastFlowLM's Qwen3.5-9B-NPU2 states no tie_word_embeddings and ships a separate head (engine#1786): the + # forward must not take the absent key for a tie. Strip it from the container the forward reads. + cp = os.path.join(out, "config.json") + with open(cp) as f: + c = json.load(f) + c.pop("tie_word_embeddings", None) + with open(cp, "w") as f: + json.dump(c, f) env = dict(os.environ, ONEBIT_TEST_Q4NX=out) logits_out = os.path.join(tmp, "sequential.f32") - if family in ("lfm3", "qwen2", "qwen2vl", "hy_v3"): + if family in ("lfm3", "qwen2", "qwen2vl", "qwen2_untied", "hy_v3"): env["ONEBIT_TEST_LOGITS_OUT"] = logits_out env.pop("NPU_INFER_HOST_GEMM", None) # init_host is host-GEMM already; nothing else should steer it r = subprocess.run([sys.argv[1]], env=env, capture_output=True, text=True, timeout=110) @@ -541,7 +550,7 @@ def main(): # arm, not a reason to skip it. print("FAIL batch_check_state_test skipped on the fabricated container (init_host refused it)") return 1 - if family in ("lfm3", "qwen2", "qwen2vl", "hy_v3") and r.returncode == 0: + if family in ("lfm3", "qwen2", "qwen2vl", "qwen2_untied", "hy_v3") and r.returncode == 0: ours = np.fromfile(logits_out, dtype=np.float32)[:VOCAB] ref = {"lfm3": lfm3_reference, "hy_v3": hyv3_reference}.get(family, qwen2_reference)(out, np, [1, 2, 3, 4, 5, 6], shapes) corr = float(np.corrcoef(ours, ref)[0, 1])