From a522e7a62fedf8ad25461ca17893c6f06cfceae8 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 12 Nov 2026 17:29:30 -0400 Subject: [PATCH 1/2] forward: an unstated tie_word_embeddings is decided from the two tables (refs #1786) FastFlowLM's Qwen3.5-9B-NPU2 omits tie_word_embeddings, ships a separate lm_head.weight and is untied (upstream: false). The forward read the absent key as "tied", took the logits off the embedding, and answered noise on the host and on the NPU although every weight decodes correctly. When the key is absent and the container holds both tables, 16 rows of each are compared: a tied head is the embedding's quantised copy (corr ~1), an untied one is unrelated (0.0142 on the 9B). A stated key is obeyed as before. batch_check_state_qwen2_untied: the Qwen2 arm with the key removed from the container must match its lm_head reference (corr 1.0000000); with the check disabled it reads corr 0.03, argmax 6 vs 7. Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 6 +++++ docs/npu.md | 11 ++++++++ npu/forward/include/q4nx_semantics.h | 4 +++ npu/forward/src/q4nx_forward.cpp | 39 ++++++++++++++++++++++++++++ npu/forward/src/q4nx_semantics.c | 3 ++- tests/batch_check_state_ci.py | 19 ++++++++++---- 6 files changed, 76 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 47413431c..c4578c4f1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2219,6 +2219,12 @@ if(ONEBIT_NPU_HOST AND NOT ONEBIT_NPU AND NOT WIN32 AND Python3_Interpreter_FOUN COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/batch_check_state_ci.py $ qwen2vl) set_tests_properties(batch_check_state_qwen2vl PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 120) + # The same Qwen2 with tie_word_embeddings removed from the container's config.json, as FastFlowLM's + # Qwen3.5-9B-NPU2 ships it: the separate lm_head must still be the head (engine#1786). + add_test(NAME batch_check_state_qwen2_untied + COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/batch_check_state_ci.py + $ qwen2_untied) + set_tests_properties(batch_check_state_qwen2_untied PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 120) # Hy-Memory (hy_v3, engine#1786): sigmoid routing with a selection bias, router [H][E], a leading dense layer, # [N][K] expert slices under FLM names, expert_hidden_dim, against a numpy hy_v3 reference. add_test(NAME batch_check_state_hy_v3 diff --git a/docs/npu.md b/docs/npu.md index 0477c7725..10efc60ca 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -2451,6 +2451,17 @@ one request still different is `Me gustaría una taza de café…` here against the same code three requests of the known-answer prompt returned `Paris` three times, and decode was 2.3 to 2.8 tok/s (32 layers on the per-op route) against FastFlowLM's 50 to 53. +**`Qwen3.5-9B-NPU2`: a head the config does not declare.** This pack's `config.json` has no +`tie_word_embeddings` key, and the forward read an absent key as "tied", the usual default. So it took the logits +off the embedding, although the pack ships a separate `lm_head.weight` and the model is untied (upstream states +`false`). The answer was noise on the host and on the NPU, from weights that all decode correctly. When the key is +absent and the container holds both tables, the forward now samples 16 rows of each and lets the weights decide. +A tied head is the embedding's quantised copy (correlation near 1), and an untied one is unrelated. The load says +which, for example `tie_word_embeddings not stated: lm_head.weight vs the embedding on 16 rows, corr 0.0142 -> +untied (the separate head is used)`. A stated key is still obeyed. `batch_check_state_qwen2_untied` pins this: the +Qwen2 container with the key removed must match the numpy reference that uses its `lm_head` (corr 1.0000000). With +the check disabled it reads corr 0.03 and argmax 6 against 7. + Not established by any row here: that a pack goes through the importer (`npu-pack` still reports these containers as recognised, not importable), anything about a vision or audio tower, and any pack whose geometry has no kernel set on the box. Of the 57 `*-NPU2` repos with a readable geometry on 2026-11-11, 19 had all four diff --git a/npu/forward/include/q4nx_semantics.h b/npu/forward/include/q4nx_semantics.h index 9dcfc27e2..a5906aa31 100644 --- a/npu/forward/include/q4nx_semantics.h +++ b/npu/forward/include/q4nx_semantics.h @@ -209,6 +209,10 @@ typedef struct { * logits from the embedding gives a completely different distribution * (corr 0.013 with the real head). Source: tie_word_embeddings. */ int tie_embeddings; + /* Whether config.json states tie_word_embeddings at all. FastFlowLM's Qwen3.5-9B-NPU2 omits it although the + * model is untied (its lm_head.weight is not the embedding), so an absent key is not evidence of a tie: the + * forward then decides from the container's own two tables (engine#1786). */ + int tie_stated; } Q4nxSemantics; /* Identifier for each flag, for the auditable field mapping. */ diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 6fc4e52a9..d890b384b 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -1757,6 +1757,45 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, // (packed I8); dequantize it once here. Nanbeige4.1-3B is such a model and // its head is unrelated to the embedding (corr 0.013), so decoding logits // from the embedding returns '的' instead of 'Paris'. + // engine#1786: a config that does not state tie_word_embeddings is not evidence of a tie. FastFlowLM's + // Qwen3.5-9B-NPU2 omits the key, ships a separate lm_head.weight and is untied (upstream says false), so reading + // the logits off the embedding answered noise on the host and on the NPU. When the key is absent and the + // container carries both tables, sample rows of each and let the weights say: a tied head is the embedding's + // quantised copy (corr ~1), an untied one is unrelated (measured 0.00x on the 9B). + // With no embedding to compare against, embed_at() reads 0.0f, the variance is zero and the default stays. + if (!sem_.tie_stated && sem_.tie_embeddings) { + TensorDesc* ltd = model_tensor_by_name(mw_, "lm_head.weight"); + TensorDesc* etd = &mw_->embed_tokens; + const bool both = ltd && ltd->data_size && etd->data_size && etd->name[0] != '\0' && strcmp(ltd->name, etd->name); + if (both && kpad_ > 0 && kpad_ % Q4NX_TILE_COLS == 0) { + const int64_t head_shape[3] = {ltd->shape[0], ltd->ndim >= 2 ? ltd->shape[1] : 0, ltd->ndim >= 3 ? ltd->shape[2] : 0}; + const int head_rows = padded_head_rows(head_shape, ltd->ndim, kpad_, NV); + const int cb = q4nx_tile_row_kind((size_t)ltd->data_size, head_rows, kpad_); + const uint8_t* lp = (const uint8_t*)model_tensor_data(mw_, ltd); + if (cb != 0 && lp) { + std::vector row((size_t)kpad_); + double sx = 0, sy = 0, sxx = 0, syy = 0, sxy = 0, n = 0; + const int samples = 16; + for (int i = 0; i < samples; i++) { + const int r = (int)((int64_t)i * (NV - 1) / (samples - 1)); + if (q4nx_dequant_tile_row(lp, (size_t)ltd->data_size, head_rows, kpad_, cb, r, row.data()) != 0) { n = 0; break; } + for (int k = 0; k < H; k++) { + const double x = row[(size_t)k], y = embed_at(r, k); + sx += x; sy += y; sxx += x * x; syy += y * y; sxy += x * y; n += 1; + } + } + if (n > 0) { + const double cov = sxy - sx * sy / n, vx = sxx - sx * sx / n, vy = syy - sy * sy / n; + const double corr = (vx > 0 && vy > 0) ? cov / std::sqrt(vx * vy) : 1.0; + const bool untied = corr < 0.5; + std::fprintf(stderr, " [forward] tie_word_embeddings not stated: lm_head.weight vs the embedding on %d " + "rows, corr %.4f -> %s\n", samples, corr, + untied ? "untied (the separate head is used)" : "tied (the embedding is the head)"); + if (untied) sem_.tie_embeddings = 0; + } + } + } + } if (!sem_.tie_embeddings) { TensorDesc* ltd = model_tensor_by_name(mw_, "lm_head.weight"); if (ltd) { diff --git a/npu/forward/src/q4nx_semantics.c b/npu/forward/src/q4nx_semantics.c index 1f02ea20d..acacd0a0e 100644 --- a/npu/forward/src/q4nx_semantics.c +++ b/npu/forward/src/q4nx_semantics.c @@ -431,7 +431,8 @@ int q4nx_semantics_from_config(const char* json_path, const char* model_dir, /* LM head: tied to the token embedding (the common case) or a SEPARATE * lm_head.weight (Nanbeige4.1-3B sets tie_word_embeddings = false). */ s->tie_embeddings = 1; - if (get_num(json, "tie_word_embeddings", &d)) s->tie_embeddings = d != 0; + s->tie_stated = 0; + if (get_num(json, "tie_word_embeddings", &d)) { s->tie_embeddings = d != 0; s->tie_stated = 1; } /* layer_types -> the per-layer attention kind the forward dispatches on */ { diff --git a/tests/batch_check_state_ci.py b/tests/batch_check_state_ci.py index 1edbd2146..9b2d59d44 100755 --- a/tests/batch_check_state_ci.py +++ b/tests/batch_check_state_ci.py @@ -15,7 +15,7 @@ # limitations under the License. """The batch-check state pin (tests/batch_check_state_test.cpp, engine#1905/#1946) on a container made here. -usage: tests/batch_check_state_ci.py [lfm2|lfm3|qwen2|qwen2vl|qwen3_5|hy_v3] (needs numpy) +usage: tests/batch_check_state_ci.py [lfm2|lfm3|qwen2|qwen2vl|qwen2_untied|qwen3_5|hy_v3] (needs numpy) batch_check_state_test needs a Q4NX container, and CI has none, so on its own it skips (77) there and a regression of #1909 -- the NPU_BATCH_CHECK oracle resetting the prefix it replays from -- lands green. This writes a two-layer qwen3 @@ -107,7 +107,7 @@ def make_model(d, np, family="qwen3"): rng = np.random.default_rng(1946) lfm2 = family == "lfm2" lfm3 = family == "lfm3" - qwen2 = family in ("qwen2", "qwen2vl") + qwen2 = family in ("qwen2", "qwen2vl", "qwen2_untied") q35 = family == "qwen3_5" layers = 3 if lfm2 else (4 if lfm3 else LAYERS) conv_layers = (0, 2) if (lfm2 or lfm3) else () @@ -501,7 +501,7 @@ def tmp_root(): def main(): - if len(sys.argv) not in (2, 3) or (len(sys.argv) == 3 and sys.argv[2] not in ("lfm2", "lfm3", "qwen2", "qwen2vl", "qwen3_5", "hy_v3")): + if len(sys.argv) not in (2, 3) or (len(sys.argv) == 3 and sys.argv[2] not in ("lfm2", "lfm3", "qwen2", "qwen2vl", "qwen2_untied", "qwen3_5", "hy_v3")): print(__doc__.strip().splitlines()[2]) return 2 family = sys.argv[2] if len(sys.argv) == 3 else "qwen3" @@ -526,9 +526,18 @@ def main(): redeclare_3d(out, {"model.layer.1.mlp.gate_exps_proj.weight": (c["E"] * c["EIM"] // 32, c["H"] // 256), "model.layer.1.mlp.up_exps_proj.weight": (c["E"] * c["EIM"] // 32, c["H"] // 256), "model.layer.1.mlp.down_exps_proj.weight": (c["E"] * c["H"] // 32, c["EIM"] // 256)}) + if family == "qwen2_untied": + # FastFlowLM's Qwen3.5-9B-NPU2 states no tie_word_embeddings and ships a separate head (engine#1786): the + # forward must not take the absent key for a tie. Strip it from the container the forward reads. + cp = os.path.join(out, "config.json") + with open(cp) as f: + c = json.load(f) + c.pop("tie_word_embeddings", None) + with open(cp, "w") as f: + json.dump(c, f) env = dict(os.environ, ONEBIT_TEST_Q4NX=out) logits_out = os.path.join(tmp, "sequential.f32") - if family in ("lfm3", "qwen2", "qwen2vl", "hy_v3"): + if family in ("lfm3", "qwen2", "qwen2vl", "qwen2_untied", "hy_v3"): env["ONEBIT_TEST_LOGITS_OUT"] = logits_out env.pop("NPU_INFER_HOST_GEMM", None) # init_host is host-GEMM already; nothing else should steer it r = subprocess.run([sys.argv[1]], env=env, capture_output=True, text=True, timeout=110) @@ -541,7 +550,7 @@ def main(): # arm, not a reason to skip it. print("FAIL batch_check_state_test skipped on the fabricated container (init_host refused it)") return 1 - if family in ("lfm3", "qwen2", "qwen2vl", "hy_v3") and r.returncode == 0: + if family in ("lfm3", "qwen2", "qwen2vl", "qwen2_untied", "hy_v3") and r.returncode == 0: ours = np.fromfile(logits_out, dtype=np.float32)[:VOCAB] ref = {"lfm3": lfm3_reference, "hy_v3": hyv3_reference}.get(family, qwen2_reference)(out, np, [1, 2, 3, 4, 5, 6], shapes) corr = float(np.corrcoef(ours, ref)[0, 1]) From 57e5a0e8b8d73a16d07f928dfc0264f58ddf8b72 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 12 Nov 2026 18:04:30 -0400 Subject: [PATCH 2/2] forward: every un-taken tie measurement says why and what it keeps (refs #1786) Review of #2309: three paths skipped the measurement silently and the zero-variance case reported corr 1.0, so the old "absent means tied" assumption could still land with nothing on stderr. Each path now prints one line naming the condition (no separate lm_head, the head is the only table, kpad off the 256 grid, no row-readable chunk kind, a row that did not decode, zero variance as "corr n/a") and "keeping tied". Co-Authored-By: Claude Opus 5.5 --- npu/forward/src/q4nx_forward.cpp | 48 ++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index d890b384b..e0e8e0e6e 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -1762,39 +1762,59 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, // the logits off the embedding answered noise on the host and on the NPU. When the key is absent and the // container carries both tables, sample rows of each and let the weights say: a tied head is the embedding's // quantised copy (corr ~1), an untied one is unrelated (measured 0.00x on the 9B). - // With no embedding to compare against, embed_at() reads 0.0f, the variance is zero and the default stays. + // Every way this check can end prints one line: an un-taken measurement names the condition that stopped it and + // the default it keeps, so the assumption this replaces can never come back silently (review of #2309). if (!sem_.tie_stated && sem_.tie_embeddings) { TensorDesc* ltd = model_tensor_by_name(mw_, "lm_head.weight"); TensorDesc* etd = &mw_->embed_tokens; - const bool both = ltd && ltd->data_size && etd->data_size && etd->name[0] != '\0' && strcmp(ltd->name, etd->name); - if (both && kpad_ > 0 && kpad_ % Q4NX_TILE_COLS == 0) { + char why[160] = ""; + if (!ltd || !ltd->data_size) + snprintf(why, sizeof(why), "no separate lm_head.weight"); + else if (!etd->data_size || etd->name[0] == '\0' || !strcmp(ltd->name, etd->name)) + snprintf(why, sizeof(why), "lm_head.weight is the only table (it is also the embedding)"); + else if (kpad_ <= 0 || kpad_ % Q4NX_TILE_COLS != 0) + snprintf(why, sizeof(why), "lm_head.weight present but kpad %d is not a multiple of %d", kpad_, Q4NX_TILE_COLS); + const int samples = 16; + double corr = 0.0; + bool measured = false; + if (!why[0]) { const int64_t head_shape[3] = {ltd->shape[0], ltd->ndim >= 2 ? ltd->shape[1] : 0, ltd->ndim >= 3 ? ltd->shape[2] : 0}; const int head_rows = padded_head_rows(head_shape, ltd->ndim, kpad_, NV); const int cb = q4nx_tile_row_kind((size_t)ltd->data_size, head_rows, kpad_); const uint8_t* lp = (const uint8_t*)model_tensor_data(mw_, ltd); - if (cb != 0 && lp) { + if (cb == 0 || !lp) + snprintf(why, sizeof(why), "lm_head.weight (%llu bytes) has no row-readable chunk kind on a %d x %d grid", + (unsigned long long)ltd->data_size, head_rows, kpad_); + else { std::vector row((size_t)kpad_); double sx = 0, sy = 0, sxx = 0, syy = 0, sxy = 0, n = 0; - const int samples = 16; - for (int i = 0; i < samples; i++) { + for (int i = 0; i < samples && !why[0]; i++) { const int r = (int)((int64_t)i * (NV - 1) / (samples - 1)); - if (q4nx_dequant_tile_row(lp, (size_t)ltd->data_size, head_rows, kpad_, cb, r, row.data()) != 0) { n = 0; break; } + if (q4nx_dequant_tile_row(lp, (size_t)ltd->data_size, head_rows, kpad_, cb, r, row.data()) != 0) { + snprintf(why, sizeof(why), "lm_head.weight row %d did not decode", r); + break; + } for (int k = 0; k < H; k++) { const double x = row[(size_t)k], y = embed_at(r, k); sx += x; sy += y; sxx += x * x; syy += y * y; sxy += x * y; n += 1; } } - if (n > 0) { + if (!why[0]) { const double cov = sxy - sx * sy / n, vx = sxx - sx * sx / n, vy = syy - sy * sy / n; - const double corr = (vx > 0 && vy > 0) ? cov / std::sqrt(vx * vy) : 1.0; - const bool untied = corr < 0.5; - std::fprintf(stderr, " [forward] tie_word_embeddings not stated: lm_head.weight vs the embedding on %d " - "rows, corr %.4f -> %s\n", samples, corr, - untied ? "untied (the separate head is used)" : "tied (the embedding is the head)"); - if (untied) sem_.tie_embeddings = 0; + if (vx > 0 && vy > 0) { corr = cov / std::sqrt(vx * vy); measured = true; } + else snprintf(why, sizeof(why), "corr n/a (zero variance: %s)", vy > 0 ? "lm_head rows" : "embedding rows"); } } } + if (measured) { + const bool untied = corr < 0.5; + std::fprintf(stderr, " [forward] tie_word_embeddings not stated: lm_head.weight vs the embedding on %d rows, " + "corr %.4f -> %s\n", samples, corr, + untied ? "untied (the separate head is used)" : "tied (the embedding is the head)"); + if (untied) sem_.tie_embeddings = 0; + } else { + std::fprintf(stderr, " [forward] tie_word_embeddings not stated: %s, keeping tied\n", why); + } } if (!sem_.tie_embeddings) { TensorDesc* ltd = model_tensor_by_name(mw_, "lm_head.weight");