Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2219,6 +2219,12 @@ if(ONEBIT_NPU_HOST AND NOT ONEBIT_NPU AND NOT WIN32 AND Python3_Interpreter_FOUN
COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/batch_check_state_ci.py
$<TARGET_FILE:batch_check_state_test> qwen2vl)
set_tests_properties(batch_check_state_qwen2vl PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 120)
# The same Qwen2 with tie_word_embeddings removed from the container's config.json, as FastFlowLM's
# Qwen3.5-9B-NPU2 ships it: the separate lm_head must still be the head (engine#1786).
add_test(NAME batch_check_state_qwen2_untied
COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/batch_check_state_ci.py
$<TARGET_FILE:batch_check_state_test> qwen2_untied)
set_tests_properties(batch_check_state_qwen2_untied PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 120)
# Hy-Memory (hy_v3, engine#1786): sigmoid routing with a selection bias, router [H][E], a leading dense layer,
# [N][K] expert slices under FLM names, expert_hidden_dim, against a numpy hy_v3 reference.
add_test(NAME batch_check_state_hy_v3
Expand Down
11 changes: 11 additions & 0 deletions docs/npu.md
Original file line number Diff line number Diff line change
Expand Up @@ -2451,6 +2451,17 @@ one request still different is `Me gustaría una taza de café…` here against
the same code three requests of the known-answer prompt returned `Paris` three times, and decode was 2.3 to 2.8
tok/s (32 layers on the per-op route) against FastFlowLM's 50 to 53.

**`Qwen3.5-9B-NPU2`: a head the config does not declare.** This pack's `config.json` has no
`tie_word_embeddings` key, and the forward read an absent key as "tied", the usual default. So it took the logits
off the embedding, although the pack ships a separate `lm_head.weight` and the model is untied (upstream states
`false`). The answer was noise on the host and on the NPU, from weights that all decode correctly. When the key is
absent and the container holds both tables, the forward now samples 16 rows of each and lets the weights decide.
A tied head is the embedding's quantised copy (correlation near 1), and an untied one is unrelated. The load says
which, for example `tie_word_embeddings not stated: lm_head.weight vs the embedding on 16 rows, corr 0.0142 ->
untied (the separate head is used)`. A stated key is still obeyed. `batch_check_state_qwen2_untied` pins this: the
Qwen2 container with the key removed must match the numpy reference that uses its `lm_head` (corr 1.0000000). With
the check disabled it reads corr 0.03 and argmax 6 against 7.

Not established by any row here: that a pack goes through the importer (`npu-pack` still reports these
containers as recognised, not importable), anything about a vision or audio tower, and any pack whose geometry
has no kernel set on the box. Of the 57 `*-NPU2` repos with a readable geometry on 2026-11-11, 19 had all four
Expand Down
4 changes: 4 additions & 0 deletions npu/forward/include/q4nx_semantics.h
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,10 @@ typedef struct {
* logits from the embedding gives a completely different distribution
* (corr 0.013 with the real head). Source: tie_word_embeddings. */
int tie_embeddings;
/* Whether config.json states tie_word_embeddings at all. FastFlowLM's Qwen3.5-9B-NPU2 omits it although the
* model is untied (its lm_head.weight is not the embedding), so an absent key is not evidence of a tie: the
* forward then decides from the container's own two tables (engine#1786). */
int tie_stated;
} Q4nxSemantics;

/* Identifier for each flag, for the auditable field mapping. */
Expand Down
59 changes: 59 additions & 0 deletions npu/forward/src/q4nx_forward.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1757,6 +1757,65 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path,
// (packed I8); dequantize it once here. Nanbeige4.1-3B is such a model and
// its head is unrelated to the embedding (corr 0.013), so decoding logits
// from the embedding returns '的' instead of 'Paris'.
// engine#1786: a config that does not state tie_word_embeddings is not evidence of a tie. FastFlowLM's
// Qwen3.5-9B-NPU2 omits the key, ships a separate lm_head.weight and is untied (upstream says false), so reading
// the logits off the embedding answered noise on the host and on the NPU. When the key is absent and the
// container carries both tables, sample rows of each and let the weights say: a tied head is the embedding's
// quantised copy (corr ~1), an untied one is unrelated (measured 0.00x on the 9B).
// Every way this check can end prints one line: an un-taken measurement names the condition that stopped it and
// the default it keeps, so the assumption this replaces can never come back silently (review of #2309).
if (!sem_.tie_stated && sem_.tie_embeddings) {
TensorDesc* ltd = model_tensor_by_name(mw_, "lm_head.weight");
TensorDesc* etd = &mw_->embed_tokens;
char why[160] = "";
if (!ltd || !ltd->data_size)
snprintf(why, sizeof(why), "no separate lm_head.weight");
else if (!etd->data_size || etd->name[0] == '\0' || !strcmp(ltd->name, etd->name))
snprintf(why, sizeof(why), "lm_head.weight is the only table (it is also the embedding)");
else if (kpad_ <= 0 || kpad_ % Q4NX_TILE_COLS != 0)
snprintf(why, sizeof(why), "lm_head.weight present but kpad %d is not a multiple of %d", kpad_, Q4NX_TILE_COLS);
const int samples = 16;
double corr = 0.0;
bool measured = false;
if (!why[0]) {
const int64_t head_shape[3] = {ltd->shape[0], ltd->ndim >= 2 ? ltd->shape[1] : 0, ltd->ndim >= 3 ? ltd->shape[2] : 0};
const int head_rows = padded_head_rows(head_shape, ltd->ndim, kpad_, NV);
const int cb = q4nx_tile_row_kind((size_t)ltd->data_size, head_rows, kpad_);
const uint8_t* lp = (const uint8_t*)model_tensor_data(mw_, ltd);
if (cb == 0 || !lp)
snprintf(why, sizeof(why), "lm_head.weight (%llu bytes) has no row-readable chunk kind on a %d x %d grid",
(unsigned long long)ltd->data_size, head_rows, kpad_);
else {
std::vector<float> row((size_t)kpad_);
double sx = 0, sy = 0, sxx = 0, syy = 0, sxy = 0, n = 0;
for (int i = 0; i < samples && !why[0]; i++) {
const int r = (int)((int64_t)i * (NV - 1) / (samples - 1));
if (q4nx_dequant_tile_row(lp, (size_t)ltd->data_size, head_rows, kpad_, cb, r, row.data()) != 0) {
snprintf(why, sizeof(why), "lm_head.weight row %d did not decode", r);
break;
}
for (int k = 0; k < H; k++) {
const double x = row[(size_t)k], y = embed_at(r, k);
sx += x; sy += y; sxx += x * x; syy += y * y; sxy += x * y; n += 1;
}
}
if (!why[0]) {
const double cov = sxy - sx * sy / n, vx = sxx - sx * sx / n, vy = syy - sy * sy / n;
if (vx > 0 && vy > 0) { corr = cov / std::sqrt(vx * vy); measured = true; }
else snprintf(why, sizeof(why), "corr n/a (zero variance: %s)", vy > 0 ? "lm_head rows" : "embedding rows");
}
}
}
if (measured) {
const bool untied = corr < 0.5;
std::fprintf(stderr, " [forward] tie_word_embeddings not stated: lm_head.weight vs the embedding on %d rows, "
"corr %.4f -> %s\n", samples, corr,
untied ? "untied (the separate head is used)" : "tied (the embedding is the head)");
if (untied) sem_.tie_embeddings = 0;
} else {
std::fprintf(stderr, " [forward] tie_word_embeddings not stated: %s, keeping tied\n", why);
}
}
if (!sem_.tie_embeddings) {
TensorDesc* ltd = model_tensor_by_name(mw_, "lm_head.weight");
if (ltd) {
Expand Down
3 changes: 2 additions & 1 deletion npu/forward/src/q4nx_semantics.c
Original file line number Diff line number Diff line change
Expand Up @@ -431,7 +431,8 @@ int q4nx_semantics_from_config(const char* json_path, const char* model_dir,
/* LM head: tied to the token embedding (the common case) or a SEPARATE
* lm_head.weight (Nanbeige4.1-3B sets tie_word_embeddings = false). */
s->tie_embeddings = 1;
if (get_num(json, "tie_word_embeddings", &d)) s->tie_embeddings = d != 0;
s->tie_stated = 0;
if (get_num(json, "tie_word_embeddings", &d)) { s->tie_embeddings = d != 0; s->tie_stated = 1; }

/* layer_types -> the per-layer attention kind the forward dispatches on */
{
Expand Down
19 changes: 14 additions & 5 deletions tests/batch_check_state_ci.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
# limitations under the License.
"""The batch-check state pin (tests/batch_check_state_test.cpp, engine#1905/#1946) on a container made here.

usage: tests/batch_check_state_ci.py <batch_check_state_test binary> [lfm2|lfm3|qwen2|qwen2vl|qwen3_5|hy_v3] (needs numpy)
usage: tests/batch_check_state_ci.py <batch_check_state_test binary> [lfm2|lfm3|qwen2|qwen2vl|qwen2_untied|qwen3_5|hy_v3] (needs numpy)

batch_check_state_test needs a Q4NX container, and CI has none, so on its own it skips (77) there and a regression of
#1909 -- the NPU_BATCH_CHECK oracle resetting the prefix it replays from -- lands green. This writes a two-layer qwen3
Expand Down Expand Up @@ -107,7 +107,7 @@ def make_model(d, np, family="qwen3"):
rng = np.random.default_rng(1946)
lfm2 = family == "lfm2"
lfm3 = family == "lfm3"
qwen2 = family in ("qwen2", "qwen2vl")
qwen2 = family in ("qwen2", "qwen2vl", "qwen2_untied")
q35 = family == "qwen3_5"
layers = 3 if lfm2 else (4 if lfm3 else LAYERS)
conv_layers = (0, 2) if (lfm2 or lfm3) else ()
Expand Down Expand Up @@ -501,7 +501,7 @@ def tmp_root():


def main():
if len(sys.argv) not in (2, 3) or (len(sys.argv) == 3 and sys.argv[2] not in ("lfm2", "lfm3", "qwen2", "qwen2vl", "qwen3_5", "hy_v3")):
if len(sys.argv) not in (2, 3) or (len(sys.argv) == 3 and sys.argv[2] not in ("lfm2", "lfm3", "qwen2", "qwen2vl", "qwen2_untied", "qwen3_5", "hy_v3")):
print(__doc__.strip().splitlines()[2])
return 2
family = sys.argv[2] if len(sys.argv) == 3 else "qwen3"
Expand All @@ -526,9 +526,18 @@ def main():
redeclare_3d(out, {"model.layer.1.mlp.gate_exps_proj.weight": (c["E"] * c["EIM"] // 32, c["H"] // 256),
"model.layer.1.mlp.up_exps_proj.weight": (c["E"] * c["EIM"] // 32, c["H"] // 256),
"model.layer.1.mlp.down_exps_proj.weight": (c["E"] * c["H"] // 32, c["EIM"] // 256)})
if family == "qwen2_untied":
# FastFlowLM's Qwen3.5-9B-NPU2 states no tie_word_embeddings and ships a separate head (engine#1786): the
# forward must not take the absent key for a tie. Strip it from the container the forward reads.
cp = os.path.join(out, "config.json")
with open(cp) as f:
c = json.load(f)
c.pop("tie_word_embeddings", None)
with open(cp, "w") as f:
json.dump(c, f)
env = dict(os.environ, ONEBIT_TEST_Q4NX=out)
logits_out = os.path.join(tmp, "sequential.f32")
if family in ("lfm3", "qwen2", "qwen2vl", "hy_v3"):
if family in ("lfm3", "qwen2", "qwen2vl", "qwen2_untied", "hy_v3"):
env["ONEBIT_TEST_LOGITS_OUT"] = logits_out
env.pop("NPU_INFER_HOST_GEMM", None) # init_host is host-GEMM already; nothing else should steer it
r = subprocess.run([sys.argv[1]], env=env, capture_output=True, text=True, timeout=110)
Expand All @@ -541,7 +550,7 @@ def main():
# arm, not a reason to skip it.
print("FAIL batch_check_state_test skipped on the fabricated container (init_host refused it)")
return 1
if family in ("lfm3", "qwen2", "qwen2vl", "hy_v3") and r.returncode == 0:
if family in ("lfm3", "qwen2", "qwen2vl", "qwen2_untied", "hy_v3") and r.returncode == 0:
ours = np.fromfile(logits_out, dtype=np.float32)[:VOCAB]
ref = {"lfm3": lfm3_reference, "hy_v3": hyv3_reference}.get(family, qwen2_reference)(out, np, [1, 2, 3, 4, 5, 6], shapes)
corr = float(np.corrcoef(ours, ref)[0, 1])
Expand Down
Loading