Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -581,6 +581,11 @@ add_test(NAME serve_watchdog COMMAND serve_watchdog_test)
# app/npu_grammar.h, docs/npu.md "Sampling and structured output"); header only, no device.
# engine#1781: the comparator behind NPU_BATCH_CHECK -- pure arithmetic over two float arrays, so it runs with no model,
# no device and no runtime. It is a header, so the test includes it directly and needs no link libraries.
# engine#1786: an unquantised BF16 matrix in either of FastFlowLM's two orientations (npu/forward/include/bf16_matrix.h).
add_executable(bf16_matrix_test tests/bf16_matrix_test.cpp)
target_include_directories(bf16_matrix_test PRIVATE npu/forward/include)
add_test(NAME bf16_matrix COMMAND bf16_matrix_test)
add_dependencies(onebit bf16_matrix_test) # ci.yml builds onebit: this is how the arm reaches CI's ctest
add_executable(npu_sampling_test tests/npu_sampling_test.cpp)
# The shape rule for a 1.8 external-data initializer (npu/pack_initializer_shape.h, engine#1503's graph-initializer
# reader). Those packs zero every `dims`, so a byte length has to become a shape -- and a shape inferred wrongly imports
Expand Down Expand Up @@ -2245,6 +2250,12 @@ if(ONEBIT_NPU_HOST AND NOT ONEBIT_NPU AND NOT WIN32 AND Python3_Interpreter_FOUN
COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/batch_check_state_ci.py
$<TARGET_FILE:batch_check_state_test> hy_v3)
set_tests_properties(batch_check_state_hy_v3 PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 120)
# The same hy_v3 with a `shared_expert_gate.weight` beside the router, as FastFlowLM's Qwen3.6-35B-A3B-NPU2 names its
# shared expert's sigmoid gate: the gate must be read and applied (engine#1786).
add_test(NAME batch_check_state_hy_v3_sgate
COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/batch_check_state_ci.py
$<TARGET_FILE:batch_check_state_test> hy_v3_sgate)
set_tests_properties(batch_check_state_hy_v3_sgate PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 120)
# GPT-OSS (engine#1786): a container written in FastFlowLM's D2560/MX2560 form, sinks, o bias, yarn, the top-k
# MoE with clamped SwiGLU, against a numpy GPT-OSS forward on the same bytes.
add_test(NAME gptoss_forward
Expand Down
16 changes: 16 additions & 0 deletions docs/npu.md
Original file line number Diff line number Diff line change
Expand Up @@ -2513,6 +2513,22 @@ untied (the separate head is used)`. A stated key is still obeyed. `batch_check_
Qwen2 container with the key removed must match the numpy reference that uses its `lm_head` (corr 1.0000000). With
the check disabled it reads corr 0.03 and argmax 6 against 7.

**`Qwen3.6-35B-A3B-NPU2`: two spellings of the same model.** This pack differs from the hub's Qwen3.5 dense packs
in two places, and each made the reply noise on the host:
- **`ssm_alpha_proj` / `ssm_beta_proj` are BF16 `[2048, 32]`**, i.e. `[in, out]`, like the pack's `[H][E]` router.
Against the GGUF of the same checkpoint (unsloth's Q8_0), read transposed they match at corr 0.99999; the GGUF
orders the 32 value heads interleaved. `npu/forward/include/bf16_matrix.h` reads a BF16 matrix whose declared shape
is exactly the transpose of the grid asked for that way, and a square one as `[out, in]`.
- **The shared expert's sigmoid gate is `shared_expert_gate.weight`**, beside the router, where the FastFlowLM name
table had `mlp.share_router.weight`. The gate was never read, so the shared expert was added ungated in every layer.
The forward reads either name.

With both fixes the host forward reasons ("Thinking Process: 1. **…"). The other tensor kinds decode against the GGUF:
routed expert 0's gate 0.9958 and the shared gate 1.0000; `ssm_out` and the GDN `qkv` match per value head. Pinned by
`bf16_matrix_test` (both orientations, the padded grid, the refusals) and `batch_check_state_hy_v3_sgate`: the hy_v3
fixture with the gate tensor, against a reference that applies it. With the gate name missed, that arm reads corr
0.989.

Not established by any row here: that a pack goes through the importer (`npu-pack` still reports these
containers as recognised, not importable), anything about a vision or audio tower, and any pack whose geometry
has no kernel set on the box. Of the 57 `*-NPU2` repos with a readable geometry on 2026-11-11, 19 had all four
Expand Down
52 changes: 52 additions & 0 deletions npu/forward/include/bf16_matrix.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// engine#1786: an unquantised BF16 matrix read into the forward's [rows][cols] grid, header-only so its arm
// (tests/bf16_matrix_test.cpp) drives it without a container.
//
// Two orientations reach the forward. FastFlowLM's Embedding-Gemma-300M-NPU2 ships every projection as HF's [out, in]
// (= [rows, cols]), possibly narrower than a grid padded to 256 columns (IM 1152 -> 1280): copied, the pad zero.
// Its Qwen3.6-35B-A3B-NPU2 stores a small matrix as [in, out] (ssm_alpha/beta_proj [2048, 32], like its [H][E]
// router): against the GGUF of the same checkpoint, read transposed, corr 0.99999. A declared shape that is exactly
// the transpose of the grid asked for is read that way; a square one cannot be told from its shape and is read as
// [out, in]. Anything else -- wider than the grid, short of bytes -- is refused (-1), never guessed.
#pragma once

#include <cstddef>
#include <cstdint>
#include <cstring>

inline float q4nx_bf16_word_to_f32(uint16_t b) {
const uint32_t u = (uint32_t)b << 16;
float f;
std::memcpy(&f, &u, sizeof f);
return f;
}

inline int q4nx_bf16_matrix_into(const uint16_t* w, size_t nbytes, int64_t r0, int64_t c0, int rows, int cols,
float* out) {
if (!w || !out || rows <= 0 || cols <= 0 || r0 <= 0 || c0 <= 0) return -1;
if (nbytes < (size_t)r0 * (size_t)c0 * 2) return -1;
if (r0 == cols && c0 == rows && rows != cols) {
for (int r = 0; r < rows; r++)
for (int c = 0; c < cols; c++) out[(size_t)r * cols + c] = q4nx_bf16_word_to_f32(w[(size_t)c * (size_t)c0 + r]);
return 0;
}
if (r0 > rows || c0 > cols) return -1;
for (int r = 0; r < rows; r++)
for (int c = 0; c < cols; c++)
out[(size_t)r * cols + c] = (r < r0 && c < c0) ? q4nx_bf16_word_to_f32(w[(size_t)r * (size_t)c0 + c]) : 0.0f;
return 0;
}
31 changes: 17 additions & 14 deletions npu/forward/src/q4nx_forward.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
#include "weight_keys.h" // the per-layer weight-cache keys, held distinct at compile time
#include "layer_taps.h" // engine#1780: where a tapped layer's rows go in the sink
#include "pad_head_rows.h" // engine#1990: the padded-head row rule, header-only for its arm
#include "bf16_matrix.h" // engine#1786: an unquantised BF16 matrix in either orientation
#include "batch_width.h"
#include "ffn_cache_policy.h"
#include "lfm2_shortconv.h" // LFM2 conv layers: the host part between the two projections
Expand Down Expand Up @@ -199,18 +200,10 @@ static int64_t declared_row_bytes(const TensorDesc* d) {

static int dequant_any(const void* data, size_t nbytes, int rows, int cols, float* out, const TensorDesc* desc = nullptr) {
if (!data || !out || rows <= 0 || cols <= 0) return -1;
// engine#1786: an unquantised projection (FastFlowLM's Embedding-Gemma-300M-NPU2 ships every weight as BF16
// [out, in]). Copied into the forward's grid, which may be padded past the declared shape (IM 1152 -> 1280): the
// pad rows and columns are zero, the layout every chunked reader below writes.
if (desc && desc->ndim == 2 && !strcmp(desc->dtype, "BF16")) {
const int64_t r0 = desc->shape[0], c0 = desc->shape[1];
if (r0 <= 0 || c0 <= 0 || r0 > rows || c0 > cols || nbytes < (size_t)r0 * (size_t)c0 * 2) return -1;
const uint16_t* w = (const uint16_t*)data;
for (int r = 0; r < rows; r++)
for (int c = 0; c < cols; c++)
out[(size_t)r * cols + c] = (r < r0 && c < c0) ? bf16_to_f32(w[(size_t)r * (size_t)c0 + (size_t)c]) : 0.0f;
return 0;
}
// engine#1786: an unquantised BF16 matrix, [out, in] (Embedding-Gemma's projections, into a grid that may be padded
// past the declared shape, pad zero) or [in, out] (Qwen3.6-35B-A3B's ssm_alpha/beta): bf16_matrix.h says which.
if (desc && desc->ndim == 2 && !strcmp(desc->dtype, "BF16"))
return q4nx_bf16_matrix_into((const uint16_t*)data, nbytes, desc->shape[0], desc->shape[1], rows, cols, out);
const int64_t decl_row_bytes = declared_row_bytes(desc);
if (decl_row_bytes == Q4NX_CHUNK_D2560) {
// GPT-OSS: [RB, CB, S, 2560]; the container's K (CB*128) may be padded past the model's, and is read into
Expand Down Expand Up @@ -2678,7 +2671,12 @@ bool Q4nxNpuForward::load_layer(int l) {
if (!load_share(mn.share_gate, share_g_, expert_im_, H)) share_g_.clear();
if (!load_share(mn.share_up, share_u_, expert_im_, H)) { share_g_.clear(); share_u_.clear(); }
if (!load_share(mn.share_down, share_d_, H, expert_im_)) { share_g_.clear(); share_u_.clear(); }
if (!load_bf16_named(mn.share_router, share_router_, H)) share_router_.clear();
// The shared expert's sigmoid gate: `mlp.share_router.weight` in the FLM table, `shared_expert_gate.weight`
// (at the layer, beside the router) in FastFlowLM's Qwen3.6-35B-A3B-NPU2 -- missed, the shared expert was added
// ungated in every layer and the reply was noise (engine#1786).
if (!load_bf16_named(mn.share_router, share_router_, H) &&
!load_bf16_named("shared_expert_gate.weight", share_router_, H))
share_router_.clear();
return true;
}
if (sem_.gpt_oss) return true; // gpt_oss: the MoE reads its MXFP4 experts in place (gptoss_moe)
Expand Down Expand Up @@ -2859,7 +2857,12 @@ bool Q4nxNpuForward::load_gdn_layer(int l) {
if (!load_share(mn.share_gate, share_g_, expert_im_, H)) share_g_.clear();
if (!load_share(mn.share_up, share_u_, expert_im_, H)) { share_g_.clear(); share_u_.clear(); }
if (!load_share(mn.share_down, share_d_, H, expert_im_)) { share_g_.clear(); share_u_.clear(); }
if (!load_bf16_named(mn.share_router, share_router_, H)) share_router_.clear();
// The shared expert's sigmoid gate: `mlp.share_router.weight` in the FLM table, `shared_expert_gate.weight`
// (at the layer, beside the router) in FastFlowLM's Qwen3.6-35B-A3B-NPU2 -- missed, the shared expert was added
// ungated in every layer and the reply was noise (engine#1786).
if (!load_bf16_named(mn.share_router, share_router_, H) &&
!load_bf16_named("shared_expert_gate.weight", share_router_, H))
share_router_.clear();
return true;
}
if (!ffn_tail_cached()) {
Expand Down
27 changes: 17 additions & 10 deletions tests/batch_check_state_ci.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
# limitations under the License.
"""The batch-check state pin (tests/batch_check_state_test.cpp, engine#1905/#1946) on a container made here.

usage: tests/batch_check_state_ci.py <batch_check_state_test binary> [lfm2|lfm3|qwen2|qwen2vl|qwen2_untied|qwen3_5|hy_v3] (needs numpy)
usage: tests/batch_check_state_ci.py <batch_check_state_test binary> [lfm2|lfm3|qwen2|qwen2vl|qwen2_untied|qwen3_5|hy_v3|hy_v3_sgate] (needs numpy)

batch_check_state_test needs a Q4NX container, and CI has none, so on its own it skips (77) there and a regression of
#1909 -- the NPU_BATCH_CHECK oracle resetting the prefix it replays from -- lands green. This writes a two-layer qwen3
Expand Down Expand Up @@ -359,7 +359,7 @@ def rope(x):
HY = dict(H=256, NH=2, NKV=1, HD=128, E=4, TOPK=2, EIM=512, DIM=768, SCALE=2.5)


def make_hyv3(d, np):
def make_hyv3(d, np, sgate=False):
"""A two-layer hy_v3 in FastFlowLM's naming; returns (weights for the reference, names to re-declare 3-D)."""
rng = np.random.default_rng(1786)
c = HY
Expand All @@ -384,6 +384,10 @@ def make_hyv3(d, np):
p + "mlp.down_exps_proj.weight": w(E * H, EIM), # stacked [N][K] per expert
p + "mlp.share_gate_exps_proj.weight": w(EIM, H), p + "mlp.share_up_exps_proj.weight": w(EIM, H),
p + "mlp.share_down_exps_proj.weight": w(H, EIM)})
if sgate:
# engine#1786: FastFlowLM's Qwen3.6-35B-A3B-NPU2 gates its shared expert with sigmoid(h . g), the vector
# stored as `shared_expert_gate.weight` beside the router; read under any other name, it was dropped
W[p + "shared_expert_gate.weight"] = w(H, sc=0.5)
os.makedirs(d)
write_safetensors(os.path.join(d, "model.safetensors"), W)
cfg = {"architectures": ["HYV3ForCausalLM"], "model_type": "hy_v3", "hidden_size": H, "num_hidden_layers": 2,
Expand Down Expand Up @@ -477,8 +481,11 @@ def rope(x):
for e, wt in zip(sel, wts):
g, u, dn = G[e * EIM:(e + 1) * EIM], U[e * EIM:(e + 1) * EIM], D[e * H:(e + 1) * H]
out[t] += wt * ((silu(h[t] @ g.T) * (h[t] @ u.T)) @ dn.T)
out += (silu(h @ W(p + "mlp.share_gate_exps_proj.weight").T) * (h @ W(p + "mlp.share_up_exps_proj.weight").T)) @ \
sh = (silu(h @ W(p + "mlp.share_gate_exps_proj.weight").T) * (h @ W(p + "mlp.share_up_exps_proj.weight").T)) @ \
W(p + "mlp.share_down_exps_proj.weight").T
if p + "shared_expert_gate.weight" in table:
sh = sh * (1.0 / (1.0 + np.exp(-(h @ W(p + "shared_expert_gate.weight")))))[:, None]
out += sh
x = x + out
return rms(x, W("model.norm.weight"))[-1] @ W("lm_head.weight").T

Expand All @@ -501,7 +508,7 @@ def tmp_root():


def main():
if len(sys.argv) not in (2, 3) or (len(sys.argv) == 3 and sys.argv[2] not in ("lfm2", "lfm3", "qwen2", "qwen2vl", "qwen2_untied", "qwen3_5", "hy_v3")):
if len(sys.argv) not in (2, 3) or (len(sys.argv) == 3 and sys.argv[2] not in ("lfm2", "lfm3", "qwen2", "qwen2vl", "qwen2_untied", "qwen3_5", "hy_v3", "hy_v3_sgate")):
print(__doc__.strip().splitlines()[2])
return 2
family = sys.argv[2] if len(sys.argv) == 3 else "qwen3"
Expand All @@ -512,16 +519,16 @@ def main():
return 77
with tempfile.TemporaryDirectory(prefix="batch_check_state_ci.", dir=tmp_root()) as tmp:
src, out = os.path.join(tmp, "hf"), os.path.join(tmp, "q4nx")
if family == "hy_v3":
Whyv = make_hyv3(src, np)
if family in ("hy_v3", "hy_v3_sgate"):
Whyv = make_hyv3(src, np, sgate=family == "hy_v3_sgate")
shapes = {k: v.shape for k, v in Whyv.items()}
else:
shapes = make_model(src, np, family)
r = subprocess.run([sys.executable, TOOL, "pack", src, out], capture_output=True, text=True)
if r.returncode != 0:
print(f"FAIL packing the fabricated container: {r.stderr[-600:]}")
return 1
if family == "hy_v3":
if family in ("hy_v3", "hy_v3_sgate"):
c = HY
redeclare_3d(out, {"model.layer.1.mlp.gate_exps_proj.weight": (c["E"] * c["EIM"] // 32, c["H"] // 256),
"model.layer.1.mlp.up_exps_proj.weight": (c["E"] * c["EIM"] // 32, c["H"] // 256),
Expand All @@ -537,7 +544,7 @@ def main():
json.dump(c, f)
env = dict(os.environ, ONEBIT_TEST_Q4NX=out)
logits_out = os.path.join(tmp, "sequential.f32")
if family in ("lfm3", "qwen2", "qwen2vl", "qwen2_untied", "hy_v3"):
if family in ("lfm3", "qwen2", "qwen2vl", "qwen2_untied", "hy_v3", "hy_v3_sgate"):
env["ONEBIT_TEST_LOGITS_OUT"] = logits_out
env.pop("NPU_INFER_HOST_GEMM", None) # init_host is host-GEMM already; nothing else should steer it
r = subprocess.run([sys.argv[1]], env=env, capture_output=True, text=True, timeout=110)
Expand All @@ -550,9 +557,9 @@ def main():
# arm, not a reason to skip it.
print("FAIL batch_check_state_test skipped on the fabricated container (init_host refused it)")
return 1
if family in ("lfm3", "qwen2", "qwen2vl", "qwen2_untied", "hy_v3") and r.returncode == 0:
if family in ("lfm3", "qwen2", "qwen2vl", "qwen2_untied", "hy_v3", "hy_v3_sgate") and r.returncode == 0:
ours = np.fromfile(logits_out, dtype=np.float32)[:VOCAB]
ref = {"lfm3": lfm3_reference, "hy_v3": hyv3_reference}.get(family, qwen2_reference)(out, np, [1, 2, 3, 4, 5, 6], shapes)
ref = {"lfm3": lfm3_reference, "hy_v3": hyv3_reference, "hy_v3_sgate": hyv3_reference}.get(family, qwen2_reference)(out, np, [1, 2, 3, 4, 5, 6], shapes)
corr = float(np.corrcoef(ours, ref)[0, 1])
ok = (corr >= LFM3_MIN_CORR and float(np.abs(ours - ref).max()) <= LFM3_MAX_DIFF * float(np.abs(ref).max())
and int(np.argmax(ours)) == int(np.argmax(ref)))
Expand Down
Loading
Loading