diff --git a/CMakeLists.txt b/CMakeLists.txt index e867c07ba..7f360bc35 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -581,6 +581,11 @@ add_test(NAME serve_watchdog COMMAND serve_watchdog_test) # app/npu_grammar.h, docs/npu.md "Sampling and structured output"); header only, no device. # engine#1781: the comparator behind NPU_BATCH_CHECK -- pure arithmetic over two float arrays, so it runs with no model, # no device and no runtime. It is a header, so the test includes it directly and needs no link libraries. +# engine#1786: an unquantised BF16 matrix in either of FastFlowLM's two orientations (npu/forward/include/bf16_matrix.h). +add_executable(bf16_matrix_test tests/bf16_matrix_test.cpp) +target_include_directories(bf16_matrix_test PRIVATE npu/forward/include) +add_test(NAME bf16_matrix COMMAND bf16_matrix_test) +add_dependencies(onebit bf16_matrix_test) # ci.yml builds onebit: this is how the arm reaches CI's ctest add_executable(npu_sampling_test tests/npu_sampling_test.cpp) # The shape rule for a 1.8 external-data initializer (npu/pack_initializer_shape.h, engine#1503's graph-initializer # reader). Those packs zero every `dims`, so a byte length has to become a shape -- and a shape inferred wrongly imports @@ -2245,6 +2250,12 @@ if(ONEBIT_NPU_HOST AND NOT ONEBIT_NPU AND NOT WIN32 AND Python3_Interpreter_FOUN COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/batch_check_state_ci.py $ hy_v3) set_tests_properties(batch_check_state_hy_v3 PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 120) + # The same hy_v3 with a `shared_expert_gate.weight` beside the router, as FastFlowLM's Qwen3.6-35B-A3B-NPU2 names its + # shared expert's sigmoid gate: the gate must be read and applied (engine#1786). + add_test(NAME batch_check_state_hy_v3_sgate + COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/batch_check_state_ci.py + $ hy_v3_sgate) + set_tests_properties(batch_check_state_hy_v3_sgate PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 120) # GPT-OSS (engine#1786): a container written in FastFlowLM's D2560/MX2560 form, sinks, o bias, yarn, the top-k # MoE with clamped SwiGLU, against a numpy GPT-OSS forward on the same bytes. add_test(NAME gptoss_forward diff --git a/docs/npu.md b/docs/npu.md index fa58927d7..efca2dd34 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -2513,6 +2513,22 @@ untied (the separate head is used)`. A stated key is still obeyed. `batch_check_ Qwen2 container with the key removed must match the numpy reference that uses its `lm_head` (corr 1.0000000). With the check disabled it reads corr 0.03 and argmax 6 against 7. +**`Qwen3.6-35B-A3B-NPU2`: two spellings of the same model.** This pack differs from the hub's Qwen3.5 dense packs +in two places, and each made the reply noise on the host: +- **`ssm_alpha_proj` / `ssm_beta_proj` are BF16 `[2048, 32]`**, i.e. `[in, out]`, like the pack's `[H][E]` router. + Against the GGUF of the same checkpoint (unsloth's Q8_0), read transposed they match at corr 0.99999; the GGUF + orders the 32 value heads interleaved. `npu/forward/include/bf16_matrix.h` reads a BF16 matrix whose declared shape + is exactly the transpose of the grid asked for that way, and a square one as `[out, in]`. +- **The shared expert's sigmoid gate is `shared_expert_gate.weight`**, beside the router, where the FastFlowLM name + table had `mlp.share_router.weight`. The gate was never read, so the shared expert was added ungated in every layer. + The forward reads either name. + +With both fixes the host forward reasons ("Thinking Process: 1. **…"). The other tensor kinds decode against the GGUF: +routed expert 0's gate 0.9958 and the shared gate 1.0000; `ssm_out` and the GDN `qkv` match per value head. Pinned by +`bf16_matrix_test` (both orientations, the padded grid, the refusals) and `batch_check_state_hy_v3_sgate`: the hy_v3 +fixture with the gate tensor, against a reference that applies it. With the gate name missed, that arm reads corr +0.989. + Not established by any row here: that a pack goes through the importer (`npu-pack` still reports these containers as recognised, not importable), anything about a vision or audio tower, and any pack whose geometry has no kernel set on the box. Of the 57 `*-NPU2` repos with a readable geometry on 2026-11-11, 19 had all four diff --git a/npu/forward/include/bf16_matrix.h b/npu/forward/include/bf16_matrix.h new file mode 100644 index 000000000..a1a00b2f2 --- /dev/null +++ b/npu/forward/include/bf16_matrix.h @@ -0,0 +1,52 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// engine#1786: an unquantised BF16 matrix read into the forward's [rows][cols] grid, header-only so its arm +// (tests/bf16_matrix_test.cpp) drives it without a container. +// +// Two orientations reach the forward. FastFlowLM's Embedding-Gemma-300M-NPU2 ships every projection as HF's [out, in] +// (= [rows, cols]), possibly narrower than a grid padded to 256 columns (IM 1152 -> 1280): copied, the pad zero. +// Its Qwen3.6-35B-A3B-NPU2 stores a small matrix as [in, out] (ssm_alpha/beta_proj [2048, 32], like its [H][E] +// router): against the GGUF of the same checkpoint, read transposed, corr 0.99999. A declared shape that is exactly +// the transpose of the grid asked for is read that way; a square one cannot be told from its shape and is read as +// [out, in]. Anything else -- wider than the grid, short of bytes -- is refused (-1), never guessed. +#pragma once + +#include +#include +#include + +inline float q4nx_bf16_word_to_f32(uint16_t b) { + const uint32_t u = (uint32_t)b << 16; + float f; + std::memcpy(&f, &u, sizeof f); + return f; +} + +inline int q4nx_bf16_matrix_into(const uint16_t* w, size_t nbytes, int64_t r0, int64_t c0, int rows, int cols, + float* out) { + if (!w || !out || rows <= 0 || cols <= 0 || r0 <= 0 || c0 <= 0) return -1; + if (nbytes < (size_t)r0 * (size_t)c0 * 2) return -1; + if (r0 == cols && c0 == rows && rows != cols) { + for (int r = 0; r < rows; r++) + for (int c = 0; c < cols; c++) out[(size_t)r * cols + c] = q4nx_bf16_word_to_f32(w[(size_t)c * (size_t)c0 + r]); + return 0; + } + if (r0 > rows || c0 > cols) return -1; + for (int r = 0; r < rows; r++) + for (int c = 0; c < cols; c++) + out[(size_t)r * cols + c] = (r < r0 && c < c0) ? q4nx_bf16_word_to_f32(w[(size_t)r * (size_t)c0 + c]) : 0.0f; + return 0; +} diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 4337652a5..1e2549145 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -28,6 +28,7 @@ #include "weight_keys.h" // the per-layer weight-cache keys, held distinct at compile time #include "layer_taps.h" // engine#1780: where a tapped layer's rows go in the sink #include "pad_head_rows.h" // engine#1990: the padded-head row rule, header-only for its arm +#include "bf16_matrix.h" // engine#1786: an unquantised BF16 matrix in either orientation #include "batch_width.h" #include "ffn_cache_policy.h" #include "lfm2_shortconv.h" // LFM2 conv layers: the host part between the two projections @@ -199,18 +200,10 @@ static int64_t declared_row_bytes(const TensorDesc* d) { static int dequant_any(const void* data, size_t nbytes, int rows, int cols, float* out, const TensorDesc* desc = nullptr) { if (!data || !out || rows <= 0 || cols <= 0) return -1; - // engine#1786: an unquantised projection (FastFlowLM's Embedding-Gemma-300M-NPU2 ships every weight as BF16 - // [out, in]). Copied into the forward's grid, which may be padded past the declared shape (IM 1152 -> 1280): the - // pad rows and columns are zero, the layout every chunked reader below writes. - if (desc && desc->ndim == 2 && !strcmp(desc->dtype, "BF16")) { - const int64_t r0 = desc->shape[0], c0 = desc->shape[1]; - if (r0 <= 0 || c0 <= 0 || r0 > rows || c0 > cols || nbytes < (size_t)r0 * (size_t)c0 * 2) return -1; - const uint16_t* w = (const uint16_t*)data; - for (int r = 0; r < rows; r++) - for (int c = 0; c < cols; c++) - out[(size_t)r * cols + c] = (r < r0 && c < c0) ? bf16_to_f32(w[(size_t)r * (size_t)c0 + (size_t)c]) : 0.0f; - return 0; - } + // engine#1786: an unquantised BF16 matrix, [out, in] (Embedding-Gemma's projections, into a grid that may be padded + // past the declared shape, pad zero) or [in, out] (Qwen3.6-35B-A3B's ssm_alpha/beta): bf16_matrix.h says which. + if (desc && desc->ndim == 2 && !strcmp(desc->dtype, "BF16")) + return q4nx_bf16_matrix_into((const uint16_t*)data, nbytes, desc->shape[0], desc->shape[1], rows, cols, out); const int64_t decl_row_bytes = declared_row_bytes(desc); if (decl_row_bytes == Q4NX_CHUNK_D2560) { // GPT-OSS: [RB, CB, S, 2560]; the container's K (CB*128) may be padded past the model's, and is read into @@ -2678,7 +2671,12 @@ bool Q4nxNpuForward::load_layer(int l) { if (!load_share(mn.share_gate, share_g_, expert_im_, H)) share_g_.clear(); if (!load_share(mn.share_up, share_u_, expert_im_, H)) { share_g_.clear(); share_u_.clear(); } if (!load_share(mn.share_down, share_d_, H, expert_im_)) { share_g_.clear(); share_u_.clear(); } - if (!load_bf16_named(mn.share_router, share_router_, H)) share_router_.clear(); + // The shared expert's sigmoid gate: `mlp.share_router.weight` in the FLM table, `shared_expert_gate.weight` + // (at the layer, beside the router) in FastFlowLM's Qwen3.6-35B-A3B-NPU2 -- missed, the shared expert was added + // ungated in every layer and the reply was noise (engine#1786). + if (!load_bf16_named(mn.share_router, share_router_, H) && + !load_bf16_named("shared_expert_gate.weight", share_router_, H)) + share_router_.clear(); return true; } if (sem_.gpt_oss) return true; // gpt_oss: the MoE reads its MXFP4 experts in place (gptoss_moe) @@ -2859,7 +2857,12 @@ bool Q4nxNpuForward::load_gdn_layer(int l) { if (!load_share(mn.share_gate, share_g_, expert_im_, H)) share_g_.clear(); if (!load_share(mn.share_up, share_u_, expert_im_, H)) { share_g_.clear(); share_u_.clear(); } if (!load_share(mn.share_down, share_d_, H, expert_im_)) { share_g_.clear(); share_u_.clear(); } - if (!load_bf16_named(mn.share_router, share_router_, H)) share_router_.clear(); + // The shared expert's sigmoid gate: `mlp.share_router.weight` in the FLM table, `shared_expert_gate.weight` + // (at the layer, beside the router) in FastFlowLM's Qwen3.6-35B-A3B-NPU2 -- missed, the shared expert was added + // ungated in every layer and the reply was noise (engine#1786). + if (!load_bf16_named(mn.share_router, share_router_, H) && + !load_bf16_named("shared_expert_gate.weight", share_router_, H)) + share_router_.clear(); return true; } if (!ffn_tail_cached()) { diff --git a/tests/batch_check_state_ci.py b/tests/batch_check_state_ci.py index 9b2d59d44..66a00ca52 100755 --- a/tests/batch_check_state_ci.py +++ b/tests/batch_check_state_ci.py @@ -15,7 +15,7 @@ # limitations under the License. """The batch-check state pin (tests/batch_check_state_test.cpp, engine#1905/#1946) on a container made here. -usage: tests/batch_check_state_ci.py [lfm2|lfm3|qwen2|qwen2vl|qwen2_untied|qwen3_5|hy_v3] (needs numpy) +usage: tests/batch_check_state_ci.py [lfm2|lfm3|qwen2|qwen2vl|qwen2_untied|qwen3_5|hy_v3|hy_v3_sgate] (needs numpy) batch_check_state_test needs a Q4NX container, and CI has none, so on its own it skips (77) there and a regression of #1909 -- the NPU_BATCH_CHECK oracle resetting the prefix it replays from -- lands green. This writes a two-layer qwen3 @@ -359,7 +359,7 @@ def rope(x): HY = dict(H=256, NH=2, NKV=1, HD=128, E=4, TOPK=2, EIM=512, DIM=768, SCALE=2.5) -def make_hyv3(d, np): +def make_hyv3(d, np, sgate=False): """A two-layer hy_v3 in FastFlowLM's naming; returns (weights for the reference, names to re-declare 3-D).""" rng = np.random.default_rng(1786) c = HY @@ -384,6 +384,10 @@ def make_hyv3(d, np): p + "mlp.down_exps_proj.weight": w(E * H, EIM), # stacked [N][K] per expert p + "mlp.share_gate_exps_proj.weight": w(EIM, H), p + "mlp.share_up_exps_proj.weight": w(EIM, H), p + "mlp.share_down_exps_proj.weight": w(H, EIM)}) + if sgate: + # engine#1786: FastFlowLM's Qwen3.6-35B-A3B-NPU2 gates its shared expert with sigmoid(h . g), the vector + # stored as `shared_expert_gate.weight` beside the router; read under any other name, it was dropped + W[p + "shared_expert_gate.weight"] = w(H, sc=0.5) os.makedirs(d) write_safetensors(os.path.join(d, "model.safetensors"), W) cfg = {"architectures": ["HYV3ForCausalLM"], "model_type": "hy_v3", "hidden_size": H, "num_hidden_layers": 2, @@ -477,8 +481,11 @@ def rope(x): for e, wt in zip(sel, wts): g, u, dn = G[e * EIM:(e + 1) * EIM], U[e * EIM:(e + 1) * EIM], D[e * H:(e + 1) * H] out[t] += wt * ((silu(h[t] @ g.T) * (h[t] @ u.T)) @ dn.T) - out += (silu(h @ W(p + "mlp.share_gate_exps_proj.weight").T) * (h @ W(p + "mlp.share_up_exps_proj.weight").T)) @ \ + sh = (silu(h @ W(p + "mlp.share_gate_exps_proj.weight").T) * (h @ W(p + "mlp.share_up_exps_proj.weight").T)) @ \ W(p + "mlp.share_down_exps_proj.weight").T + if p + "shared_expert_gate.weight" in table: + sh = sh * (1.0 / (1.0 + np.exp(-(h @ W(p + "shared_expert_gate.weight")))))[:, None] + out += sh x = x + out return rms(x, W("model.norm.weight"))[-1] @ W("lm_head.weight").T @@ -501,7 +508,7 @@ def tmp_root(): def main(): - if len(sys.argv) not in (2, 3) or (len(sys.argv) == 3 and sys.argv[2] not in ("lfm2", "lfm3", "qwen2", "qwen2vl", "qwen2_untied", "qwen3_5", "hy_v3")): + if len(sys.argv) not in (2, 3) or (len(sys.argv) == 3 and sys.argv[2] not in ("lfm2", "lfm3", "qwen2", "qwen2vl", "qwen2_untied", "qwen3_5", "hy_v3", "hy_v3_sgate")): print(__doc__.strip().splitlines()[2]) return 2 family = sys.argv[2] if len(sys.argv) == 3 else "qwen3" @@ -512,8 +519,8 @@ def main(): return 77 with tempfile.TemporaryDirectory(prefix="batch_check_state_ci.", dir=tmp_root()) as tmp: src, out = os.path.join(tmp, "hf"), os.path.join(tmp, "q4nx") - if family == "hy_v3": - Whyv = make_hyv3(src, np) + if family in ("hy_v3", "hy_v3_sgate"): + Whyv = make_hyv3(src, np, sgate=family == "hy_v3_sgate") shapes = {k: v.shape for k, v in Whyv.items()} else: shapes = make_model(src, np, family) @@ -521,7 +528,7 @@ def main(): if r.returncode != 0: print(f"FAIL packing the fabricated container: {r.stderr[-600:]}") return 1 - if family == "hy_v3": + if family in ("hy_v3", "hy_v3_sgate"): c = HY redeclare_3d(out, {"model.layer.1.mlp.gate_exps_proj.weight": (c["E"] * c["EIM"] // 32, c["H"] // 256), "model.layer.1.mlp.up_exps_proj.weight": (c["E"] * c["EIM"] // 32, c["H"] // 256), @@ -537,7 +544,7 @@ def main(): json.dump(c, f) env = dict(os.environ, ONEBIT_TEST_Q4NX=out) logits_out = os.path.join(tmp, "sequential.f32") - if family in ("lfm3", "qwen2", "qwen2vl", "qwen2_untied", "hy_v3"): + if family in ("lfm3", "qwen2", "qwen2vl", "qwen2_untied", "hy_v3", "hy_v3_sgate"): env["ONEBIT_TEST_LOGITS_OUT"] = logits_out env.pop("NPU_INFER_HOST_GEMM", None) # init_host is host-GEMM already; nothing else should steer it r = subprocess.run([sys.argv[1]], env=env, capture_output=True, text=True, timeout=110) @@ -550,9 +557,9 @@ def main(): # arm, not a reason to skip it. print("FAIL batch_check_state_test skipped on the fabricated container (init_host refused it)") return 1 - if family in ("lfm3", "qwen2", "qwen2vl", "qwen2_untied", "hy_v3") and r.returncode == 0: + if family in ("lfm3", "qwen2", "qwen2vl", "qwen2_untied", "hy_v3", "hy_v3_sgate") and r.returncode == 0: ours = np.fromfile(logits_out, dtype=np.float32)[:VOCAB] - ref = {"lfm3": lfm3_reference, "hy_v3": hyv3_reference}.get(family, qwen2_reference)(out, np, [1, 2, 3, 4, 5, 6], shapes) + ref = {"lfm3": lfm3_reference, "hy_v3": hyv3_reference, "hy_v3_sgate": hyv3_reference}.get(family, qwen2_reference)(out, np, [1, 2, 3, 4, 5, 6], shapes) corr = float(np.corrcoef(ours, ref)[0, 1]) ok = (corr >= LFM3_MIN_CORR and float(np.abs(ours - ref).max()) <= LFM3_MAX_DIFF * float(np.abs(ref).max()) and int(np.argmax(ours)) == int(np.argmax(ref))) diff --git a/tests/bf16_matrix_test.cpp b/tests/bf16_matrix_test.cpp new file mode 100644 index 000000000..93794cf4c --- /dev/null +++ b/tests/bf16_matrix_test.cpp @@ -0,0 +1,70 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// engine#1786: q4nx_bf16_matrix_into (npu/forward/include/bf16_matrix.h) -- an unquantised BF16 matrix in either of the +// two orientations FastFlowLM's packs use, into the forward's [rows][cols] grid. Each value is its own (row, col) +// written as a bf16 word, so a wrong orientation or a wrong pad shows as a wrong number at a named place. +#include "bf16_matrix.h" + +#include +#include + +static int failures = 0; +static void check(bool ok, const char* what) { + if (!ok) { std::printf("FAIL %s\n", what); failures++; } +} +// bf16 holds small integers exactly: encode value v = 100*r + c (r < 64, c < 100). +static uint16_t enc(float f) { uint32_t u; std::memcpy(&u, &f, 4); return (uint16_t)(u >> 16); } + +int main() { + // [out, in] = [3, 5] into a [3, 8] grid: copied, columns 5..7 zero. + { + std::vector w(3 * 5); + for (int r = 0; r < 3; r++) for (int c = 0; c < 5; c++) w[r * 5 + c] = enc(100.0f * r + c); + std::vector o(3 * 8, -1.0f); + check(q4nx_bf16_matrix_into(w.data(), w.size() * 2, 3, 5, 3, 8, o.data()) == 0, "[out, in] into a padded grid is read"); + bool ok = true; + for (int r = 0; r < 3; r++) for (int c = 0; c < 8; c++) ok &= o[r * 8 + c] == (c < 5 ? 100.0f * r + c : 0.0f); + check(ok, "[out, in]: each value at its own place, the pad columns zero"); + } + // [in, out] = [5, 3] asked as a [3, 5] grid: read transposed (Qwen3.6-35B-A3B's ssm_alpha [2048, 32]). + { + std::vector w(5 * 3); + for (int i = 0; i < 5; i++) for (int o_ = 0; o_ < 3; o_++) w[i * 3 + o_] = enc(100.0f * o_ + i); // value of (row o_, col i) + std::vector o(3 * 5, -1.0f); + check(q4nx_bf16_matrix_into(w.data(), w.size() * 2, 5, 3, 3, 5, o.data()) == 0, "[in, out] is read"); + bool ok = true; + for (int r = 0; r < 3; r++) for (int c = 0; c < 5; c++) ok &= o[r * 5 + c] == 100.0f * r + c; + check(ok, "[in, out]: read transposed, each value at its own place"); + } + // Square: the shape cannot say, so it is [out, in]. + { + std::vector w(4 * 4); + for (int r = 0; r < 4; r++) for (int c = 0; c < 4; c++) w[r * 4 + c] = enc(100.0f * r + c); + std::vector o(16); + check(q4nx_bf16_matrix_into(w.data(), w.size() * 2, 4, 4, 4, 4, o.data()) == 0 && o[1] == 1.0f && o[4] == 100.0f, + "a square matrix is read as [out, in]"); + } + // Refusals: wider than the grid, and short of bytes. + { + std::vector w(6 * 6, 0); + std::vector o(16); + check(q4nx_bf16_matrix_into(w.data(), w.size() * 2, 6, 6, 4, 4, o.data()) == -1, "a matrix wider than the grid is refused"); + check(q4nx_bf16_matrix_into(w.data(), 10, 3, 5, 3, 8, o.data()) == -1, "a matrix short of bytes is refused"); + } + if (failures) { std::printf("%d failure(s)\n", failures); return 1; } + std::printf("ok: both orientations, the padded grid and the refusals\n"); + return 0; +}