From 7eb633ca602a7b98f8c1cd7fcef7b26021bc666c Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Wed, 11 Nov 2026 05:53:03 -0400 Subject: [PATCH] npu: LFM2's gated short convolution as host arithmetic, with known answers (Refs #1786) npu/forward/include/lfm2_shortconv.h: the block between an LFM2 conv layer's two projections (B*X, a causal depthwise convolution over the last L-1 values, gated by C), with its per-layer state, defined once for decode, batched prefill and state snapshots. Not wired into the forward yet; this is the reference the wiring will be checked against. Co-Authored-By: Claude Fable 5.2 --- CMakeLists.txt | 6 ++ npu/forward/include/lfm2_shortconv.h | 94 +++++++++++++++++ tests/lfm2_shortconv_test.cpp | 152 +++++++++++++++++++++++++++ 3 files changed, 252 insertions(+) create mode 100644 npu/forward/include/lfm2_shortconv.h create mode 100644 tests/lfm2_shortconv_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 2746671bd..2ee02342b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1374,6 +1374,12 @@ add_executable(ffn_cache_policy_test tests/ffn_cache_policy_test.cpp) target_include_directories(ffn_cache_policy_test PRIVATE npu/forward/include) add_test(NAME ffn_cache_policy COMMAND ffn_cache_policy_test) add_dependencies(onebit ffn_cache_policy_test) +# LFM2's gated short convolution as host arithmetic (npu/forward/include/lfm2_shortconv.h, engine#1786): the batched +# form must equal the token-by-token form bit for bit. Header-only, no model, no device. +add_executable(lfm2_shortconv_test tests/lfm2_shortconv_test.cpp) +target_include_directories(lfm2_shortconv_test PRIVATE npu/forward/include) +add_test(NAME lfm2_shortconv COMMAND lfm2_shortconv_test) +add_dependencies(onebit lfm2_shortconv_test) # The device weight layout contract (q4nx_weight_layout): pure index math, no device # and no runtime. CI builds an explicit target list, so the check compiles itself with diff --git a/npu/forward/include/lfm2_shortconv.h b/npu/forward/include/lfm2_shortconv.h new file mode 100644 index 000000000..88d2466b2 --- /dev/null +++ b/npu/forward/include/lfm2_shortconv.h @@ -0,0 +1,94 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// lfm2_shortconv.h -- the gated short convolution of an LFM2 layer, as host arithmetic (engine#1786). +// +// LFM2 is a hybrid: ten of LFM2-1.2B's sixteen layers have no attention at all. In place of it they run +// +// bcx = in_proj(norm(x)) H -> 3H, the three thirds are B, C and X +// bx = B * X elementwise +// conv[h] = sum_j taps[h][j] * seq[t + j][h] depthwise, causal, L taps: seq is the last L-1 values of bx +// followed by this token's, so taps[h][L-1] meets the present +// y = out_proj(C * conv) H -> H +// +// and then the same residual and feed-forward as an attention layer. The only state a layer carries from one token +// to the next is the last L-1 values of bx, per channel: (L-1) * H floats, where an attention layer carries a KV row. +// +// The two projections are ordinary GEMMs and belong to the device. What is here is everything between them, written +// once so that the three places that will need it (token-by-token decode, a batched prefill, and a snapshot of the +// state for the prefix cache) share one definition, and tested without a model: the batched form must equal the +// token-by-token form bit for bit, because a prefill that disagrees with decode is wrong at every later position. +// +// The definition follows the graph our own llama.cpp fork builds for this family (src/models/lfm2.cpp, +// build_shortconv_block), not any vendor's code. Layout of `taps`: [H][L] row-major, the container's +// `shortconv.conv.weight` after its bf16 decode. + +#pragma once + +#include +#include +#include + +namespace onebit::npu { + +// The conv state of one layer: the last L-1 values of bx, oldest first, [L-1][H] row-major. All zeros at position 0. +struct ShortConvState { + int hidden = 0; + int taps = 0; // L, the kernel length (config.json conv_L_cache) + std::vector history; // (L-1) * hidden + + void reset(int hidden_size, int kernel_len) { + hidden = hidden_size; + taps = kernel_len; + history.assign(size_t(kernel_len > 1 ? kernel_len - 1 : 0) * size_t(hidden_size), 0.0f); + } + bool ready() const { return hidden > 0 && taps > 0 && history.size() == size_t(taps - 1) * size_t(hidden); } +}; + +// One token. `bcx` is in_proj's output, 3*H values [B | C | X]; `taps` is [H][L]; `y` receives H values, the input of +// out_proj. The state is advanced by one position. Returns false, touching nothing, when the state is not sized. +inline bool shortconv_step(ShortConvState& st, const float* bcx, const float* taps, float* y) { + if (!st.ready() || bcx == nullptr || taps == nullptr || y == nullptr) return false; + const int H = st.hidden, L = st.taps; + const float* B = bcx; + const float* C = bcx + H; + const float* X = bcx + 2 * H; + for (int h = 0; h < H; ++h) { + const float bx = B[h] * X[h]; + const float* k = taps + size_t(h) * size_t(L); + float acc = 0.0f; + for (int j = 0; j < L - 1; ++j) acc += k[j] * st.history[size_t(j) * size_t(H) + size_t(h)]; + acc += k[L - 1] * bx; + y[h] = C[h] * acc; + } + // slide the window: drop the oldest row, append this token's bx + if (L > 1) { + if (L > 2) std::memmove(st.history.data(), st.history.data() + H, size_t(L - 2) * size_t(H) * sizeof(float)); + float* last = st.history.data() + size_t(L - 2) * size_t(H); + for (int h = 0; h < H; ++h) last[h] = B[h] * X[h]; + } + return true; +} + +// M tokens in order: `bcx` is [M][3H] row-major, `y` is [M][H]. Defined as M calls of shortconv_step(), in that +// order, so there is nothing for it to disagree with. +inline bool shortconv_rows(ShortConvState& st, const float* bcx, int rows, const float* taps, float* y) { + if (rows < 0) return false; + for (int r = 0; r < rows; ++r) + if (!shortconv_step(st, bcx + size_t(r) * 3 * size_t(st.hidden), taps, y + size_t(r) * size_t(st.hidden))) return false; + return true; +} + +} // namespace onebit::npu diff --git a/tests/lfm2_shortconv_test.cpp b/tests/lfm2_shortconv_test.cpp new file mode 100644 index 000000000..fd6e1c28f --- /dev/null +++ b/tests/lfm2_shortconv_test.cpp @@ -0,0 +1,152 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// lfm2_shortconv_test: the gated short convolution (npu/forward/include/lfm2_shortconv.h) against the definition +// written out directly, with no model: a causal depthwise convolution over the whole sequence, zero history before +// position 0. Token-by-token, in one batch, and in uneven batches must all give the same bits. + +#include "lfm2_shortconv.h" + +#include +#include +#include +#include +#include + +using onebit::npu::ShortConvState; +using onebit::npu::shortconv_rows; +using onebit::npu::shortconv_step; + +static int failures = 0; +static void check(bool ok, const std::string& what) { + if (!ok) { + std::printf("FAIL %s\n", what.c_str()); + ++failures; + } +} + +// Deterministic inputs with a little of everything: both signs, zeros, different scales per channel. +static float bcx_of(int t, int i, int H) { + const int third = i / H, h = i % H; + float v = std::sin(0.37f * float(t + 1) * float(h + 1) + 1.3f * float(third)); + if ((t + h + third) % 7 == 0) v = 0.0f; + return v * (1.0f + 0.25f * float(h % 4)); +} +static float tap_of(int h, int j) { return 0.5f * std::cos(0.9f * float(h + 1) + 1.7f * float(j)) + (j == 0 ? 0.125f : 0.0f); } + +// The definition, on the whole sequence at once: y[t][h] = C[t][h] * sum_j taps[h][j] * bx[t - (L-1) + j][h], bx = 0 before 0. +// The sum runs in the same order as the code under test, so equality can be exact. +static std::vector reference(int T, int H, int L, const std::vector& bcx, const std::vector& taps) { + std::vector y(size_t(T) * size_t(H)); + for (int t = 0; t < T; ++t) + for (int h = 0; h < H; ++h) { + float acc = 0.0f; + for (int j = 0; j < L; ++j) { + const int src = t - (L - 1) + j; + const float bx = src < 0 ? 0.0f : bcx[size_t(src) * 3 * H + h] * bcx[size_t(src) * 3 * H + 2 * H + h]; + acc += taps[size_t(h) * L + j] * bx; + } + y[size_t(t) * H + h] = bcx[size_t(t) * 3 * H + H + h] * acc; + } + return y; +} + +int main() { + for (int L : {1, 2, 3, 4}) { + const int H = 37, T = 23; + std::vector bcx(size_t(T) * 3 * H), taps(size_t(H) * L); + for (int t = 0; t < T; ++t) + for (int i = 0; i < 3 * H; ++i) bcx[size_t(t) * 3 * H + i] = bcx_of(t, i, H); + for (int h = 0; h < H; ++h) + for (int j = 0; j < L; ++j) taps[size_t(h) * L + j] = tap_of(h, j); + const std::vector want = reference(T, H, L, bcx, taps); + const std::string tag = " (L=" + std::to_string(L) + ")"; + + ShortConvState one; + one.reset(H, L); + check(one.ready() && one.history.size() == size_t(L - 1) * H, "the state is L-1 rows of H" + tag); + std::vector got(size_t(T) * H); + bool ok = true; + for (int t = 0; t < T; ++t) ok = ok && shortconv_step(one, &bcx[size_t(t) * 3 * H], taps.data(), &got[size_t(t) * H]); + check(ok && std::memcmp(got.data(), want.data(), want.size() * sizeof(float)) == 0, + "token by token it is the causal depthwise convolution of B*X, gated by C, bit for bit" + tag); + + ShortConvState all; + all.reset(H, L); + std::vector batched(size_t(T) * H); + check(shortconv_rows(all, bcx.data(), T, taps.data(), batched.data()) && + std::memcmp(batched.data(), want.data(), want.size() * sizeof(float)) == 0, + "one batch of every token gives the same bits" + tag); + check(all.history == one.history, "and leaves the same state behind" + tag); + + ShortConvState mixed; // a prefill in uneven chunks, then decode: 5 + 1 + 9 + the rest one at a time + mixed.reset(H, L); + std::vector m(size_t(T) * H); + int at = 0; + bool mok = true; + for (int chunk : {5, 1, 9}) { + mok = mok && shortconv_rows(mixed, &bcx[size_t(at) * 3 * H], chunk, taps.data(), &m[size_t(at) * H]); + at += chunk; + } + for (; at < T; ++at) mok = mok && shortconv_step(mixed, &bcx[size_t(at) * 3 * H], taps.data(), &m[size_t(at) * H]); + check(mok && std::memcmp(m.data(), want.data(), want.size() * sizeof(float)) == 0 && mixed.history == one.history, + "uneven prefill chunks followed by decode steps give the same bits and state" + tag); + + // a snapshot of the state is all that a resume needs: copy it at position 11, replay from there + ShortConvState first; + first.reset(H, L); + std::vector scratch(size_t(T) * H); + shortconv_rows(first, bcx.data(), 11, taps.data(), scratch.data()); + ShortConvState resumed = first; + std::vector tail(size_t(T - 11) * H); + check(shortconv_rows(resumed, &bcx[size_t(11) * 3 * H], T - 11, taps.data(), tail.data()) && + std::memcmp(tail.data(), &want[size_t(11) * H], tail.size() * sizeof(float)) == 0, + "a copy of the state at position 11 resumes to the same bits" + tag); + + // the history really is the last L-1 values of B*X, oldest first + bool hist = true; + for (int j = 0; j < L - 1 && hist; ++j) + for (int h = 0; h < H; ++h) { + const int src = T - (L - 1) + j; + const float bx = bcx[size_t(src) * 3 * H + h] * bcx[size_t(src) * 3 * H + 2 * H + h]; + if (one.history[size_t(j) * H + h] != bx) { hist = false; break; } + } + check(hist, "the state is the last L-1 values of B*X, oldest first" + tag); + } + { // the present token meets the LAST tap, and the gate C multiplies the result, not the input + ShortConvState st; + st.reset(1, 3); + const float taps[3] = {100.0f, 10.0f, 1.0f}; + float y = 0.0f; + const float t0[3] = {2.0f, 5.0f, 3.0f}; // B, C, X -> bx 6 + check(shortconv_step(st, t0, taps, &y) && y == 5.0f * (1.0f * 6.0f), "at position 0 only the last tap sees anything"); + const float t1[3] = {1.0f, 2.0f, 4.0f}; // bx 4 + check(shortconv_step(st, t1, taps, &y) && y == 2.0f * (10.0f * 6.0f + 1.0f * 4.0f), "the token before meets the middle tap"); + const float t2[3] = {1.0f, 1.0f, 1.0f}; // bx 1 + check(shortconv_step(st, t2, taps, &y) && y == 1.0f * (100.0f * 6.0f + 10.0f * 4.0f + 1.0f * 1.0f), + "and the one before that the first tap"); + } + { // refusals: an unsized state, null buffers, a negative row count + ShortConvState st; + float b[3] = {1, 1, 1}, k[1] = {1}, y = 7.0f; + check(!shortconv_step(st, b, k, &y) && y == 7.0f, "an unsized state is refused and nothing is written"); + st.reset(1, 1); + check(!shortconv_step(st, nullptr, k, &y) && !shortconv_step(st, b, nullptr, &y) && !shortconv_step(st, b, k, nullptr), + "null buffers are refused"); + check(!shortconv_rows(st, b, -1, k, &y) && shortconv_rows(st, b, 0, k, &y), "a negative row count is refused; zero rows is a no-op"); + } + if (failures == 0) std::printf("lfm2_shortconv: ok\n"); + return failures == 0 ? 0 : 1; +}