Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1374,6 +1374,12 @@ add_executable(ffn_cache_policy_test tests/ffn_cache_policy_test.cpp)
target_include_directories(ffn_cache_policy_test PRIVATE npu/forward/include)
add_test(NAME ffn_cache_policy COMMAND ffn_cache_policy_test)
add_dependencies(onebit ffn_cache_policy_test)
# LFM2's gated short convolution as host arithmetic (npu/forward/include/lfm2_shortconv.h, engine#1786): the batched
# form must equal the token-by-token form bit for bit. Header-only, no model, no device.
add_executable(lfm2_shortconv_test tests/lfm2_shortconv_test.cpp)
target_include_directories(lfm2_shortconv_test PRIVATE npu/forward/include)
add_test(NAME lfm2_shortconv COMMAND lfm2_shortconv_test)
add_dependencies(onebit lfm2_shortconv_test)

# The device weight layout contract (q4nx_weight_layout): pure index math, no device
# and no runtime. CI builds an explicit target list, so the check compiles itself with
Expand Down
94 changes: 94 additions & 0 deletions npu/forward/include/lfm2_shortconv.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// lfm2_shortconv.h -- the gated short convolution of an LFM2 layer, as host arithmetic (engine#1786).
//
// LFM2 is a hybrid: ten of LFM2-1.2B's sixteen layers have no attention at all. In place of it they run
//
// bcx = in_proj(norm(x)) H -> 3H, the three thirds are B, C and X
// bx = B * X elementwise
// conv[h] = sum_j taps[h][j] * seq[t + j][h] depthwise, causal, L taps: seq is the last L-1 values of bx
// followed by this token's, so taps[h][L-1] meets the present
// y = out_proj(C * conv) H -> H
//
// and then the same residual and feed-forward as an attention layer. The only state a layer carries from one token
// to the next is the last L-1 values of bx, per channel: (L-1) * H floats, where an attention layer carries a KV row.
//
// The two projections are ordinary GEMMs and belong to the device. What is here is everything between them, written
// once so that the three places that will need it (token-by-token decode, a batched prefill, and a snapshot of the
// state for the prefix cache) share one definition, and tested without a model: the batched form must equal the
// token-by-token form bit for bit, because a prefill that disagrees with decode is wrong at every later position.
//
// The definition follows the graph our own llama.cpp fork builds for this family (src/models/lfm2.cpp,
// build_shortconv_block), not any vendor's code. Layout of `taps`: [H][L] row-major, the container's
// `shortconv.conv.weight` after its bf16 decode.

#pragma once

#include <cstddef>
#include <cstring>
#include <vector>

namespace onebit::npu {

// The conv state of one layer: the last L-1 values of bx, oldest first, [L-1][H] row-major. All zeros at position 0.
struct ShortConvState {
int hidden = 0;
int taps = 0; // L, the kernel length (config.json conv_L_cache)
std::vector<float> history; // (L-1) * hidden

void reset(int hidden_size, int kernel_len) {
hidden = hidden_size;
taps = kernel_len;
history.assign(size_t(kernel_len > 1 ? kernel_len - 1 : 0) * size_t(hidden_size), 0.0f);
}
bool ready() const { return hidden > 0 && taps > 0 && history.size() == size_t(taps - 1) * size_t(hidden); }
};

// One token. `bcx` is in_proj's output, 3*H values [B | C | X]; `taps` is [H][L]; `y` receives H values, the input of
// out_proj. The state is advanced by one position. Returns false, touching nothing, when the state is not sized.
inline bool shortconv_step(ShortConvState& st, const float* bcx, const float* taps, float* y) {
if (!st.ready() || bcx == nullptr || taps == nullptr || y == nullptr) return false;
const int H = st.hidden, L = st.taps;
const float* B = bcx;
const float* C = bcx + H;
const float* X = bcx + 2 * H;
for (int h = 0; h < H; ++h) {
const float bx = B[h] * X[h];
const float* k = taps + size_t(h) * size_t(L);
float acc = 0.0f;
for (int j = 0; j < L - 1; ++j) acc += k[j] * st.history[size_t(j) * size_t(H) + size_t(h)];
acc += k[L - 1] * bx;
y[h] = C[h] * acc;
}
// slide the window: drop the oldest row, append this token's bx
if (L > 1) {
if (L > 2) std::memmove(st.history.data(), st.history.data() + H, size_t(L - 2) * size_t(H) * sizeof(float));
float* last = st.history.data() + size_t(L - 2) * size_t(H);
for (int h = 0; h < H; ++h) last[h] = B[h] * X[h];
}
return true;
}

// M tokens in order: `bcx` is [M][3H] row-major, `y` is [M][H]. Defined as M calls of shortconv_step(), in that
// order, so there is nothing for it to disagree with.
inline bool shortconv_rows(ShortConvState& st, const float* bcx, int rows, const float* taps, float* y) {
if (rows < 0) return false;
for (int r = 0; r < rows; ++r)
if (!shortconv_step(st, bcx + size_t(r) * 3 * size_t(st.hidden), taps, y + size_t(r) * size_t(st.hidden))) return false;
return true;
}

} // namespace onebit::npu
152 changes: 152 additions & 0 deletions tests/lfm2_shortconv_test.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// lfm2_shortconv_test: the gated short convolution (npu/forward/include/lfm2_shortconv.h) against the definition
// written out directly, with no model: a causal depthwise convolution over the whole sequence, zero history before
// position 0. Token-by-token, in one batch, and in uneven batches must all give the same bits.

#include "lfm2_shortconv.h"

#include <cmath>
#include <cstdio>
#include <cstring>
#include <string>
#include <vector>

using onebit::npu::ShortConvState;
using onebit::npu::shortconv_rows;
using onebit::npu::shortconv_step;

static int failures = 0;
static void check(bool ok, const std::string& what) {
if (!ok) {
std::printf("FAIL %s\n", what.c_str());
++failures;
}
}

// Deterministic inputs with a little of everything: both signs, zeros, different scales per channel.
static float bcx_of(int t, int i, int H) {
const int third = i / H, h = i % H;
float v = std::sin(0.37f * float(t + 1) * float(h + 1) + 1.3f * float(third));
if ((t + h + third) % 7 == 0) v = 0.0f;
return v * (1.0f + 0.25f * float(h % 4));
}
static float tap_of(int h, int j) { return 0.5f * std::cos(0.9f * float(h + 1) + 1.7f * float(j)) + (j == 0 ? 0.125f : 0.0f); }

// The definition, on the whole sequence at once: y[t][h] = C[t][h] * sum_j taps[h][j] * bx[t - (L-1) + j][h], bx = 0 before 0.
// The sum runs in the same order as the code under test, so equality can be exact.
static std::vector<float> reference(int T, int H, int L, const std::vector<float>& bcx, const std::vector<float>& taps) {
std::vector<float> y(size_t(T) * size_t(H));
for (int t = 0; t < T; ++t)
for (int h = 0; h < H; ++h) {
float acc = 0.0f;
for (int j = 0; j < L; ++j) {
const int src = t - (L - 1) + j;
const float bx = src < 0 ? 0.0f : bcx[size_t(src) * 3 * H + h] * bcx[size_t(src) * 3 * H + 2 * H + h];
acc += taps[size_t(h) * L + j] * bx;
}
y[size_t(t) * H + h] = bcx[size_t(t) * 3 * H + H + h] * acc;
}
return y;
}

int main() {
for (int L : {1, 2, 3, 4}) {
const int H = 37, T = 23;
std::vector<float> bcx(size_t(T) * 3 * H), taps(size_t(H) * L);
for (int t = 0; t < T; ++t)
for (int i = 0; i < 3 * H; ++i) bcx[size_t(t) * 3 * H + i] = bcx_of(t, i, H);
for (int h = 0; h < H; ++h)
for (int j = 0; j < L; ++j) taps[size_t(h) * L + j] = tap_of(h, j);
const std::vector<float> want = reference(T, H, L, bcx, taps);
const std::string tag = " (L=" + std::to_string(L) + ")";

ShortConvState one;
one.reset(H, L);
check(one.ready() && one.history.size() == size_t(L - 1) * H, "the state is L-1 rows of H" + tag);
std::vector<float> got(size_t(T) * H);
bool ok = true;
for (int t = 0; t < T; ++t) ok = ok && shortconv_step(one, &bcx[size_t(t) * 3 * H], taps.data(), &got[size_t(t) * H]);
check(ok && std::memcmp(got.data(), want.data(), want.size() * sizeof(float)) == 0,
"token by token it is the causal depthwise convolution of B*X, gated by C, bit for bit" + tag);

ShortConvState all;
all.reset(H, L);
std::vector<float> batched(size_t(T) * H);
check(shortconv_rows(all, bcx.data(), T, taps.data(), batched.data()) &&
std::memcmp(batched.data(), want.data(), want.size() * sizeof(float)) == 0,
"one batch of every token gives the same bits" + tag);
check(all.history == one.history, "and leaves the same state behind" + tag);

ShortConvState mixed; // a prefill in uneven chunks, then decode: 5 + 1 + 9 + the rest one at a time
mixed.reset(H, L);
std::vector<float> m(size_t(T) * H);
int at = 0;
bool mok = true;
for (int chunk : {5, 1, 9}) {
mok = mok && shortconv_rows(mixed, &bcx[size_t(at) * 3 * H], chunk, taps.data(), &m[size_t(at) * H]);
at += chunk;
}
for (; at < T; ++at) mok = mok && shortconv_step(mixed, &bcx[size_t(at) * 3 * H], taps.data(), &m[size_t(at) * H]);
check(mok && std::memcmp(m.data(), want.data(), want.size() * sizeof(float)) == 0 && mixed.history == one.history,
"uneven prefill chunks followed by decode steps give the same bits and state" + tag);

// a snapshot of the state is all that a resume needs: copy it at position 11, replay from there
ShortConvState first;
first.reset(H, L);
std::vector<float> scratch(size_t(T) * H);
shortconv_rows(first, bcx.data(), 11, taps.data(), scratch.data());
ShortConvState resumed = first;
std::vector<float> tail(size_t(T - 11) * H);
check(shortconv_rows(resumed, &bcx[size_t(11) * 3 * H], T - 11, taps.data(), tail.data()) &&
std::memcmp(tail.data(), &want[size_t(11) * H], tail.size() * sizeof(float)) == 0,
"a copy of the state at position 11 resumes to the same bits" + tag);

// the history really is the last L-1 values of B*X, oldest first
bool hist = true;
for (int j = 0; j < L - 1 && hist; ++j)
for (int h = 0; h < H; ++h) {
const int src = T - (L - 1) + j;
const float bx = bcx[size_t(src) * 3 * H + h] * bcx[size_t(src) * 3 * H + 2 * H + h];
if (one.history[size_t(j) * H + h] != bx) { hist = false; break; }
}
check(hist, "the state is the last L-1 values of B*X, oldest first" + tag);
}
{ // the present token meets the LAST tap, and the gate C multiplies the result, not the input
ShortConvState st;
st.reset(1, 3);
const float taps[3] = {100.0f, 10.0f, 1.0f};
float y = 0.0f;
const float t0[3] = {2.0f, 5.0f, 3.0f}; // B, C, X -> bx 6
check(shortconv_step(st, t0, taps, &y) && y == 5.0f * (1.0f * 6.0f), "at position 0 only the last tap sees anything");
const float t1[3] = {1.0f, 2.0f, 4.0f}; // bx 4
check(shortconv_step(st, t1, taps, &y) && y == 2.0f * (10.0f * 6.0f + 1.0f * 4.0f), "the token before meets the middle tap");
const float t2[3] = {1.0f, 1.0f, 1.0f}; // bx 1
check(shortconv_step(st, t2, taps, &y) && y == 1.0f * (100.0f * 6.0f + 10.0f * 4.0f + 1.0f * 1.0f),
"and the one before that the first tap");
}
{ // refusals: an unsized state, null buffers, a negative row count
ShortConvState st;
float b[3] = {1, 1, 1}, k[1] = {1}, y = 7.0f;
check(!shortconv_step(st, b, k, &y) && y == 7.0f, "an unsized state is refused and nothing is written");
st.reset(1, 1);
check(!shortconv_step(st, nullptr, k, &y) && !shortconv_step(st, b, nullptr, &y) && !shortconv_step(st, b, k, nullptr),
"null buffers are refused");
check(!shortconv_rows(st, b, -1, k, &y) && shortconv_rows(st, b, 0, k, &y), "a negative row count is refused; zero rows is a no-op");
}
if (failures == 0) std::printf("lfm2_shortconv: ok\n");
return failures == 0 ? 0 : 1;
}
Loading