Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1318,6 +1318,12 @@ add_executable(ops_dump_test tests/ops_dump_test.cpp)
target_include_directories(ops_dump_test PRIVATE npu/forward/include)
add_test(NAME ops_dump COMMAND ops_dump_test)
add_dependencies(onebit ops_dump_test)
# When the dense-tail cache is on by default (npu/forward/include/ffn_cache_policy.h): the size rule and the two
# models it was measured on, one on each side of the old floor. Header-only, no device.
add_executable(ffn_cache_policy_test tests/ffn_cache_policy_test.cpp)
target_include_directories(ffn_cache_policy_test PRIVATE npu/forward/include)
add_test(NAME ffn_cache_policy COMMAND ffn_cache_policy_test)
add_dependencies(onebit ffn_cache_policy_test)

# The device weight layout contract (q4nx_weight_layout): pure index math, no device
# and no runtime. CI builds an explicit target list, so the check compiles itself with
Expand Down
7 changes: 5 additions & 2 deletions docs/npu.md
Original file line number Diff line number Diff line change
Expand Up @@ -1845,12 +1845,15 @@ re-dequantised each layer's 17408-wide dense FFN tail on every token even after
packed on the device. `ffn_tail_cached()` (`npu/forward/src/q4nx_forward.cpp`) closes that: once a
layer's packed gate/up/down BOs are resident, the f32 `wg_`/`wu_`/`wd_` buffers are no longer read,
so the reload is skipped. It is on by default for a GDN model on the NPU route, when the packed
weight cache is on, the run is not `NPU_INFER_HOST_GEMM`, and the tail is at least 256 MB per layer
(the size guard; below that the reload is too small to be worth a behaviour change). `NPU_FFN_CACHE=0`
weight cache is on, the run is not `NPU_INFER_HOST_GEMM`, and the tail is at least 128 MB per layer
(the size guard, set just under the smallest tail measured; `npu/forward/include/ffn_cache_policy.h`). `NPU_FFN_CACHE=0`
or `=1` forces it either way, and the resolved state is logged once as
`[ffn] dense-tail cache on|off (default|NPU_FFN_CACHE; lin_kd=… IM=… H=…)`. On Qwen3.8-27B-NPU2 an
interleaved 5-round A/B (`~/lb/587-cache-20261030`) measured an 8-token request at 104.5 s against
117.4 s (**−11%**), CPU per token 22.9 s against 63.3 s (**−64%**), same text in every arm.
On FastFlowLM's Qwen3.5-2B-NPU2 (IM 6144, H 2048, a 151 MB tail that the earlier 256 MB floor left out) an A B B A
run in one ticket measured decode at 362–367 ms per token with the cache on (both arms) against 485–520 ms with
it off in the quieter of the two off arms and 478–707 ms in the other, same text in all twelve runs (engine#1785).

### The prefix cache's snapshot: two kinds of plane

Expand Down
48 changes: 48 additions & 0 deletions npu/forward/include/ffn_cache_policy.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// ffn_cache_policy.h -- when the dense-tail cache is on by default (engine#587, engine#1785).
//
// On a hybrid (GDN) model the forward used to re-dequantise each layer's dense FFN tail on every token, even after the
// packed weights were resident on the device. The dense-tail cache skips that reload. Whether it is on when
// NPU_FFN_CACHE says nothing is a size rule, and the floor of that rule is set by what has been measured, not guessed:
//
// model tail per layer decode, cache off -> on text
// Qwen3.8-27B-NPU2 (IM 17408, H 5120) 1.07 GB 117.4 -> 104.5 s per 8-token request identical (~/lb/587-cache-20261030)
// Qwen3.5-2B-NPU2 (IM 6144, H 2048) 151 MB 485-520 -> 362-367 ms per token identical in 12 of 12 runs
//
// The second row is engine#1785's measurement (A B B A, one ticket; the off figure is the quieter off arm, the other
// was 478-707): the reload was about a fifth of every decode token on a model the old 256 MB floor left out. The floor is therefore just under the smallest tail measured.
// Below it nothing has been measured, so nothing is switched on unasked; NPU_FFN_CACHE=1 still forces it.

#pragma once

namespace onebit::npu {

// Bytes of f32 the three dense-tail matrices of one layer occupy (gate, up, down: IM x H each).
inline double ffn_tail_bytes(int intermediate, int hidden) {
return 3.0 * double(intermediate) * double(hidden) * 4.0;
}

// The smallest tail the cache has been measured to pay for (see the table above).
inline constexpr double kFfnCacheFloorBytes = 128.0 * 1024.0 * 1024.0;

// The size half of the default: a tail at or above the floor. The caller adds the conditions that are about the
// run, not the model (a GDN model, the packed-weight cache on, not the host-GEMM mode).
inline bool ffn_cache_worth_it(int intermediate, int hidden) {
return intermediate > 0 && hidden > 0 && ffn_tail_bytes(intermediate, hidden) >= kFfnCacheFloorBytes;
}

} // namespace onebit::npu
11 changes: 4 additions & 7 deletions npu/forward/src/q4nx_forward.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
// element-wise.
#include "q4nx_forward.h"
#include "batch_width.h"
#include "ffn_cache_policy.h"
#include "q4nx_state_planes.h" // the plane-copy half of a snapshot, device-free and unit-tested
#include "model/op_dispatch.h" // the optional-op fallback seam (engine#803 A3)
#include <algorithm>
Expand Down Expand Up @@ -1821,15 +1822,11 @@ void Q4nxNpuForward::layer_attn_dims(int l, int& hd, int& nkv) const {
// it -- a GDN model (lin_kd_ > 0, the one case step()'s `dense_weights_cached`
// can never cover, since that guard stops at lin_kd_ == 0), packed weights
// actually being cached, and a dense tail big enough for the per-token reload to
// be material. Qwen3.8-27B-NPU2 (IM 17408 x H 5120) clears it with 1.07 GB per
// layer, and measured 117.4 -> 104.5 s wall and 63.3 -> 22.9 s CPU per token on
// an 8-token request with identical text (~/lb/587-cache-20261030).
// The floor is a size guard, not a second measurement: below it the reload is
// small enough that the behaviour change is not worth taking unasked.
// be material. The size rule and the two measurements it rests on (Qwen3.8-27B
// and, since engine#1785, Qwen3.5-2B) are in ffn_cache_policy.h.
bool Q4nxNpuForward::ffn_cache_default() const {
if (host_gemm_ || !xd_->weight_cache_on() || lin_kd_ <= 0) return false;
const double tail_bytes = 3.0 * (double)c_.IM * (double)c_.H * 4.0; // wg_+wu_+wd_
return tail_bytes >= 256.0 * 1024.0 * 1024.0;
return onebit::npu::ffn_cache_worth_it(c_.IM, c_.H);
}
// ---- layer-loop host attribution ------------------------------------------
// The `host=` residual of the [ph] line is layer time that is NOT any of the
Expand Down
53 changes: 53 additions & 0 deletions tests/ffn_cache_policy_test.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// ffn_cache_policy_test: the size rule for the dense-tail cache's default (npu/forward/include/ffn_cache_policy.h).
// The two measured models are the fixtures: both must be on, and the one engine#1785 measured is the one the
// earlier 256 MB floor left off.

#include "ffn_cache_policy.h"

#include <cstdio>
#include <string>

using onebit::npu::ffn_cache_worth_it;
using onebit::npu::ffn_tail_bytes;
using onebit::npu::kFfnCacheFloorBytes;

static int failures = 0;
static void check(bool ok, const std::string& what) {
if (!ok) {
std::printf("FAIL %s\n", what.c_str());
++failures;
}
}

int main() {
const double MB = 1024.0 * 1024.0;
check(ffn_tail_bytes(6144, 2048) == 3.0 * 6144 * 2048 * 4, "the tail is three IM x H f32 matrices");
check(ffn_tail_bytes(17408, 5120) > 1000.0 * MB && ffn_cache_worth_it(17408, 5120), "Qwen3.8-27B (1.07 GB tail): on, as before");
check(ffn_tail_bytes(6144, 2048) == 144.0 * MB, "Qwen3.5-2B's tail is 144 MiB (151 MB)");
check(ffn_cache_worth_it(6144, 2048), "Qwen3.5-2B: on -- measured 485-520 -> 362-367 ms per token, same text (engine#1785)");
check(ffn_tail_bytes(6144, 2048) < 256.0 * MB, "and it is the model the earlier 256 MB floor left off");
// Below the smallest measured tail nothing is switched on unasked.
check(!ffn_cache_worth_it(3584, 1024), "a 0.8B-class tail (42 MiB) stays off by default: not measured");
check(!ffn_cache_worth_it(4096, 2048), "a 96 MiB tail stays off by default: not measured");
check(kFfnCacheFloorBytes == 128.0 * MB, "the floor is 128 MiB");
check(!ffn_cache_worth_it(5460, 2048) && ffn_cache_worth_it(5462, 2048), "the rule turns exactly at the floor (5461.33 x 2048)");
check(!ffn_cache_worth_it(0, 2048) && !ffn_cache_worth_it(6144, 0) && !ffn_cache_worth_it(-6144, -2048),
"a geometry that is not one is never on");
if (failures == 0) std::printf("ffn_cache_policy: ok\n");
return failures == 0 ? 0 : 1;
}
Loading