From 229a8821de9562e4febe2ba0e60b3111e646a347 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 20:06:57 -0400 Subject: [PATCH] npu: the dense-tail cache is on by default from a 128 MiB tail, the size it was measured on (Refs #1785) FastFlowLM's Qwen3.5-2B (IM 6144, H 2048, 144 MiB tail) sat under the 256 MB floor, so every decode token re-dequantised the dense FFN tail. A B B A in one npu-hold ticket: 362-367 ms per token with the cache on, 485-520 ms with it off, identical text in 12 of 12 runs. The size rule moves to npu/forward/include/ffn_cache_policy.h with a test. Co-Authored-By: Claude Fable 5.2 --- CMakeLists.txt | 6 +++ docs/npu.md | 7 +++- npu/forward/include/ffn_cache_policy.h | 48 +++++++++++++++++++++++ npu/forward/src/q4nx_forward.cpp | 11 ++---- tests/ffn_cache_policy_test.cpp | 53 ++++++++++++++++++++++++++ 5 files changed, 116 insertions(+), 9 deletions(-) create mode 100644 npu/forward/include/ffn_cache_policy.h create mode 100644 tests/ffn_cache_policy_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 16cbce9bc..8b51fe299 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1252,6 +1252,12 @@ add_executable(ops_dump_test tests/ops_dump_test.cpp) target_include_directories(ops_dump_test PRIVATE npu/forward/include) add_test(NAME ops_dump COMMAND ops_dump_test) add_dependencies(onebit ops_dump_test) +# When the dense-tail cache is on by default (npu/forward/include/ffn_cache_policy.h): the size rule and the two +# models it was measured on, one on each side of the old floor. Header-only, no device. +add_executable(ffn_cache_policy_test tests/ffn_cache_policy_test.cpp) +target_include_directories(ffn_cache_policy_test PRIVATE npu/forward/include) +add_test(NAME ffn_cache_policy COMMAND ffn_cache_policy_test) +add_dependencies(onebit ffn_cache_policy_test) # The device weight layout contract (q4nx_weight_layout): pure index math, no device # and no runtime. CI builds an explicit target list, so the check compiles itself with diff --git a/docs/npu.md b/docs/npu.md index 76f584b16..c22e6b1fa 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1664,12 +1664,15 @@ re-dequantised each layer's 17408-wide dense FFN tail on every token even after packed on the device. `ffn_tail_cached()` (`npu/forward/src/q4nx_forward.cpp`) closes that: once a layer's packed gate/up/down BOs are resident, the f32 `wg_`/`wu_`/`wd_` buffers are no longer read, so the reload is skipped. It is on by default for a GDN model on the NPU route, when the packed -weight cache is on, the run is not `NPU_INFER_HOST_GEMM`, and the tail is at least 256 MB per layer -(the size guard; below that the reload is too small to be worth a behaviour change). `NPU_FFN_CACHE=0` +weight cache is on, the run is not `NPU_INFER_HOST_GEMM`, and the tail is at least 128 MB per layer +(the size guard, set just under the smallest tail measured; `npu/forward/include/ffn_cache_policy.h`). `NPU_FFN_CACHE=0` or `=1` forces it either way, and the resolved state is logged once as `[ffn] dense-tail cache on|off (default|NPU_FFN_CACHE; lin_kd=… IM=… H=…)`. On Qwen3.8-27B-NPU2 an interleaved 5-round A/B (`~/lb/587-cache-20261030`) measured an 8-token request at 104.5 s against 117.4 s (**−11%**), CPU per token 22.9 s against 63.3 s (**−64%**), same text in every arm. +On FastFlowLM's Qwen3.5-2B-NPU2 (IM 6144, H 2048, a 151 MB tail that the earlier 256 MB floor left out) an A B B A +run in one ticket measured decode at 362–367 ms per token with the cache on (both arms) against 485–520 ms with +it off in the quieter of the two off arms and 478–707 ms in the other, same text in all twelve runs (engine#1785). ### The prefix cache's snapshot: two kinds of plane diff --git a/npu/forward/include/ffn_cache_policy.h b/npu/forward/include/ffn_cache_policy.h new file mode 100644 index 000000000..7f33986a0 --- /dev/null +++ b/npu/forward/include/ffn_cache_policy.h @@ -0,0 +1,48 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ffn_cache_policy.h -- when the dense-tail cache is on by default (engine#587, engine#1785). +// +// On a hybrid (GDN) model the forward used to re-dequantise each layer's dense FFN tail on every token, even after the +// packed weights were resident on the device. The dense-tail cache skips that reload. Whether it is on when +// NPU_FFN_CACHE says nothing is a size rule, and the floor of that rule is set by what has been measured, not guessed: +// +// model tail per layer decode, cache off -> on text +// Qwen3.8-27B-NPU2 (IM 17408, H 5120) 1.07 GB 117.4 -> 104.5 s per 8-token request identical (~/lb/587-cache-20261030) +// Qwen3.5-2B-NPU2 (IM 6144, H 2048) 151 MB 485-520 -> 362-367 ms per token identical in 12 of 12 runs +// +// The second row is engine#1785's measurement (A B B A, one ticket; the off figure is the quieter off arm, the other +// was 478-707): the reload was about a fifth of every decode token on a model the old 256 MB floor left out. The floor is therefore just under the smallest tail measured. +// Below it nothing has been measured, so nothing is switched on unasked; NPU_FFN_CACHE=1 still forces it. + +#pragma once + +namespace onebit::npu { + +// Bytes of f32 the three dense-tail matrices of one layer occupy (gate, up, down: IM x H each). +inline double ffn_tail_bytes(int intermediate, int hidden) { + return 3.0 * double(intermediate) * double(hidden) * 4.0; +} + +// The smallest tail the cache has been measured to pay for (see the table above). +inline constexpr double kFfnCacheFloorBytes = 128.0 * 1024.0 * 1024.0; + +// The size half of the default: a tail at or above the floor. The caller adds the conditions that are about the +// run, not the model (a GDN model, the packed-weight cache on, not the host-GEMM mode). +inline bool ffn_cache_worth_it(int intermediate, int hidden) { + return intermediate > 0 && hidden > 0 && ffn_tail_bytes(intermediate, hidden) >= kFfnCacheFloorBytes; +} + +} // namespace onebit::npu diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 6a0a0a534..f53ab0884 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -24,6 +24,7 @@ // Numerics follow tools/q4nx_forward_ref.py so the two can be compared // element-wise. #include "q4nx_forward.h" +#include "ffn_cache_policy.h" #include "q4nx_state_planes.h" // the plane-copy half of a snapshot, device-free and unit-tested #include "model/op_dispatch.h" // the optional-op fallback seam (engine#803 A3) #include @@ -1736,15 +1737,11 @@ void Q4nxNpuForward::layer_attn_dims(int l, int& hd, int& nkv) const { // it -- a GDN model (lin_kd_ > 0, the one case step()'s `dense_weights_cached` // can never cover, since that guard stops at lin_kd_ == 0), packed weights // actually being cached, and a dense tail big enough for the per-token reload to -// be material. Qwen3.8-27B-NPU2 (IM 17408 x H 5120) clears it with 1.07 GB per -// layer, and measured 117.4 -> 104.5 s wall and 63.3 -> 22.9 s CPU per token on -// an 8-token request with identical text (~/lb/587-cache-20261030). -// The floor is a size guard, not a second measurement: below it the reload is -// small enough that the behaviour change is not worth taking unasked. +// be material. The size rule and the two measurements it rests on (Qwen3.8-27B +// and, since engine#1785, Qwen3.5-2B) are in ffn_cache_policy.h. bool Q4nxNpuForward::ffn_cache_default() const { if (host_gemm_ || !xd_->weight_cache_on() || lin_kd_ <= 0) return false; - const double tail_bytes = 3.0 * (double)c_.IM * (double)c_.H * 4.0; // wg_+wu_+wd_ - return tail_bytes >= 256.0 * 1024.0 * 1024.0; + return onebit::npu::ffn_cache_worth_it(c_.IM, c_.H); } // ---- layer-loop host attribution ------------------------------------------ // The `host=` residual of the [ph] line is layer time that is NOT any of the diff --git a/tests/ffn_cache_policy_test.cpp b/tests/ffn_cache_policy_test.cpp new file mode 100644 index 000000000..c0a7d6e1b --- /dev/null +++ b/tests/ffn_cache_policy_test.cpp @@ -0,0 +1,53 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ffn_cache_policy_test: the size rule for the dense-tail cache's default (npu/forward/include/ffn_cache_policy.h). +// The two measured models are the fixtures: both must be on, and the one engine#1785 measured is the one the +// earlier 256 MB floor left off. + +#include "ffn_cache_policy.h" + +#include +#include + +using onebit::npu::ffn_cache_worth_it; +using onebit::npu::ffn_tail_bytes; +using onebit::npu::kFfnCacheFloorBytes; + +static int failures = 0; +static void check(bool ok, const std::string& what) { + if (!ok) { + std::printf("FAIL %s\n", what.c_str()); + ++failures; + } +} + +int main() { + const double MB = 1024.0 * 1024.0; + check(ffn_tail_bytes(6144, 2048) == 3.0 * 6144 * 2048 * 4, "the tail is three IM x H f32 matrices"); + check(ffn_tail_bytes(17408, 5120) > 1000.0 * MB && ffn_cache_worth_it(17408, 5120), "Qwen3.8-27B (1.07 GB tail): on, as before"); + check(ffn_tail_bytes(6144, 2048) == 144.0 * MB, "Qwen3.5-2B's tail is 144 MiB (151 MB)"); + check(ffn_cache_worth_it(6144, 2048), "Qwen3.5-2B: on -- measured 485-520 -> 362-367 ms per token, same text (engine#1785)"); + check(ffn_tail_bytes(6144, 2048) < 256.0 * MB, "and it is the model the earlier 256 MB floor left off"); + // Below the smallest measured tail nothing is switched on unasked. + check(!ffn_cache_worth_it(3584, 1024), "a 0.8B-class tail (42 MiB) stays off by default: not measured"); + check(!ffn_cache_worth_it(4096, 2048), "a 96 MiB tail stays off by default: not measured"); + check(kFfnCacheFloorBytes == 128.0 * MB, "the floor is 128 MiB"); + check(!ffn_cache_worth_it(5460, 2048) && ffn_cache_worth_it(5462, 2048), "the rule turns exactly at the floor (5461.33 x 2048)"); + check(!ffn_cache_worth_it(0, 2048) && !ffn_cache_worth_it(6144, 0) && !ffn_cache_worth_it(-6144, -2048), + "a geometry that is not one is never on"); + if (failures == 0) std::printf("ffn_cache_policy: ok\n"); + return failures == 0 ? 0 : 1; +}