From 6d5703f229f8b1c918a529a25261795203dd726f Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 17:56:03 -0400 Subject: [PATCH 1/2] act_outliers: assert the tau/kmax defaults in a test (Refs #2576) Answers the CHANGES verdict on #2585 (merged): the tau-20 default was stated in two places and asserted in none. - act_outliers.h: kActOutlierTauDefault (20) / kActOutlierKmaxDefault (16) plus act_outlier_tau_from / act_outlier_kmax_from, the env readers init_impl now calls (same behaviour: unset -> default, "0" -> off, kmax clamped to >= 1). - act_outliers_test: test_defaults asserts unset -> 20, "0" -> 0, kmax unset -> 16, "0" -> 1, and that a ~28.5*rms channel is split at the default but not at tau 40. Mutant default 40 fails 2 checks. - docs/npu.md: prefill cost worded as time (lower is better), the 13.7 % spread of the tau-20 samples, and the trade stated next to the price. Co-Authored-By: Claude Opus 5.5 --- docs/npu.md | 4 +++- npu/forward/include/act_outliers.h | 15 +++++++++++++++ npu/forward/include/q4nx_forward.h | 4 ++-- npu/forward/src/q4nx_forward.cpp | 7 +++---- tests/act_outliers_test.cpp | 24 ++++++++++++++++++++++++ 5 files changed, 47 insertions(+), 7 deletions(-) diff --git a/docs/npu.md b/docs/npu.md index 2921a8416..03a61bc06 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -3343,7 +3343,9 @@ At the default tau 40 the same container answers `'Sure! 12 times 7 is 84.'` and CodeLlama speed, tau 0 vs 20, A B A B in one ticket: - **Decode (64 tok):** 0.997 / 1.046 tok/s vs 1.030 / 1.001 tok/s, unchanged. -- **Prefill (520 tok):** 14,260 / 14,400 ms vs 15,165 / 17,244 ms, **+6 % to +20 %**. At tau 20 more prefill rows split, and each new channel costs one column-block gather on first use. +- **Prefill (520 tok, ms, lower is better):** 14,260 / 14,400 vs 15,165 / 17,244, so **tau 20 costs 6-20 % more prefill time**. The two tau-20 samples differ from each other by 13.7 %, so treat the upper end as noise. At tau 20 more prefill rows split, and each new channel costs one column-block gather on first use. + +The trade: 6-20 % more prefill buys EOS correctness on 1.7.1 hybrid Llama-2, where host f32 gives EOS and tau 40 does not. The default lives in one place, `kActOutlierTauDefault` in `npu/forward/include/act_outliers.h`, and `act_outliers_test` asserts it (unset -> 20, `0` -> off). The older table below was measured before gate/up were covered and is kept as it was measured. diff --git a/npu/forward/include/act_outliers.h b/npu/forward/include/act_outliers.h index ea024cfc7..bfd02214d 100644 --- a/npu/forward/include/act_outliers.h +++ b/npu/forward/include/act_outliers.h @@ -32,12 +32,27 @@ #include #include #include +#include #include #include #include namespace onebit::npu { +// The defaults, in one place so act_outliers_test can assert them (engine#2585 review): tau 20 (docs/npu.md has the +// table; 40 left Llama-2-7b-chat 1.7.1 without its EOS once gate/up were covered) and at most 16 channels per row. +constexpr float kActOutlierTauDefault = 20.0f; +constexpr int kActOutlierKmaxDefault = 16; + +// NPU_ACT_OUTLIER_TAU as read: unset gives the default, "0" turns the split off. +inline float act_outlier_tau_from(const char* env) { return env ? (float)std::atof(env) : kActOutlierTauDefault; } +// NPU_ACT_OUTLIER_KMAX as read: unset gives the default; anything below 1 is clamped to 1. +inline int act_outlier_kmax_from(const char* env) { + if (!env) return kActOutlierKmaxDefault; + const int k = std::atoi(env); + return k < 1 ? 1 : k; +} + struct ActOutlier { int row; // activation row int ch; // input channel (the weight column) diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index 9dbbcc55e..22c9226b8 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -478,8 +478,8 @@ class Q4nxNpuForward { int impad_ = 0; // c_.IM on the container's K grid // engine#2536: outlier-channel split for the int8 ffn_down GEMM (NPU_ACT_OUTLIER_TAU, act_outliers.h). Columns of // the layer's down_proj are decoded one 256-wide block at a time and cached per (layer, channel). - float outl_tau_ = 0.0f; // set in init_impl: 20 unless NPU_ACT_OUTLIER_TAU says otherwise - int outl_kmax_ = 16; + float outl_tau_ = 0.0f; // set in init_impl from act_outlier_tau_from (act_outliers.h holds the default) + int outl_kmax_ = 16; // set in init_impl from act_outlier_kmax_from bool outl_busy_ = false; bool outl_gather_ok_ = true; std::unordered_map> outl_cols_; diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index c285f02e1..c1ff4d17d 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -703,11 +703,10 @@ bool Q4nxNpuForward::open_decode_family_local(bool want_m16, xrt::device* dev, c bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, const std::string& dir) { - // engine#2536 / #2576: on by default at tau 20 (the measured table is in docs/npu.md: 40 left Llama-2-7b-chat's EOS + // engine#2536 / #2576: on by default (kActOutlierTauDefault in act_outliers.h; the measured table is in docs/npu.md: 40 left Llama-2-7b-chat's EOS // lost once gate/up were covered); NPU_ACT_OUTLIER_TAU=0 turns it off, NPU_ACT_OUTLIER_KMAX caps channels per row. - outl_tau_ = 20.0f; - if (const char* t = getenv("NPU_ACT_OUTLIER_TAU")) outl_tau_ = (float)atof(t); - if (const char* k = getenv("NPU_ACT_OUTLIER_KMAX")) outl_kmax_ = std::max(1, atoi(k)); + outl_tau_ = onebit::npu::act_outlier_tau_from(getenv("NPU_ACT_OUTLIER_TAU")); + outl_kmax_ = onebit::npu::act_outlier_kmax_from(getenv("NPU_ACT_OUTLIER_KMAX")); if (const char* hg = getenv("NPU_INFER_HOST_GEMM")) { host_gemm_ = true; // engine#1808: "i8" selects the integer arm, whose int32 accumulation is the only thing a diff --git a/tests/act_outliers_test.cpp b/tests/act_outliers_test.cpp index 2d8970dd6..1f0a94da8 100644 --- a/tests/act_outliers_test.cpp +++ b/tests/act_outliers_test.cpp @@ -45,6 +45,9 @@ static int failures = 0; } while (0) using onebit::npu::ActOutlier; +using onebit::npu::act_outlier_kmax_from; +using onebit::npu::act_outlier_pick; +using onebit::npu::act_outlier_tau_from; // The per-row int8 GEMM the i8 path computes: one scale amax/127 per row, codes in [-127, 127], f32 weights. static void gemm_i8_row(const std::vector& x, const std::vector& W, int K, int N, std::vector& y) { @@ -232,7 +235,28 @@ static void test_fused_gate_up() { CHECK(guard == split, "fused: an offset at N writes nothing"); } +// engine#2585 review: the default is asserted, not only stated. Unset -> 20 (a 30*rms channel is split, which 40 would +// leave in int8), "0" -> off, explicit values pass through; kmax unset -> 16, below 1 -> 1. +static void test_defaults() { + CHECK(act_outlier_tau_from(nullptr) == 20.0f, "tau default %g, want 20", act_outlier_tau_from(nullptr)); + CHECK(act_outlier_tau_from("0") == 0.0f, "tau \"0\" gives %g, want 0 (off)", act_outlier_tau_from("0")); + CHECK(act_outlier_tau_from("40") == 40.0f, "tau \"40\" gives %g", act_outlier_tau_from("40")); + CHECK(act_outlier_kmax_from(nullptr) == 16, "kmax default %d, want 16", act_outlier_kmax_from(nullptr)); + CHECK(act_outlier_kmax_from("0") == 1, "kmax \"0\" gives %d, want 1", act_outlier_kmax_from("0")); + CHECK(act_outlier_kmax_from("4") == 4, "kmax \"4\" gives %d", act_outlier_kmax_from("4")); + // One row of 4096 ones with one channel at 30 * rms: the default picks it, tau 40 does not. + const int w = 4096; + std::vector a((size_t)w, 1.0f); + a[100] = 30.0f * 1.06f; // rms of the row is ~1.06 with this channel in it, so x/rms is just under 30 + std::vector p20, p40; + act_outlier_pick(a.data(), 1, w, w, act_outlier_tau_from(nullptr), act_outlier_kmax_from(nullptr), p20); + act_outlier_pick(a.data(), 1, w, w, 40.0f, 16, p40); + CHECK(p20.size() == 1 && p20[0].ch == 100, "default tau picked %zu channels, want channel 100", p20.size()); + CHECK(p40.empty(), "tau 40 picked %zu channels, want none", p40.size()); +} + int main() { + test_defaults(); test_pick(); test_arithmetic(); test_gather(); From 3ff926ee62a0b0324518ce15cc0b4ec01936256e Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 18:12:05 -0400 Subject: [PATCH 2/2] docs/npu: state the tau/kmax env contract and that no measured model prefers 40 (Refs #2576) Answers the #2587 review: one sentence for the NPU_ACT_OUTLIER_TAU / _KMAX contract the test asserts, and the per-model reason 20 is a default. Co-Authored-By: Claude Opus 5.5 --- docs/npu.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/npu.md b/docs/npu.md index 03a61bc06..cd92a1c68 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -3345,7 +3345,7 @@ CodeLlama speed, tau 0 vs 20, A B A B in one ticket: - **Decode (64 tok):** 0.997 / 1.046 tok/s vs 1.030 / 1.001 tok/s, unchanged. - **Prefill (520 tok, ms, lower is better):** 14,260 / 14,400 vs 15,165 / 17,244, so **tau 20 costs 6-20 % more prefill time**. The two tau-20 samples differ from each other by 13.7 %, so treat the upper end as noise. At tau 20 more prefill rows split, and each new channel costs one column-block gather on first use. -The trade: 6-20 % more prefill buys EOS correctness on 1.7.1 hybrid Llama-2, where host f32 gives EOS and tau 40 does not. The default lives in one place, `kActOutlierTauDefault` in `npu/forward/include/act_outliers.h`, and `act_outliers_test` asserts it (unset -> 20, `0` -> off). +The trade: 6-20 % more prefill buys EOS correctness on 1.7.1 hybrid Llama-2, where host f32 gives EOS and tau 40 does not. The default lives in one place, `kActOutlierTauDefault` in `npu/forward/include/act_outliers.h`, and `act_outliers_test` asserts it. The env contract: `NPU_ACT_OUTLIER_TAU` unset means 20, `0` turns the split off, and any other value is used as given. `NPU_ACT_OUTLIER_KMAX` unset means 16, and a value below 1 is clamped to 1. No model in the table above does better at 40 than at 20 (Qwen2.5-3B 58/64 at both, Qwen2.5-0.5B 52 vs 51, SmolLM2 54 vs 53), so 20 costs the others prefill time only, not correctness. The older table below was measured before gate/up were covered and is kept as it was measured.