Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion docs/npu.md
Original file line number Diff line number Diff line change
Expand Up @@ -3343,7 +3343,9 @@ At the default tau 40 the same container answers `'Sure! 12 times 7 is 84.'` and
CodeLlama speed, tau 0 vs 20, A B A B in one ticket:

- **Decode (64 tok):** 0.997 / 1.046 tok/s vs 1.030 / 1.001 tok/s, unchanged.
- **Prefill (520 tok):** 14,260 / 14,400 ms vs 15,165 / 17,244 ms, **+6 % to +20 %**. At tau 20 more prefill rows split, and each new channel costs one column-block gather on first use.
- **Prefill (520 tok, ms, lower is better):** 14,260 / 14,400 vs 15,165 / 17,244, so **tau 20 costs 6-20 % more prefill time**. The two tau-20 samples differ from each other by 13.7 %, so treat the upper end as noise. At tau 20 more prefill rows split, and each new channel costs one column-block gather on first use.

The trade: 6-20 % more prefill buys EOS correctness on 1.7.1 hybrid Llama-2, where host f32 gives EOS and tau 40 does not. The default lives in one place, `kActOutlierTauDefault` in `npu/forward/include/act_outliers.h`, and `act_outliers_test` asserts it. The env contract: `NPU_ACT_OUTLIER_TAU` unset means 20, `0` turns the split off, and any other value is used as given. `NPU_ACT_OUTLIER_KMAX` unset means 16, and a value below 1 is clamped to 1. No model in the table above does better at 40 than at 20 (Qwen2.5-3B 58/64 at both, Qwen2.5-0.5B 52 vs 51, SmolLM2 54 vs 53), so 20 costs the others prefill time only, not correctness.

The older table below was measured before gate/up were covered and is kept as it was measured.

Expand Down
15 changes: 15 additions & 0 deletions npu/forward/include/act_outliers.h
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,27 @@
#include <algorithm>
#include <cmath>
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <utility>
#include <vector>

namespace onebit::npu {

// The defaults, in one place so act_outliers_test can assert them (engine#2585 review): tau 20 (docs/npu.md has the
// table; 40 left Llama-2-7b-chat 1.7.1 without its EOS once gate/up were covered) and at most 16 channels per row.
constexpr float kActOutlierTauDefault = 20.0f;
constexpr int kActOutlierKmaxDefault = 16;

// NPU_ACT_OUTLIER_TAU as read: unset gives the default, "0" turns the split off.
inline float act_outlier_tau_from(const char* env) { return env ? (float)std::atof(env) : kActOutlierTauDefault; }
// NPU_ACT_OUTLIER_KMAX as read: unset gives the default; anything below 1 is clamped to 1.
inline int act_outlier_kmax_from(const char* env) {
if (!env) return kActOutlierKmaxDefault;
const int k = std::atoi(env);
return k < 1 ? 1 : k;
}

struct ActOutlier {
int row; // activation row
int ch; // input channel (the weight column)
Expand Down
4 changes: 2 additions & 2 deletions npu/forward/include/q4nx_forward.h
Original file line number Diff line number Diff line change
Expand Up @@ -478,8 +478,8 @@ class Q4nxNpuForward {
int impad_ = 0; // c_.IM on the container's K grid
// engine#2536: outlier-channel split for the int8 ffn_down GEMM (NPU_ACT_OUTLIER_TAU, act_outliers.h). Columns of
// the layer's down_proj are decoded one 256-wide block at a time and cached per (layer, channel).
float outl_tau_ = 0.0f; // set in init_impl: 20 unless NPU_ACT_OUTLIER_TAU says otherwise
int outl_kmax_ = 16;
float outl_tau_ = 0.0f; // set in init_impl from act_outlier_tau_from (act_outliers.h holds the default)
int outl_kmax_ = 16; // set in init_impl from act_outlier_kmax_from
bool outl_busy_ = false;
bool outl_gather_ok_ = true;
std::unordered_map<uint64_t, std::vector<float>> outl_cols_;
Expand Down
7 changes: 3 additions & 4 deletions npu/forward/src/q4nx_forward.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -703,11 +703,10 @@ bool Q4nxNpuForward::open_decode_family_local(bool want_m16, xrt::device* dev, c

bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path,
const std::string& dir) {
// engine#2536 / #2576: on by default at tau 20 (the measured table is in docs/npu.md: 40 left Llama-2-7b-chat's EOS
// engine#2536 / #2576: on by default (kActOutlierTauDefault in act_outliers.h; the measured table is in docs/npu.md: 40 left Llama-2-7b-chat's EOS
// lost once gate/up were covered); NPU_ACT_OUTLIER_TAU=0 turns it off, NPU_ACT_OUTLIER_KMAX caps channels per row.
outl_tau_ = 20.0f;
if (const char* t = getenv("NPU_ACT_OUTLIER_TAU")) outl_tau_ = (float)atof(t);
if (const char* k = getenv("NPU_ACT_OUTLIER_KMAX")) outl_kmax_ = std::max(1, atoi(k));
outl_tau_ = onebit::npu::act_outlier_tau_from(getenv("NPU_ACT_OUTLIER_TAU"));
outl_kmax_ = onebit::npu::act_outlier_kmax_from(getenv("NPU_ACT_OUTLIER_KMAX"));
if (const char* hg = getenv("NPU_INFER_HOST_GEMM")) {
host_gemm_ = true;
// engine#1808: "i8" selects the integer arm, whose int32 accumulation is the only thing a
Expand Down
24 changes: 24 additions & 0 deletions tests/act_outliers_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,9 @@ static int failures = 0;
} while (0)

using onebit::npu::ActOutlier;
using onebit::npu::act_outlier_kmax_from;
using onebit::npu::act_outlier_pick;
using onebit::npu::act_outlier_tau_from;

// The per-row int8 GEMM the i8 path computes: one scale amax/127 per row, codes in [-127, 127], f32 weights.
static void gemm_i8_row(const std::vector<float>& x, const std::vector<float>& W, int K, int N, std::vector<float>& y) {
Expand Down Expand Up @@ -232,7 +235,28 @@ static void test_fused_gate_up() {
CHECK(guard == split, "fused: an offset at N writes nothing");
}

// engine#2585 review: the default is asserted, not only stated. Unset -> 20 (a 30*rms channel is split, which 40 would
// leave in int8), "0" -> off, explicit values pass through; kmax unset -> 16, below 1 -> 1.
static void test_defaults() {
CHECK(act_outlier_tau_from(nullptr) == 20.0f, "tau default %g, want 20", act_outlier_tau_from(nullptr));
CHECK(act_outlier_tau_from("0") == 0.0f, "tau \"0\" gives %g, want 0 (off)", act_outlier_tau_from("0"));
CHECK(act_outlier_tau_from("40") == 40.0f, "tau \"40\" gives %g", act_outlier_tau_from("40"));
CHECK(act_outlier_kmax_from(nullptr) == 16, "kmax default %d, want 16", act_outlier_kmax_from(nullptr));
CHECK(act_outlier_kmax_from("0") == 1, "kmax \"0\" gives %d, want 1", act_outlier_kmax_from("0"));
CHECK(act_outlier_kmax_from("4") == 4, "kmax \"4\" gives %d", act_outlier_kmax_from("4"));
// One row of 4096 ones with one channel at 30 * rms: the default picks it, tau 40 does not.
const int w = 4096;
std::vector<float> a((size_t)w, 1.0f);
a[100] = 30.0f * 1.06f; // rms of the row is ~1.06 with this channel in it, so x/rms is just under 30
std::vector<ActOutlier> p20, p40;
act_outlier_pick(a.data(), 1, w, w, act_outlier_tau_from(nullptr), act_outlier_kmax_from(nullptr), p20);
act_outlier_pick(a.data(), 1, w, w, 40.0f, 16, p40);
CHECK(p20.size() == 1 && p20[0].ch == 100, "default tau picked %zu channels, want channel 100", p20.size());
CHECK(p40.empty(), "tau 40 picked %zu channels, want none", p40.size());
}

int main() {
test_defaults();
test_pick();
test_arithmetic();
test_gather();
Expand Down
Loading