diff --git a/docs/npu.md b/docs/npu.md index 8405e96f7..2921a8416 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -3330,7 +3330,24 @@ At that position, host f32 for `ffn_gate_up` alone restored EOS. So did a fully At the default tau 40 the same container answers `'Sure! 12 times 7 is 84.'` and stops after 14 tokens. -**Default: on, `tau` 40.** `NPU_ACT_OUTLIER_TAU=0` turns it off. Measured with `serve --device npu`, using teacher-forced +**Default: on, `tau` 20 (since engine#2576; it was 40 while only `ffn_down` was split).** With gate/up covered, tau 40 still left Llama-2-7b-chat (1.7.1 hybrid) answering `'… 84. Unterscheidung between …'`, where host f32 gives EOS. Tau 20, 10 and 5 all give EOS. The same tickets measured, with gate/up included: + +| model | tau 0 | tau 20 | tau 40 | reply at tau 20 | +|---|---|---|---|---| +| Llama-2-7b-chat 1.7.1 | — | — | — | `Great! 12 times 7 is equal to 84.` (stop, 16 tok); EOS at the failing prefix | +| CodeLlama-7b-Instruct 1.7.1 | — | — | — | EOS at the failing prefix (also at 40) | +| Qwen2.5-3B-Instruct 1.7.1 | — | 58/64 | 58/64 | `… July 20, 1969 …`; the `…, 1` prefix gives `96` | +| Qwen2.5-0.5B-Instruct 1.9.0 | 53/64 | 52/64 | 51/64 | answers unchanged | +| SmolLM2-135M-Instruct 1.9.0 | 53/64 | 54/64 | 53/64 | answers unchanged | + +CodeLlama speed, tau 0 vs 20, A B A B in one ticket: + +- **Decode (64 tok):** 0.997 / 1.046 tok/s vs 1.030 / 1.001 tok/s, unchanged. +- **Prefill (520 tok):** 14,260 / 14,400 ms vs 15,165 / 17,244 ms, **+6 % to +20 %**. At tau 20 more prefill rows split, and each new channel costs one column-block gather on first use. + +The older table below was measured before gate/up were covered and is kept as it was measured. + +**Before engine#2576 (ffn_down only): default tau 40.** `NPU_ACT_OUTLIER_TAU=0` turns it off. Measured with `serve --device npu`, using teacher-forced top-1 agreement against HF fp32 on 64 positions (the same probes for every column): | model | off | tau 20 | tau 40 | reply | diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index f902ffa20..9dbbcc55e 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -478,7 +478,7 @@ class Q4nxNpuForward { int impad_ = 0; // c_.IM on the container's K grid // engine#2536: outlier-channel split for the int8 ffn_down GEMM (NPU_ACT_OUTLIER_TAU, act_outliers.h). Columns of // the layer's down_proj are decoded one 256-wide block at a time and cached per (layer, channel). - float outl_tau_ = 0.0f; // set in init_impl: 40 unless NPU_ACT_OUTLIER_TAU says otherwise + float outl_tau_ = 0.0f; // set in init_impl: 20 unless NPU_ACT_OUTLIER_TAU says otherwise int outl_kmax_ = 16; bool outl_busy_ = false; bool outl_gather_ok_ = true; diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index d58151aff..c285f02e1 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -703,9 +703,9 @@ bool Q4nxNpuForward::open_decode_family_local(bool want_m16, xrt::device* dev, c bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, const std::string& dir) { - // engine#2536: on by default at tau 40 (the measured table is in docs/npu.md); NPU_ACT_OUTLIER_TAU=0 turns it off, - // NPU_ACT_OUTLIER_KMAX caps the channels split out of one row. - outl_tau_ = 40.0f; + // engine#2536 / #2576: on by default at tau 20 (the measured table is in docs/npu.md: 40 left Llama-2-7b-chat's EOS + // lost once gate/up were covered); NPU_ACT_OUTLIER_TAU=0 turns it off, NPU_ACT_OUTLIER_KMAX caps channels per row. + outl_tau_ = 20.0f; if (const char* t = getenv("NPU_ACT_OUTLIER_TAU")) outl_tau_ = (float)atof(t); if (const char* k = getenv("NPU_ACT_OUTLIER_KMAX")) outl_kmax_ = std::max(1, atoi(k)); if (const char* hg = getenv("NPU_INFER_HOST_GEMM")) {