diff --git a/docs/npu.md b/docs/npu.md index 029f7ed5a..8405e96f7 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -3311,7 +3311,7 @@ failing position: - the host i8 arm and the device both said ` lick`; - host f32 for `ffn_down` alone, or `ffn_gate_up` alone, fixed it, while `attn_qkv` and `attn_output` did not. -The split (`npu/forward/include/act_outliers.h`, wired in `ffn_down_outliers()`) works per row: +The split (`npu/forward/include/act_outliers.h`, wired in `ffn_outliers()`) works per row: 1. The channels with `|x| > tau * rms` (at most `NPU_ACT_OUTLIER_KMAX`, default 16) are zeroed in the int8 activation, so the scale is set by the remaining channels. @@ -3322,6 +3322,14 @@ checked bit-for-bit against a full decode; a difference turns the gather off and channel runs the plain launch, so a model without outliers pays one scan per row. Host-GEMM runs (`NPU_INFER_HOST_GEMM`) are not changed. +**It covers the gate/up GEMMs too (engine#2576).** CodeLlama-7b-Instruct (1.7.1 hybrid) answered `'Sure! 12 times 7 is 84. package.'`, then newlines to the length cap: `` was lost. + +At that position, host f32 for `ffn_gate_up` alone restored EOS. So did a fully f32 host run. `ffn_down`, `attn_qkv` and `attn_output` alone did not, and lowering tau on the `ffn_down` split did not either (20, 10 and 5). The int8 loss there is in the gate/up input, the post-attention norm output. + +`ffn_outliers()` therefore applies the same split to `ffn_gate_up` and to split `ffn_gate`/`ffn_up` designs, with columns from `gate_proj`/`up_proj`. For the fused design the corrections land at their own offsets: the outputs are `[gate | up]` along N, as `q4nx_assemble_B` lays them, so `up` starts at `gate_rows`. + +At the default tau 40 the same container answers `'Sure! 12 times 7 is 84.'` and stops after 14 tokens. + **Default: on, `tau` 40.** `NPU_ACT_OUTLIER_TAU=0` turns it off. Measured with `serve --device npu`, using teacher-forced top-1 agreement against HF fp32 on 64 positions (the same probes for every column): diff --git a/npu/forward/include/act_outliers.h b/npu/forward/include/act_outliers.h index b9ee4a15d..ea024cfc7 100644 --- a/npu/forward/include/act_outliers.h +++ b/npu/forward/include/act_outliers.h @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. // -// act_outliers.h -- the outlier-channel split for the int8 ffn_down GEMM (engine#2536). +// act_outliers.h -- the outlier-channel split for the int8 FFN GEMMs (engine#2536 ffn_down, engine#2576 gate/up). // // The i8 GEMM path quantises each activation row with ONE scale, amax/127. On Qwen2.5-3B the ffn_down input rows have // amax/rms of 38 (median over the 36 layers) to 100 (layer 35, one channel at 1200), so a typical channel keeps 1-3 int8 @@ -75,4 +75,20 @@ inline void act_outlier_gather_block(const uint8_t* raw, int TR, int TC, int cb, std::memcpy(&buf[(size_t)tr * (size_t)cb], raw + ((size_t)tr * (size_t)TC + (size_t)tc) * (size_t)cb, (size_t)cb); } +// Add the split-out channels back on the host: for every pick, out[row][out_off + n] += x * col(ch)[n], n < rows (and +// within N). `out_off` is where this tensor's outputs start in the GEMM's N: 0 for a single projection, gate_rows for +// the `up` half of a fused [gate | up] design (engine#2576). +template +inline void act_outlier_correct(float* out, int N, const std::vector& picks, int out_off, int rows, + ColFn col) { + if (!out || N <= 0 || out_off < 0 || out_off >= N) return; + const int nn = std::min(rows, N - out_off); + for (const auto& pk : picks) { + const float* c = col(pk.ch); + if (!c) continue; + float* y = out + (size_t)pk.row * (size_t)N + (size_t)out_off; + for (int n = 0; n < nn; n++) y[n] += pk.x * c[n]; + } +} + } // namespace onebit::npu diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index ad2738f11..f902ffa20 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -483,10 +483,10 @@ class Q4nxNpuForward { bool outl_busy_ = false; bool outl_gather_ok_ = true; std::unordered_map> outl_cols_; - std::unordered_set outl_checked_; + std::unordered_set outl_checked_; // (layer * 8 + tensor id) whose gather was checked std::vector outl_scratch_; - bool ffn_down_outliers(const float* A, int M, int a_k, int K, int N, const float* Bmat, std::vector& out, - int wkey, int role, bool rows_form, bool decode_twin); + bool ffn_outliers(const char* sc, const float* A, int M, int a_k, int K, int N, const float* Bmat, + std::vector& out, int wkey, int role, bool rows_form, bool decode_twin); // Row stride of the f32 table embed_at()/lm_at() read: the padded width when the table was // dequantised from a container tensor, c_.H when it is the raw BF16 embedding. int lm_w_ = 0; diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index d30a1dc6b..d58151aff 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -4129,32 +4129,56 @@ static inline void op_line(const char* tag, int layer, int rows, const std::vect onebit::npu::ops_dump(tag, layer, rows, out.data(), n); } -// engine#2536: ffn_down with its outlier channels split out (act_outliers.h). Runs on the device path only (a host-GEMM -// run returns before the hook). Returns false when no row has an outlier: the caller then runs the plain launch, so a -// model without outliers pays one scan of its rows. -bool Q4nxNpuForward::ffn_down_outliers(const float* A, int M, int a_k, int K, int N, const float* Bmat, - std::vector& out, int wkey, int role, bool rows_form, bool decode_twin) { +// engine#2536 / #2576: an FFN projection with its outlier channels split out (act_outliers.h). Covers `ffn_down` (whose +// input is SiLU(gate)*up -- Qwen2.5-3B) and the gate/up GEMMs (whose input is the post-attention norm output -- +// CodeLlama/Llama-2), fused (`ffn_gate_up`: outputs [gate | up] along N, as q4nx_assemble_B lays them) or split +// (`ffn_gate`, `ffn_up`). Runs on the device path only (a host-GEMM run returns before the hook). Returns false when the +// tag is not covered or no row has an outlier: the caller then runs the plain launch, so a model without outliers pays +// one scan of its rows. +bool Q4nxNpuForward::ffn_outliers(const char* sc, const float* A, int M, int a_k, int K, int N, const float* Bmat, + std::vector& out, int wkey, int role, bool rows_form, bool decode_twin) { + struct Part { TensorDesc* t; int rows, cols, out_off, id; }; + std::vector parts; + const int L = cur_layer_; + if (!mw_ || L < 0) return false; + auto& lw = mw_->layers[L]; + const Q4nxDesignGeom* g = (role >= 0 && role < ndesigns_) ? &designs_[role] : nullptr; + const int gate_rows = (g && g->gate_rows > 0) ? g->gate_rows : c_.IM; + const int up_rows = (g && g->up_rows > 0) ? g->up_rows : c_.IM; + if (std::strcmp(sc, "ffn_down") == 0) { + parts.push_back({&lw.down_proj_weight, c_.H, q4nx_pad_k(c_.IM), 0, 0}); + } else if (std::strcmp(sc, "ffn_gate_up") == 0) { + parts.push_back({&lw.gate_proj_weight, gate_rows, kpad_, 0, 1}); + parts.push_back({&lw.up_proj_weight, up_rows, kpad_, gate_rows, 2}); + } else if (std::strcmp(sc, "ffn_gate") == 0) { + parts.push_back({&lw.gate_proj_weight, gate_rows, kpad_, 0, 1}); + } else if (std::strcmp(sc, "ffn_up") == 0) { + parts.push_back({&lw.up_proj_weight, up_rows, kpad_, 0, 2}); + } else { + return false; + } const int w = (a_k > 0 && a_k < K) ? a_k : K; const int stride = rows_form ? onebit::npu::row_stride(a_k, K) : w; std::vector picks; onebit::npu::act_outlier_pick(A, M, stride, w, outl_tau_, outl_kmax_, picks); if (picks.empty()) return false; - const int L = cur_layer_, rowsW = c_.H, colsW = q4nx_pad_k(c_.IM); - auto key_of = [L](int ch) { return ((uint64_t)(uint32_t)L << 32) | (uint32_t)ch; }; - std::vector blocks; - for (const auto& pk : picks) - if (!outl_cols_.count(key_of(pk.ch))) blocks.push_back(pk.ch / Q4NX_TILE_COLS); - std::sort(blocks.begin(), blocks.end()); - blocks.erase(std::unique(blocks.begin(), blocks.end()), blocks.end()); - if (!blocks.empty()) { - TensorDesc* d = &mw_->layers[L].down_proj_weight; + auto key_of = [L](int id, int ch) { return ((uint64_t)(uint32_t)L << 36) | ((uint64_t)id << 32) | (uint32_t)ch; }; + if (outl_cols_.size() > 8192) outl_cols_.clear(); // 8192 columns x rows floats: 64 MB at 2048 rows + for (const Part& part : parts) { + const int rowsW = part.rows, colsW = part.cols; + std::vector blocks; + for (const auto& pk : picks) + if (!outl_cols_.count(key_of(part.id, pk.ch))) blocks.push_back(pk.ch / Q4NX_TILE_COLS); + std::sort(blocks.begin(), blocks.end()); + blocks.erase(std::unique(blocks.begin(), blocks.end()), blocks.end()); + if (blocks.empty()) continue; + TensorDesc* d = part.t; const uint8_t* raw = (const uint8_t*)model_tensor_data(mw_, d); if (!raw) return false; - if (outl_cols_.size() > 8192) outl_cols_.clear(); // 8192 columns x H floats: 64 MB at H 2048 // A column of a [rowsW, colsW] tile tensor lives in one 256-wide block (tile (tr, c/256) of every tile-row). // For uniform chunked tiles in row-major tile order, that block decodes on its own as a [rowsW, 256] tensor. - // The first block of each layer is checked bit-for-bit against the full decode; a difference turns the - // gather off for good and says so, and every other layout takes the full decode. + // The first block of each (layer, tensor) is checked bit-for-bit against the full decode; a difference turns + // the gather off for good and says so, and every other layout takes the full decode. const int TR = rowsW / Q4NX_TILE_ROWS, TC = colsW / Q4NX_TILE_COLS; const size_t ntiles = (size_t)TR * (size_t)TC; const int cb = (ntiles && rowsW % Q4NX_TILE_ROWS == 0 && colsW % Q4NX_TILE_COLS == 0) @@ -4172,6 +4196,7 @@ bool Q4nxNpuForward::ffn_down_outliers(const float* A, int M, int a_k, int K, in }; std::vector gbuf; std::vector bout; + const int check_key = L * 8 + part.id; for (int tc : blocks) { const float* src = nullptr; int ld = colsW, off = tc * Q4NX_TILE_COLS; @@ -4180,17 +4205,17 @@ bool Q4nxNpuForward::ffn_down_outliers(const float* A, int M, int a_k, int K, in bout.resize((size_t)rowsW * Q4NX_TILE_COLS); if (q4nx_dequant_tensor_chunked(gbuf.data(), gbuf.size(), rowsW, Q4NX_TILE_COLS, cb, bout.data()) != 0) return false; - if (!outl_checked_.count(L)) { + if (!outl_checked_.count(check_key)) { if (!full_decode()) return false; bool same = true; for (int n = 0; n < rowsW && same; n++) same = std::memcmp(&bout[(size_t)n * Q4NX_TILE_COLS], &outl_scratch_[(size_t)n * colsW + off], Q4NX_TILE_COLS * sizeof(float)) == 0; - outl_checked_.insert(L); + outl_checked_.insert(check_key); if (!same) { outl_gather_ok_ = false; - std::fprintf(stderr, " [outlier] layer %d: a gathered down_proj column block differs from the " - "full decode; every later column takes the full decode\n", L); + std::fprintf(stderr, " [outlier] layer %d (%s): a gathered column block differs from the full " + "decode; every later column takes the full decode\n", L, sc); } } if (outl_gather_ok_) { src = bout.data(); ld = Q4NX_TILE_COLS; off = 0; } @@ -4200,27 +4225,26 @@ bool Q4nxNpuForward::ffn_down_outliers(const float* A, int M, int a_k, int K, in src = outl_scratch_.data(); } for (const auto& pk : picks) { - if (pk.ch / Q4NX_TILE_COLS != tc || outl_cols_.count(key_of(pk.ch))) continue; + if (pk.ch / Q4NX_TILE_COLS != tc || outl_cols_.count(key_of(part.id, pk.ch))) continue; std::vector col((size_t)rowsW); const int cc = off + pk.ch % Q4NX_TILE_COLS; for (int n = 0; n < rowsW; n++) col[(size_t)n] = src[(size_t)n * ld + cc]; - outl_cols_.emplace(key_of(pk.ch), std::move(col)); + outl_cols_.emplace(key_of(part.id, pk.ch), std::move(col)); } } } std::vector Az(A, A + (size_t)(M - 1) * (size_t)stride + (size_t)w); for (const auto& pk : picks) Az[(size_t)pk.row * (size_t)stride + (size_t)pk.ch] = 0.0f; outl_busy_ = true; - if (rows_form) npu_gemm_rows(role, Az.data(), M, K, N, Bmat, out, wkey, "ffn_down", a_k, decode_twin); - else npu_gemm(role, Az.data(), K, N, Bmat, out, wkey, "ffn_down", a_k); + if (rows_form) npu_gemm_rows(role, Az.data(), M, K, N, Bmat, out, wkey, sc, a_k, decode_twin); + else npu_gemm(role, Az.data(), K, N, Bmat, out, wkey, sc, a_k); outl_busy_ = false; if (out.size() < (size_t)M * (size_t)N) return true; // the launch failed and set err_; keep its message - const int nn = std::min(N, rowsW); - for (const auto& pk : picks) { - const std::vector& col = outl_cols_[key_of(pk.ch)]; - float* y = &out[(size_t)pk.row * (size_t)N]; - for (int n = 0; n < nn; n++) y[n] += pk.x * col[(size_t)n]; - } + for (const Part& part : parts) + onebit::npu::act_outlier_correct(out.data(), N, picks, part.out_off, part.rows, [&](int ch) -> const float* { + auto it = outl_cols_.find(key_of(part.id, ch)); + return it == outl_cols_.end() ? nullptr : it->second.data(); + }); return true; } @@ -4241,8 +4265,8 @@ void Q4nxNpuForward::npu_gemm(int role, const float* A, int K, int N, const floa op_line(sc, cur_layer_, 1, out, N); return; } - if (outl_tau_ > 0.0f && !outl_busy_ && sc && std::strcmp(sc, "ffn_down") == 0 && - ffn_down_outliers(A, 1, a_k, K, N, Bmat, out, wkey, role, false, false)) + if (outl_tau_ > 0.0f && !outl_busy_ && sc && std::strncmp(sc, "ffn_", 4) == 0 && + ffn_outliers(sc, A, 1, a_k, K, N, Bmat, out, wkey, role, false, false)) return; npu::target::GemvArgs a; a.role = role; a.layer = cur_layer_; a.M = 1; a.K = K; a.N = N; @@ -4281,8 +4305,8 @@ void Q4nxNpuForward::npu_gemm_rows(int role, const float* A, int M, int K, int N // does not -- the designs that do not set real_k. So the stride is a_k only when it is a real, narrower width: // the same rule the target applies (xdna_target.cpp: astride). Walking by `a_k` itself made every block after // the first read block 0's rows on a design with real_k == 0 (engine#1959 follow-up, found on a t2s pack). - if (outl_tau_ > 0.0f && !outl_busy_ && sc && std::strcmp(sc, "ffn_down") == 0 && - ffn_down_outliers(A, M, a_k, K, N, Bmat, out, wkey, role, true, decode_twin)) + if (outl_tau_ > 0.0f && !outl_busy_ && sc && std::strncmp(sc, "ffn_", 4) == 0 && + ffn_outliers(sc, A, M, a_k, K, N, Bmat, out, wkey, role, true, decode_twin)) return; const int astride = onebit::npu::row_stride(a_k, K); const int blk = max_batch_rows(); diff --git a/tests/act_outliers_test.cpp b/tests/act_outliers_test.cpp index 3ea2e4df3..2d8970dd6 100644 --- a/tests/act_outliers_test.cpp +++ b/tests/act_outliers_test.cpp @@ -188,10 +188,55 @@ static void test_gather() { "block 1 decoded equal to block 0: the gather check cannot fail"); } +// engine#2576: a FUSED [gate | up] GEMM (outputs gate rows first, then up rows, as q4nx_assemble_B lays them). The +// outlier channel feeds both halves, so each half's correction must land at its own offset. A correction applied with +// the up half at offset 0 (on top of gate) must not pass. +static void test_fused_gate_up() { + const int K = 2048, R = 128, N = 2 * R; + std::mt19937 rng(23); + std::normal_distribution g(0.0f, 1.0f); + std::vector x((size_t)K), Wg((size_t)R * K), Wu((size_t)R * K); + for (auto& v : x) v = g(rng); + for (auto& v : Wg) v = 0.02f * g(rng); + for (auto& v : Wu) v = 0.02f * g(rng); + x[777] = 90.0f; // one channel ~90x the row's rms: the post-attention norm output on CodeLlama + std::vector W((size_t)N * K); // fused rows: [gate; up] + std::memcpy(W.data(), Wg.data(), Wg.size() * sizeof(float)); + std::memcpy(W.data() + Wg.size(), Wu.data(), Wu.size() * sizeof(float)); + std::vector ref((size_t)N, 0.0f); + for (int n = 0; n < N; n++) + for (int k = 0; k < K; k++) ref[(size_t)n] += x[(size_t)k] * W[(size_t)n * K + k]; + std::vector p; + onebit::npu::act_outlier_pick(x.data(), 1, K, K, 40.0f, 16, p); + CHECK(p.size() == 1 && p[0].ch == 777, "fused: the split finds channel 777 at tau 40"); + std::vector xz = x; + for (const auto& e : p) xz[(size_t)e.ch] = 0.0f; + std::vector colg((size_t)R), colu((size_t)R); + for (int n = 0; n < R; n++) { colg[(size_t)n] = Wg[(size_t)n * K + 777]; colu[(size_t)n] = Wu[(size_t)n * K + 777]; } + std::vector plain, split, wrong; + gemm_i8_row(x, W, K, N, plain); + gemm_i8_row(xz, W, K, N, split); + wrong = split; + onebit::npu::act_outlier_correct(split.data(), N, p, 0, R, [&](int) { return colg.data(); }); + onebit::npu::act_outlier_correct(split.data(), N, p, R, R, [&](int) { return colu.data(); }); + onebit::npu::act_outlier_correct(wrong.data(), N, p, 0, R, [&](int) { return colg.data(); }); + onebit::npu::act_outlier_correct(wrong.data(), N, p, 0, R, [&](int) { return colu.data(); }); // up at the WRONG offset + const double ep = rel_err(plain, ref), es = rel_err(split, ref), ew = rel_err(wrong, ref); + std::printf(" fused gate|up vs f32: plain %.4f, split %.4f, up-at-offset-0 %.4f\n", ep, es, ew); + CHECK(es < 0.02, "fused: split rel_err %.4f is not at int8 noise", es); + CHECK(es * 5.0 < ep, "fused: split (%.4f) is not 5x better than plain (%.4f)", es, ep); + CHECK(ew > 10.0 * es, "fused: the up half at offset 0 (%.4f) must be far worse than the right offset (%.4f)", ew, es); + // out_off past N is a no-op, not a write past the row + std::vector guard = split; + onebit::npu::act_outlier_correct(guard.data(), N, p, N, R, [&](int) { return colu.data(); }); + CHECK(guard == split, "fused: an offset at N writes nothing"); +} + int main() { test_pick(); test_arithmetic(); test_gather(); + test_fused_gate_up(); if (failures) { std::fprintf(stderr, "act_outliers_test: %d failure(s)\n", failures); return 1;