From 2add34c6bdb3fe3b57f5517ecd6bf2c038a393bc Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Fri, 13 Nov 2026 07:10:03 -0400 Subject: [PATCH] npu pack: the blocked int8 packer vectorises and runs its amax pass in parallel, bit-identical (#1783) q4nx_pack_B_int8_blocked packs every weight into the NPU's b8 image, and on the encoder route it also packs the activation operands of the attention products on every call. Two things kept it slow: the column amax pass was serial, and std::roundf / std::isfinite kept the quantise loop scalar. - amax runs over row chunks in parallel, per-thread arrays merged after (a max is exact and order-independent); - the code is computed by truncation plus a step on the exact fraction, which is roundf's half-away-from-zero for |y| <= 127 and vectorises; `v - v == 0` replaces isfinite; - one blocked_index per 8-column microtile run instead of per element. Output is unchanged: codes, scales and the scale sum equal the previous packer's on 20 shapes x 2 magnitudes, 1/8/16 threads. A new arm of npu_blocked_layout_test pins the rounding against the row-major reference on every k +- 0.5 tie and its float neighbours, the +-127 clamp and NaN/inf; a ties-toward-zero mutant passes the old test and fails this one. Co-Authored-By: Claude Opus 5.5 --- npu/forward/src/q4nx_pack_blocked.cpp | 50 ++++++++++++++++++++------- tests/npu_blocked_layout_test.cpp | 43 +++++++++++++++++++++++ 2 files changed, 81 insertions(+), 12 deletions(-) diff --git a/npu/forward/src/q4nx_pack_blocked.cpp b/npu/forward/src/q4nx_pack_blocked.cpp index 6f045632b..b6813db02 100644 --- a/npu/forward/src/q4nx_pack_blocked.cpp +++ b/npu/forward/src/q4nx_pack_blocked.cpp @@ -37,6 +37,21 @@ inline size_t bpos(size_t i, size_t j, int KD, int ND, int blocked) { : q4nx_layout::rowmajor_index(i, j, KD, ND); } +// One code: roundf(v * tis) clamped to [-127, 127], a non-finite v as 0 -- the expression this packer has always used, +// written so the loop vectorises (engine#1783: std::roundf and std::isfinite kept it scalar, ~3 ns per element, and a +// 1536 x 1536 activation is packed on every attention call). Exact, not approximate: y = v * tis is finite with +// |y| <= 127 (v is bounded by its column's amax, or zeroed), so the truncation t fits an int and y - t is exact; rounding +// half away from zero is then t + 1 for a fraction >= 0.5 and t - 1 for one <= -0.5. `v - v == 0` holds for finite v only. +inline int8_t quant_code(float v, float tis) { + const float y = (v - v == 0.0f) ? v * tis : 0.0f; + int x = (int)y; + const float f = y - (float)x; + x += (f >= 0.5f) - (f <= -0.5f); + x = x > 127 ? 127 : x; + x = x < -127 ? -127 : x; + return (int8_t)x; +} + } // namespace extern "C" int q4nx_pack_B_int8_blocked(int8_t* Bm, const float* w, int K, int N, int KD, int ND, int blocked, @@ -52,14 +67,25 @@ extern "C" int q4nx_pack_B_int8_blocked(int8_t* Bm, const float* w, int K, int N // The scale sum is accumulated in a serial loop over j rather than inside a `reduction(+:ssum)` parallel loop. That is // deliberate: the mean column scale becomes order-independent, where the parallel reduction left its last bits to the // schedule. Codes and scales are unchanged (the amax of a column is a max over rows, and the rounding is per element). + // Column amax in parallel over row chunks, each thread into its own array, merged after: a max is exact and + // order-independent, so the scales are the serial loop's bit for bit (engine#1783: the serial pass was most of a + // 1536 x 1536 pack once the write below was parallel). std::vector amax((size_t)N, 0.0f); std::vector tis((size_t)N, 0.0f); - for (int i = 0; i < K; i++) { - const float* row = w + (size_t)i * (size_t)N; - for (int j = 0; j < N; j++) { - const float a = std::fabs(row[j]); - if (std::isfinite(a) && a > amax[(size_t)j]) amax[(size_t)j] = a; +#pragma omp parallel + { + std::vector mine((size_t)N, 0.0f); +#pragma omp for schedule(static) nowait + for (int i = 0; i < K; i++) { + const float* row = w + (size_t)i * (size_t)N; + for (int j = 0; j < N; j++) { + const float a = std::fabs(row[j]); + if (std::isfinite(a) && a > mine[(size_t)j]) mine[(size_t)j] = a; + } } +#pragma omp critical(q4nx_pack_amax) + for (int j = 0; j < N; j++) + if (mine[(size_t)j] > amax[(size_t)j]) amax[(size_t)j] = mine[(size_t)j]; } double ssum = 0.0; for (int j = 0; j < N; j++) { @@ -69,16 +95,16 @@ extern "C" int q4nx_pack_B_int8_blocked(int8_t* Bm, const float* w, int K, int N tis[(size_t)j] = 127.0f / am; ssum += (double)(am / 127.0f); } + // The write, one address per 8-column run. Inside the blocked layout, columns j0..j0+7 of a row with j0 % 8 == 0 sit + // in one 8x8 microtile row, so they are 8 consecutive bytes from blocked_index(i, j0) (q4nx_weight_layout.h, the + // image_index/copy note); row-major is consecutive anyway. The element arithmetic is unchanged. #pragma omp parallel for schedule(static) for (int i = 0; i < K; i++) { const float* row = w + (size_t)i * (size_t)N; - for (int j = 0; j < N; j++) { - float v = row[j]; - if (!std::isfinite(v)) v = 0.0f; - int x = (int)std::roundf(v * tis[(size_t)j]); - if (x > 127) x = 127; - else if (x < -127) x = -127; - Bm[bpos((size_t)i, (size_t)j, KD, ND, blocked)] = (int8_t)x; + for (int j0 = 0; j0 < N; j0 += 8) { + int8_t* dst = Bm + bpos((size_t)i, (size_t)j0, KD, ND, blocked); + const int jn = N - j0 < 8 ? N - j0 : 8; + for (int c = 0; c < jn; c++) dst[c] = quant_code(row[j0 + c], tis[(size_t)(j0 + c)]); } } *ssum_out = ssum; diff --git a/tests/npu_blocked_layout_test.cpp b/tests/npu_blocked_layout_test.cpp index 3489594fa..2fd6fbbd1 100644 --- a/tests/npu_blocked_layout_test.cpp +++ b/tests/npu_blocked_layout_test.cpp @@ -209,6 +209,48 @@ void test_production_packer() { std::printf(" production packer: %zu codes, %d scales, blocked == bpos(rowmajor)\n", (size_t)K * N, N); } +// engine#1783: the production packer's rounding is written so it vectorises (truncate, then step on the fraction), and +// that has to be roundf's half-away-from-zero exactly. Columns whose amax is 127 make the scale exactly 1, so each value +// below reaches the rounding unchanged: every k + 0.5 and k - 0.5 tie, the floats one ulp either side of a tie, the +// +-127 clamp, and NaN / +-inf (zeroed). The row-major reference (q4nx_pack_B_int8, roundf) is the oracle. +void test_packer_rounding_edges() { + const int K = 64, N = 128; + std::vector vals; + for (int k = -127; k <= 126; ++k) { + const float tie = float(k) + 0.5f; + vals.push_back(tie); + vals.push_back(std::nextafter(tie, 0.0f)); + vals.push_back(std::nextafter(tie, tie > 0 ? 1e9f : -1e9f)); + } + for (float v : {0.0f, -0.0f, 0.49999997f, -0.49999997f, 127.0f, -127.0f, 126.99999f, -126.99999f, 1e-30f, -1e-30f}) + vals.push_back(v); + vals.push_back(0.0f / 0.0f); + vals.push_back(1.0f / 0.0f); + vals.push_back(-1.0f / 0.0f); + std::vector w((size_t)K * N, 0.0f); + for (int j = 0; j < N; ++j) w[(size_t)j] = (j % 2) ? 127.0f : -127.0f; // row 0: amax 127 in every column + size_t at = 0; + for (int i = 1; i < K && at < vals.size(); ++i) + for (int j = 0; j < N && at < vals.size(); ++j) w[(size_t)i * N + (size_t)j] = vals[at++]; + check(at == vals.size(), "packer rounding: every edge value placed"); + std::vector rowmaj((size_t)K * N), blocked((size_t)K * N); + std::vector sc_ref((size_t)N), sc_blk((size_t)N); + q4nx_pack_B_int8(w.data(), K, N, rowmaj.data(), N, sc_ref.data()); + double ssum = 0; + check(q4nx_pack_B_int8_blocked(blocked.data(), w.data(), K, N, K, N, 1, sc_blk.data(), &ssum) == 0, "packer rounding: runs"); + check(sc_blk == sc_ref, "packer rounding: scales equal the reference's"); + size_t mismatched = 0, ties_up = 0; + for (int i = 0; i < K; ++i) + for (int j = 0; j < N; ++j) { + const int8_t got = blocked[q4nx_layout::blocked_index((size_t)i, (size_t)j, K, N)]; + if (got != rowmaj[(size_t)i * N + (size_t)j]) mismatched++; + ties_up += w[(size_t)i * N + (size_t)j] == 2.5f && got == 3; + } + check(mismatched == 0, "packer rounding: every code equals roundf's, ties and neighbours included"); + check(ties_up == 1, "packer rounding: 2.5 rounds away from zero (3), not to even"); + std::printf(" packer rounding: %zu edge values, codes == roundf reference\n", vals.size()); +} + // engine#1994: the integer host arm packs UNPADDED images (KD == K, ND == N) into K*N bytes. blocked_index() addresses // whole 64x128 tiles, so a geometry that is not whole tiles has to be packed and read row-major, or both the packer and // the MAC walk past the buffer. This pins the rule the arm uses -- blocked = blocked_fits(K, N), read through @@ -291,6 +333,7 @@ int main() { } test_refusals(); test_production_packer(); + test_packer_rounding_edges(); test_row_major_unchanged(); test_pack_geometries(); test_unpadded_image(3072, 7488, false); // phi-3-mini's last vocabulary block (32064 % 8192)