diff --git a/npu/forward/src/q4nx_pack_blocked.cpp b/npu/forward/src/q4nx_pack_blocked.cpp index 6f045632b..b6813db02 100644 --- a/npu/forward/src/q4nx_pack_blocked.cpp +++ b/npu/forward/src/q4nx_pack_blocked.cpp @@ -37,6 +37,21 @@ inline size_t bpos(size_t i, size_t j, int KD, int ND, int blocked) { : q4nx_layout::rowmajor_index(i, j, KD, ND); } +// One code: roundf(v * tis) clamped to [-127, 127], a non-finite v as 0 -- the expression this packer has always used, +// written so the loop vectorises (engine#1783: std::roundf and std::isfinite kept it scalar, ~3 ns per element, and a +// 1536 x 1536 activation is packed on every attention call). Exact, not approximate: y = v * tis is finite with +// |y| <= 127 (v is bounded by its column's amax, or zeroed), so the truncation t fits an int and y - t is exact; rounding +// half away from zero is then t + 1 for a fraction >= 0.5 and t - 1 for one <= -0.5. `v - v == 0` holds for finite v only. +inline int8_t quant_code(float v, float tis) { + const float y = (v - v == 0.0f) ? v * tis : 0.0f; + int x = (int)y; + const float f = y - (float)x; + x += (f >= 0.5f) - (f <= -0.5f); + x = x > 127 ? 127 : x; + x = x < -127 ? -127 : x; + return (int8_t)x; +} + } // namespace extern "C" int q4nx_pack_B_int8_blocked(int8_t* Bm, const float* w, int K, int N, int KD, int ND, int blocked, @@ -52,14 +67,25 @@ extern "C" int q4nx_pack_B_int8_blocked(int8_t* Bm, const float* w, int K, int N // The scale sum is accumulated in a serial loop over j rather than inside a `reduction(+:ssum)` parallel loop. That is // deliberate: the mean column scale becomes order-independent, where the parallel reduction left its last bits to the // schedule. Codes and scales are unchanged (the amax of a column is a max over rows, and the rounding is per element). + // Column amax in parallel over row chunks, each thread into its own array, merged after: a max is exact and + // order-independent, so the scales are the serial loop's bit for bit (engine#1783: the serial pass was most of a + // 1536 x 1536 pack once the write below was parallel). std::vector amax((size_t)N, 0.0f); std::vector tis((size_t)N, 0.0f); - for (int i = 0; i < K; i++) { - const float* row = w + (size_t)i * (size_t)N; - for (int j = 0; j < N; j++) { - const float a = std::fabs(row[j]); - if (std::isfinite(a) && a > amax[(size_t)j]) amax[(size_t)j] = a; +#pragma omp parallel + { + std::vector mine((size_t)N, 0.0f); +#pragma omp for schedule(static) nowait + for (int i = 0; i < K; i++) { + const float* row = w + (size_t)i * (size_t)N; + for (int j = 0; j < N; j++) { + const float a = std::fabs(row[j]); + if (std::isfinite(a) && a > mine[(size_t)j]) mine[(size_t)j] = a; + } } +#pragma omp critical(q4nx_pack_amax) + for (int j = 0; j < N; j++) + if (mine[(size_t)j] > amax[(size_t)j]) amax[(size_t)j] = mine[(size_t)j]; } double ssum = 0.0; for (int j = 0; j < N; j++) { @@ -69,16 +95,16 @@ extern "C" int q4nx_pack_B_int8_blocked(int8_t* Bm, const float* w, int K, int N tis[(size_t)j] = 127.0f / am; ssum += (double)(am / 127.0f); } + // The write, one address per 8-column run. Inside the blocked layout, columns j0..j0+7 of a row with j0 % 8 == 0 sit + // in one 8x8 microtile row, so they are 8 consecutive bytes from blocked_index(i, j0) (q4nx_weight_layout.h, the + // image_index/copy note); row-major is consecutive anyway. The element arithmetic is unchanged. #pragma omp parallel for schedule(static) for (int i = 0; i < K; i++) { const float* row = w + (size_t)i * (size_t)N; - for (int j = 0; j < N; j++) { - float v = row[j]; - if (!std::isfinite(v)) v = 0.0f; - int x = (int)std::roundf(v * tis[(size_t)j]); - if (x > 127) x = 127; - else if (x < -127) x = -127; - Bm[bpos((size_t)i, (size_t)j, KD, ND, blocked)] = (int8_t)x; + for (int j0 = 0; j0 < N; j0 += 8) { + int8_t* dst = Bm + bpos((size_t)i, (size_t)j0, KD, ND, blocked); + const int jn = N - j0 < 8 ? N - j0 : 8; + for (int c = 0; c < jn; c++) dst[c] = quant_code(row[j0 + c], tis[(size_t)(j0 + c)]); } } *ssum_out = ssum; diff --git a/tests/npu_blocked_layout_test.cpp b/tests/npu_blocked_layout_test.cpp index 3489594fa..2fd6fbbd1 100644 --- a/tests/npu_blocked_layout_test.cpp +++ b/tests/npu_blocked_layout_test.cpp @@ -209,6 +209,48 @@ void test_production_packer() { std::printf(" production packer: %zu codes, %d scales, blocked == bpos(rowmajor)\n", (size_t)K * N, N); } +// engine#1783: the production packer's rounding is written so it vectorises (truncate, then step on the fraction), and +// that has to be roundf's half-away-from-zero exactly. Columns whose amax is 127 make the scale exactly 1, so each value +// below reaches the rounding unchanged: every k + 0.5 and k - 0.5 tie, the floats one ulp either side of a tie, the +// +-127 clamp, and NaN / +-inf (zeroed). The row-major reference (q4nx_pack_B_int8, roundf) is the oracle. +void test_packer_rounding_edges() { + const int K = 64, N = 128; + std::vector vals; + for (int k = -127; k <= 126; ++k) { + const float tie = float(k) + 0.5f; + vals.push_back(tie); + vals.push_back(std::nextafter(tie, 0.0f)); + vals.push_back(std::nextafter(tie, tie > 0 ? 1e9f : -1e9f)); + } + for (float v : {0.0f, -0.0f, 0.49999997f, -0.49999997f, 127.0f, -127.0f, 126.99999f, -126.99999f, 1e-30f, -1e-30f}) + vals.push_back(v); + vals.push_back(0.0f / 0.0f); + vals.push_back(1.0f / 0.0f); + vals.push_back(-1.0f / 0.0f); + std::vector w((size_t)K * N, 0.0f); + for (int j = 0; j < N; ++j) w[(size_t)j] = (j % 2) ? 127.0f : -127.0f; // row 0: amax 127 in every column + size_t at = 0; + for (int i = 1; i < K && at < vals.size(); ++i) + for (int j = 0; j < N && at < vals.size(); ++j) w[(size_t)i * N + (size_t)j] = vals[at++]; + check(at == vals.size(), "packer rounding: every edge value placed"); + std::vector rowmaj((size_t)K * N), blocked((size_t)K * N); + std::vector sc_ref((size_t)N), sc_blk((size_t)N); + q4nx_pack_B_int8(w.data(), K, N, rowmaj.data(), N, sc_ref.data()); + double ssum = 0; + check(q4nx_pack_B_int8_blocked(blocked.data(), w.data(), K, N, K, N, 1, sc_blk.data(), &ssum) == 0, "packer rounding: runs"); + check(sc_blk == sc_ref, "packer rounding: scales equal the reference's"); + size_t mismatched = 0, ties_up = 0; + for (int i = 0; i < K; ++i) + for (int j = 0; j < N; ++j) { + const int8_t got = blocked[q4nx_layout::blocked_index((size_t)i, (size_t)j, K, N)]; + if (got != rowmaj[(size_t)i * N + (size_t)j]) mismatched++; + ties_up += w[(size_t)i * N + (size_t)j] == 2.5f && got == 3; + } + check(mismatched == 0, "packer rounding: every code equals roundf's, ties and neighbours included"); + check(ties_up == 1, "packer rounding: 2.5 rounds away from zero (3), not to even"); + std::printf(" packer rounding: %zu edge values, codes == roundf reference\n", vals.size()); +} + // engine#1994: the integer host arm packs UNPADDED images (KD == K, ND == N) into K*N bytes. blocked_index() addresses // whole 64x128 tiles, so a geometry that is not whole tiles has to be packed and read row-major, or both the packer and // the MAC walk past the buffer. This pins the rule the arm uses -- blocked = blocked_fits(K, N), read through @@ -291,6 +333,7 @@ int main() { } test_refusals(); test_production_packer(); + test_packer_rounding_edges(); test_row_major_unchanged(); test_pack_geometries(); test_unpadded_image(3072, 7488, false); // phi-3-mini's last vocabulary block (32064 % 8192)