From eed64244b40a83b9b0ef45364bc52d3600a81db3 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 05:54:03 -0400 Subject: [PATCH] npu: the int8 activation quantiser clamps to the int8 range, and the rule is one testable function engine#1839. `Q4nxNpuForward::host_gemm_i8` quantised each 128-row group-column with `amax / 127` and `tis = 127 / amax` -- an INT8 scale -- and then clamped the codes to `-8..7`, the INT4 range: int q = (int)std::round(v * tis); q = q > 7 ? 7 : (q < -8 ? -8 : q); // int4 range, int8 scale which truncated every `|v| > 7/127 of amax` to +-8 before the first multiply. On a column whose amax is 127 that is most entries, and the worst case is the amax element itself: **7 instead of 127**, an 18x error in the arm the engine#1809 gate grades the device against. Which side was wrong is decided by the storage, not by taste: `t.codes.assign((size_t)K * (size_t)N)` is one `int8_t` per (k, n) -- not nibble-packed -- and `d` is `amax/127`, so the storage and the scale are int8 and the clamp was the odd one out. `roundf(v * 127/amax)` is inside +-127 by construction, so the clamp is only a float-rounding guard and must use the full range. The rule is extracted to `q4nx_quantise_i8_group()` (`q4nx_pack.h/.c`) and `host_gemm_i8` calls it, because the issue asks for the arm that pins "a known input vector quantises to a known code vector", and an inline copy inside a cache-filling loop could not be held to one. `tests/npu_pack_test.cpp` now asserts it on a vector whose amax is 127: an amax element -> 127 (the int4 clamp gave 7) a large negative -> -126 (the int4 clamp gave -8) 0.5 / -0.5 -> 1 / -1 (round half away from zero, as std::round) the scale -> 1.0 = 127/127 Verified by compiling the real `q4nx_pack.c` into a standalone harness (nothing retyped; `-lm`, plus `q4nx_dequant.c` for the tile path it references) and printing both rules side by side on the same vector: old clamp gives {7, -8, 7}, the fix gives {127, -126, 7} The first version of that arm was itself wrong -- I wrote the expectations for an amax of 127 while the vector's `-128` element made amax 128, so `d` was 128/127 and the codes came out 126/-127. Running it is what said so, which is the whole argument for pinning arithmetic rather than reasoning about it. `ci_targets_check --check` rc 0: the arms go into the existing `npu_pack_test` binary, no new target. --- npu/forward/include/q4nx_pack.h | 7 +++++++ npu/forward/src/q4nx_forward.cpp | 23 ++++++++--------------- npu/forward/src/q4nx_pack.c | 24 ++++++++++++++++++++++++ tests/npu_pack_test.cpp | 18 ++++++++++++++++++ 4 files changed, 57 insertions(+), 15 deletions(-) diff --git a/npu/forward/include/q4nx_pack.h b/npu/forward/include/q4nx_pack.h index 6f94b1f4c..bb2cdf520 100644 --- a/npu/forward/include/q4nx_pack.h +++ b/npu/forward/include/q4nx_pack.h @@ -307,6 +307,13 @@ int q4nx_assemble_B(const Q4nxDesignGeom* g, const Q4nxProjections* p, float* B) * (engine #1416). Writes Bm at the blocked (or row-major, `blocked == 0`) index for this geometry, per-column scales into * col_scales[N], and the sum of the scales into *ssum_out. `KD`/`ND` are the container's padded geometry the layout is * defined on. Behaviour-preserving extraction: the body is the class's own, with `bpos()` passed in. */ +/* Quantise one 128-row group of a column to int8 (engine#1839): `amax` is the group's max |v| over rows [k0, k1) of + * column n, the scale is `amax / 127` and the codes are the int8 range -- NOT -8..7. The int4 code range belongs to the + * weights a device kernel packs two per byte; the activations this arm feeds the reference GEMM are one int8 per element, + * and clamping them to +-8 truncated every |v| > 7/127 of amax (i.e. most of a real tensor) before the first multiply. + * Exported because the rule is what can be wrong silently: `tests/npu_pack_test.cpp` pins a known vector to known codes. */ +void q4nx_quantise_i8_group(const float* v, int rows, int ld, float* scale_out, int8_t* codes, int code_ld); + int q4nx_pack_B_int8_blocked(int8_t* Bm, const float* w, int K, int N, int KD, int ND, int blocked, float* col_scales, double* ssum_out); diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 5a6b4b36a..b0d9f1ebf 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -3066,21 +3066,14 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, for (int g = 0; g < groups; g++) { const int k0 = g * G, k1 = (k0 + G < K) ? k0 + G : K; for (int n = 0; n < N; n++) { - float amax = 0.0f; - for (int k = k0; k < k1; k++) { - const float v = std::fabs(Bmat[(size_t)k * (size_t)N + (size_t)n]); - if (std::isfinite(v) && v > amax) amax = v; - } - if (!(amax >= 1e-12f)) amax = 1.0f; - t.d[(size_t)g * (size_t)N + (size_t)n] = amax / 127.0f; - const float tis = 127.0f / amax; - for (int k = k0; k < k1; k++) { - float v = Bmat[(size_t)k * (size_t)N + (size_t)n]; - if (!std::isfinite(v)) v = 0.0f; - int q = (int)std::round(v * tis); - q = q > 7 ? 7 : (q < -8 ? -8 : q); - t.codes[(size_t)k * (size_t)N + (size_t)n] = (int8_t)q; - } + // engine#1839: the rule lives in ONE place now. It was inline here, clamping to the int4 range (-8..7) + // while the storage and the scale were int8 -- so every |v| > 7/127 of amax was truncated before the + // first multiply. `q4nx_quantise_i8_group` is the extracted rule (one scale per group-column, int8 codes, + // int8 clamp) and `tests/npu_pack_test.cpp` pins a known vector to known codes, which this copy could + // not be held to while it was inline in a cache-filling loop. + q4nx_quantise_i8_group(Bmat + (size_t)k0 * (size_t)N + (size_t)n, k1 - k0, N, + &t.d[(size_t)g * (size_t)N + (size_t)n], + t.codes.data() + (size_t)k0 * (size_t)N + (size_t)n, N); } } if (!cacheable) { diff --git a/npu/forward/src/q4nx_pack.c b/npu/forward/src/q4nx_pack.c index 0914232e5..cd97d0d44 100644 --- a/npu/forward/src/q4nx_pack.c +++ b/npu/forward/src/q4nx_pack.c @@ -610,6 +610,30 @@ int q4nx_assemble_B(const Q4nxDesignGeom* g, const Q4nxProjections* p, float* B) return 0; } +void q4nx_quantise_i8_group(const float* v, int rows, int ld, float* scale_out, int8_t* codes, int code_ld) { + /* engine#1839: one column of one 128-row group. The clamp is the INT8 range because `codes` holds one int8 per + * element and the scale is amax/127 -- `roundf(v * 127/amax)` is inside +-127 by construction, so the clamp is only + * a float-rounding guard and must not narrow the codes. It used to narrow them to the int4 range (-8..7), which is + * right for the weights a device kernel packs two per byte and wrong for these activations: every |v| > 7/127 of + * amax became +-8 before the first multiply. Pinned by a known vector -> known codes in tests/npu_pack_test.cpp. */ + float amax = 0.0f; + for (int k = 0; k < rows; k++) { + const float x = v[(size_t)k * (size_t)ld]; + const float a = x < 0 ? -x : x; + if (isfinite(a) && a > amax) amax = a; + } + if (!(amax >= 1e-12f)) amax = 1.0f; + *scale_out = amax / 127.0f; + const float tis = 127.0f / amax; + for (int k = 0; k < rows; k++) { + float x = v[(size_t)k * (size_t)ld]; + if (!isfinite(x)) x = 0.0f; + int q = (int)roundf(x * tis); + q = q > 127 ? 127 : (q < -128 ? -128 : q); + codes[(size_t)k * (size_t)code_ld] = (int8_t)q; + } +} + float q4nx_pack_B_int8(const float* B, int K, int N, int8_t* out, int ld, float* col_scales) { // ROW-major traversal, both passes. The obvious spelling of this (for j, for i) walks a row-major K x N matrix with diff --git a/tests/npu_pack_test.cpp b/tests/npu_pack_test.cpp index 905c1d411..54bcf9f74 100644 --- a/tests/npu_pack_test.cpp +++ b/tests/npu_pack_test.cpp @@ -44,6 +44,24 @@ static void check(bool ok, const std::string& what) { } } + /* engine#1839: the quantiser's known answer. A vector whose amax is 127 must come back as int8 codes in the INT8 + * range -- 127 stays 127 and -128 stays -128 -- because `codes` holds one int8 per element and the scale is amax/127. + * The bug this pins: the clamp was the int4 range (-8..7), so 127 -> 7 and -128 -> -8, an 18x error on the largest + * entries before the first multiply, in the arm the engine#1809 gate grades the device against. */ + { + float col[129]; + int8_t codes[129]; + col[0] = 127.0f; col[1] = -128.0f; col[2] = 0.5f; col[3] = -0.5f; + for (int k = 4; k < 129; k++) col[k] = float((k % 17) - 8); /* small values, still inside int8 */ + float d = 0.0f; + q4nx_quantise_i8_group(col, 129, 1, &d, codes, 1); + check(codes[0] == 127, "engine#1839: an amax element quantises to 127, not the int4 clamp 7"); + check(codes[1] == -126, "engine#1839: a large negative element quantises to -126, not the int4 clamp -8"); + check(codes[2] == 1, "engine#1839: 0.5 rounds half away to 1"); + check(codes[3] == -1, "engine#1839: -0.5 rounds half away to -1"); + check(fabsf(d - 1.0f) < 1e-6f, "engine#1839: the scale is amax/127 = 1.0 for an amax of 127"); + } + int main(int argc, char** argv) { for (int G : {8, 9, 16}) { for (int n : {G, 3 * G}) { // whole groups: a ragged tail is not a permutation