From 8aca2f01ab16b7ef3d18a39f13ee0dcfa144e84d Mon Sep 17 00:00:00 2001 From: t Date: Tue, 10 Nov 2026 03:27:29 -0400 Subject: [PATCH] npu: the int4 codes and their scale are one pair, so the activation-quantiser sites stop pairing amax/127 with [-8,7] (engine#1839) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both places that quantise float weights into the container's **int4** codes used an **int8** scale: t.d = amax / 127.0f; tis = 127.0f / amax; ... q clamped to [-8, 7] so every code but the very largest saturated at +-7 and the dequantised weight came out ~18x too small. One of the two sites is `q4nx_host_gemm_i8`'s weight cache -- the host arm the device gate compares against -- which is the worst place for it: a wrong quantisation there is folded into every activation before the first multiply, so a correlation number averages over it instead of failing on it. Measured on the old pairing: v = 0.1 with amax = 1 quantises to **7** where int4 expects **1**. Fix: `q4nx_quant_int4_scale()` / `q4nx_quant_int4_code()` in `npu/forward/src/q4nx_pack.c` own the pair, and both sites call them, so they cannot drift apart again. The scale is `amax / 7` -- 7 rather than 8 so that no code saturates (`round(v * 7/amax)` lands in [-7, 7] for |v| <= amax, and the -8 slot stays a guard instead of an asymmetry). The activation side was already correct (`amax/127` with a [-127, 127] clamp) and is untouched. Test, folded into `tests/npu_pack_test.cpp` (that binary is already built by ci.yml; a new target would need a workflow edit this lane's token cannot push -- the same wall as #1795/#1799): 11 arms, and the two that matter are the ones the old pairing fails: * a **ladder**: each seventh of amax comes back as its own code (`k/7*amax -> k` for k = -7..7). Under `amax/127` every one of those saturates to +-7, so this arm distinguishes the pairings rather than asserting a property the clamp makes unfailable (an earlier "no code leaves [-8,7]" arm was exactly that and was replaced); * a **known value**: `0.1` at `amax=1` is code 1 (the 127 scale gives 7). I also proved the arms can fail: injecting `== 7` on the known-value arm made it report `FAIL int4 pair: a small value is 1, not saturated` and exit FAILED, then the revert restored `ok (pass …)`. **Numbers change.** Anything measured through `NPU_INFER_HOST_GEMM=i8` before this carries the 18x weight error; the device route is unaffected (it reads the container's own codes and scales). Re-measuring that arm is the next step on the issue, and no device measurement in this lane's F4 work (#1795, #1807) went through this path. Suite 172/172; `tools/copyright.py --check` clean. Refs #1839. --- npu/forward/include/q4nx_pack.h | 11 ++++++++ npu/forward/src/q4nx_forward.cpp | 19 +++++++------- npu/forward/src/q4nx_pack.c | 26 +++++++++++++------ tests/npu_pack_test.cpp | 43 ++++++++++++++++++++++++++++++++ 4 files changed, 82 insertions(+), 17 deletions(-) diff --git a/npu/forward/include/q4nx_pack.h b/npu/forward/include/q4nx_pack.h index 6f94b1f4c..70dd13248 100644 --- a/npu/forward/include/q4nx_pack.h +++ b/npu/forward/include/q4nx_pack.h @@ -177,6 +177,17 @@ void q4nx_host_gemm_f32_padded(const float* A, int a_k, int M, int K, int N, con * holding an int4 value in [-8, 7], exactly as the importer's packB lays it down. */ void q4nx_host_gemm_i8(const int8_t* a8, int K, const int8_t* codes, int N, int32_t* out32); +/* engine#1839: the int4 code range and its scale, together, because they are only correct as a pair. + * + * A symmetric int4 code spans [-8, 7], so the scale that maps a group's `amax` onto that range is `amax / 7`, and the + * code is `round(v * 7 / amax)` clamped to [-8, 7]. The two places that quantise float weights into the container's + * int4 codes used to pair `amax / 127` (an INT8 scale) with the int4 clamp, so every code but the very largest + * saturated at +-7 and the dequantised weight was ~18x too small -- in the host arm the gate compares the device + * against, which is the worst place for it. 7 rather than 8 so that no code saturates: round(v * 7 / amax) lands in + * [-7, 7] for |v| <= amax, and the -8 slot stays a guard rather than an asymmetry. */ +float q4nx_quant_int4_scale(float amax); +int8_t q4nx_quant_int4_code(float v, float amax); + /* The whole GEMM in int8 x int4 for the gate's `i8` host arm, quantising both sides here with the engine's * own rule. It cannot see a wrong weight LAYOUT (the f32 arm covers that); what it can see is a wrong int4 * quantisation, a wrong 128-K group boundary, an int8 intermediate or a saturated accumulator -- i.e. the diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 95ec35e94..34550ea89 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -3056,16 +3056,15 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, const float v = std::fabs(Bmat[(size_t)k * (size_t)N + (size_t)n]); if (std::isfinite(v) && v > amax) amax = v; } - if (!(amax >= 1e-12f)) amax = 1.0f; - t.d[(size_t)g * (size_t)N + (size_t)n] = amax / 127.0f; - const float tis = 127.0f / amax; - for (int k = k0; k < k1; k++) { - float v = Bmat[(size_t)k * (size_t)N + (size_t)n]; - if (!std::isfinite(v)) v = 0.0f; - int q = (int)std::round(v * tis); - q = q > 7 ? 7 : (q < -8 ? -8 : q); - t.codes[(size_t)k * (size_t)N + (size_t)n] = (int8_t)q; - } + /* engine#1839: the weights here are the container's INT4 codes, so the scale that maps amax onto + * that range is amax/7 -- pairing amax/127 (an int8 scale) with the [-8, 7] clamp saturated every code + * but the largest and made the host reference's weights ~18x too small. q4nx_quant_int4_*() own the + * pair so this site and q4nx_pack.c's cannot drift apart again. */ + const float amax_eff = (amax >= 1e-12f) ? amax : 1.0f; + t.d[(size_t)g * (size_t)N + (size_t)n] = q4nx_quant_int4_scale(amax_eff); + for (int k = k0; k < k1; k++) + t.codes[(size_t)k * (size_t)N + (size_t)n] = + q4nx_quant_int4_code(Bmat[(size_t)k * (size_t)N + (size_t)n], amax_eff); } } it = host_i8_cache_.emplace(wkey, std::move(t)).first; diff --git a/npu/forward/src/q4nx_pack.c b/npu/forward/src/q4nx_pack.c index 0914232e5..533996f52 100644 --- a/npu/forward/src/q4nx_pack.c +++ b/npu/forward/src/q4nx_pack.c @@ -194,14 +194,12 @@ void q4nx_host_gemm_i8_from_f32(const float* A, int a_k, int M, int K, int N, co const float v = fabsf(Bmat[(size_t)k * (size_t)N + (size_t)n]); if (v > amax && isfinite(v)) amax = v; } - if (!(amax >= 1e-12f)) amax = 1.0f; - const float tis = 127.0f / amax; + /* engine#1839: the scale and the clamp describe the SAME range -- int4, [-8, 7] -- or the codes + * saturate and the dequantised weight is ~18x too small. q4nx_quant_int4_*() own the pair. */ + const float amax_eff = (amax >= 1e-12f) ? amax : 1.0f; for (int k = k0; k < k1; k++) { - float v = Bmat[(size_t)k * (size_t)N + (size_t)n]; - if (!isfinite(v)) v = 0.0f; - int q = (int)roundf(v * tis); - if (q > 7) q = 7; - else if (q < -8) q = -8; /* int4 codes are [-8, 7] in this format */ + const float v = Bmat[(size_t)k * (size_t)N + (size_t)n]; + const int q = q4nx_quant_int4_code(v, amax_eff); int aq = (int)roundf(a[k] * 127.0f); if (aq > 127) aq = 127; else if (aq < -127) aq = -127; @@ -212,6 +210,20 @@ void q4nx_host_gemm_i8_from_f32(const float* A, int a_k, int M, int K, int N, co } } +float q4nx_quant_int4_scale(float amax) { + if (!(amax >= 1e-12f) || !isfinite(amax)) return 1.0f; /* an all-zero group still needs a usable scale */ + return amax / 7.0f; +} + +int8_t q4nx_quant_int4_code(float v, float amax) { + if (!isfinite(v)) v = 0.0f; + const float s = q4nx_quant_int4_scale(amax); + int q = (int)roundf(v / s); + if (q > 7) q = 7; + else if (q < -8) q = -8; + return (int8_t)q; +} + void q4nx_host_gemm_i8(const int8_t* a8, int K, const int8_t* codes, int N, int32_t* out32) { for (int n = 0; n < N; n++) out32[n] = 0; for (int k = 0; k < K; k++) { diff --git a/tests/npu_pack_test.cpp b/tests/npu_pack_test.cpp index 905c1d411..a544d9601 100644 --- a/tests/npu_pack_test.cpp +++ b/tests/npu_pack_test.cpp @@ -44,7 +44,50 @@ static void check(bool ok, const std::string& what) { } } +/* engine#1839: an int4 code and its scale are only correct as a PAIR. The defect this pins: both sites that quantise + * float weights into the container's int4 codes used `amax / 127` (an INT8 scale) with the int4 clamp [-8, 7], so every + * code but the very largest saturated and the dequantised weight was ~18x too small — in the host arm the device gate + * compares against. The arms below are a known input -> known code (not a property any pairing satisfies), the no- + * saturation property at full range, and the two degenerate cases the floor exists for. */ +static void quant_int4_arms() { + const float amax = 7.0f; + check(q4nx_quant_int4_scale(7.0f) == 1.0f, "int4 pair: amax 7 takes scale 1 (so the codes ARE the values)"); + check(q4nx_quant_int4_code(7.0f, amax) == 7, "int4 pair: +amax quantises to +7"); + check(q4nx_quant_int4_code(-7.0f, amax) == -7, "int4 pair: -amax quantises to -7"); + check(q4nx_quant_int4_code(0.0f, amax) == 0, "int4 pair: zero is zero"); + check(q4nx_quant_int4_code(3.5f, amax) == 4, "int4 pair: half of amax rounds to 4 (round-half-away)"); + /* The arm the old pairing fails: at amax/127 a mid-range value saturates. */ + check(q4nx_quant_int4_code(0.1f, 1.0f) == 1, "int4 pair: a small value is 1, not saturated — the 127 scale gives 13"); + /* THE LADDER: the sevenths of amax must come back as their own code number. This is the arm that can fail for the + * wrong scale -- under amax/127 every one of these saturates to +-7 (checked: v=0.1 at amax=1 gives 7 where int4 + * expects 1), so it distinguishes the pairings rather than asserting a property the clamp makes unfailable. */ + { + const float amax_l = 3.25f; + int wrong = 0; + for (int k = -7; k <= 7; k++) { + const float v = (float)k / 7.0f * amax_l; + if (q4nx_quant_int4_code(v, amax_l) != k) wrong++; + } + check(wrong == 0, "int4 pair: each seventh of amax comes back as its own code (k/7*amax -> k, k=-7..7)"); + } + /* Degenerate groups: the floor keeps the scale usable and the code zero. */ + check(q4nx_quant_int4_scale(0.0f) == 1.0f, "int4 pair: an all-zero group floors the scale at 1"); + check(q4nx_quant_int4_code(0.0f, 0.0f) == 0, "int4 pair: and its code is 0, not a division by zero"); + check(q4nx_quant_int4_code(NAN, 1.0f) == 0, "int4 pair: a NaN value quantises to 0 rather than an undefined code"); + /* The dequantised value round-trips inside half a step: that is the accuracy the kernel's epilogue assumes. */ + int worst_ok = 1; + for (int i = -127; i <= 127; i++) { + const float v = (float)i / 127.0f * 2.0f; /* amax 2.0 */ + const float s = q4nx_quant_int4_scale(2.0f); + const float back = (float)q4nx_quant_int4_code(v, 2.0f) * s; + if (std::fabs(back - v) > s / 2.0f + 1e-6f) worst_ok = 0; + } + check(worst_ok, "int4 pair: every value round-trips within half a step of amax/7"); + std::printf(" int4 pair: 11 arms (known codes, ladder, floor, NaN, round-trip)\n"); +} + int main(int argc, char** argv) { + quant_int4_arms(); for (int G : {8, 9, 16}) { for (int n : {G, 3 * G}) { // whole groups: a ragged tail is not a permutation // Tile i holds the value i in its first word; after the reorder every