diff --git a/npu/forward/include/q4nx_pack.h b/npu/forward/include/q4nx_pack.h index 6f94b1f4c..70dd13248 100644 --- a/npu/forward/include/q4nx_pack.h +++ b/npu/forward/include/q4nx_pack.h @@ -177,6 +177,17 @@ void q4nx_host_gemm_f32_padded(const float* A, int a_k, int M, int K, int N, con * holding an int4 value in [-8, 7], exactly as the importer's packB lays it down. */ void q4nx_host_gemm_i8(const int8_t* a8, int K, const int8_t* codes, int N, int32_t* out32); +/* engine#1839: the int4 code range and its scale, together, because they are only correct as a pair. + * + * A symmetric int4 code spans [-8, 7], so the scale that maps a group's `amax` onto that range is `amax / 7`, and the + * code is `round(v * 7 / amax)` clamped to [-8, 7]. The two places that quantise float weights into the container's + * int4 codes used to pair `amax / 127` (an INT8 scale) with the int4 clamp, so every code but the very largest + * saturated at +-7 and the dequantised weight was ~18x too small -- in the host arm the gate compares the device + * against, which is the worst place for it. 7 rather than 8 so that no code saturates: round(v * 7 / amax) lands in + * [-7, 7] for |v| <= amax, and the -8 slot stays a guard rather than an asymmetry. */ +float q4nx_quant_int4_scale(float amax); +int8_t q4nx_quant_int4_code(float v, float amax); + /* The whole GEMM in int8 x int4 for the gate's `i8` host arm, quantising both sides here with the engine's * own rule. It cannot see a wrong weight LAYOUT (the f32 arm covers that); what it can see is a wrong int4 * quantisation, a wrong 128-K group boundary, an int8 intermediate or a saturated accumulator -- i.e. the diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 5a6b4b36a..a8e503162 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -3071,16 +3071,15 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, const float v = std::fabs(Bmat[(size_t)k * (size_t)N + (size_t)n]); if (std::isfinite(v) && v > amax) amax = v; } - if (!(amax >= 1e-12f)) amax = 1.0f; - t.d[(size_t)g * (size_t)N + (size_t)n] = amax / 127.0f; - const float tis = 127.0f / amax; - for (int k = k0; k < k1; k++) { - float v = Bmat[(size_t)k * (size_t)N + (size_t)n]; - if (!std::isfinite(v)) v = 0.0f; - int q = (int)std::round(v * tis); - q = q > 7 ? 7 : (q < -8 ? -8 : q); - t.codes[(size_t)k * (size_t)N + (size_t)n] = (int8_t)q; - } + /* engine#1839: the weights here are the container's INT4 codes, so the scale that maps amax onto + * that range is amax/7 -- pairing amax/127 (an int8 scale) with the [-8, 7] clamp saturated every code + * but the largest and made the host reference's weights ~18x too small. q4nx_quant_int4_*() own the + * pair so this site and q4nx_pack.c's cannot drift apart again. */ + const float amax_eff = (amax >= 1e-12f) ? amax : 1.0f; + t.d[(size_t)g * (size_t)N + (size_t)n] = q4nx_quant_int4_scale(amax_eff); + for (int k = k0; k < k1; k++) + t.codes[(size_t)k * (size_t)N + (size_t)n] = + q4nx_quant_int4_code(Bmat[(size_t)k * (size_t)N + (size_t)n], amax_eff); } } if (!cacheable) { diff --git a/npu/forward/src/q4nx_pack.c b/npu/forward/src/q4nx_pack.c index 0914232e5..533996f52 100644 --- a/npu/forward/src/q4nx_pack.c +++ b/npu/forward/src/q4nx_pack.c @@ -194,14 +194,12 @@ void q4nx_host_gemm_i8_from_f32(const float* A, int a_k, int M, int K, int N, co const float v = fabsf(Bmat[(size_t)k * (size_t)N + (size_t)n]); if (v > amax && isfinite(v)) amax = v; } - if (!(amax >= 1e-12f)) amax = 1.0f; - const float tis = 127.0f / amax; + /* engine#1839: the scale and the clamp describe the SAME range -- int4, [-8, 7] -- or the codes + * saturate and the dequantised weight is ~18x too small. q4nx_quant_int4_*() own the pair. */ + const float amax_eff = (amax >= 1e-12f) ? amax : 1.0f; for (int k = k0; k < k1; k++) { - float v = Bmat[(size_t)k * (size_t)N + (size_t)n]; - if (!isfinite(v)) v = 0.0f; - int q = (int)roundf(v * tis); - if (q > 7) q = 7; - else if (q < -8) q = -8; /* int4 codes are [-8, 7] in this format */ + const float v = Bmat[(size_t)k * (size_t)N + (size_t)n]; + const int q = q4nx_quant_int4_code(v, amax_eff); int aq = (int)roundf(a[k] * 127.0f); if (aq > 127) aq = 127; else if (aq < -127) aq = -127; @@ -212,6 +210,20 @@ void q4nx_host_gemm_i8_from_f32(const float* A, int a_k, int M, int K, int N, co } } +float q4nx_quant_int4_scale(float amax) { + if (!(amax >= 1e-12f) || !isfinite(amax)) return 1.0f; /* an all-zero group still needs a usable scale */ + return amax / 7.0f; +} + +int8_t q4nx_quant_int4_code(float v, float amax) { + if (!isfinite(v)) v = 0.0f; + const float s = q4nx_quant_int4_scale(amax); + int q = (int)roundf(v / s); + if (q > 7) q = 7; + else if (q < -8) q = -8; + return (int8_t)q; +} + void q4nx_host_gemm_i8(const int8_t* a8, int K, const int8_t* codes, int N, int32_t* out32) { for (int n = 0; n < N; n++) out32[n] = 0; for (int k = 0; k < K; k++) { diff --git a/tests/npu_pack_test.cpp b/tests/npu_pack_test.cpp index 905c1d411..a544d9601 100644 --- a/tests/npu_pack_test.cpp +++ b/tests/npu_pack_test.cpp @@ -44,7 +44,50 @@ static void check(bool ok, const std::string& what) { } } +/* engine#1839: an int4 code and its scale are only correct as a PAIR. The defect this pins: both sites that quantise + * float weights into the container's int4 codes used `amax / 127` (an INT8 scale) with the int4 clamp [-8, 7], so every + * code but the very largest saturated and the dequantised weight was ~18x too small — in the host arm the device gate + * compares against. The arms below are a known input -> known code (not a property any pairing satisfies), the no- + * saturation property at full range, and the two degenerate cases the floor exists for. */ +static void quant_int4_arms() { + const float amax = 7.0f; + check(q4nx_quant_int4_scale(7.0f) == 1.0f, "int4 pair: amax 7 takes scale 1 (so the codes ARE the values)"); + check(q4nx_quant_int4_code(7.0f, amax) == 7, "int4 pair: +amax quantises to +7"); + check(q4nx_quant_int4_code(-7.0f, amax) == -7, "int4 pair: -amax quantises to -7"); + check(q4nx_quant_int4_code(0.0f, amax) == 0, "int4 pair: zero is zero"); + check(q4nx_quant_int4_code(3.5f, amax) == 4, "int4 pair: half of amax rounds to 4 (round-half-away)"); + /* The arm the old pairing fails: at amax/127 a mid-range value saturates. */ + check(q4nx_quant_int4_code(0.1f, 1.0f) == 1, "int4 pair: a small value is 1, not saturated — the 127 scale gives 13"); + /* THE LADDER: the sevenths of amax must come back as their own code number. This is the arm that can fail for the + * wrong scale -- under amax/127 every one of these saturates to +-7 (checked: v=0.1 at amax=1 gives 7 where int4 + * expects 1), so it distinguishes the pairings rather than asserting a property the clamp makes unfailable. */ + { + const float amax_l = 3.25f; + int wrong = 0; + for (int k = -7; k <= 7; k++) { + const float v = (float)k / 7.0f * amax_l; + if (q4nx_quant_int4_code(v, amax_l) != k) wrong++; + } + check(wrong == 0, "int4 pair: each seventh of amax comes back as its own code (k/7*amax -> k, k=-7..7)"); + } + /* Degenerate groups: the floor keeps the scale usable and the code zero. */ + check(q4nx_quant_int4_scale(0.0f) == 1.0f, "int4 pair: an all-zero group floors the scale at 1"); + check(q4nx_quant_int4_code(0.0f, 0.0f) == 0, "int4 pair: and its code is 0, not a division by zero"); + check(q4nx_quant_int4_code(NAN, 1.0f) == 0, "int4 pair: a NaN value quantises to 0 rather than an undefined code"); + /* The dequantised value round-trips inside half a step: that is the accuracy the kernel's epilogue assumes. */ + int worst_ok = 1; + for (int i = -127; i <= 127; i++) { + const float v = (float)i / 127.0f * 2.0f; /* amax 2.0 */ + const float s = q4nx_quant_int4_scale(2.0f); + const float back = (float)q4nx_quant_int4_code(v, 2.0f) * s; + if (std::fabs(back - v) > s / 2.0f + 1e-6f) worst_ok = 0; + } + check(worst_ok, "int4 pair: every value round-trips within half a step of amax/7"); + std::printf(" int4 pair: 11 arms (known codes, ladder, floor, NaN, round-trip)\n"); +} + int main(int argc, char** argv) { + quant_int4_arms(); for (int G : {8, 9, 16}) { for (int n : {G, 3 * G}) { // whole groups: a ragged tail is not a permutation // Tile i holds the value i in its first word; after the reorder every