Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions npu/forward/include/q4nx_pack.h
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,17 @@ void q4nx_host_gemm_f32_padded(const float* A, int a_k, int M, int K, int N, con
* holding an int4 value in [-8, 7], exactly as the importer's packB lays it down. */
void q4nx_host_gemm_i8(const int8_t* a8, int K, const int8_t* codes, int N, int32_t* out32);

/* engine#1839: the int4 code range and its scale, together, because they are only correct as a pair.
*
* A symmetric int4 code spans [-8, 7], so the scale that maps a group's `amax` onto that range is `amax / 7`, and the
* code is `round(v * 7 / amax)` clamped to [-8, 7]. The two places that quantise float weights into the container's
* int4 codes used to pair `amax / 127` (an INT8 scale) with the int4 clamp, so every code but the very largest
* saturated at +-7 and the dequantised weight was ~18x too small -- in the host arm the gate compares the device
* against, which is the worst place for it. 7 rather than 8 so that no code saturates: round(v * 7 / amax) lands in
* [-7, 7] for |v| <= amax, and the -8 slot stays a guard rather than an asymmetry. */
float q4nx_quant_int4_scale(float amax);
int8_t q4nx_quant_int4_code(float v, float amax);

/* The whole GEMM in int8 x int4 for the gate's `i8` host arm, quantising both sides here with the engine's
* own rule. It cannot see a wrong weight LAYOUT (the f32 arm covers that); what it can see is a wrong int4
* quantisation, a wrong 128-K group boundary, an int8 intermediate or a saturated accumulator -- i.e. the
Expand Down
19 changes: 9 additions & 10 deletions npu/forward/src/q4nx_forward.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3071,16 +3071,15 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N,
const float v = std::fabs(Bmat[(size_t)k * (size_t)N + (size_t)n]);
if (std::isfinite(v) && v > amax) amax = v;
}
if (!(amax >= 1e-12f)) amax = 1.0f;
t.d[(size_t)g * (size_t)N + (size_t)n] = amax / 127.0f;
const float tis = 127.0f / amax;
for (int k = k0; k < k1; k++) {
float v = Bmat[(size_t)k * (size_t)N + (size_t)n];
if (!std::isfinite(v)) v = 0.0f;
int q = (int)std::round(v * tis);
q = q > 7 ? 7 : (q < -8 ? -8 : q);
t.codes[(size_t)k * (size_t)N + (size_t)n] = (int8_t)q;
}
/* engine#1839: the weights here are the container's INT4 codes, so the scale that maps amax onto
* that range is amax/7 -- pairing amax/127 (an int8 scale) with the [-8, 7] clamp saturated every code
* but the largest and made the host reference's weights ~18x too small. q4nx_quant_int4_*() own the
* pair so this site and q4nx_pack.c's cannot drift apart again. */
const float amax_eff = (amax >= 1e-12f) ? amax : 1.0f;
t.d[(size_t)g * (size_t)N + (size_t)n] = q4nx_quant_int4_scale(amax_eff);
for (int k = k0; k < k1; k++)
t.codes[(size_t)k * (size_t)N + (size_t)n] =
q4nx_quant_int4_code(Bmat[(size_t)k * (size_t)N + (size_t)n], amax_eff);
}
}
if (!cacheable) {
Expand Down
26 changes: 19 additions & 7 deletions npu/forward/src/q4nx_pack.c
Original file line number Diff line number Diff line change
Expand Up @@ -194,14 +194,12 @@ void q4nx_host_gemm_i8_from_f32(const float* A, int a_k, int M, int K, int N, co
const float v = fabsf(Bmat[(size_t)k * (size_t)N + (size_t)n]);
if (v > amax && isfinite(v)) amax = v;
}
if (!(amax >= 1e-12f)) amax = 1.0f;
const float tis = 127.0f / amax;
/* engine#1839: the scale and the clamp describe the SAME range -- int4, [-8, 7] -- or the codes
* saturate and the dequantised weight is ~18x too small. q4nx_quant_int4_*() own the pair. */
const float amax_eff = (amax >= 1e-12f) ? amax : 1.0f;
for (int k = k0; k < k1; k++) {
float v = Bmat[(size_t)k * (size_t)N + (size_t)n];
if (!isfinite(v)) v = 0.0f;
int q = (int)roundf(v * tis);
if (q > 7) q = 7;
else if (q < -8) q = -8; /* int4 codes are [-8, 7] in this format */
const float v = Bmat[(size_t)k * (size_t)N + (size_t)n];
const int q = q4nx_quant_int4_code(v, amax_eff);
int aq = (int)roundf(a[k] * 127.0f);
if (aq > 127) aq = 127;
else if (aq < -127) aq = -127;
Expand All @@ -212,6 +210,20 @@ void q4nx_host_gemm_i8_from_f32(const float* A, int a_k, int M, int K, int N, co
}
}

float q4nx_quant_int4_scale(float amax) {
if (!(amax >= 1e-12f) || !isfinite(amax)) return 1.0f; /* an all-zero group still needs a usable scale */
return amax / 7.0f;
}

int8_t q4nx_quant_int4_code(float v, float amax) {
if (!isfinite(v)) v = 0.0f;
const float s = q4nx_quant_int4_scale(amax);
int q = (int)roundf(v / s);
if (q > 7) q = 7;
else if (q < -8) q = -8;
return (int8_t)q;
}

void q4nx_host_gemm_i8(const int8_t* a8, int K, const int8_t* codes, int N, int32_t* out32) {
for (int n = 0; n < N; n++) out32[n] = 0;
for (int k = 0; k < K; k++) {
Expand Down
43 changes: 43 additions & 0 deletions tests/npu_pack_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,50 @@ static void check(bool ok, const std::string& what) {
}
}

/* engine#1839: an int4 code and its scale are only correct as a PAIR. The defect this pins: both sites that quantise
* float weights into the container's int4 codes used `amax / 127` (an INT8 scale) with the int4 clamp [-8, 7], so every
* code but the very largest saturated and the dequantised weight was ~18x too small — in the host arm the device gate
* compares against. The arms below are a known input -> known code (not a property any pairing satisfies), the no-
* saturation property at full range, and the two degenerate cases the floor exists for. */
static void quant_int4_arms() {
const float amax = 7.0f;
check(q4nx_quant_int4_scale(7.0f) == 1.0f, "int4 pair: amax 7 takes scale 1 (so the codes ARE the values)");
check(q4nx_quant_int4_code(7.0f, amax) == 7, "int4 pair: +amax quantises to +7");
check(q4nx_quant_int4_code(-7.0f, amax) == -7, "int4 pair: -amax quantises to -7");
check(q4nx_quant_int4_code(0.0f, amax) == 0, "int4 pair: zero is zero");
check(q4nx_quant_int4_code(3.5f, amax) == 4, "int4 pair: half of amax rounds to 4 (round-half-away)");
/* The arm the old pairing fails: at amax/127 a mid-range value saturates. */
check(q4nx_quant_int4_code(0.1f, 1.0f) == 1, "int4 pair: a small value is 1, not saturated — the 127 scale gives 13");
/* THE LADDER: the sevenths of amax must come back as their own code number. This is the arm that can fail for the
* wrong scale -- under amax/127 every one of these saturates to +-7 (checked: v=0.1 at amax=1 gives 7 where int4
* expects 1), so it distinguishes the pairings rather than asserting a property the clamp makes unfailable. */
{
const float amax_l = 3.25f;
int wrong = 0;
for (int k = -7; k <= 7; k++) {
const float v = (float)k / 7.0f * amax_l;
if (q4nx_quant_int4_code(v, amax_l) != k) wrong++;
}
check(wrong == 0, "int4 pair: each seventh of amax comes back as its own code (k/7*amax -> k, k=-7..7)");
}
/* Degenerate groups: the floor keeps the scale usable and the code zero. */
check(q4nx_quant_int4_scale(0.0f) == 1.0f, "int4 pair: an all-zero group floors the scale at 1");
check(q4nx_quant_int4_code(0.0f, 0.0f) == 0, "int4 pair: and its code is 0, not a division by zero");
check(q4nx_quant_int4_code(NAN, 1.0f) == 0, "int4 pair: a NaN value quantises to 0 rather than an undefined code");
/* The dequantised value round-trips inside half a step: that is the accuracy the kernel's epilogue assumes. */
int worst_ok = 1;
for (int i = -127; i <= 127; i++) {
const float v = (float)i / 127.0f * 2.0f; /* amax 2.0 */
const float s = q4nx_quant_int4_scale(2.0f);
const float back = (float)q4nx_quant_int4_code(v, 2.0f) * s;
if (std::fabs(back - v) > s / 2.0f + 1e-6f) worst_ok = 0;
}
check(worst_ok, "int4 pair: every value round-trips within half a step of amax/7");
std::printf(" int4 pair: 11 arms (known codes, ladder, floor, NaN, round-trip)\n");
}

int main(int argc, char** argv) {
quant_int4_arms();
for (int G : {8, 9, 16}) {
for (int n : {G, 3 * G}) { // whole groups: a ragged tail is not a permutation
// Tile i holds the value i in its first word; after the reorder every
Expand Down
Loading