Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions npu/forward/include/q4nx_pack.h
Original file line number Diff line number Diff line change
Expand Up @@ -307,6 +307,13 @@ int q4nx_assemble_B(const Q4nxDesignGeom* g, const Q4nxProjections* p, float* B)
* (engine #1416). Writes Bm at the blocked (or row-major, `blocked == 0`) index for this geometry, per-column scales into
* col_scales[N], and the sum of the scales into *ssum_out. `KD`/`ND` are the container's padded geometry the layout is
* defined on. Behaviour-preserving extraction: the body is the class's own, with `bpos()` passed in. */
/* Quantise one 128-row group of a column to int8 (engine#1839): `amax` is the group's max |v| over rows [k0, k1) of
* column n, the scale is `amax / 127` and the codes are the int8 range -- NOT -8..7. The int4 code range belongs to the
* weights a device kernel packs two per byte; the activations this arm feeds the reference GEMM are one int8 per element,
* and clamping them to +-8 truncated every |v| > 7/127 of amax (i.e. most of a real tensor) before the first multiply.
* Exported because the rule is what can be wrong silently: `tests/npu_pack_test.cpp` pins a known vector to known codes. */
void q4nx_quantise_i8_group(const float* v, int rows, int ld, float* scale_out, int8_t* codes, int code_ld);

int q4nx_pack_B_int8_blocked(int8_t* Bm, const float* w, int K, int N, int KD, int ND, int blocked,
float* col_scales, double* ssum_out);

Expand Down
23 changes: 8 additions & 15 deletions npu/forward/src/q4nx_forward.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3066,21 +3066,14 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N,
for (int g = 0; g < groups; g++) {
const int k0 = g * G, k1 = (k0 + G < K) ? k0 + G : K;
for (int n = 0; n < N; n++) {
float amax = 0.0f;
for (int k = k0; k < k1; k++) {
const float v = std::fabs(Bmat[(size_t)k * (size_t)N + (size_t)n]);
if (std::isfinite(v) && v > amax) amax = v;
}
if (!(amax >= 1e-12f)) amax = 1.0f;
t.d[(size_t)g * (size_t)N + (size_t)n] = amax / 127.0f;
const float tis = 127.0f / amax;
for (int k = k0; k < k1; k++) {
float v = Bmat[(size_t)k * (size_t)N + (size_t)n];
if (!std::isfinite(v)) v = 0.0f;
int q = (int)std::round(v * tis);
q = q > 7 ? 7 : (q < -8 ? -8 : q);
t.codes[(size_t)k * (size_t)N + (size_t)n] = (int8_t)q;
}
// engine#1839: the rule lives in ONE place now. It was inline here, clamping to the int4 range (-8..7)
// while the storage and the scale were int8 -- so every |v| > 7/127 of amax was truncated before the
// first multiply. `q4nx_quantise_i8_group` is the extracted rule (one scale per group-column, int8 codes,
// int8 clamp) and `tests/npu_pack_test.cpp` pins a known vector to known codes, which this copy could
// not be held to while it was inline in a cache-filling loop.
q4nx_quantise_i8_group(Bmat + (size_t)k0 * (size_t)N + (size_t)n, k1 - k0, N,
&t.d[(size_t)g * (size_t)N + (size_t)n],
t.codes.data() + (size_t)k0 * (size_t)N + (size_t)n, N);
}
}
if (!cacheable) {
Expand Down
24 changes: 24 additions & 0 deletions npu/forward/src/q4nx_pack.c
Original file line number Diff line number Diff line change
Expand Up @@ -610,6 +610,30 @@ int q4nx_assemble_B(const Q4nxDesignGeom* g, const Q4nxProjections* p, float* B)
return 0;
}

void q4nx_quantise_i8_group(const float* v, int rows, int ld, float* scale_out, int8_t* codes, int code_ld) {
/* engine#1839: one column of one 128-row group. The clamp is the INT8 range because `codes` holds one int8 per
* element and the scale is amax/127 -- `roundf(v * 127/amax)` is inside +-127 by construction, so the clamp is only
* a float-rounding guard and must not narrow the codes. It used to narrow them to the int4 range (-8..7), which is
* right for the weights a device kernel packs two per byte and wrong for these activations: every |v| > 7/127 of
* amax became +-8 before the first multiply. Pinned by a known vector -> known codes in tests/npu_pack_test.cpp. */
float amax = 0.0f;
for (int k = 0; k < rows; k++) {
const float x = v[(size_t)k * (size_t)ld];
const float a = x < 0 ? -x : x;
if (isfinite(a) && a > amax) amax = a;
}
if (!(amax >= 1e-12f)) amax = 1.0f;
*scale_out = amax / 127.0f;
const float tis = 127.0f / amax;
for (int k = 0; k < rows; k++) {
float x = v[(size_t)k * (size_t)ld];
if (!isfinite(x)) x = 0.0f;
int q = (int)roundf(x * tis);
q = q > 127 ? 127 : (q < -128 ? -128 : q);
codes[(size_t)k * (size_t)code_ld] = (int8_t)q;
}
}

float q4nx_pack_B_int8(const float* B, int K, int N, int8_t* out, int ld,
float* col_scales) {
// ROW-major traversal, both passes. The obvious spelling of this (for j, for i) walks a row-major K x N matrix with
Expand Down
18 changes: 18 additions & 0 deletions tests/npu_pack_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,24 @@ static void check(bool ok, const std::string& what) {
}
}

/* engine#1839: the quantiser's known answer. A vector whose amax is 127 must come back as int8 codes in the INT8
* range -- 127 stays 127 and -128 stays -128 -- because `codes` holds one int8 per element and the scale is amax/127.
* The bug this pins: the clamp was the int4 range (-8..7), so 127 -> 7 and -128 -> -8, an 18x error on the largest
* entries before the first multiply, in the arm the engine#1809 gate grades the device against. */
{
float col[129];
int8_t codes[129];
col[0] = 127.0f; col[1] = -128.0f; col[2] = 0.5f; col[3] = -0.5f;
for (int k = 4; k < 129; k++) col[k] = float((k % 17) - 8); /* small values, still inside int8 */
float d = 0.0f;
q4nx_quantise_i8_group(col, 129, 1, &d, codes, 1);
check(codes[0] == 127, "engine#1839: an amax element quantises to 127, not the int4 clamp 7");
check(codes[1] == -126, "engine#1839: a large negative element quantises to -126, not the int4 clamp -8");
check(codes[2] == 1, "engine#1839: 0.5 rounds half away to 1");
check(codes[3] == -1, "engine#1839: -0.5 rounds half away to -1");
check(fabsf(d - 1.0f) < 1e-6f, "engine#1839: the scale is amax/127 = 1.0 for an amax of 127");
}

int main(int argc, char** argv) {
for (int G : {8, 9, 16}) {
for (int n : {G, 3 * G}) { // whole groups: a ragged tail is not a permutation
Expand Down
Loading