Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 38 additions & 12 deletions npu/forward/src/q4nx_pack_blocked.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,21 @@ inline size_t bpos(size_t i, size_t j, int KD, int ND, int blocked) {
: q4nx_layout::rowmajor_index(i, j, KD, ND);
}

// One code: roundf(v * tis) clamped to [-127, 127], a non-finite v as 0 -- the expression this packer has always used,
// written so the loop vectorises (engine#1783: std::roundf and std::isfinite kept it scalar, ~3 ns per element, and a
// 1536 x 1536 activation is packed on every attention call). Exact, not approximate: y = v * tis is finite with
// |y| <= 127 (v is bounded by its column's amax, or zeroed), so the truncation t fits an int and y - t is exact; rounding
// half away from zero is then t + 1 for a fraction >= 0.5 and t - 1 for one <= -0.5. `v - v == 0` holds for finite v only.
inline int8_t quant_code(float v, float tis) {
const float y = (v - v == 0.0f) ? v * tis : 0.0f;
int x = (int)y;
const float f = y - (float)x;
x += (f >= 0.5f) - (f <= -0.5f);
x = x > 127 ? 127 : x;
x = x < -127 ? -127 : x;
return (int8_t)x;
}

} // namespace

extern "C" int q4nx_pack_B_int8_blocked(int8_t* Bm, const float* w, int K, int N, int KD, int ND, int blocked,
Expand All @@ -52,14 +67,25 @@ extern "C" int q4nx_pack_B_int8_blocked(int8_t* Bm, const float* w, int K, int N
// The scale sum is accumulated in a serial loop over j rather than inside a `reduction(+:ssum)` parallel loop. That is
// deliberate: the mean column scale becomes order-independent, where the parallel reduction left its last bits to the
// schedule. Codes and scales are unchanged (the amax of a column is a max over rows, and the rounding is per element).
// Column amax in parallel over row chunks, each thread into its own array, merged after: a max is exact and
// order-independent, so the scales are the serial loop's bit for bit (engine#1783: the serial pass was most of a
// 1536 x 1536 pack once the write below was parallel).
std::vector<float> amax((size_t)N, 0.0f);
std::vector<float> tis((size_t)N, 0.0f);
for (int i = 0; i < K; i++) {
const float* row = w + (size_t)i * (size_t)N;
for (int j = 0; j < N; j++) {
const float a = std::fabs(row[j]);
if (std::isfinite(a) && a > amax[(size_t)j]) amax[(size_t)j] = a;
#pragma omp parallel
{
std::vector<float> mine((size_t)N, 0.0f);
#pragma omp for schedule(static) nowait
for (int i = 0; i < K; i++) {
const float* row = w + (size_t)i * (size_t)N;
for (int j = 0; j < N; j++) {
const float a = std::fabs(row[j]);
if (std::isfinite(a) && a > mine[(size_t)j]) mine[(size_t)j] = a;
}
}
#pragma omp critical(q4nx_pack_amax)
for (int j = 0; j < N; j++)
if (mine[(size_t)j] > amax[(size_t)j]) amax[(size_t)j] = mine[(size_t)j];
}
double ssum = 0.0;
for (int j = 0; j < N; j++) {
Expand All @@ -69,16 +95,16 @@ extern "C" int q4nx_pack_B_int8_blocked(int8_t* Bm, const float* w, int K, int N
tis[(size_t)j] = 127.0f / am;
ssum += (double)(am / 127.0f);
}
// The write, one address per 8-column run. Inside the blocked layout, columns j0..j0+7 of a row with j0 % 8 == 0 sit
// in one 8x8 microtile row, so they are 8 consecutive bytes from blocked_index(i, j0) (q4nx_weight_layout.h, the
// image_index/copy note); row-major is consecutive anyway. The element arithmetic is unchanged.
#pragma omp parallel for schedule(static)
for (int i = 0; i < K; i++) {
const float* row = w + (size_t)i * (size_t)N;
for (int j = 0; j < N; j++) {
float v = row[j];
if (!std::isfinite(v)) v = 0.0f;
int x = (int)std::roundf(v * tis[(size_t)j]);
if (x > 127) x = 127;
else if (x < -127) x = -127;
Bm[bpos((size_t)i, (size_t)j, KD, ND, blocked)] = (int8_t)x;
for (int j0 = 0; j0 < N; j0 += 8) {
int8_t* dst = Bm + bpos((size_t)i, (size_t)j0, KD, ND, blocked);
const int jn = N - j0 < 8 ? N - j0 : 8;
for (int c = 0; c < jn; c++) dst[c] = quant_code(row[j0 + c], tis[(size_t)(j0 + c)]);
}
}
*ssum_out = ssum;
Expand Down
43 changes: 43 additions & 0 deletions tests/npu_blocked_layout_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,48 @@ void test_production_packer() {
std::printf(" production packer: %zu codes, %d scales, blocked == bpos(rowmajor)\n", (size_t)K * N, N);
}

// engine#1783: the production packer's rounding is written so it vectorises (truncate, then step on the fraction), and
// that has to be roundf's half-away-from-zero exactly. Columns whose amax is 127 make the scale exactly 1, so each value
// below reaches the rounding unchanged: every k + 0.5 and k - 0.5 tie, the floats one ulp either side of a tie, the
// +-127 clamp, and NaN / +-inf (zeroed). The row-major reference (q4nx_pack_B_int8, roundf) is the oracle.
void test_packer_rounding_edges() {
const int K = 64, N = 128;
std::vector<float> vals;
for (int k = -127; k <= 126; ++k) {
const float tie = float(k) + 0.5f;
vals.push_back(tie);
vals.push_back(std::nextafter(tie, 0.0f));
vals.push_back(std::nextafter(tie, tie > 0 ? 1e9f : -1e9f));
}
for (float v : {0.0f, -0.0f, 0.49999997f, -0.49999997f, 127.0f, -127.0f, 126.99999f, -126.99999f, 1e-30f, -1e-30f})
vals.push_back(v);
vals.push_back(0.0f / 0.0f);
vals.push_back(1.0f / 0.0f);
vals.push_back(-1.0f / 0.0f);
std::vector<float> w((size_t)K * N, 0.0f);
for (int j = 0; j < N; ++j) w[(size_t)j] = (j % 2) ? 127.0f : -127.0f; // row 0: amax 127 in every column
size_t at = 0;
for (int i = 1; i < K && at < vals.size(); ++i)
for (int j = 0; j < N && at < vals.size(); ++j) w[(size_t)i * N + (size_t)j] = vals[at++];
check(at == vals.size(), "packer rounding: every edge value placed");
std::vector<int8_t> rowmaj((size_t)K * N), blocked((size_t)K * N);
std::vector<float> sc_ref((size_t)N), sc_blk((size_t)N);
q4nx_pack_B_int8(w.data(), K, N, rowmaj.data(), N, sc_ref.data());
double ssum = 0;
check(q4nx_pack_B_int8_blocked(blocked.data(), w.data(), K, N, K, N, 1, sc_blk.data(), &ssum) == 0, "packer rounding: runs");
check(sc_blk == sc_ref, "packer rounding: scales equal the reference's");
size_t mismatched = 0, ties_up = 0;
for (int i = 0; i < K; ++i)
for (int j = 0; j < N; ++j) {
const int8_t got = blocked[q4nx_layout::blocked_index((size_t)i, (size_t)j, K, N)];
if (got != rowmaj[(size_t)i * N + (size_t)j]) mismatched++;
ties_up += w[(size_t)i * N + (size_t)j] == 2.5f && got == 3;
}
check(mismatched == 0, "packer rounding: every code equals roundf's, ties and neighbours included");
check(ties_up == 1, "packer rounding: 2.5 rounds away from zero (3), not to even");
std::printf(" packer rounding: %zu edge values, codes == roundf reference\n", vals.size());
}

// engine#1994: the integer host arm packs UNPADDED images (KD == K, ND == N) into K*N bytes. blocked_index() addresses
// whole 64x128 tiles, so a geometry that is not whole tiles has to be packed and read row-major, or both the packer and
// the MAC walk past the buffer. This pins the rule the arm uses -- blocked = blocked_fits(K, N), read through
Expand Down Expand Up @@ -291,6 +333,7 @@ int main() {
}
test_refusals();
test_production_packer();
test_packer_rounding_edges();
test_row_major_unchanged();
test_pack_geometries();
test_unpadded_image(3072, 7488, false); // phi-3-mini's last vocabulary block (32064 % 8192)
Expand Down
Loading