From 40a2e899116c642109567ca2a3576b7958592e06 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 18:14:06 -0400 Subject: [PATCH 1/2] npu: a prefill launch's rows are rotated by one thread team, not one after another (Refs #1990) The batched folded-basis path rotated M rows serially on one thread before every launch (copy, Hadamard, amax -> int8 scale). q4nx_hadamard::rotate_rows() runs the same per-row statements in a thread team; the rows share nothing, so the result is bit-identical for any thread count. NPU_ROT_ROWS=0 keeps the serial form. tests/hadamard_rotate_test holds rotate_rows to the serial loop bit for bit at 1/2/5/16 threads (now linked with OpenMP). Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 5 ++++ npu/forward/include/hadamard.h | 24 +++++++++++++++++ npu/target/xdna/xdna_target.cpp | 28 ++++++++++++-------- tests/hadamard_rotate_test.cpp | 46 +++++++++++++++++++++++++++++++++ 4 files changed, 92 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fb9feacfd..5376c67cf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1316,6 +1316,11 @@ add_dependencies(npu_sampling_test npu_rope_llama3_test) add_executable(hadamard_rotate_test tests/hadamard_rotate_test.cpp) target_include_directories(hadamard_rotate_test PRIVATE npu/forward/include) add_test(NAME hadamard_rotate COMMAND hadamard_rotate_test) +# engine#1990: rotate_rows() rotates a launch's rows in one thread team, and the test holds it to the serial loop bit +# for bit; that comparison is only about threads when the test has them. +if(TARGET OpenMP::OpenMP_CXX) + target_link_libraries(hadamard_rotate_test PRIVATE OpenMP::OpenMP_CXX) +endif() # The arithmetic P7 (#806) part A rests on: (QH)(KH)^T = QK^T to f32 rounding, and what the rotation is # worth against an f16 KV cache (attention-weight KLD) with and without outlier channels. Header only, no # device, no model: runs in CI. diff --git a/npu/forward/include/hadamard.h b/npu/forward/include/hadamard.h index c44aac01b..13856f3e9 100644 --- a/npu/forward/include/hadamard.h +++ b/npu/forward/include/hadamard.h @@ -146,4 +146,28 @@ inline void rotate(const float* signs, float* a, int n, int block) { rotate_scalar(signs, a, n, block); } +// engine#1990: a batched launch's M rows -- each copied from `src` into `dst`, rotated, and given its int8 scale from +// the rotated vector (amax / 127, or 1.0 for an all-zero row). XdnaTarget::npu_gemm_rows did this one row after +// another on one thread, about 45-50 ms of a 16-row prefill block on bonsai2-27b-t2s2. The rows share nothing, so a +// thread team over them runs exactly the per-row statements of the serial loop and the result is bit-identical for +// any thread count (pinned by tests/hadamard_rotate_test.cpp). `threads` <= 1 is the serial loop. +inline void rotate_rows(const float* signs, const float* src, float* dst, int M, int K, int block, float* ascales, + int threads) { +#if defined(_OPENMP) +#pragma omp parallel for schedule(static) num_threads(threads > 1 ? threads : 1) if (threads > 1 && M > 1) +#endif + for (int m = 0; m < M; m++) { + float* row = dst + (size_t)m * (size_t)K; + const float* in = src + (size_t)m * (size_t)K; + for (int k = 0; k < K; k++) row[k] = in[k]; + rotate(signs, row, K, block); + float am = 0.0f; + for (int k = 0; k < K; k++) { + const float a = std::fabs(row[k]); + if (a > am) am = a; + } + ascales[m] = am > 0.0f ? am / 127.0f : 1.0f; + } +} + } // namespace q4nx_hadamard diff --git a/npu/target/xdna/xdna_target.cpp b/npu/target/xdna/xdna_target.cpp index 5864d12e2..e5c9c34ca 100644 --- a/npu/target/xdna/xdna_target.cpp +++ b/npu/target/xdna/xdna_target.cpp @@ -31,6 +31,9 @@ #include #include #include // mkstemp/close: the pb16i activation probe's dump +#if defined(_OPENMP) +#include // omp_get_max_threads: the batched rotation's thread team (engine#1990) +#endif #include "hadamard.h" // the activation rotation: one implementation, shared with the model #include "npu_engine_i8ctx_inc.h" // I8Ctx @@ -1057,17 +1060,20 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector& out, std: return; } const auto h0 = std::chrono::steady_clock::now(); - had_buf_.assign(A, A + (size_t)M * (size_t)K); - for (int m = 0; m < M; m++) { - float* row = had_buf_.data() + (size_t)m * (size_t)K; - q4nx_hadamard::rotate(hs->second.data(), row, K, had_block_); - float am = 0.0f; - for (int k = 0; k < K; k++) { - const float a = std::fabs(row[(size_t)k]); - if (a > am) am = a; - } - ascales[(size_t)m] = am > 0.0f ? am / 127.0f : 1.0f; - } + // engine#1990: the rows are rotated by one thread team (bit-identical to one after another; see + // q4nx_hadamard::rotate_rows). NPU_ROT_ROWS=0 keeps the serial form for an A/B. + static const int rot_threads = [] { + const char* e = getenv("NPU_ROT_ROWS"); + if (e && e[0] == '0') return 1; +#if defined(_OPENMP) + return omp_get_max_threads(); +#else + return 1; +#endif + }(); + had_buf_.resize((size_t)M * (size_t)K); + q4nx_hadamard::rotate_rows(hs->second.data(), A, had_buf_.data(), M, K, had_block_, ascales.data(), + rot_threads); Ab = had_buf_.data(); had_ms_ += std::chrono::duration( std::chrono::steady_clock::now() - h0).count(); diff --git a/tests/hadamard_rotate_test.cpp b/tests/hadamard_rotate_test.cpp index bf0ea9c5e..120321fa7 100644 --- a/tests/hadamard_rotate_test.cpp +++ b/tests/hadamard_rotate_test.cpp @@ -113,6 +113,52 @@ int main() { break; } } + // engine#1990: rotate_rows() is the batched launch's rows, rotated by one thread team. Each row must come out + // exactly as the serial loop it replaced makes it (copy, rotate, amax / 127 -- written out below as that loop), + // for every thread count, with the all-zero row's scale 1.0 and the input left untouched. + { + const int Ks[] = {5120, 17408}; + const int threads[] = {1, 2, 5, 16}; + size_t compared = 0; + for (int K : Ks) { + const int M = 16; + std::mt19937 rng(77u + (unsigned)K); + std::uniform_real_distribution dist(-3.0f, 3.0f); + std::vector signs((size_t)K), src((size_t)M * (size_t)K); + for (int i = 0; i < K; i++) signs[(size_t)i] = (i % 5 == 0) ? -1.0f : 1.0f; + for (float& v : src) v = dist(rng); + for (int k = 0; k < K; k++) src[(size_t)3 * (size_t)K + (size_t)k] = 0.0f; // row 3: all zero + const std::vector src_copy = src; + std::vector want((size_t)M * (size_t)K), want_sc((size_t)M); + for (int m = 0; m < M; m++) { // the serial loop XdnaTarget::npu_gemm_rows ran before + float* row = want.data() + (size_t)m * (size_t)K; + std::memcpy(row, src.data() + (size_t)m * (size_t)K, sizeof(float) * (size_t)K); + q4nx_hadamard::rotate(signs.data(), row, K, 1024); + float am = 0.0f; + for (int k = 0; k < K; k++) am = std::fabs(row[k]) > am ? std::fabs(row[k]) : am; + want_sc[(size_t)m] = am > 0.0f ? am / 127.0f : 1.0f; + } + for (int t : threads) { + std::vector got((size_t)M * (size_t)K, -9.0f), sc((size_t)M, -9.0f); + q4nx_hadamard::rotate_rows(signs.data(), src.data(), got.data(), M, K, 1024, sc.data(), t); + char what[160]; + std::snprintf(what, sizeof what, "rotate_rows K=%d threads=%d: rows differ from the serial loop", K, t); + check(std::memcmp(got.data(), want.data(), sizeof(float) * got.size()) == 0, what); + std::snprintf(what, sizeof what, "rotate_rows K=%d threads=%d: scales differ from the serial loop", K, t); + check(std::memcmp(sc.data(), want_sc.data(), sizeof(float) * sc.size()) == 0, what); + compared += got.size() + sc.size(); + } + check(want_sc[3] == 1.0f, "rotate_rows: an all-zero row's scale is 1.0"); + check(src == src_copy, "rotate_rows: the input rows are not modified"); + } +#if defined(_OPENMP) + std::printf(" rotate_rows: %zu values bit-identical to the serial loop (OpenMP build: threads 1, 2, 5, 16)\n", + compared); +#else + std::printf(" rotate_rows: %zu values bit-identical to the serial loop (no OpenMP: every count is serial)\n", + compared); +#endif + } if (failures) { std::printf("%d failure(s)\n", failures); return 1; From de8fa3481f2bfec96578ad742ef1efdf45384441 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 18:53:49 -0400 Subject: [PATCH 2/2] npu: say why the rotation buffer's resize is safe, and when NPU_ROT_ROWS is read (review of #2047) Co-Authored-By: Claude Opus 5.5 --- npu/target/xdna/xdna_target.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/npu/target/xdna/xdna_target.cpp b/npu/target/xdna/xdna_target.cpp index e5c9c34ca..065dfd758 100644 --- a/npu/target/xdna/xdna_target.cpp +++ b/npu/target/xdna/xdna_target.cpp @@ -1061,7 +1061,10 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector& out, std: } const auto h0 = std::chrono::steady_clock::now(); // engine#1990: the rows are rotated by one thread team (bit-identical to one after another; see - // q4nx_hadamard::rotate_rows). NPU_ROT_ROWS=0 keeps the serial form for an A/B. + // q4nx_hadamard::rotate_rows). NPU_ROT_ROWS=0 keeps the serial form for an A/B; it is read once, at the + // first batched launch, so the A/B is per process start, not switchable mid-serve. + // `resize` (not `assign`) is safe only because `A` is the caller's activation and never points into + // `had_buf_`: a growing resize would free the buffer `A` reads from. rotate_rows writes every element. static const int rot_threads = [] { const char* e = getenv("NPU_ROT_ROWS"); if (e && e[0] == '0') return 1;