Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1316,6 +1316,11 @@ add_dependencies(npu_sampling_test npu_rope_llama3_test)
add_executable(hadamard_rotate_test tests/hadamard_rotate_test.cpp)
target_include_directories(hadamard_rotate_test PRIVATE npu/forward/include)
add_test(NAME hadamard_rotate COMMAND hadamard_rotate_test)
# engine#1990: rotate_rows() rotates a launch's rows in one thread team, and the test holds it to the serial loop bit
# for bit; that comparison is only about threads when the test has them.
if(TARGET OpenMP::OpenMP_CXX)
target_link_libraries(hadamard_rotate_test PRIVATE OpenMP::OpenMP_CXX)
endif()
# The arithmetic P7 (#806) part A rests on: (QH)(KH)^T = QK^T to f32 rounding, and what the rotation is
# worth against an f16 KV cache (attention-weight KLD) with and without outlier channels. Header only, no
# device, no model: runs in CI.
Expand Down
24 changes: 24 additions & 0 deletions npu/forward/include/hadamard.h
Original file line number Diff line number Diff line change
Expand Up @@ -146,4 +146,28 @@ inline void rotate(const float* signs, float* a, int n, int block) {
rotate_scalar(signs, a, n, block);
}

// engine#1990: a batched launch's M rows -- each copied from `src` into `dst`, rotated, and given its int8 scale from
// the rotated vector (amax / 127, or 1.0 for an all-zero row). XdnaTarget::npu_gemm_rows did this one row after
// another on one thread, about 45-50 ms of a 16-row prefill block on bonsai2-27b-t2s2. The rows share nothing, so a
// thread team over them runs exactly the per-row statements of the serial loop and the result is bit-identical for
// any thread count (pinned by tests/hadamard_rotate_test.cpp). `threads` <= 1 is the serial loop.
inline void rotate_rows(const float* signs, const float* src, float* dst, int M, int K, int block, float* ascales,
int threads) {
#if defined(_OPENMP)
#pragma omp parallel for schedule(static) num_threads(threads > 1 ? threads : 1) if (threads > 1 && M > 1)
#endif
for (int m = 0; m < M; m++) {
float* row = dst + (size_t)m * (size_t)K;
const float* in = src + (size_t)m * (size_t)K;
for (int k = 0; k < K; k++) row[k] = in[k];
rotate(signs, row, K, block);
float am = 0.0f;
for (int k = 0; k < K; k++) {
const float a = std::fabs(row[k]);
if (a > am) am = a;
}
ascales[m] = am > 0.0f ? am / 127.0f : 1.0f;
}
}

} // namespace q4nx_hadamard
31 changes: 20 additions & 11 deletions npu/target/xdna/xdna_target.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,9 @@
#include <fcntl.h>
#include <fstream>
#include <unistd.h> // mkstemp/close: the pb16i activation probe's dump
#if defined(_OPENMP)
#include <omp.h> // omp_get_max_threads: the batched rotation's thread team (engine#1990)
#endif

#include "hadamard.h" // the activation rotation: one implementation, shared with the model
#include "npu_engine_i8ctx_inc.h" // I8Ctx
Expand Down Expand Up @@ -1057,17 +1060,23 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector<float>& out, std:
return;
}
const auto h0 = std::chrono::steady_clock::now();
had_buf_.assign(A, A + (size_t)M * (size_t)K);
for (int m = 0; m < M; m++) {
float* row = had_buf_.data() + (size_t)m * (size_t)K;
q4nx_hadamard::rotate(hs->second.data(), row, K, had_block_);
float am = 0.0f;
for (int k = 0; k < K; k++) {
const float a = std::fabs(row[(size_t)k]);
if (a > am) am = a;
}
ascales[(size_t)m] = am > 0.0f ? am / 127.0f : 1.0f;
}
// engine#1990: the rows are rotated by one thread team (bit-identical to one after another; see
// q4nx_hadamard::rotate_rows). NPU_ROT_ROWS=0 keeps the serial form for an A/B; it is read once, at the
// first batched launch, so the A/B is per process start, not switchable mid-serve.
// `resize` (not `assign`) is safe only because `A` is the caller's activation and never points into
// `had_buf_`: a growing resize would free the buffer `A` reads from. rotate_rows writes every element.
static const int rot_threads = [] {
const char* e = getenv("NPU_ROT_ROWS");
if (e && e[0] == '0') return 1;
#if defined(_OPENMP)
return omp_get_max_threads();
#else
return 1;
#endif
}();
had_buf_.resize((size_t)M * (size_t)K);
q4nx_hadamard::rotate_rows(hs->second.data(), A, had_buf_.data(), M, K, had_block_, ascales.data(),
rot_threads);
Ab = had_buf_.data();
had_ms_ += std::chrono::duration<double, std::milli>(
std::chrono::steady_clock::now() - h0).count();
Expand Down
46 changes: 46 additions & 0 deletions tests/hadamard_rotate_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,52 @@ int main() {
break;
}
}
// engine#1990: rotate_rows() is the batched launch's rows, rotated by one thread team. Each row must come out
// exactly as the serial loop it replaced makes it (copy, rotate, amax / 127 -- written out below as that loop),
// for every thread count, with the all-zero row's scale 1.0 and the input left untouched.
{
const int Ks[] = {5120, 17408};
const int threads[] = {1, 2, 5, 16};
size_t compared = 0;
for (int K : Ks) {
const int M = 16;
std::mt19937 rng(77u + (unsigned)K);
std::uniform_real_distribution<float> dist(-3.0f, 3.0f);
std::vector<float> signs((size_t)K), src((size_t)M * (size_t)K);
for (int i = 0; i < K; i++) signs[(size_t)i] = (i % 5 == 0) ? -1.0f : 1.0f;
for (float& v : src) v = dist(rng);
for (int k = 0; k < K; k++) src[(size_t)3 * (size_t)K + (size_t)k] = 0.0f; // row 3: all zero
const std::vector<float> src_copy = src;
std::vector<float> want((size_t)M * (size_t)K), want_sc((size_t)M);
for (int m = 0; m < M; m++) { // the serial loop XdnaTarget::npu_gemm_rows ran before
float* row = want.data() + (size_t)m * (size_t)K;
std::memcpy(row, src.data() + (size_t)m * (size_t)K, sizeof(float) * (size_t)K);
q4nx_hadamard::rotate(signs.data(), row, K, 1024);
float am = 0.0f;
for (int k = 0; k < K; k++) am = std::fabs(row[k]) > am ? std::fabs(row[k]) : am;
want_sc[(size_t)m] = am > 0.0f ? am / 127.0f : 1.0f;
}
for (int t : threads) {
std::vector<float> got((size_t)M * (size_t)K, -9.0f), sc((size_t)M, -9.0f);
q4nx_hadamard::rotate_rows(signs.data(), src.data(), got.data(), M, K, 1024, sc.data(), t);
char what[160];
std::snprintf(what, sizeof what, "rotate_rows K=%d threads=%d: rows differ from the serial loop", K, t);
check(std::memcmp(got.data(), want.data(), sizeof(float) * got.size()) == 0, what);
std::snprintf(what, sizeof what, "rotate_rows K=%d threads=%d: scales differ from the serial loop", K, t);
check(std::memcmp(sc.data(), want_sc.data(), sizeof(float) * sc.size()) == 0, what);
compared += got.size() + sc.size();
}
check(want_sc[3] == 1.0f, "rotate_rows: an all-zero row's scale is 1.0");
check(src == src_copy, "rotate_rows: the input rows are not modified");
}
#if defined(_OPENMP)
std::printf(" rotate_rows: %zu values bit-identical to the serial loop (OpenMP build: threads 1, 2, 5, 16)\n",
compared);
#else
std::printf(" rotate_rows: %zu values bit-identical to the serial loop (no OpenMP: every count is serial)\n",
compared);
#endif
}
if (failures) {
std::printf("%d failure(s)\n", failures);
return 1;
Expand Down
Loading