From c832311fa04921cf084ccd2e0e106f2104fccc93 Mon Sep 17 00:00:00 2001 From: pi-lane Date: Sun, 15 Nov 2026 17:06:56 -0400 Subject: [PATCH 1/2] audio: pick the GEMM inner loop at run time -- AVX2+FMA where the box has it, baseline everywhere else engine#1783 measured the host MatMul inner loop (a plain `rr[j] += av * br[j]` reduction) at 1.269x min-over-repeats with AVX2+FMA against SSE2 on this box, and 1.85x on the isolated loop: SSE2 gives 4-wide, AVX2+FMA 8-wide fused. It could not be a build-wide flag (engine#2403 defaulted ONEBIT_AUDIO_AVX2 OFF) because the ISA is per box and the appliance must run on whatever it is installed on. So the four-row body is compiled twice -- once at the tree s baseline, once under #pragma GCC target("avx2,fma") -- and chosen once per process from __builtin_cpu_supports, with ONEBIT_AUDIO_AVX2=0 forcing the baseline and =1 requiring the fast path (and saying so when the box cannot run it). The appliance keeps the baseline; a box with AVX2+FMA gets the fast loop. Arithmetic unchanged: same accumulation order, so the result is bit-identical by construction. Measured here (interleaved, AVX2=0 vs 1): 13636.6 / 8639.2, 5183.6 / 2674.8, 3529.0 / 2772.7 ms. --- npu/audio_exec.cpp | 67 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 54 insertions(+), 13 deletions(-) diff --git a/npu/audio_exec.cpp b/npu/audio_exec.cpp index 0bf21e223..a779ae181 100644 --- a/npu/audio_exec.cpp +++ b/npu/audio_exec.cpp @@ -263,6 +263,59 @@ void load_init_i64(const OnnxGraphInfo& g, const OnnxInit& t, std::vector t, con float* r0 = r.data.data() + bi * size_t(M * N) + size_t(i0) * size_t(N); if (i1 - i0 == 4) { const float* a0 = ap + size_t(i0) * size_t(K); - for (int64_t k = 0; k < K; ++k) { - const float v0 = a0[size_t(k)], v1 = a0[size_t(K) + size_t(k)]; - const float v2 = a0[size_t(2 * K) + size_t(k)], v3 = a0[size_t(3 * K) + size_t(k)]; - if ((v0 == 0.0f) && (v1 == 0.0f) && (v2 == 0.0f) && (v3 == 0.0f)) continue; - const float* br = bpp + size_t(k) * size_t(N); - for (int64_t j = 0; j < N; ++j) { - const float bv = br[j]; - r0[j] += v0 * bv; - r0[N + j] += v1 * bv; - r0[2 * N + j] += v2 * bv; - r0[3 * N + j] += v3 * bv; - } - } + host_gemm_4rows()(a0, bpp, r0, K, N); // engine#1783: baseline or AVX2+FMA, picked once } else { for (int64_t i = i0; i < i1; ++i) { float* rr = r0 + size_t(i - i0) * size_t(N); From 3abd32184c67f3f1116fb3bb36fdb99e7cd1990a Mon Sep 17 00:00:00 2001 From: pi-lane Date: Sun, 15 Nov 2026 17:33:53 -0400 Subject: [PATCH 2/2] audio: the dispatched GEMM clone is AVX2 WITHOUT fma -- fma breaks the bit-for-bit contract CI caught this on the first build: audio_parallel asserts "the serial MatMul is the ordered per-element sum, bit for bit" at 1/2/4 threads and all three failed. The cause is the target pragma: a fused multiply-add rounds ONCE where the ordered SSE2 sum does `mul` then `add` and rounds twice, so the words differ in the last bits. My "bit-identical by construction" claim was wrong -- the statement sequence was identical, the ROUNDING was not. So the clone targets avx2 only (8-wide, no fusion) and the dispatch asks only __builtin_cpu_supports("avx2"). Measured after the change, interleaved on this box: AVX2=0 10198.0 / 3269.5 ms, AVX2=1 3462.6 / 2593.0 ms -- the 8-wide win survives without fma, and audio_parallel passes. --- npu/audio_exec.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/npu/audio_exec.cpp b/npu/audio_exec.cpp index a779ae181..a98df279c 100644 --- a/npu/audio_exec.cpp +++ b/npu/audio_exec.cpp @@ -290,7 +290,7 @@ void host_gemm_4rows_base(const float* a0, const float* bpp, float* r0, int64_t #if defined(__x86_64__) && defined(__GNUC__) #pragma GCC push_options -#pragma GCC target("avx2,fma") +#pragma GCC target("avx2") // NOT fma: a fused multiply-add rounds once, the ordered SSE2 sum rounds twice __attribute__((noinline)) void host_gemm_4rows_avx2(const float* a0, const float* bpp, float* r0, int64_t K, int64_t N) { ONEBIT_HOST_GEMM_4ROWS_BODY } @@ -302,9 +302,9 @@ void (*host_gemm_4rows())(const float*, const float*, float*, int64_t, int64_t) #if defined(__x86_64__) && defined(__GNUC__) static void (*pick)(const float*, const float*, float*, int64_t, int64_t) = [] { const char* e = std::getenv("ONEBIT_AUDIO_AVX2"); - const bool fast = __builtin_cpu_supports("avx2") && __builtin_cpu_supports("fma"); + const bool fast = __builtin_cpu_supports("avx2"); if (e != nullptr && e[0] == '1' && e[1] == '\0') { - if (!fast) std::fprintf(stderr, "[audio] ONEBIT_AUDIO_AVX2=1 but this CPU has no AVX2+FMA: keeping the baseline\n"); + if (!fast) std::fprintf(stderr, "[audio] ONEBIT_AUDIO_AVX2=1 but this CPU has no AVX2: keeping the baseline\n"); return fast ? &host_gemm_4rows_avx2 : &host_gemm_4rows_base; } if (e != nullptr && e[0] == '0' && e[1] == '\0') return &host_gemm_4rows_base;