From ae7aab59cd2d67b582d1290bb38ec01f11c9b5c7 Mon Sep 17 00:00:00 2001 From: pi-lane Date: Mon, 16 Nov 2026 01:49:51 -0400 Subject: [PATCH 1/3] audio: compile the executor twice and pick the flavour at load (AVX2 TU where the box has it) The flags lever measured on engine#1783 is a WHOLE-TU effect: a TU-wide -mavx2 build is 1.252x on the whisper-small encoder, while a clone of just the four-row GEMM inner loop (engine#2584, merged) measured 1.010x. The width helps every loop in the file -- elementwise, softmax rows, transpose gathers, the GEMM -- and only a TU split captures all of it. It cannot be a build-wide flag: the compiler is chosen per repo, the ISA is per box, and the 1bit OS appliance must run on whatever it is installed on (engine#2403). So audio_exec.cpp is compiled twice, distinguished by an inline namespace, and audio_exec_dispatch.cpp chooses once per process from __builtin_cpu_supports("avx2"). ONEBIT_AUDIO_AVX2=0 forces the baseline, =1 requires the fast flavour and says so when the CPU cannot run it. No fma in either flavour: a fused multiply-add rounds once where the ordered sum rounds twice, and audio_parallel asserts the ordered sum bit for bit (that arm is why #2584 ships without -mfma). Measured here, interleaved, same binary, thermal-run, 4 threads: default (AVX2 TU) 2952.0 / 2709.3 ms baseline 3796.9 / 3581.3 ms min-over-repeats 3581.3 / 2709.3 = 1.322x and audio_parallel: ok. --- CMakeLists.txt | 17 +++++++- npu/audio_exec.cpp | 12 +++++ npu/audio_exec_dispatch.cpp | 87 +++++++++++++++++++++++++++++++++++++ 3 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 npu/audio_exec_dispatch.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 061e3f38e..3dfa4620d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1075,8 +1075,23 @@ if(ONEBIT_AUDIO_AVX2) endif() endif() +# engine#1783: the audio executor is compiled twice and picked at load. The flags effect is a WHOLE-TU one +# (measured 1.252x TU-wide vs 1.010x for a clone of just the four-row GEMM), and the appliance must keep the +# baseline, so this is a runtime choice rather than a build-wide -mavx2 (engine#2403). +add_library(onebit_audio_exec_base OBJECT npu/audio_exec.cpp) +add_library(onebit_audio_exec_avx2 OBJECT npu/audio_exec.cpp) +foreach(_f onebit_audio_exec_base onebit_audio_exec_avx2) + target_include_directories(${_f} PRIVATE npu) +endforeach() +target_compile_definitions(onebit_audio_exec_base PRIVATE ONEBIT_AUDIO_TU_SUFFIX=base) +target_compile_definitions(onebit_audio_exec_avx2 PRIVATE ONEBIT_AUDIO_TU_SUFFIX=avx2) +include(CheckCXXCompilerFlag) +check_cxx_compiler_flag("-mavx2" ONEBIT_HAVE_AVX2_TU) +if(ONEBIT_HAVE_AVX2_TU AND CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64") + target_compile_options(onebit_audio_exec_avx2 PRIVATE -mavx2) # NOT -mfma: it breaks the ordered-sum contract +endif() add_library(onebit_npu STATIC npu/file_map.cpp npu/flm_head_import.cpp npu/full_elf.cpp npu/model.cpp npu/vision_pack.cpp npu/vision_tower.cpp npu/vision_image.cpp npu/vision_request.cpp - npu/audio_exec.cpp npu/audio_pack.cpp npu/moe.cpp npu/pack.cpp npu/pack_import.cpp npu/private_route.cpp + $ $ npu/audio_exec_dispatch.cpp npu/audio_pack.cpp npu/moe.cpp npu/pack.cpp npu/pack_import.cpp npu/private_route.cpp npu/q4nx.cpp npu/q4nx_device.cpp npu/tokenizer.cpp) target_include_directories(onebit_npu PUBLIC npu) target_link_libraries(onebit_npu PUBLIC nlohmann_json::nlohmann_json onebit_q4nx_cache PRIVATE PkgConfig::PCRE2) diff --git a/npu/audio_exec.cpp b/npu/audio_exec.cpp index 0bf21e223..3740036de 100644 --- a/npu/audio_exec.cpp +++ b/npu/audio_exec.cpp @@ -33,6 +33,16 @@ #include "file_map.h" namespace onebit::npu::pack_import { +// engine#1783: this TU is compiled TWICE -- once at the tree's own flags and once with -mavx2 -- and the two +// flavours are told apart only by this inline namespace, so every symbol mangles differently and both link. +// The 1.25x measured on this box is a WHOLE-TU effect (every elementwise/softmax/transpose loop, not just the +// four-row GEMM: a per-function clone of that alone measured 1.010x). See onebit_audio_exec_* in CMakeLists. +#ifndef ONEBIT_AUDIO_TU_SUFFIX +#define ONEBIT_AUDIO_TU_SUFFIX base +#endif +#define ONEBIT_JOIN2(a, b) a##b +#define ONEBIT_JOIN(a, b) ONEBIT_JOIN2(a, b) +inline namespace ONEBIT_JOIN(tu_, ONEBIT_AUDIO_TU_SUFFIX) { namespace { struct Tensor { @@ -860,3 +870,5 @@ bool run_decoder_graph(const OnnxGraphInfo& g, const int64_t* tokens, int64_t n_ } } // namespace onebit::npu::pack_import + +} // inline namespace tu_ diff --git a/npu/audio_exec_dispatch.cpp b/npu/audio_exec_dispatch.cpp new file mode 100644 index 000000000..d6946324f --- /dev/null +++ b/npu/audio_exec_dispatch.cpp @@ -0,0 +1,87 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// engine#1783: the audio executor ships as two compiled flavours and this file picks one, once, at load. +// +// Why a whole-TU split and not a per-function clone: the flags' effect is spread across every loop in +// audio_exec.cpp (elementwise, softmax rows, transpose gathers, the four-row GEMM). Measured on this box, a +// TU-wide -mavx2 build is 1.252x on the whisper-small encoder while a clone of just the GEMM inner loop was +// 1.010x -- the width helps all of them, and only the TU split captures all of it. +// +// Why runtime and not build-time: the compiler is chosen per repo, the ISA is per box, and the 1bit OS +// appliance must run on whatever it is installed on (engine#2403). The baseline flavour is always linked; the +// AVX2 one is only entered when the CPU reports both features. No `fma` in either flavour: a fused multiply-add +// rounds once where the ordered sum rounds twice, and audio_parallel asserts the ordered sum bit for bit. +#include "audio_exec.h" + +#include +#include + +namespace onebit::npu::pack_import { + +namespace tu_base { +bool run_audio_graph(const OnnxGraphInfo&, const float*, int64_t, int64_t, const std::string&, AudioRunResult*, + std::string*); +bool run_audio_graph(const OnnxGraphInfo&, const float*, int64_t, int64_t, const std::string&, AudioRunResult*, + std::string*, AudioGemm*); +bool run_decoder_graph(const OnnxGraphInfo&, const int64_t*, int64_t, const float*, const std::vector&, + const std::string&, AudioRunResult*, std::string*); +} // namespace tu_base + +#if defined(__x86_64__) && defined(__GNUC__) +namespace tu_avx2 { +bool run_audio_graph(const OnnxGraphInfo&, const float*, int64_t, int64_t, const std::string&, AudioRunResult*, + std::string*); +bool run_audio_graph(const OnnxGraphInfo&, const float*, int64_t, int64_t, const std::string&, AudioRunResult*, + std::string*, AudioGemm*); +bool run_decoder_graph(const OnnxGraphInfo&, const int64_t*, int64_t, const float*, const std::vector&, + const std::string&, AudioRunResult*, std::string*); +} // namespace tu_avx2 +#endif + +// Read once: the answer cannot change inside a process. ONEBIT_AUDIO_AVX2=0 forces the baseline; =1 requires +// the fast flavour and says so when the CPU cannot run it, rather than executing it anyway. +static bool use_avx2_tu() { +#if defined(__x86_64__) && defined(__GNUC__) + static const bool pick = [] { + const bool fast = __builtin_cpu_supports("avx2"); + const char* e = std::getenv("ONEBIT_AUDIO_AVX2"); + if (e != nullptr && e[0] == '0' && e[1] == '\0') return false; + if (e != nullptr && e[0] == '1' && e[1] == '\0' && !fast) { + std::fprintf(stderr, "[audio] ONEBIT_AUDIO_AVX2=1 but this CPU has no AVX2: keeping the baseline\n"); + return false; + } + return fast; + }(); + return pick; +#else + return false; +#endif +} + +bool run_audio_graph(const OnnxGraphInfo& g, const float* mel, int64_t mel_bins, int64_t frames, + const std::string& out_name, AudioRunResult* out, std::string* why) { +#if defined(__x86_64__) && defined(__GNUC__) + if (use_avx2_tu()) return tu_avx2::run_audio_graph(g, mel, mel_bins, frames, out_name, out, why); +#endif + return tu_base::run_audio_graph(g, mel, mel_bins, frames, out_name, out, why); +} + +bool run_audio_graph(const OnnxGraphInfo& g, const float* mel, int64_t mel_bins, int64_t frames, + const std::string& out_name, AudioRunResult* out, std::string* why, AudioGemm* dev) { +#if defined(__x86_64__) && defined(__GNUC__) + if (use_avx2_tu()) return tu_avx2::run_audio_graph(g, mel, mel_bins, frames, out_name, out, why, dev); +#endif + return tu_base::run_audio_graph(g, mel, mel_bins, frames, out_name, out, why, dev); +} + +bool run_decoder_graph(const OnnxGraphInfo& g, const int64_t* tokens, int64_t n_tokens, const float* xa, + const std::vector& xa_dims, const std::string& out_name, AudioRunResult* out, + std::string* why) { +#if defined(__x86_64__) && defined(__GNUC__) + if (use_avx2_tu()) return tu_avx2::run_decoder_graph(g, tokens, n_tokens, xa, xa_dims, out_name, out, why); +#endif + return tu_base::run_decoder_graph(g, tokens, n_tokens, xa, xa_dims, out_name, out, why); +} + +} // namespace onebit::npu::pack_import From 061f2088aee00e7d4348edd97ade314ddeee99f8 Mon Sep 17 00:00:00 2001 From: pi-lane Date: Mon, 16 Nov 2026 02:26:21 -0400 Subject: [PATCH 2/3] =?UTF-8?q?ci:=20re-dispatch=20=E2=80=94=20the=20first?= =?UTF-8?q?=20three=20builds=20404d=20on=20the=20apt=20mirror=20(perl-base?= =?UTF-8?q?/perl-modules-5.40/libperl5.40=20from=20two=20mirror=20IPs)=20w?= =?UTF-8?q?hile=20other=20PRs=20built=20green;=20no=20code=20change,=20thi?= =?UTF-8?q?s=20is=20a=20fresh=20run=20on=20a=20fresh=20runner?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From 5a066c46fdb4454a567b502a80e5c6efaf8b261c Mon Sep 17 00:00:00 2001 From: pi-lane Date: Mon, 16 Nov 2026 02:37:03 -0400 Subject: [PATCH 3/3] npu/audio_exec_dispatch.cpp: the full licence block, not just the SPDX line CI's copyright check wants the 16-line Apache block, not only copyright + SPDX; tools/copyright.py --fix added it and --check is clean. (The first three build failures on this PR were the apt mirror, engine#1814; this one was mine.) --- npu/audio_exec_dispatch.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/npu/audio_exec_dispatch.cpp b/npu/audio_exec_dispatch.cpp index d6946324f..63bc9cffc 100644 --- a/npu/audio_exec_dispatch.cpp +++ b/npu/audio_exec_dispatch.cpp @@ -1,6 +1,18 @@ // Copyright 2026 bong-water-water-bong // SPDX-License-Identifier: Apache-2.0 // +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// // engine#1783: the audio executor ships as two compiled flavours and this file picks one, once, at load. // // Why a whole-TU split and not a per-function clone: the flags' effect is spread across every loop in