diff --git a/CMakeLists.txt b/CMakeLists.txt index 061e3f38e..3dfa4620d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1075,8 +1075,23 @@ if(ONEBIT_AUDIO_AVX2) endif() endif() +# engine#1783: the audio executor is compiled twice and picked at load. The flags effect is a WHOLE-TU one +# (measured 1.252x TU-wide vs 1.010x for a clone of just the four-row GEMM), and the appliance must keep the +# baseline, so this is a runtime choice rather than a build-wide -mavx2 (engine#2403). +add_library(onebit_audio_exec_base OBJECT npu/audio_exec.cpp) +add_library(onebit_audio_exec_avx2 OBJECT npu/audio_exec.cpp) +foreach(_f onebit_audio_exec_base onebit_audio_exec_avx2) + target_include_directories(${_f} PRIVATE npu) +endforeach() +target_compile_definitions(onebit_audio_exec_base PRIVATE ONEBIT_AUDIO_TU_SUFFIX=base) +target_compile_definitions(onebit_audio_exec_avx2 PRIVATE ONEBIT_AUDIO_TU_SUFFIX=avx2) +include(CheckCXXCompilerFlag) +check_cxx_compiler_flag("-mavx2" ONEBIT_HAVE_AVX2_TU) +if(ONEBIT_HAVE_AVX2_TU AND CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64") + target_compile_options(onebit_audio_exec_avx2 PRIVATE -mavx2) # NOT -mfma: it breaks the ordered-sum contract +endif() add_library(onebit_npu STATIC npu/file_map.cpp npu/flm_head_import.cpp npu/full_elf.cpp npu/model.cpp npu/vision_pack.cpp npu/vision_tower.cpp npu/vision_image.cpp npu/vision_request.cpp - npu/audio_exec.cpp npu/audio_pack.cpp npu/moe.cpp npu/pack.cpp npu/pack_import.cpp npu/private_route.cpp + $ $ npu/audio_exec_dispatch.cpp npu/audio_pack.cpp npu/moe.cpp npu/pack.cpp npu/pack_import.cpp npu/private_route.cpp npu/q4nx.cpp npu/q4nx_device.cpp npu/tokenizer.cpp) target_include_directories(onebit_npu PUBLIC npu) target_link_libraries(onebit_npu PUBLIC nlohmann_json::nlohmann_json onebit_q4nx_cache PRIVATE PkgConfig::PCRE2) diff --git a/npu/audio_exec.cpp b/npu/audio_exec.cpp index 0bf21e223..3740036de 100644 --- a/npu/audio_exec.cpp +++ b/npu/audio_exec.cpp @@ -33,6 +33,16 @@ #include "file_map.h" namespace onebit::npu::pack_import { +// engine#1783: this TU is compiled TWICE -- once at the tree's own flags and once with -mavx2 -- and the two +// flavours are told apart only by this inline namespace, so every symbol mangles differently and both link. +// The 1.25x measured on this box is a WHOLE-TU effect (every elementwise/softmax/transpose loop, not just the +// four-row GEMM: a per-function clone of that alone measured 1.010x). See onebit_audio_exec_* in CMakeLists. +#ifndef ONEBIT_AUDIO_TU_SUFFIX +#define ONEBIT_AUDIO_TU_SUFFIX base +#endif +#define ONEBIT_JOIN2(a, b) a##b +#define ONEBIT_JOIN(a, b) ONEBIT_JOIN2(a, b) +inline namespace ONEBIT_JOIN(tu_, ONEBIT_AUDIO_TU_SUFFIX) { namespace { struct Tensor { @@ -860,3 +870,5 @@ bool run_decoder_graph(const OnnxGraphInfo& g, const int64_t* tokens, int64_t n_ } } // namespace onebit::npu::pack_import + +} // inline namespace tu_ diff --git a/npu/audio_exec_dispatch.cpp b/npu/audio_exec_dispatch.cpp new file mode 100644 index 000000000..63bc9cffc --- /dev/null +++ b/npu/audio_exec_dispatch.cpp @@ -0,0 +1,99 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// engine#1783: the audio executor ships as two compiled flavours and this file picks one, once, at load. +// +// Why a whole-TU split and not a per-function clone: the flags' effect is spread across every loop in +// audio_exec.cpp (elementwise, softmax rows, transpose gathers, the four-row GEMM). Measured on this box, a +// TU-wide -mavx2 build is 1.252x on the whisper-small encoder while a clone of just the GEMM inner loop was +// 1.010x -- the width helps all of them, and only the TU split captures all of it. +// +// Why runtime and not build-time: the compiler is chosen per repo, the ISA is per box, and the 1bit OS +// appliance must run on whatever it is installed on (engine#2403). The baseline flavour is always linked; the +// AVX2 one is only entered when the CPU reports both features. No `fma` in either flavour: a fused multiply-add +// rounds once where the ordered sum rounds twice, and audio_parallel asserts the ordered sum bit for bit. +#include "audio_exec.h" + +#include +#include + +namespace onebit::npu::pack_import { + +namespace tu_base { +bool run_audio_graph(const OnnxGraphInfo&, const float*, int64_t, int64_t, const std::string&, AudioRunResult*, + std::string*); +bool run_audio_graph(const OnnxGraphInfo&, const float*, int64_t, int64_t, const std::string&, AudioRunResult*, + std::string*, AudioGemm*); +bool run_decoder_graph(const OnnxGraphInfo&, const int64_t*, int64_t, const float*, const std::vector&, + const std::string&, AudioRunResult*, std::string*); +} // namespace tu_base + +#if defined(__x86_64__) && defined(__GNUC__) +namespace tu_avx2 { +bool run_audio_graph(const OnnxGraphInfo&, const float*, int64_t, int64_t, const std::string&, AudioRunResult*, + std::string*); +bool run_audio_graph(const OnnxGraphInfo&, const float*, int64_t, int64_t, const std::string&, AudioRunResult*, + std::string*, AudioGemm*); +bool run_decoder_graph(const OnnxGraphInfo&, const int64_t*, int64_t, const float*, const std::vector&, + const std::string&, AudioRunResult*, std::string*); +} // namespace tu_avx2 +#endif + +// Read once: the answer cannot change inside a process. ONEBIT_AUDIO_AVX2=0 forces the baseline; =1 requires +// the fast flavour and says so when the CPU cannot run it, rather than executing it anyway. +static bool use_avx2_tu() { +#if defined(__x86_64__) && defined(__GNUC__) + static const bool pick = [] { + const bool fast = __builtin_cpu_supports("avx2"); + const char* e = std::getenv("ONEBIT_AUDIO_AVX2"); + if (e != nullptr && e[0] == '0' && e[1] == '\0') return false; + if (e != nullptr && e[0] == '1' && e[1] == '\0' && !fast) { + std::fprintf(stderr, "[audio] ONEBIT_AUDIO_AVX2=1 but this CPU has no AVX2: keeping the baseline\n"); + return false; + } + return fast; + }(); + return pick; +#else + return false; +#endif +} + +bool run_audio_graph(const OnnxGraphInfo& g, const float* mel, int64_t mel_bins, int64_t frames, + const std::string& out_name, AudioRunResult* out, std::string* why) { +#if defined(__x86_64__) && defined(__GNUC__) + if (use_avx2_tu()) return tu_avx2::run_audio_graph(g, mel, mel_bins, frames, out_name, out, why); +#endif + return tu_base::run_audio_graph(g, mel, mel_bins, frames, out_name, out, why); +} + +bool run_audio_graph(const OnnxGraphInfo& g, const float* mel, int64_t mel_bins, int64_t frames, + const std::string& out_name, AudioRunResult* out, std::string* why, AudioGemm* dev) { +#if defined(__x86_64__) && defined(__GNUC__) + if (use_avx2_tu()) return tu_avx2::run_audio_graph(g, mel, mel_bins, frames, out_name, out, why, dev); +#endif + return tu_base::run_audio_graph(g, mel, mel_bins, frames, out_name, out, why, dev); +} + +bool run_decoder_graph(const OnnxGraphInfo& g, const int64_t* tokens, int64_t n_tokens, const float* xa, + const std::vector& xa_dims, const std::string& out_name, AudioRunResult* out, + std::string* why) { +#if defined(__x86_64__) && defined(__GNUC__) + if (use_avx2_tu()) return tu_avx2::run_decoder_graph(g, tokens, n_tokens, xa, xa_dims, out_name, out, why); +#endif + return tu_base::run_decoder_graph(g, tokens, n_tokens, xa, xa_dims, out_name, out, why); +} + +} // namespace onebit::npu::pack_import