Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 16 additions & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1075,8 +1075,23 @@ if(ONEBIT_AUDIO_AVX2)
endif()
endif()

# engine#1783: the audio executor is compiled twice and picked at load. The flags effect is a WHOLE-TU one
# (measured 1.252x TU-wide vs 1.010x for a clone of just the four-row GEMM), and the appliance must keep the
# baseline, so this is a runtime choice rather than a build-wide -mavx2 (engine#2403).
add_library(onebit_audio_exec_base OBJECT npu/audio_exec.cpp)
add_library(onebit_audio_exec_avx2 OBJECT npu/audio_exec.cpp)
foreach(_f onebit_audio_exec_base onebit_audio_exec_avx2)
target_include_directories(${_f} PRIVATE npu)
endforeach()
target_compile_definitions(onebit_audio_exec_base PRIVATE ONEBIT_AUDIO_TU_SUFFIX=base)
target_compile_definitions(onebit_audio_exec_avx2 PRIVATE ONEBIT_AUDIO_TU_SUFFIX=avx2)
include(CheckCXXCompilerFlag)
check_cxx_compiler_flag("-mavx2" ONEBIT_HAVE_AVX2_TU)
if(ONEBIT_HAVE_AVX2_TU AND CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64")
target_compile_options(onebit_audio_exec_avx2 PRIVATE -mavx2) # NOT -mfma: it breaks the ordered-sum contract
endif()
add_library(onebit_npu STATIC npu/file_map.cpp npu/flm_head_import.cpp npu/full_elf.cpp npu/model.cpp npu/vision_pack.cpp npu/vision_tower.cpp npu/vision_image.cpp npu/vision_request.cpp
npu/audio_exec.cpp npu/audio_pack.cpp npu/moe.cpp npu/pack.cpp npu/pack_import.cpp npu/private_route.cpp
$<TARGET_OBJECTS:onebit_audio_exec_base> $<TARGET_OBJECTS:onebit_audio_exec_avx2> npu/audio_exec_dispatch.cpp npu/audio_pack.cpp npu/moe.cpp npu/pack.cpp npu/pack_import.cpp npu/private_route.cpp
npu/q4nx.cpp npu/q4nx_device.cpp npu/tokenizer.cpp)
target_include_directories(onebit_npu PUBLIC npu)
target_link_libraries(onebit_npu PUBLIC nlohmann_json::nlohmann_json onebit_q4nx_cache PRIVATE PkgConfig::PCRE2)
Expand Down
12 changes: 12 additions & 0 deletions npu/audio_exec.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,16 @@
#include "file_map.h"

namespace onebit::npu::pack_import {
// engine#1783: this TU is compiled TWICE -- once at the tree's own flags and once with -mavx2 -- and the two
// flavours are told apart only by this inline namespace, so every symbol mangles differently and both link.
// The 1.25x measured on this box is a WHOLE-TU effect (every elementwise/softmax/transpose loop, not just the
// four-row GEMM: a per-function clone of that alone measured 1.010x). See onebit_audio_exec_* in CMakeLists.
#ifndef ONEBIT_AUDIO_TU_SUFFIX
#define ONEBIT_AUDIO_TU_SUFFIX base
#endif
#define ONEBIT_JOIN2(a, b) a##b
#define ONEBIT_JOIN(a, b) ONEBIT_JOIN2(a, b)
inline namespace ONEBIT_JOIN(tu_, ONEBIT_AUDIO_TU_SUFFIX) {
namespace {

struct Tensor {
Expand Down Expand Up @@ -860,3 +870,5 @@ bool run_decoder_graph(const OnnxGraphInfo& g, const int64_t* tokens, int64_t n_
}

} // namespace onebit::npu::pack_import

} // inline namespace tu_<suffix>
99 changes: 99 additions & 0 deletions npu/audio_exec_dispatch.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// engine#1783: the audio executor ships as two compiled flavours and this file picks one, once, at load.
//
// Why a whole-TU split and not a per-function clone: the flags' effect is spread across every loop in
// audio_exec.cpp (elementwise, softmax rows, transpose gathers, the four-row GEMM). Measured on this box, a
// TU-wide -mavx2 build is 1.252x on the whisper-small encoder while a clone of just the GEMM inner loop was
// 1.010x -- the width helps all of them, and only the TU split captures all of it.
//
// Why runtime and not build-time: the compiler is chosen per repo, the ISA is per box, and the 1bit OS
// appliance must run on whatever it is installed on (engine#2403). The baseline flavour is always linked; the
// AVX2 one is only entered when the CPU reports both features. No `fma` in either flavour: a fused multiply-add
// rounds once where the ordered sum rounds twice, and audio_parallel asserts the ordered sum bit for bit.
#include "audio_exec.h"

#include <cstdio>
#include <cstdlib>

namespace onebit::npu::pack_import {

namespace tu_base {
bool run_audio_graph(const OnnxGraphInfo&, const float*, int64_t, int64_t, const std::string&, AudioRunResult*,
std::string*);
bool run_audio_graph(const OnnxGraphInfo&, const float*, int64_t, int64_t, const std::string&, AudioRunResult*,
std::string*, AudioGemm*);
bool run_decoder_graph(const OnnxGraphInfo&, const int64_t*, int64_t, const float*, const std::vector<int64_t>&,
const std::string&, AudioRunResult*, std::string*);
} // namespace tu_base

#if defined(__x86_64__) && defined(__GNUC__)
namespace tu_avx2 {
bool run_audio_graph(const OnnxGraphInfo&, const float*, int64_t, int64_t, const std::string&, AudioRunResult*,
std::string*);
bool run_audio_graph(const OnnxGraphInfo&, const float*, int64_t, int64_t, const std::string&, AudioRunResult*,
std::string*, AudioGemm*);
bool run_decoder_graph(const OnnxGraphInfo&, const int64_t*, int64_t, const float*, const std::vector<int64_t>&,
const std::string&, AudioRunResult*, std::string*);
} // namespace tu_avx2
#endif

// Read once: the answer cannot change inside a process. ONEBIT_AUDIO_AVX2=0 forces the baseline; =1 requires
// the fast flavour and says so when the CPU cannot run it, rather than executing it anyway.
static bool use_avx2_tu() {
#if defined(__x86_64__) && defined(__GNUC__)
static const bool pick = [] {
const bool fast = __builtin_cpu_supports("avx2");
const char* e = std::getenv("ONEBIT_AUDIO_AVX2");
if (e != nullptr && e[0] == '0' && e[1] == '\0') return false;
if (e != nullptr && e[0] == '1' && e[1] == '\0' && !fast) {
std::fprintf(stderr, "[audio] ONEBIT_AUDIO_AVX2=1 but this CPU has no AVX2: keeping the baseline\n");
return false;
}
return fast;
}();
return pick;
#else
return false;
#endif
}

bool run_audio_graph(const OnnxGraphInfo& g, const float* mel, int64_t mel_bins, int64_t frames,
const std::string& out_name, AudioRunResult* out, std::string* why) {
#if defined(__x86_64__) && defined(__GNUC__)
if (use_avx2_tu()) return tu_avx2::run_audio_graph(g, mel, mel_bins, frames, out_name, out, why);
#endif
return tu_base::run_audio_graph(g, mel, mel_bins, frames, out_name, out, why);
}

bool run_audio_graph(const OnnxGraphInfo& g, const float* mel, int64_t mel_bins, int64_t frames,
const std::string& out_name, AudioRunResult* out, std::string* why, AudioGemm* dev) {
#if defined(__x86_64__) && defined(__GNUC__)
if (use_avx2_tu()) return tu_avx2::run_audio_graph(g, mel, mel_bins, frames, out_name, out, why, dev);
#endif
return tu_base::run_audio_graph(g, mel, mel_bins, frames, out_name, out, why, dev);
}

bool run_decoder_graph(const OnnxGraphInfo& g, const int64_t* tokens, int64_t n_tokens, const float* xa,
const std::vector<int64_t>& xa_dims, const std::string& out_name, AudioRunResult* out,
std::string* why) {
#if defined(__x86_64__) && defined(__GNUC__)
if (use_avx2_tu()) return tu_avx2::run_decoder_graph(g, tokens, n_tokens, xa, xa_dims, out_name, out, why);
#endif
return tu_base::run_decoder_graph(g, tokens, n_tokens, xa, xa_dims, out_name, out, why);
}

} // namespace onebit::npu::pack_import
Loading