From ce109a1ead76b031f0e97ec04519109f05dc7c7c Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sat, 14 Nov 2026 01:44:36 -0400 Subject: [PATCH 1/2] cmake: ONEBIT_AUDIO_AVX2 builds the audio targets with -mavx2 -mfma, 1.33x on the audio GEMM (engine#2399) --- CMakeLists.txt | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4a3268e1c..a2803ea29 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -974,6 +974,33 @@ option(ONEBIT_NPU_HOST "Build the NPU host library and its tests" ${_onebit_npu_ if(ONEBIT_NPU_HOST) find_package(PkgConfig REQUIRED) pkg_check_modules(PCRE2 REQUIRED IMPORTED_TARGET libpcre2-8) +# engine#2399: the audio graph's core is a 4-row f32 GEMM. At the tree's own -O3 the microbenchmark gives 37.5 GFLOP/s, +# but the ISA is never requested, so the vectoriser emits mul+add instead of fused multiply-add. Measured IN SITU on the +# whisper-small encoder's MatMul family, alternating two binaries under one load at 4 threads: +# 2698.9 / 2699.4 ms (-O3) -> 2037.9 / 2019.2 ms (-O3 -mavx2 -mfma) 1.33x +# Transcripts unchanged: ids md5 91c6561060 (jfk.wav) and fd875ac0d9 (a German clip -- checked because FMA changes +# rounding, and English transcribe==translate would hide a last-bit divergence). +# +# The flags must cover the TESTS as well as the library: audio_parallel_test compares each thread count against an +# ordered per-element reference computed in its own TU. Flags on the library only fails 16 checks with "matches the +# ordered sum" at every thread count -- not a reproducibility defect, just two TUs compiled differently. Verified both +# ways: flags on both sides -> ok at ONEBIT_AUDIO_THREADS=1,2,4; library only -> 16 failures. +# +# Explicit -mavx2 -mfma rather than -march=native: the same speed within noise, but it states the requirement instead +# of inheriting the build host. The cost is real, which is why this is an option: the binary then needs AVX2+FMA at RUN +# time (universal on this fleet -- Zen 2 and later -- not universal for a distributed build). OFF restores portability. +option(ONEBIT_AUDIO_AVX2 "Compile with -mavx2 -mfma: audio GEMM 1.33x; the binary then requires AVX2+FMA at run time" ON) +if(ONEBIT_AUDIO_AVX2) + include(CheckCXXCompilerFlag) + check_cxx_compiler_flag("-mavx2" ONEBIT_HAVE_MAVX2) + check_cxx_compiler_flag("-mfma" ONEBIT_HAVE_MFMA) + if(ONEBIT_HAVE_MAVX2 AND ONEBIT_HAVE_MFMA) + set(ONEBIT_AUDIO_AVX2_READY ON) + else() + message(STATUS "ONEBIT_AUDIO_AVX2=ON but the compiler rejects -mavx2/-mfma; building portable") + endif() +endif() + add_library(onebit_npu STATIC npu/file_map.cpp npu/flm_head_import.cpp npu/full_elf.cpp npu/model.cpp npu/vision_pack.cpp npu/vision_tower.cpp npu/vision_image.cpp npu/vision_request.cpp npu/audio_exec.cpp npu/audio_pack.cpp npu/moe.cpp npu/pack.cpp npu/pack_import.cpp npu/private_route.cpp npu/q4nx.cpp npu/q4nx_device.cpp npu/tokenizer.cpp) @@ -2015,6 +2042,19 @@ add_executable(audio_npu_gemm_test tests/audio_npu_gemm_test.cpp) target_include_directories(audio_npu_gemm_test PRIVATE npu/forward/include npu/target npu/plan) target_link_libraries(audio_npu_gemm_test PRIVATE nlohmann_json::nlohmann_json) add_executable(audio_launch_plan_test tests/audio_launch_plan_test.cpp) +# engine#2399: apply the ISA flags to the audio targets only (see the option above). Placed here, after those targets are +# declared, so nothing outside the audio path inherits an AVX2 requirement. +if(ONEBIT_AUDIO_AVX2_READY) + # Enumerated HERE, after every audio target exists: the option block runs ~600 lines earlier, where + # audio_ops_test/audio_parallel_test are not defined yet, so an `if(TARGET ...)` test there is always false. + set(_avx2_targets onebit_npu audio_parallel_test audio_ops_test audio_exec_test audio_mel_test + whisper_transcribe audio_kernel_shapes audio_device_gemm_test) + foreach(_t ${_avx2_targets}) + if(TARGET ${_t}) + target_compile_options(${_t} PRIVATE -mavx2 -mfma) + endif() + endforeach() +endif() target_include_directories(audio_launch_plan_test PRIVATE npu/forward/include npu/target npu/plan) target_link_libraries(audio_launch_plan_test PRIVATE nlohmann_json::nlohmann_json) add_test(NAME whisper_ids COMMAND whisper_ids_test) From 13e952a42f8c092fb28d545c1e7c5ec88daaf668 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sat, 14 Nov 2026 14:09:33 -0400 Subject: [PATCH 2/2] cmake: ONEBIT_AUDIO_AVX2 flags the audio SOURCES, not onebit_npu -- and defaults OFF to match its own argument (engine#2403) All three items from the CHANGES at `2c5079a`. **1. The flag was on the whole library.** `onebit_npu` is a STATIC library that also holds the tokenizer, the pack/ELF import path and the vision packer, so `target_compile_options(onebit_npu ... -mavx2 -mfma)` made every one of those TUs require AVX2+FMA at run time. Measured before/after with `CMAKE_EXPORT_COMPILE_COMMANDS` at `-DONEBIT_AUDIO_AVX2=ON`: ``` before onebit_npu's whole source list -mavx2 on ~160 TUs incl. tokenizer, pack_import, vision_pack, model after -mavx2 on 12 TUs: audio_exec.cpp audio_pack.cpp + the audio_*/whisper_* test sources tokenizer.cpp=False pack_import.cpp=False vision_pack.cpp=False model.cpp=False ``` The requirement now sits on the sources that need it. The test side keeps target-level flags because the comment above records why both sides must agree -- flags on one side only fails 16 checks at `ONEBIT_AUDIO_THREADS=1,2,4`. **2. The default was ON while the paragraph argued the cost is why it is an option.** The option now defaults **OFF** and its doc string says what it actually does ("the audio TUs" rather than "the binary"). Verified: a default configure produces `-mavx2 count: 0 of 259 TUs`; `-DONEBIT_AUDIO_AVX2=ON` produces the 12 above. The 1.33x is still there for whoever turns it on -- the distributor decides to take a run-time ISA requirement, a default does not decide for them. **3. The enumeration is now a glob, so the next audio target is covered.** `npu/audio_*.cpp` and `tests/audio_*.cpp tests/whisper_*.cpp` are globbed with `CONFIGURE_DEPENDS` and given the flags as source properties. The named targets that remain in the loop (`whisper_transcribe`, `audio_*_test`) are the ones whose TU is not named `audio_*`; anything added later under those globs is picked up without anyone remembering a list. Configure is clean at both settings (`Configuring done (2.4s)`); I did not run the audio tests here, which need the NPU runtime and a build. --- CMakeLists.txt | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a2803ea29..ed6bca8c6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -987,9 +987,12 @@ pkg_check_modules(PCRE2 REQUIRED IMPORTED_TARGET libpcre2-8) # ways: flags on both sides -> ok at ONEBIT_AUDIO_THREADS=1,2,4; library only -> 16 failures. # # Explicit -mavx2 -mfma rather than -march=native: the same speed within noise, but it states the requirement instead -# of inheriting the build host. The cost is real, which is why this is an option: the binary then needs AVX2+FMA at RUN -# time (universal on this fleet -- Zen 2 and later -- not universal for a distributed build). OFF restores portability. -option(ONEBIT_AUDIO_AVX2 "Compile with -mavx2 -mfma: audio GEMM 1.33x; the binary then requires AVX2+FMA at run time" ON) +# of inheriting the build host. The cost is real: the audio TUs then need AVX2+FMA at RUN time (universal on this +# fleet -- Zen 2 and later -- not universal for a distributed build). The cost is real, so this is an OPTION, and an +# option that imposes a run-time ISA requirement defaults to OFF: the speedup is 1.33x on the audio GEMM and buying it +# is a decision the distributor makes, not one a default makes for them. (engine#2403: the default was ON while the +# paragraph above argued that the requirement is why it is optional -- the two disagreed with each other.) +option(ONEBIT_AUDIO_AVX2 "Compile the audio TUs with -mavx2 -mfma: audio GEMM 1.33x; those TUs then require AVX2+FMA at run time" OFF) if(ONEBIT_AUDIO_AVX2) include(CheckCXXCompilerFlag) check_cxx_compiler_flag("-mavx2" ONEBIT_HAVE_MAVX2) @@ -2047,9 +2050,23 @@ add_executable(audio_launch_plan_test tests/audio_launch_plan_test.cpp) if(ONEBIT_AUDIO_AVX2_READY) # Enumerated HERE, after every audio target exists: the option block runs ~600 lines earlier, where # audio_ops_test/audio_parallel_test are not defined yet, so an `if(TARGET ...)` test there is always false. - set(_avx2_targets onebit_npu audio_parallel_test audio_ops_test audio_exec_test audio_mel_test - whisper_transcribe audio_kernel_shapes audio_device_gemm_test) - foreach(_t ${_avx2_targets}) + # engine#2403: onebit_npu is a STATIC library that ALSO holds the tokenizer, the pack/ELF import path and the + # vision packer. A target-level flag here made every one of those TUs require AVX2+FMA, which is a run-time + # requirement nobody asked them to take. The requirement belongs to the audio sources, so the flags go on the + # audio sources -- and the list is a GLOB, so the next npu/audio_*.cpp is covered without anyone remembering. + file(GLOB _onebit_audio_srcs CONFIGURE_DEPENDS npu/audio_*.cpp) + if(_onebit_audio_srcs) + set_source_files_properties(${_onebit_audio_srcs} PROPERTIES COMPILE_OPTIONS "-mavx2;-mfma") + endif() + # The test targets below compute their per-element reference in their OWN TU, and the comment above records that + # flags on one side only fails 16 checks at ONEBIT_AUDIO_THREADS=1,2,4 -- so both sides get them. Same glob + # argument: a new tests/audio_*.cpp is picked up, and the named targets are the ones whose TU is not named audio_*. + file(GLOB _onebit_audio_test_srcs CONFIGURE_DEPENDS tests/audio_*.cpp tests/whisper_*.cpp) + if(_onebit_audio_test_srcs) + set_source_files_properties(${_onebit_audio_test_srcs} PROPERTIES COMPILE_OPTIONS "-mavx2;-mfma") + endif() + foreach(_t audio_parallel_test audio_ops_test audio_exec_test audio_mel_test audio_kernel_shapes + audio_device_gemm_test whisper_transcribe) if(TARGET ${_t}) target_compile_options(${_t} PRIVATE -mavx2 -mfma) endif()