Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 57 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -974,6 +974,36 @@ option(ONEBIT_NPU_HOST "Build the NPU host library and its tests" ${_onebit_npu_
if(ONEBIT_NPU_HOST)
find_package(PkgConfig REQUIRED)
pkg_check_modules(PCRE2 REQUIRED IMPORTED_TARGET libpcre2-8)
# engine#2399: the audio graph's core is a 4-row f32 GEMM. At the tree's own -O3 the microbenchmark gives 37.5 GFLOP/s,
# but the ISA is never requested, so the vectoriser emits mul+add instead of fused multiply-add. Measured IN SITU on the
# whisper-small encoder's MatMul family, alternating two binaries under one load at 4 threads:
# 2698.9 / 2699.4 ms (-O3) -> 2037.9 / 2019.2 ms (-O3 -mavx2 -mfma) 1.33x
# Transcripts unchanged: ids md5 91c6561060 (jfk.wav) and fd875ac0d9 (a German clip -- checked because FMA changes
# rounding, and English transcribe==translate would hide a last-bit divergence).
#
# The flags must cover the TESTS as well as the library: audio_parallel_test compares each thread count against an
# ordered per-element reference computed in its own TU. Flags on the library only fails 16 checks with "matches the
# ordered sum" at every thread count -- not a reproducibility defect, just two TUs compiled differently. Verified both
# ways: flags on both sides -> ok at ONEBIT_AUDIO_THREADS=1,2,4; library only -> 16 failures.
#
# Explicit -mavx2 -mfma rather than -march=native: the same speed within noise, but it states the requirement instead
# of inheriting the build host. The cost is real: the audio TUs then need AVX2+FMA at RUN time (universal on this
# fleet -- Zen 2 and later -- not universal for a distributed build). The cost is real, so this is an OPTION, and an
# option that imposes a run-time ISA requirement defaults to OFF: the speedup is 1.33x on the audio GEMM and buying it
# is a decision the distributor makes, not one a default makes for them. (engine#2403: the default was ON while the
# paragraph above argued that the requirement is why it is optional -- the two disagreed with each other.)
option(ONEBIT_AUDIO_AVX2 "Compile the audio TUs with -mavx2 -mfma: audio GEMM 1.33x; those TUs then require AVX2+FMA at run time" OFF)
if(ONEBIT_AUDIO_AVX2)
include(CheckCXXCompilerFlag)
check_cxx_compiler_flag("-mavx2" ONEBIT_HAVE_MAVX2)
check_cxx_compiler_flag("-mfma" ONEBIT_HAVE_MFMA)
if(ONEBIT_HAVE_MAVX2 AND ONEBIT_HAVE_MFMA)
set(ONEBIT_AUDIO_AVX2_READY ON)
else()
message(STATUS "ONEBIT_AUDIO_AVX2=ON but the compiler rejects -mavx2/-mfma; building portable")
endif()
endif()

add_library(onebit_npu STATIC npu/file_map.cpp npu/flm_head_import.cpp npu/full_elf.cpp npu/model.cpp npu/vision_pack.cpp npu/vision_tower.cpp npu/vision_image.cpp npu/vision_request.cpp
npu/audio_exec.cpp npu/audio_pack.cpp npu/moe.cpp npu/pack.cpp npu/pack_import.cpp npu/private_route.cpp
npu/q4nx.cpp npu/q4nx_device.cpp npu/tokenizer.cpp)
Expand Down Expand Up @@ -2015,6 +2045,33 @@ add_executable(audio_npu_gemm_test tests/audio_npu_gemm_test.cpp)
target_include_directories(audio_npu_gemm_test PRIVATE npu/forward/include npu/target npu/plan)
target_link_libraries(audio_npu_gemm_test PRIVATE nlohmann_json::nlohmann_json)
add_executable(audio_launch_plan_test tests/audio_launch_plan_test.cpp)
# engine#2399: apply the ISA flags to the audio targets only (see the option above). Placed here, after those targets are
# declared, so nothing outside the audio path inherits an AVX2 requirement.
if(ONEBIT_AUDIO_AVX2_READY)
# Enumerated HERE, after every audio target exists: the option block runs ~600 lines earlier, where
# audio_ops_test/audio_parallel_test are not defined yet, so an `if(TARGET ...)` test there is always false.
# engine#2403: onebit_npu is a STATIC library that ALSO holds the tokenizer, the pack/ELF import path and the
# vision packer. A target-level flag here made every one of those TUs require AVX2+FMA, which is a run-time
# requirement nobody asked them to take. The requirement belongs to the audio sources, so the flags go on the
# audio sources -- and the list is a GLOB, so the next npu/audio_*.cpp is covered without anyone remembering.
file(GLOB _onebit_audio_srcs CONFIGURE_DEPENDS npu/audio_*.cpp)
if(_onebit_audio_srcs)
set_source_files_properties(${_onebit_audio_srcs} PROPERTIES COMPILE_OPTIONS "-mavx2;-mfma")
endif()
# The test targets below compute their per-element reference in their OWN TU, and the comment above records that
# flags on one side only fails 16 checks at ONEBIT_AUDIO_THREADS=1,2,4 -- so both sides get them. Same glob
# argument: a new tests/audio_*.cpp is picked up, and the named targets are the ones whose TU is not named audio_*.
file(GLOB _onebit_audio_test_srcs CONFIGURE_DEPENDS tests/audio_*.cpp tests/whisper_*.cpp)
if(_onebit_audio_test_srcs)
set_source_files_properties(${_onebit_audio_test_srcs} PROPERTIES COMPILE_OPTIONS "-mavx2;-mfma")
endif()
foreach(_t audio_parallel_test audio_ops_test audio_exec_test audio_mel_test audio_kernel_shapes
audio_device_gemm_test whisper_transcribe)
if(TARGET ${_t})
target_compile_options(${_t} PRIVATE -mavx2 -mfma)
endif()
endforeach()
endif()
target_include_directories(audio_launch_plan_test PRIVATE npu/forward/include npu/target npu/plan)
target_link_libraries(audio_launch_plan_test PRIVATE nlohmann_json::nlohmann_json)
add_test(NAME whisper_ids COMMAND whisper_ids_test)
Expand Down
Loading