Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 37 additions & 2 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -925,7 +925,7 @@ if(ONEBIT_NPU_HOST)
find_package(PkgConfig REQUIRED)
pkg_check_modules(PCRE2 REQUIRED IMPORTED_TARGET libpcre2-8)
add_library(onebit_npu STATIC npu/file_map.cpp npu/flm_head_import.cpp npu/full_elf.cpp npu/model.cpp npu/vision_pack.cpp npu/vision_tower.cpp npu/vision_image.cpp npu/vision_request.cpp
npu/audio_exec.cpp npu/moe.cpp npu/pack.cpp npu/pack_import.cpp npu/private_route.cpp
npu/audio_exec.cpp npu/audio_pack.cpp npu/moe.cpp npu/pack.cpp npu/pack_import.cpp npu/private_route.cpp
npu/q4nx.cpp npu/q4nx_device.cpp npu/tokenizer.cpp)
target_include_directories(onebit_npu PUBLIC npu)
target_link_libraries(onebit_npu PUBLIC nlohmann_json::nlohmann_json onebit_q4nx_cache PRIVATE PkgConfig::PCRE2)
Expand Down Expand Up @@ -1560,7 +1560,8 @@ add_executable(audio_parallel_test tests/audio_parallel_test.cpp)
target_link_libraries(audio_parallel_test PRIVATE onebit_npu Threads::Threads)
# ci.yml's build list is not ours to edit without `workflow` scope, so these reach CI the way ops_dump_test does: as build
# dependencies of a target it already builds (tools/ci/ci_targets_check.py follows this edge).
add_dependencies(onebit audio_ops_test audio_mel_test onnx_graph_test audio_exec_test audio_parallel_test)
# One line, deliberately: tools/ci/ci_targets_check.py follows `add_dependencies` line by line.
add_dependencies(onebit audio_ops_test audio_mel_test onnx_graph_test audio_exec_test audio_parallel_test audio_pack_test audio_kernel_roles_test whisper_ids_test audio_device_gemm_test audio_launch_plan_test audio_kernel_shapes whisper_transcribe)
# A DFlash draft package as a model source (engine#1780): config + container -> a checked model whose tensors decode
# one at a time. The loader is compiled here and not into onebit_npu because it needs the q4_1 tile decoder, which
# lives with the forward (npu/forward/src/q4nx_dequant.c); it moves there with the draft's forward. Synthetic package,
Expand Down Expand Up @@ -1943,6 +1944,40 @@ add_test(NAME audio_mel COMMAND audio_mel_test)
set_tests_properties(audio_ops PROPERTIES SKIP_RETURN_CODE 77)
add_test(NAME onnx_graph COMMAND onnx_graph_test ${ONEBIT_AUDIO_ONNX})
add_test(NAME audio_exec COMMAND audio_exec_test ${ONEBIT_AUDIO_ONNX})
# engine#1783: the audio pack -- two graphs by their exported names (never `decoder_with_past`), mel bins from the encoder's
# own first Conv, config.json as a cross-check.
add_executable(audio_pack_test tests/audio_pack_test.cpp)
target_include_directories(audio_pack_test PRIVATE npu/forward/include)
target_link_libraries(audio_pack_test PRIVATE onebit_npu)
add_executable(audio_kernel_roles_test tests/audio_kernel_roles_test.cpp)
target_include_directories(audio_kernel_roles_test PRIVATE npu/forward/include)
add_executable(whisper_ids_test tests/whisper_ids_test.cpp)
target_include_directories(whisper_ids_test PRIVATE npu/forward/include)
add_executable(audio_kernel_shapes tools/audio_kernel_shapes.cpp)
target_include_directories(audio_kernel_shapes PRIVATE npu/forward/include)
target_link_libraries(audio_kernel_shapes PRIVATE onebit_npu)
add_executable(audio_device_gemm_test tests/audio_device_gemm_test.cpp)
target_include_directories(audio_device_gemm_test PRIVATE npu/forward/include npu/target npu/plan)
target_link_libraries(audio_device_gemm_test PRIVATE nlohmann_json::nlohmann_json)
add_executable(audio_launch_plan_test tests/audio_launch_plan_test.cpp)
target_include_directories(audio_launch_plan_test PRIVATE npu/forward/include npu/target npu/plan)
target_link_libraries(audio_launch_plan_test PRIVATE nlohmann_json::nlohmann_json)
add_test(NAME whisper_ids COMMAND whisper_ids_test)
add_test(NAME audio_kernel_roles COMMAND audio_kernel_roles_test)
add_test(NAME audio_pack COMMAND audio_pack_test)
add_test(NAME audio_device_gemm COMMAND audio_device_gemm_test)
add_test(NAME audio_launch_plan COMMAND audio_launch_plan_test)
add_test(NAME whisper_transcribe_jfk COMMAND whisper_transcribe)
set_tests_properties(whisper_transcribe_jfk PROPERTIES
ENVIRONMENT "ONEBIT_TEST_WHISPER_IDS_FILE=${CMAKE_SOURCE_DIR}/tests/data/whisper-jfk-ids.json"
SKIP_RETURN_CODE 77 TIMEOUT 1800)
add_test(NAME whisper_transcribe_jfk_lv3
COMMAND ${CMAKE_SOURCE_DIR}/tests/whisper_lv3_row.sh $<TARGET_FILE:whisper_transcribe>
${CMAKE_SOURCE_DIR}/tests/data/whisper-jfk-ids.json)
set_tests_properties(whisper_transcribe_jfk_lv3 PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 3600)
add_test(NAME audio_kernel_plan
COMMAND bash ${CMAKE_SOURCE_DIR}/tests/audio_kernel_plan.sh $<TARGET_FILE:audio_kernel_shapes>)
set_tests_properties(audio_kernel_plan PROPERTIES SKIP_RETURN_CODE 77)
add_test(NAME audio_parallel COMMAND audio_parallel_test)
set_tests_properties(audio_exec PROPERTIES SKIP_RETURN_CODE 77)
set_tests_properties(onnx_graph PROPERTIES SKIP_RETURN_CODE 77)
Expand Down
190 changes: 190 additions & 0 deletions npu/audio_pack.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,190 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// audio_pack.cpp -- see audio_pack.h. The discovery is deliberately boring: known names, in a fixed order, and a refusal that
// names what is missing. The one interesting step is the mel-bin count, which comes from the encoder graph rather than a config.

#include "audio_pack.h"

#include <algorithm>
#include <cstdio>
#include <cstdlib>
#include <filesystem>

#include "onnx_graph.h"

namespace fs = std::filesystem;

namespace onebit::audio {
namespace {

bool is_file(const std::string& p) {
std::error_code ec;
return fs::is_regular_file(fs::path(p), ec);
}

// The names exporters actually use, most specific first. A pack that carries both spellings is a pack someone assembled by
// hand; the first match wins and the other is ignored, which is why `dir` is recorded.
const char* kEncoderNames[] = {"encoder_model.onnx", "encoder.onnx"};
const char* kDecoderNames[] = {"decoder_model.onnx", "decoder.onnx"};
const char* kTokenizerNames[] = {"tokenizer.json", "tokenizer_config.json"};

int mel_bins_from_encoder(const std::string& path, std::string* why) {
using namespace onebit::npu::pack_import;
OnnxGraphInfo g;
try {
g = parse_onnx_graph(path);
} catch (const std::exception& e) {
if (why != nullptr) *why = e.what();
return 0;
}
// The mel front-end is the graph's first Conv; its input-channel count is the bin count the encoder was exported for.
for (const auto& n : g.nodes) {
if (n.op != "Conv" || n.ins.size() < 2) continue;
for (const auto& ti : g.init_tensors) {
if (ti.name != n.ins[1] || ti.dims.size() != 3) continue;
return int(ti.dims[1]);
}
}
if (why != nullptr) *why = "no Conv with a weight tensor found, so the mel-bin count cannot be derived";
return 0;
}

} // namespace

bool discover_audio_pack(const std::string& dir, AudioPack* out, std::string* why) {
if (out == nullptr) return false;
*out = AudioPack{};
out->dir = dir;
std::error_code ec;
if (!fs::is_directory(fs::path(dir), ec)) {
if (why != nullptr) *why = dir + ": not a directory";
return false;
}
for (const char* name : kEncoderNames) {
const std::string p = dir + "/" + name;
if (is_file(p)) { out->encoder = p; break; }
}
for (const char* name : kDecoderNames) {
const std::string p = dir + "/" + name;
if (is_file(p)) { out->decoder = p; break; }
}
for (const char* name : kTokenizerNames) {
const std::string p = dir + "/" + name;
if (is_file(p)) { out->tokenizer_json = p; break; }
}
// A pack an operator actually has may name its graphs after the model (large_v3_encoder.onnx), so after the exact names a
// scan is allowed -- but it never picks a `with_past` decoder, which is a different graph with a different input list, and
// whatever it picks is reported as a concern so the choice is visible rather than silent.
if (out->encoder.empty() || out->decoder.empty()) {
std::vector<std::string> candidates;
for (const auto& entry : fs::directory_iterator(fs::path(dir), ec)) {
if (!entry.is_regular_file(ec)) continue;
const std::string name = entry.path().filename().string();
if (name.size() < 6 || name.compare(name.size() - 5, 5, ".onnx") != 0) continue;
candidates.push_back(name);
}
std::sort(candidates.begin(), candidates.end());
const auto pick = [&](const char* needle, bool exclude_with_past) -> std::string {
for (const auto& name : candidates) {
if (name.find(needle) == std::string::npos) continue;
if (exclude_with_past && name.find("with_past") != std::string::npos) continue;
return dir + "/" + name;
}
return std::string();
};
if (out->encoder.empty()) {
out->encoder = pick("encoder", false);
if (!out->encoder.empty()) {
out->concern += "encoder graph taken from its name: " + out->encoder;
}
}
if (out->decoder.empty()) {
out->decoder = pick("decoder", true);
if (!out->decoder.empty()) {
out->concern += (out->concern.empty() ? "" : "; ") + std::string("decoder graph taken from its name: ") + out->decoder;
}
}
}
if (out->encoder.empty()) out->missing.push_back("an encoder graph (encoder_model.onnx, encoder.onnx, or a *encoder*.onnx)");
if (out->decoder.empty()) out->missing.push_back("a decoder graph (decoder_model.onnx, decoder.onnx, or a *decoder*.onnx without with_past)");
if (!out->missing.empty()) {
if (why != nullptr) *why = dir + ": missing " + out->missing.front();
return false;
}

std::string graph_why;
const int from_graph = mel_bins_from_encoder(out->encoder, &graph_why);
if (from_graph > 0) {
out->mel_bins = from_graph;
out->mel_bins_source = "encoder graph";
} else {
out->concern += (out->concern.empty() ? std::string() : std::string("; ")) +
std::string("the encoder graph could not be read for its mel-bin count (") + graph_why + ")";
}

// config.json is a cross-check, never the authority: a disagreement is reported, and the graph wins because it is what the
// weights were exported with.
const std::string cfg_path = dir + "/config.json";
if (is_file(cfg_path)) {
const std::string text = [&] {
std::string body;
FILE* f = std::fopen(cfg_path.c_str(), "rb");
if (f == nullptr) return body;
char buf[4096];
size_t n;
while ((n = std::fread(buf, 1, sizeof buf, f)) > 0) body.append(buf, n);
std::fclose(f);
return body;
}();
const auto number_after = [&](const char* key) -> int {
const std::string k = std::string("\"") + key + "\"";
const size_t at = text.find(k);
if (at == std::string::npos) return 0;
const size_t colon = text.find(':', at);
if (colon == std::string::npos) return 0;
return std::atoi(text.c_str() + colon + 1);
};
const int cfg_bins = number_after("num_mel_bins");
if (cfg_bins > 0) {
if (out->mel_bins == 0) {
out->mel_bins = cfg_bins;
out->mel_bins_source = "config.json";
} else if (cfg_bins != out->mel_bins) {
out->concern += (out->concern.empty() ? "" : "; ") + std::string("config.json says ") +
std::to_string(cfg_bins) + " mel bins but the encoder graph's Conv takes " +
std::to_string(out->mel_bins) + " (the graph is what the weights were exported with)";
}
}
const int sr = number_after("sampling_rate");
if (sr > 0) out->sample_rate = sr;
const int fft = number_after("n_fft");
if (fft > 0) out->n_fft = fft;
const int hop = number_after("hop_length");
if (hop > 0) out->hop = hop;
const int frames = number_after("nb_max_frames");
if (frames > 0) out->frames = frames;
}
if (out->mel_bins == 0) {
if (why != nullptr) {
*why = dir + ": the mel-bin count is in neither the encoder graph nor config.json, and guessing it is how a 128-bin "
"window reaches an 80-bin encoder";
}
return false;
}
return true;
}

} // namespace onebit::audio
53 changes: 53 additions & 0 deletions npu/audio_pack.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// audio_pack.h -- what an audio pack is, discovered rather than configured (engine#1783).
//
// Until now the host chain was driven by two file paths on a command line, which is enough for a bench and wrong for a route:
// a pack directory is what an operator has, and the two graphs inside it are named by whoever exported it. This finds them,
// derives the front-end's mel-bin count from the encoder graph's own first Conv instead of trusting a config field, and
// refuses by name when a pack is not one.
//
// The mel-bin derivation matters: whisper-small exports 80 bins and large-v3 exports 128, and feeding a 128-bin window to an
// 80-bin encoder produces a graph that runs and answers wrongly -- the failure mode this whole tree has spent the night
// removing. The Conv's input-channel count is the pack's own statement; a `config.json` is only consulted when the graph
// cannot be read, and a disagreement between the two is reported rather than resolved silently.

#pragma once

#include <string>
#include <vector>

namespace onebit::audio {

struct AudioPack {
std::string dir;
std::string encoder; // absolute paths, empty when absent
std::string decoder;
std::string tokenizer_json;
int mel_bins = 0; // from the encoder's first Conv; 0 when it could not be derived
std::string mel_bins_source; // "encoder graph" | "config.json" | "" -- so a caller can say where the number came from
int sample_rate = 16000; // the front-end's defaults, overridable by config.json
int n_fft = 400;
int hop = 160;
int frames = 3000; // the fixed 30 s window the encoder was exported for
std::vector<std::string> missing; // what a complete pack needs and this one lacks, named
std::string concern; // a disagreement or an oddity worth printing, never a refusal on its own
};

// True when the directory yields both graphs. `why` carries the first concrete reason when it does not.
bool discover_audio_pack(const std::string& dir, AudioPack* out, std::string* why);

} // namespace onebit::audio
25 changes: 24 additions & 1 deletion npu/audio_parallel.h
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,10 @@
#include <thread>
#include <vector>

#ifdef __linux__
#include <sched.h>
#endif

namespace onebit::npu::pack_import {

// How many threads the audio executor may use. ONEBIT_AUDIO_THREADS=N sets it; anything that is not a positive number
Expand All @@ -45,7 +49,26 @@ inline size_t audio_thread_count() {
const long n = std::strtol(env, nullptr, 10);
return n > 0 ? size_t(n) : 1;
}
const size_t hw = std::thread::hardware_concurrency();
// hardware_concurrency() answers "how many CPUs does this machine have", not "how many may this process use". Under a
// cpuset -- a thermal-run scope, a container, a systemd AllowedCPUs= -- it over-reports: measured on this box it says 32
// while the scope allows 4, so the capped-at-8 default spawned 8 threads for 4 CPUs. The affinity mask is what the
// scheduler enforces, so that is what this follows.
// What this is NOT: a measured speedup. Three interleaved repetitions at 4 CPUs (8 threads / 4 threads) gave encoder
// 10.38/10.40 s and decode 2.58/2.69 s per token -- within the box's own noise, and my first, non-interleaved comparison
// of 8 threads against serial (5.21 against 4.24 s/token) was load, not threading. The reason to prefer the mask is that
// oversubscribing a cpuset cannot help and can only queue, not that it measured faster.
size_t hw = 0;
#ifdef __linux__
{
cpu_set_t set;
CPU_ZERO(&set);
if (sched_getaffinity(0, sizeof set, &set) == 0) {
const int allowed = CPU_COUNT(&set);
if (allowed > 0) hw = size_t(allowed);
}
}
#endif
if (hw == 0) hw = std::thread::hardware_concurrency();
return hw == 0 ? 1 : (hw > 8 ? 8 : hw);
}

Expand Down
Loading
Loading