diff --git a/CMakeLists.txt b/CMakeLists.txt index d5d372dde..55940c9d1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -925,7 +925,7 @@ if(ONEBIT_NPU_HOST) find_package(PkgConfig REQUIRED) pkg_check_modules(PCRE2 REQUIRED IMPORTED_TARGET libpcre2-8) add_library(onebit_npu STATIC npu/file_map.cpp npu/flm_head_import.cpp npu/full_elf.cpp npu/model.cpp npu/vision_pack.cpp npu/vision_tower.cpp npu/vision_image.cpp npu/vision_request.cpp - npu/audio_exec.cpp npu/moe.cpp npu/pack.cpp npu/pack_import.cpp npu/private_route.cpp + npu/audio_exec.cpp npu/audio_pack.cpp npu/moe.cpp npu/pack.cpp npu/pack_import.cpp npu/private_route.cpp npu/q4nx.cpp npu/q4nx_device.cpp npu/tokenizer.cpp) target_include_directories(onebit_npu PUBLIC npu) target_link_libraries(onebit_npu PUBLIC nlohmann_json::nlohmann_json onebit_q4nx_cache PRIVATE PkgConfig::PCRE2) @@ -1560,7 +1560,8 @@ add_executable(audio_parallel_test tests/audio_parallel_test.cpp) target_link_libraries(audio_parallel_test PRIVATE onebit_npu Threads::Threads) # ci.yml's build list is not ours to edit without `workflow` scope, so these reach CI the way ops_dump_test does: as build # dependencies of a target it already builds (tools/ci/ci_targets_check.py follows this edge). -add_dependencies(onebit audio_ops_test audio_mel_test onnx_graph_test audio_exec_test audio_parallel_test) +# One line, deliberately: tools/ci/ci_targets_check.py follows `add_dependencies` line by line. +add_dependencies(onebit audio_ops_test audio_mel_test onnx_graph_test audio_exec_test audio_parallel_test audio_pack_test audio_kernel_roles_test whisper_ids_test audio_device_gemm_test audio_launch_plan_test audio_kernel_shapes whisper_transcribe) # A DFlash draft package as a model source (engine#1780): config + container -> a checked model whose tensors decode # one at a time. The loader is compiled here and not into onebit_npu because it needs the q4_1 tile decoder, which # lives with the forward (npu/forward/src/q4nx_dequant.c); it moves there with the draft's forward. Synthetic package, @@ -1943,6 +1944,40 @@ add_test(NAME audio_mel COMMAND audio_mel_test) set_tests_properties(audio_ops PROPERTIES SKIP_RETURN_CODE 77) add_test(NAME onnx_graph COMMAND onnx_graph_test ${ONEBIT_AUDIO_ONNX}) add_test(NAME audio_exec COMMAND audio_exec_test ${ONEBIT_AUDIO_ONNX}) +# engine#1783: the audio pack -- two graphs by their exported names (never `decoder_with_past`), mel bins from the encoder's +# own first Conv, config.json as a cross-check. +add_executable(audio_pack_test tests/audio_pack_test.cpp) +target_include_directories(audio_pack_test PRIVATE npu/forward/include) +target_link_libraries(audio_pack_test PRIVATE onebit_npu) +add_executable(audio_kernel_roles_test tests/audio_kernel_roles_test.cpp) +target_include_directories(audio_kernel_roles_test PRIVATE npu/forward/include) +add_executable(whisper_ids_test tests/whisper_ids_test.cpp) +target_include_directories(whisper_ids_test PRIVATE npu/forward/include) +add_executable(audio_kernel_shapes tools/audio_kernel_shapes.cpp) +target_include_directories(audio_kernel_shapes PRIVATE npu/forward/include) +target_link_libraries(audio_kernel_shapes PRIVATE onebit_npu) +add_executable(audio_device_gemm_test tests/audio_device_gemm_test.cpp) +target_include_directories(audio_device_gemm_test PRIVATE npu/forward/include npu/target npu/plan) +target_link_libraries(audio_device_gemm_test PRIVATE nlohmann_json::nlohmann_json) +add_executable(audio_launch_plan_test tests/audio_launch_plan_test.cpp) +target_include_directories(audio_launch_plan_test PRIVATE npu/forward/include npu/target npu/plan) +target_link_libraries(audio_launch_plan_test PRIVATE nlohmann_json::nlohmann_json) +add_test(NAME whisper_ids COMMAND whisper_ids_test) +add_test(NAME audio_kernel_roles COMMAND audio_kernel_roles_test) +add_test(NAME audio_pack COMMAND audio_pack_test) +add_test(NAME audio_device_gemm COMMAND audio_device_gemm_test) +add_test(NAME audio_launch_plan COMMAND audio_launch_plan_test) +add_test(NAME whisper_transcribe_jfk COMMAND whisper_transcribe) +set_tests_properties(whisper_transcribe_jfk PROPERTIES + ENVIRONMENT "ONEBIT_TEST_WHISPER_IDS_FILE=${CMAKE_SOURCE_DIR}/tests/data/whisper-jfk-ids.json" + SKIP_RETURN_CODE 77 TIMEOUT 1800) +add_test(NAME whisper_transcribe_jfk_lv3 + COMMAND ${CMAKE_SOURCE_DIR}/tests/whisper_lv3_row.sh $ + ${CMAKE_SOURCE_DIR}/tests/data/whisper-jfk-ids.json) +set_tests_properties(whisper_transcribe_jfk_lv3 PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 3600) +add_test(NAME audio_kernel_plan + COMMAND bash ${CMAKE_SOURCE_DIR}/tests/audio_kernel_plan.sh $) +set_tests_properties(audio_kernel_plan PROPERTIES SKIP_RETURN_CODE 77) add_test(NAME audio_parallel COMMAND audio_parallel_test) set_tests_properties(audio_exec PROPERTIES SKIP_RETURN_CODE 77) set_tests_properties(onnx_graph PROPERTIES SKIP_RETURN_CODE 77) diff --git a/npu/audio_pack.cpp b/npu/audio_pack.cpp new file mode 100644 index 000000000..af1d03efc --- /dev/null +++ b/npu/audio_pack.cpp @@ -0,0 +1,190 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// audio_pack.cpp -- see audio_pack.h. The discovery is deliberately boring: known names, in a fixed order, and a refusal that +// names what is missing. The one interesting step is the mel-bin count, which comes from the encoder graph rather than a config. + +#include "audio_pack.h" + +#include +#include +#include +#include + +#include "onnx_graph.h" + +namespace fs = std::filesystem; + +namespace onebit::audio { +namespace { + +bool is_file(const std::string& p) { + std::error_code ec; + return fs::is_regular_file(fs::path(p), ec); +} + +// The names exporters actually use, most specific first. A pack that carries both spellings is a pack someone assembled by +// hand; the first match wins and the other is ignored, which is why `dir` is recorded. +const char* kEncoderNames[] = {"encoder_model.onnx", "encoder.onnx"}; +const char* kDecoderNames[] = {"decoder_model.onnx", "decoder.onnx"}; +const char* kTokenizerNames[] = {"tokenizer.json", "tokenizer_config.json"}; + +int mel_bins_from_encoder(const std::string& path, std::string* why) { + using namespace onebit::npu::pack_import; + OnnxGraphInfo g; + try { + g = parse_onnx_graph(path); + } catch (const std::exception& e) { + if (why != nullptr) *why = e.what(); + return 0; + } + // The mel front-end is the graph's first Conv; its input-channel count is the bin count the encoder was exported for. + for (const auto& n : g.nodes) { + if (n.op != "Conv" || n.ins.size() < 2) continue; + for (const auto& ti : g.init_tensors) { + if (ti.name != n.ins[1] || ti.dims.size() != 3) continue; + return int(ti.dims[1]); + } + } + if (why != nullptr) *why = "no Conv with a weight tensor found, so the mel-bin count cannot be derived"; + return 0; +} + +} // namespace + +bool discover_audio_pack(const std::string& dir, AudioPack* out, std::string* why) { + if (out == nullptr) return false; + *out = AudioPack{}; + out->dir = dir; + std::error_code ec; + if (!fs::is_directory(fs::path(dir), ec)) { + if (why != nullptr) *why = dir + ": not a directory"; + return false; + } + for (const char* name : kEncoderNames) { + const std::string p = dir + "/" + name; + if (is_file(p)) { out->encoder = p; break; } + } + for (const char* name : kDecoderNames) { + const std::string p = dir + "/" + name; + if (is_file(p)) { out->decoder = p; break; } + } + for (const char* name : kTokenizerNames) { + const std::string p = dir + "/" + name; + if (is_file(p)) { out->tokenizer_json = p; break; } + } + // A pack an operator actually has may name its graphs after the model (large_v3_encoder.onnx), so after the exact names a + // scan is allowed -- but it never picks a `with_past` decoder, which is a different graph with a different input list, and + // whatever it picks is reported as a concern so the choice is visible rather than silent. + if (out->encoder.empty() || out->decoder.empty()) { + std::vector candidates; + for (const auto& entry : fs::directory_iterator(fs::path(dir), ec)) { + if (!entry.is_regular_file(ec)) continue; + const std::string name = entry.path().filename().string(); + if (name.size() < 6 || name.compare(name.size() - 5, 5, ".onnx") != 0) continue; + candidates.push_back(name); + } + std::sort(candidates.begin(), candidates.end()); + const auto pick = [&](const char* needle, bool exclude_with_past) -> std::string { + for (const auto& name : candidates) { + if (name.find(needle) == std::string::npos) continue; + if (exclude_with_past && name.find("with_past") != std::string::npos) continue; + return dir + "/" + name; + } + return std::string(); + }; + if (out->encoder.empty()) { + out->encoder = pick("encoder", false); + if (!out->encoder.empty()) { + out->concern += "encoder graph taken from its name: " + out->encoder; + } + } + if (out->decoder.empty()) { + out->decoder = pick("decoder", true); + if (!out->decoder.empty()) { + out->concern += (out->concern.empty() ? "" : "; ") + std::string("decoder graph taken from its name: ") + out->decoder; + } + } + } + if (out->encoder.empty()) out->missing.push_back("an encoder graph (encoder_model.onnx, encoder.onnx, or a *encoder*.onnx)"); + if (out->decoder.empty()) out->missing.push_back("a decoder graph (decoder_model.onnx, decoder.onnx, or a *decoder*.onnx without with_past)"); + if (!out->missing.empty()) { + if (why != nullptr) *why = dir + ": missing " + out->missing.front(); + return false; + } + + std::string graph_why; + const int from_graph = mel_bins_from_encoder(out->encoder, &graph_why); + if (from_graph > 0) { + out->mel_bins = from_graph; + out->mel_bins_source = "encoder graph"; + } else { + out->concern += (out->concern.empty() ? std::string() : std::string("; ")) + + std::string("the encoder graph could not be read for its mel-bin count (") + graph_why + ")"; + } + + // config.json is a cross-check, never the authority: a disagreement is reported, and the graph wins because it is what the + // weights were exported with. + const std::string cfg_path = dir + "/config.json"; + if (is_file(cfg_path)) { + const std::string text = [&] { + std::string body; + FILE* f = std::fopen(cfg_path.c_str(), "rb"); + if (f == nullptr) return body; + char buf[4096]; + size_t n; + while ((n = std::fread(buf, 1, sizeof buf, f)) > 0) body.append(buf, n); + std::fclose(f); + return body; + }(); + const auto number_after = [&](const char* key) -> int { + const std::string k = std::string("\"") + key + "\""; + const size_t at = text.find(k); + if (at == std::string::npos) return 0; + const size_t colon = text.find(':', at); + if (colon == std::string::npos) return 0; + return std::atoi(text.c_str() + colon + 1); + }; + const int cfg_bins = number_after("num_mel_bins"); + if (cfg_bins > 0) { + if (out->mel_bins == 0) { + out->mel_bins = cfg_bins; + out->mel_bins_source = "config.json"; + } else if (cfg_bins != out->mel_bins) { + out->concern += (out->concern.empty() ? "" : "; ") + std::string("config.json says ") + + std::to_string(cfg_bins) + " mel bins but the encoder graph's Conv takes " + + std::to_string(out->mel_bins) + " (the graph is what the weights were exported with)"; + } + } + const int sr = number_after("sampling_rate"); + if (sr > 0) out->sample_rate = sr; + const int fft = number_after("n_fft"); + if (fft > 0) out->n_fft = fft; + const int hop = number_after("hop_length"); + if (hop > 0) out->hop = hop; + const int frames = number_after("nb_max_frames"); + if (frames > 0) out->frames = frames; + } + if (out->mel_bins == 0) { + if (why != nullptr) { + *why = dir + ": the mel-bin count is in neither the encoder graph nor config.json, and guessing it is how a 128-bin " + "window reaches an 80-bin encoder"; + } + return false; + } + return true; +} + +} // namespace onebit::audio diff --git a/npu/audio_pack.h b/npu/audio_pack.h new file mode 100644 index 000000000..74d997d4f --- /dev/null +++ b/npu/audio_pack.h @@ -0,0 +1,53 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// audio_pack.h -- what an audio pack is, discovered rather than configured (engine#1783). +// +// Until now the host chain was driven by two file paths on a command line, which is enough for a bench and wrong for a route: +// a pack directory is what an operator has, and the two graphs inside it are named by whoever exported it. This finds them, +// derives the front-end's mel-bin count from the encoder graph's own first Conv instead of trusting a config field, and +// refuses by name when a pack is not one. +// +// The mel-bin derivation matters: whisper-small exports 80 bins and large-v3 exports 128, and feeding a 128-bin window to an +// 80-bin encoder produces a graph that runs and answers wrongly -- the failure mode this whole tree has spent the night +// removing. The Conv's input-channel count is the pack's own statement; a `config.json` is only consulted when the graph +// cannot be read, and a disagreement between the two is reported rather than resolved silently. + +#pragma once + +#include +#include + +namespace onebit::audio { + +struct AudioPack { + std::string dir; + std::string encoder; // absolute paths, empty when absent + std::string decoder; + std::string tokenizer_json; + int mel_bins = 0; // from the encoder's first Conv; 0 when it could not be derived + std::string mel_bins_source; // "encoder graph" | "config.json" | "" -- so a caller can say where the number came from + int sample_rate = 16000; // the front-end's defaults, overridable by config.json + int n_fft = 400; + int hop = 160; + int frames = 3000; // the fixed 30 s window the encoder was exported for + std::vector missing; // what a complete pack needs and this one lacks, named + std::string concern; // a disagreement or an oddity worth printing, never a refusal on its own +}; + +// True when the directory yields both graphs. `why` carries the first concrete reason when it does not. +bool discover_audio_pack(const std::string& dir, AudioPack* out, std::string* why); + +} // namespace onebit::audio diff --git a/npu/audio_parallel.h b/npu/audio_parallel.h index 30ebfc05c..e019b6d7b 100644 --- a/npu/audio_parallel.h +++ b/npu/audio_parallel.h @@ -35,6 +35,10 @@ #include #include +#ifdef __linux__ +#include +#endif + namespace onebit::npu::pack_import { // How many threads the audio executor may use. ONEBIT_AUDIO_THREADS=N sets it; anything that is not a positive number @@ -45,7 +49,26 @@ inline size_t audio_thread_count() { const long n = std::strtol(env, nullptr, 10); return n > 0 ? size_t(n) : 1; } - const size_t hw = std::thread::hardware_concurrency(); + // hardware_concurrency() answers "how many CPUs does this machine have", not "how many may this process use". Under a + // cpuset -- a thermal-run scope, a container, a systemd AllowedCPUs= -- it over-reports: measured on this box it says 32 + // while the scope allows 4, so the capped-at-8 default spawned 8 threads for 4 CPUs. The affinity mask is what the + // scheduler enforces, so that is what this follows. + // What this is NOT: a measured speedup. Three interleaved repetitions at 4 CPUs (8 threads / 4 threads) gave encoder + // 10.38/10.40 s and decode 2.58/2.69 s per token -- within the box's own noise, and my first, non-interleaved comparison + // of 8 threads against serial (5.21 against 4.24 s/token) was load, not threading. The reason to prefer the mask is that + // oversubscribing a cpuset cannot help and can only queue, not that it measured faster. + size_t hw = 0; +#ifdef __linux__ + { + cpu_set_t set; + CPU_ZERO(&set); + if (sched_getaffinity(0, sizeof set, &set) == 0) { + const int allowed = CPU_COUNT(&set); + if (allowed > 0) hw = size_t(allowed); + } + } +#endif + if (hw == 0) hw = std::thread::hardware_concurrency(); return hw == 0 ? 1 : (hw > 8 ? 8 : hw); } diff --git a/npu/forward/include/audio_device_gemm.h b/npu/forward/include/audio_device_gemm.h new file mode 100644 index 000000000..f309fdfeb --- /dev/null +++ b/npu/forward/include/audio_device_gemm.h @@ -0,0 +1,124 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// audio_device_gemm.h -- one staged projection of an audio encoder, on the NPU (engine#1783). +// +// Why this exists as a header and not as code inside the audio executor: the device side of a projection is the TARGET's job +// (`NpuTarget::gemv` takes f32 activations and f32 back, and does the per-row int8 quantisation, the b8 pack and the +// dequantisation itself), so what the executor needs is the small, testable decision layer around it -- which shape may go to +// the device, what the caller's tensor is versus what the kernel writes, and what to say when it cannot. That is a pure +// function over a `NpuTarget&`, so it is driven against a fake target with no device, the same shape as every other rule here. +// +// Two facts it encodes, both learned from measurements on this box (engine#1783): +// * the kernel is named at the ENGINE'S STAGED width -- K a multiple of 256, N of 512 -- so the kernel writes N padded +// (whisper-small's O is N768 named N1024). The caller's tensor is the model's width, so the result is de-padded here rather +// than by every caller; forgetting that is how a padded column becomes a plausible wrong activation. +// * a `Plan` that loads has told you NOTHING about its designs: `Plan::designs_agree`/`check_artifacts` both return true when +// the plan names no designs. So the shape this header trusts is the one the CALLER states (role/K/N taken from the pack), and +// it refuses a shape the engine's grid cannot name instead of deriving one. + +#pragma once + +#include +#include + +#include "audio_kernel_roles.h" +#include "target.h" // ::npu::target::{NpuTarget, GemvArgs, Caps, Op} (target.h opens npu::target at global scope) + +namespace onebit::audio { + +namespace nput = ::npu::target; + +// One projection as the ENCODER states it (the model's own K and N, not the staged widths). +struct DeviceProjection { + int role = 0; // index into the target's design table + int layer = -1; // layer in flight, for residency and profiling + int k = 0; // the model's contracted width + int n = 0; // the model's output width (the CALLER's tensor width) + int rows = 0; // M: rows to run in one launch + const float* x = nullptr; // M x k row-major f32 + int a_k = 0; // the activation's real width per row (0 = k) + const char* tag = nullptr; // container tag, for the target's log +}; + +// One projection on the device: `out` receives rows x n (the CALLER's width), de-padded from the staged write. +// Returns false and fills `why` for: a shape the engine's grid cannot name, a row count the target cannot launch, +// a target without gemv, a target failure, or an output whose size is not rows x staged_n (which is the target +// disagreeing with the plan, and is never repaired here). +inline bool device_projection(nput::NpuTarget& target, const DeviceProjection& p, std::vector* out, + std::string* why) { + if (out == nullptr) { + if (why != nullptr) *why = "no output vector"; + return false; + } + if (p.x == nullptr || p.k <= 0 || p.n <= 0 || p.rows <= 0) { + if (why != nullptr) *why = "a projection needs x, rows > 0, k > 0 and n > 0"; + return false; + } + if (!target.supports(nput::Op::gemv)) { + if (why != nullptr) *why = std::string("the target \"") + target.name() + "\" does not implement gemv"; + return false; + } + const ProjectionShape shape{p.k, p.n}; + if (role_of_shape(shape, 0) == std::string("unknown") && p.tag == nullptr) { + // Not a refusal: the caller states the role index directly. But an unnamed shape with no tag is undebuggable, and + // this header's job is to keep the device log answerable. + if (why != nullptr) *why = "an unnamed projection needs a tag so the target's log can name it"; + return false; + } + const ProjectionShape staged_shape = staged(shape); + const nput::Caps caps = target.caps(); + const bool wide = p.rows > caps.max_rows; + if (wide && p.rows > caps.prefill_rows) { + if (why != nullptr) + *why = "rows=" + std::to_string(p.rows) + " is more than the target launches in one call (prefill_rows=" + + std::to_string(caps.prefill_rows) + ", max_rows=" + std::to_string(caps.max_rows) + ")"; + return false; + } + + nput::GemvArgs a; + a.role = p.role; + a.layer = p.layer; + a.M = p.rows; + a.K = staged_shape.k; + a.N = staged_shape.n; + a.x = p.x; + a.a_k = p.a_k > 0 ? p.a_k : p.k; + a.rows = wide; + a.tag = p.tag; + + std::vector got; + std::string err; + if (!target.gemv(a, got, err)) { + if (why != nullptr) *why = err.empty() ? "the target refused the projection" : err; + return false; + } + const size_t want = size_t(p.rows) * size_t(staged_shape.n); + if (got.size() != want) { + if (why != nullptr) + *why = "the target wrote " + std::to_string(got.size()) + " floats, the plan's staged shape is " + + std::to_string(p.rows) + " x " + std::to_string(staged_shape.n) + " = " + std::to_string(want) + + " (the target and the plan disagree; this is not repaired here)"; + return false; + } + // De-pad: the kernel's rows are staged_n wide, the caller's tensor is n wide. Drop the padding columns per row rather + // than returning the staged buffer, which would hand the caller a row whose stride is not its own width. + out->assign(size_t(p.rows) * size_t(p.n), 0.0f); + for (int r = 0; r < p.rows; ++r) + for (int c = 0; c < p.n; ++c) (*out)[size_t(r) * size_t(p.n) + size_t(c)] = got[size_t(r) * size_t(staged_shape.n) + size_t(c)]; + return true; +} + +} // namespace onebit::audio diff --git a/npu/forward/include/audio_kernel_roles.h b/npu/forward/include/audio_kernel_roles.h new file mode 100644 index 000000000..bca9977b7 --- /dev/null +++ b/npu/forward/include/audio_kernel_roles.h @@ -0,0 +1,97 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// audio_kernel_roles.h -- which GEMM shape an encoder's projection is, decided by SHAPE against the hidden width (engine#1783). +// +// Why by shape and not by name: a packed export reaches its weights through generically named initializers (`val_452`) and the +// human name sits on a tensor the MatMul never touches, so a name-based classifier reports every projection as unknown. Shape is +// also what a kernel actually depends on -- one ELF serves every role with the same (K, N), because q, k, v and o are all h x h. +// +// The four roles are the ones the family generator emits (see tools/audio_kernel_shapes.cpp and the private family's +// `full_i8__K_N_b8[_m16].elf` naming): +// +// fused QKV K = h, N = 3h attention's q|k|v packed into one matrix +// O K = h, N = h q, k, v and o all land here -- one ELF serves all four +// GU K = h, N = 4h the MLP's fused gate|up (Whisper's 4x expansion) +// D K = 4h, N = h the MLP's down projection +// +// A shape that is none of these is `unknown`, and that is deliberate: an unrecognised shape is a projection nobody has built a +// kernel for, and saying so is more useful than guessing a role and asking for the wrong ELF. + +#pragma once + +#include + +namespace onebit::audio { + +struct ProjectionShape { + int k = 0; // reduction (K) + int n = 0; // output width (N) +}; + +// The hidden width, from the model's own square projections: q/k/v/o are h x h in every layer, so the most common square width +// IS h. A median over all weights would be skewed by the embedding and the 4h MLP, which is what the first version did. +// The hidden width is the MODE of the square projection widths (h x h is q/k/v/o, and there are four of them per layer). +// A tie is refused rather than broken by whichever width came first: two widths occurring equally often means the pack has +// two candidate hidden widths, and picking one silently would decide every role after it -- the same "confident guess in the +// middle of its own evidence" that a 0.5 threshold on a correlation has. Returns 0 for "no evidence" and for "ambiguous". +inline int hidden_from_square_widths(const int* widths, int count) { + int best_count = 0, hidden = 0, tied = 0; + for (int i = 0; i < count; ++i) { + int occurrences = 0; + for (int j = 0; j < count; ++j) + if (widths[j] == widths[i]) ++occurrences; + if (occurrences > best_count) { + best_count = occurrences; + hidden = widths[i]; + tied = 0; + } else if (occurrences == best_count && widths[i] != hidden) { + ++tied; + } + } + return tied == 0 ? hidden : 0; +} + +inline const char* role_of_shape(const ProjectionShape& s, int hidden) { + if (hidden <= 0) return "unknown"; + if (s.k == hidden && s.n == 3 * hidden) return "QKV"; + if (s.k == hidden && s.n == hidden) return "O"; // q, k, v and o share this shape and this ELF + if (s.k == hidden && s.n == 4 * hidden) return "GU"; + if (s.k == 4 * hidden && s.n == hidden) return "D"; + return "unknown"; +} + +// The ELF the family generator emits for this shape, whether or not that role is real: the caller decides what to do with a +// name for an `unknown` shape (the tool reports it, a gate would refuse it). +// The engine's own staging grid, from npu/forward/src/q4nx_pack.c (`q4nx_pad_k` / `q4nx_pad_n`): a container's K is a +// multiple of 256 and its N a multiple of 512, and the ELF the forward asks for is named at THOSE widths, not at the model's. +// Measured on the synth768 run (engine#1783): QKV 2304 -> the forward asks for N2560, O 768 -> N1024, and a set emitted at the +// unpadded widths is "missing full ELF" while this header said the family was covered. +constexpr int kQ4nxKGrid = 256; +constexpr int kQ4nxNGrid = 512; + +inline int q4nx_pad_k(int k) { return k <= 0 ? k : ((k + kQ4nxKGrid - 1) / kQ4nxKGrid) * kQ4nxKGrid; } +inline int q4nx_pad_n(int n) { return n <= 0 ? n : ((n + kQ4nxNGrid - 1) / kQ4nxNGrid) * kQ4nxNGrid; } + +// The shape as the forward names it: padded on the engine's grid. +inline ProjectionShape staged(const ProjectionShape& s) { return ProjectionShape{q4nx_pad_k(s.k), q4nx_pad_n(s.n)}; } + +inline std::string elf_name_for(const ProjectionShape& s, int hidden, bool m16) { + const ProjectionShape p = staged(s); + return std::string("full_i8_") + role_of_shape(s, hidden) + "_K" + std::to_string(p.k) + "_N" + std::to_string(p.n) + + "_b8" + (m16 ? "_m16" : "") + ".elf"; +} + +} // namespace onebit::audio diff --git a/npu/forward/include/audio_launch_plan.h b/npu/forward/include/audio_launch_plan.h new file mode 100644 index 000000000..9778a0a5f --- /dev/null +++ b/npu/forward/include/audio_launch_plan.h @@ -0,0 +1,129 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// audio_launch_plan.h -- walk an encoder's launch list, device first, host for what the device cannot take (engine#1783). +// +// The route this belongs to (item 2 of the issue's close condition) is not "run the encoder on the NPU": it is a loop over the +// projections in graph order that sends each one to the device when the device can take it and to the host when it cannot, and +// REPORTS WHICH. That report is the measurement the row is scored on -- `device_gemm= s + host_remainder= s` -- so the +// accounting is part of the contract, not a log line: a launch that silently fell back is a launch whose time is being counted +// in the wrong column, and the whole point of measuring the split is to know which column is worth attacking next. +// +// It is deliberately not a planner: the specs come from `audio_kernel_shapes --write-plan` (already emitted, 72 launches for +// whisper-small, dense order, staged widths), and the host path is a CALLBACK so this file neither duplicates the executor's +// GEMM nor decides what "host" means. +// +// Why a callback for the activations too: each launch reads the previous one's output, so the caller owns the buffers. A +// function that took a single big X would be a function that only works for a stack of independent projections. + +#pragma once + +#include +#include +#include +#include + +#include "audio_device_gemm.h" + +namespace onebit::audio { + +struct LaunchSpec { + int index = -1; // the plan's own `order`, so a report can be compared with the plan line for line + int role = 0; + int layer = -1; + int k = 0; + int n = 0; + int rows = 0; + const char* tag = nullptr; +}; + +struct LaunchRun { + bool device = false; // ran on the device + bool host = false; // ran on the host (either because there is no target, or after a device refusal) + std::string why; // why it did not run on the device, when it did not +}; + +struct LaunchReport { + int device_launches = 0; + int host_launches = 0; + int failed = 0; + std::vector per_launch; // parallel to the specs, so a caller can print them in order + std::vector failures; // " : / " + bool ok() const { return failed == 0; } +}; + +// A host path for one launch: writes the caller's width (n), returns false and fills why when it cannot. +using HostLaunchFn = std::function*, std::string*)>; +// The activations for one launch, supplied by the caller because each launch reads the previous one's result. +using LaunchInputFn = std::function; + +// Runs every spec in order. A launch goes to the device when `target` is not null AND `device_projection` accepts it; otherwise +// the host callback runs it and the refusal reason is kept. A launch both paths refuse is a failure and does not silently +// count as either column. +inline LaunchReport run_launches(::npu::target::NpuTarget* target, const std::vector& specs, + const LaunchInputFn& input_of, const HostLaunchFn& host) { + LaunchReport rep; + rep.per_launch.resize(specs.size()); + for (size_t i = 0; i < specs.size(); ++i) { + const LaunchSpec& s = specs[i]; + LaunchRun& r = rep.per_launch[i]; + if (target != nullptr) { + DeviceProjection p; + p.role = s.role; + p.layer = s.layer; + p.k = s.k; + p.n = s.n; + p.rows = s.rows; + p.x = input_of != nullptr ? input_of(s) : nullptr; + p.tag = s.tag; + std::vector out; + std::string why; + if (device_projection(*target, p, &out, &why)) { + r.device = true; + ++rep.device_launches; + continue; + } + r.why = why; + } else { + r.why = "no target"; + } + std::vector out; + std::string hwhy; + if (host != nullptr && host(s, &out, &hwhy)) { + r.host = true; + ++rep.host_launches; + continue; + } + ++rep.failed; + rep.failures.push_back("launch " + std::to_string(s.index) + " " + (s.tag != nullptr ? s.tag : "?") + + ": device: " + (r.why.empty() ? std::string("not attempted") : r.why) + + " / host: " + (hwhy.empty() ? std::string("no host path") : hwhy)); + } + return rep; +} + +// The bar this route is scored on, formatted from a run rather than assembled by hand at the end of a session: the issue's close +// condition asks for `device_gemm= s + host_remainder= s per 30 s window`, and a number typed from memory is not evidence. +// A launch that is in NEITHER column (failed) is printed explicitly -- a split with a silent third category is a split that adds +// up to less than the run. +inline std::string format_split_run(const LaunchReport& r, double device_ms, double host_ms) { + char buf[220]; + std::snprintf(buf, sizeof buf, + "device_gemm=%.3f s + host_remainder=%.3f s per window (%d device launch(es), %d host, %d failed)", + device_ms / 1000.0, host_ms / 1000.0, r.device_launches, r.host_launches, r.failed); + return std::string(buf); +} + +} // namespace onebit::audio diff --git a/npu/forward/include/q4nx_pack.h b/npu/forward/include/q4nx_pack.h index f1f97cb88..47a59cf7e 100644 --- a/npu/forward/include/q4nx_pack.h +++ b/npu/forward/include/q4nx_pack.h @@ -67,8 +67,21 @@ extern "C" { * groups still looped inside the ELF (so one launch per role stays the * contract). Measured cost of an M=16 launch is ~0.34 ms against 3.4-4.9 ms * for the M=128 path. */ +// Overridable at build time, because a wider prefill tier turns out to be a KNOB and not a hardware floor (engine#1783, +// measured on the Strix Halo array): M is rows x m, m is a generator parameter, and the four compute rows this part has are +// unchanged by M. Emitting the encoder's shapes at M=1536 (rows=4, m=16) and building with -DQ4NX_DESIGN_M_PREFILL=1536 takes +// whisper-small's projection GEMM from 2.05 s to 0.163 s per 30 s window -- 12x the rows for the same launch time: +// M=128 : QKV 4.468 O 4.351 GU 5.460 D 4.418 ms/launch +// M=1536: QKV 4.740 O 4.373 GU 4.662 D 4.588 ms/launch +// What keeps an override safe: whatever a build sets here must equal the M the adopted ELFs were generated with, and the target +// CHECKS that against the loaded ELFs' own MD rather than assuming it ("plan: prefill M=1536 but the adopted kernels run M=128", +// observed before the guard existed). Absent -D, both constants are exactly what they were. +#ifndef Q4NX_DESIGN_M_PREFILL #define Q4NX_DESIGN_M_PREFILL 128 +#endif +#ifndef Q4NX_DESIGN_M_DECODE #define Q4NX_DESIGN_M_DECODE 16 +#endif typedef enum { Q4NX_DESIGN_QKV = 0, @@ -299,10 +312,10 @@ const char* q4nx_design_log_tag(Q4nxDesign role); // side of the split: a duplicate would be two contracts that drift silently. const char* q4nx_pb16i_instance(Q4nxDesign role); -// The same for a SEALED int8 b8 ELF (engine#2208): one hw_context for the decode roles of a b8 container whose -// full-attention QKV is one fused role (QKV, with the output gate) and whose gate/up is one fused role (GU). -// The split G and U roles have no instance: the sealed GU instance is the fused width, so they keep their own -// kernels. NULL means the sealed ELF does not cover the role. +/* The sealed aie-instance name for an int8 role (`main:qkvfa`, `main:o`, ...), or NULL for a role the sealed family does not + * carry. Declared here because `npu/target/xdna/xdna_target.cpp` calls it and this header is what it includes; without the + * declaration that translation unit only compiled because of how the libraries happened to be assembled (a standalone build of + * the same file, e.g. one that links the target without the forward, fails with "was not declared in this scope"). */ const char* q4nx_i8_sealed_instance(Q4nxDesign role); /* Look up the entry for a role in a derived set; NULL if absent. */ diff --git a/npu/forward/include/whisper_ids.h b/npu/forward/include/whisper_ids.h new file mode 100644 index 000000000..068affae7 --- /dev/null +++ b/npu/forward/include/whisper_ids.h @@ -0,0 +1,58 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// whisper_ids.h -- the special-token ids a Whisper decoder's prompt needs, selected by VOCABULARY WIDTH (engine#1783). +// +// Why by width and not by a constant: whisper-small exports 51865 logits and large-v3 exports 51866, and the extra token shifts +// <|translate|>/<|transcribe|>/<|nospeech|>/<|notimestamps|> by one. A hard-coded 50359/50363 decodes whisper-small correctly -- +// because English audio makes <|translate|> and <|transcribe|> agree -- and makes large-v3 answer <|endoftext|> at the first step. +// Measured on both packs against onnxruntime: argmax 50257 with 50359/50363 on large-v3, 400 with 50360/50364. +// +// Neither pack ships a tokenizer or generation_config, so the width (read from the decoder's declared output shape) is the only +// fact available. A `.en` model (51864) is NOT covered: its prompt differs and nobody has measured it here, so it refuses rather +// than guessing one of the two known layouts. + +#pragma once + +#include + +namespace onebit::audio { + +struct WhisperIds { + int64_t sot = 50258; // <|startoftranscript|> + int64_t lang_en = 50259; // <|en|> + int64_t transcribe = 0; // <|transcribe|> + int64_t no_timestamps = 0; // <|notimestamps|> + int64_t eot = 50257; // <|endoftext|> +}; + +// Returns false, leaving `out` untouched, for a vocabulary this header has no measured layout for. +inline bool whisper_ids_for_vocab(int64_t vocab, WhisperIds* out) { + if (out == nullptr) return false; + WhisperIds ids; + if (vocab >= 51866) { // large-v3 and its derivatives + ids.transcribe = 50360; + ids.no_timestamps = 50364; + } else if (vocab == 51865) { // v1/v2/small/medium, multilingual + ids.transcribe = 50359; + ids.no_timestamps = 50363; + } else { + return false; // a `.en` vocabulary or an unknown family: refuse rather than guess + } + *out = ids; + return true; +} + +} // namespace onebit::audio diff --git a/npu/onnx_graph.h b/npu/onnx_graph.h index 87acc99cd..c4002e8c8 100644 --- a/npu/onnx_graph.h +++ b/npu/onnx_graph.h @@ -57,7 +57,11 @@ struct OnnxGraphInfo { std::string dir; // directory of the .onnx file, for resolving external_data std::vector init_tensors; std::vector inputs; // graph inputs, in declaration order + // The declared shape of each input, parallel to `inputs`. A decoder's token window lives here (whisper-small exports 128, + // large-v3 exports 448), and a reader that assumes one of them builds a mask the other cannot broadcast against. + std::vector> input_dims; std::vector outputs; // graph outputs + std::vector> output_dims; // parallel to `outputs`; a decoder's vocabulary is the last of these std::vector nodes; // graph order, which ONNX guarantees is topological std::vector initializers; // weight/bias names std::vector external; // initializers whose data is in a side file (location set) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 106fa8d0c..1858bc41d 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -7134,13 +7134,33 @@ OnnxGraphInfo parse_onnx_graph(const std::string& path) { OnnxGraphInfo out; const size_t slash = path.find_last_of('/'); out.dir = (slash == std::string::npos) ? std::string(".") : path.substr(0, slash); + // engine#1783: the declared SHAPE of each graph input and output beside its name -- a decoder's token window and an + // encoder's mel-bin count are facts of the graph, not constants a caller may assume (128 vs 448 positions). + const auto dims_of_vi = [](const std::vector& vi) { + std::vector dims; + if (const auto* tp = field1(vi, 2)) { + for (const auto* tt : fields(pb(tp->p, tp->n), 1)) { + const auto ttb = pb(tt->p, tt->n); + for (const auto* sh : fields(ttb, 2)) { + for (const auto* d : fields(pb(sh->p, sh->n), 1)) { + const auto db = pb(d->p, d->n); + if (const auto* v = field1(db, 1)) dims.push_back(int64_t(v->v)); + else dims.push_back(-1); // a symbolic dimension + } + } + } + } + return dims; + }; for (const auto* f : fields(g, 11)) { // graph.input const auto vi = pb(f->p, f->n); if (const auto* nm = field1(vi, 1)) out.inputs.push_back(pb_str(*nm)); + out.input_dims.push_back(dims_of_vi(vi)); } for (const auto* f : fields(g, 12)) { // graph.output const auto vi = pb(f->p, f->n); if (const auto* nm = field1(vi, 1)) out.outputs.push_back(pb_str(*nm)); + out.output_dims.push_back(dims_of_vi(vi)); } for (const auto* f : fields(g, 5)) { // graph.initializer const auto t = pb(f->p, f->n); diff --git a/scripts/npu-kernel-bitx.sh b/scripts/npu-kernel-bitx.sh new file mode 100755 index 000000000..a1304a356 --- /dev/null +++ b/scripts/npu-kernel-bitx.sh @@ -0,0 +1,136 @@ +#!/bin/bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# +# +# npu-kernel-bitx.sh: is a kernel family BIT-EXACT against the host, on the device? (engine#1783, engine#1808) +# +# npu-kernel-bitx.sh [--repeats N] [prompt ...] +# +# A kernel family's contract is not "it runs": it is bit-exact against the host reference, on the device, over more than one +# input. The engine carries the reference the device can be exact against -- NPU_INFER_HOST_GEMM=i8, whose int32 accumulation +# is the only thing an int8 kernel family can match by construction (engine#1808) -- so the check is: serve the same container +# twice, once on the NPU and once through that host arm, dump every position's raw logits (--dump-logits), and compare bytes. +# +# Why this is a script and not a ctest row: it needs the NPU, a container and a ticket. It therefore REFUSES to run outside an +# npu-hold ticket (no ~/NPU-TOKEN) rather than starting a bare serve on the device, and it releases what it starts. +# +# --repeats N runs the DEVICE arm N times and requires every run's dumps to be byte-identical to each other as well as to the host: +# the family contract is "bit-exact against the host reference, and 3 repeats identical", and doing the repeats by hand in a second +# script is how the second half gets skipped. +# +# Measured with it (whisper's 768 family, 2026-11-12): four prompts -- a single token, a three-token, a five-token and an +# eight-token one, so position-0 and repeated-token edges are covered -- all IDENTICAL, 4096 B each, and three repeats identical. +set -u +bin=${ONEBIT_BIN:-} +if [ -z "$bin" ]; then + for c in "$(dirname "$0")/../build/1bit" "$HOME/.local/share/1bit/1bit" "$(command -v 1bit 2>/dev/null)"; do + [ -n "$c" ] && [ -x "$c" ] && { bin=$c; break; } + done +fi +[ -n "$bin" ] && [ -x "$bin" ] || { echo "npu-kernel-bitx: no 1bit binary (set ONEBIT_BIN=)" >&2; exit 2; } +if [ ! -e "$HOME/NPU-TOKEN" ]; then + echo "npu-kernel-bitx: not inside an npu-hold ticket (no ~/NPU-TOKEN); refusing to start a serve on the device" >&2 + exit 2 +fi +container=${1:-}; kernels=${2:-} +[ -n "$container" ] && [ -n "$kernels" ] || { + echo "usage: $0 [prompt ...]" >&2; exit 2; } +shift 2 +repeats=1 +args=() +while [ $# -gt 0 ]; do + case "$1" in + --repeats) repeats=${2:-1}; shift 2 ;; + *) args+=("$1"); shift ;; + esac +done +if ! [ "$repeats" -ge 1 ] 2>/dev/null; then echo "npu-kernel-bitx: --repeats needs a count >= 1" >&2; exit 2; fi +prompts=(${args[@]+"${args[@]}"}) +[ ${#prompts[@]} -gt 0 ] || prompts=('<1>' '<7> <9> <11>' '<300> <301> <302> <303> <304>' '<1000> <1000> <2> <3> <4> <5> <6> <7>') + +out=${BITX_OUT:-$HOME/.cache/scratch-bitx/$(basename "$container" .q4nx)} +rm -rf "$out"; mkdir -p "$out/dev" "$out/host" + +serve() { # [env ...] + local dir=$1 port=$2; shift 2 + env "$@" "$bin" forward-serve -m "$container" --kernels "$kernels" --prefix-slots 0 \ + -p "$port" --dump-logits "$dir" > "$dir.log" 2>&1 & + local pid=$! + local up=0 + for _ in $(seq 1 60); do + curl -sf "http://127.0.0.1:$port/health" >/dev/null 2>&1 && { up=1; break; } + kill -0 "$pid" 2>/dev/null || break + sleep 1 + done + if [ "$up" = 0 ]; then + echo " $dir: serve did not come up -- $(grep -E 'init_elf failed|missing|error' "$dir.log" | tail -1)" >&2 + kill "$pid" 2>/dev/null; wait "$pid" 2>/dev/null; return 1 + fi + local i=0 rc=0 + for p in "${prompts[@]}"; do + i=$((i + 1)) + code=$(curl -s -o /dev/null -w '%{http_code}' -X POST "http://127.0.0.1:$port/v1/completions" \ + -H 'Content-Type: application/json' \ + -d "{\"prompt\":\"$p\",\"max_tokens\":1,\"temperature\":0}" -m 240) + [ "$code" = 200 ] || { echo " $dir: prompt $i -> http $code" >&2; rc=1; } + done + kill "$pid" 2>/dev/null; wait "$pid" 2>/dev/null + return $rc +} + +echo "device :" +for r in $(seq 1 "$repeats"); do + d=$( [ "$repeats" = 1 ] && printf '%s' "$out/dev" || printf '%s/dev%s' "$out" "$r" ) + mkdir -p "$d" + serve "$d" $((18150 + r - 1)) || exit 1 +done +echo "host :"; serve "$out/host" 18151 NPU_INFER_HOST_GEMM=i8 || exit 1 + +bad=0 n=0 +# Every device run must match the host; then the repeats must match each other. Comparison loops over the run dirs actually made. +devdirs="" +for r in $(seq 1 "$repeats"); do devdirs="$devdirs $( [ "$repeats" = 1 ] && printf '%s' "$out/dev" || printf '%s/dev%s' "$out" "$r" )"; done +first=$(printf '%s' "$devdirs" | awk '{print $1}') +for d in $devdirs; do + for f in "$d"/step_*_0.bin; do + [ -e "$f" ] || continue + b=$(basename "$f"); n=$((n + 1)) + if [ ! -f "$out/host/$b" ]; then echo " $d/$b: the host run wrote no counterpart"; bad=$((bad + 1)); continue; fi + if cmp -s "$f" "$out/host/$b"; then echo " $d/$b IDENTICAL to host ($(stat -c%s "$f") B)" + else echo " $d/$b DIFFERS from host ($(cmp -l "$f" "$out/host/$b" | wc -l) bytes)"; bad=$((bad + 1)); fi + done +done +if [ "$repeats" -gt 1 ]; then + drift=0 total=0 + for f in "$first"/step_*_0.bin; do + [ -e "$f" ] || continue + b=$(basename "$f"); total=$((total + 1)) + for d in $devdirs; do + [ "$d" = "$first" ] && continue + if ! cmp -s "$f" "$d/$b"; then + echo " $b: $first and $d differ -- the family is not deterministic on device" >&2 + drift=$((drift + 1)); break + fi + done + done + [ "$drift" = 0 ] && echo " all $total position(s) identical across $repeats device run(s)" \ + || { echo "npu-kernel-bitx: $drift of $total position(s) differ between repeats" >&2; bad=$((bad + drift)); } +fi +[ "$n" -gt 0 ] || { echo "npu-kernel-bitx: no logits were dumped -- the run proved nothing" >&2; exit 1; } +if [ "$bad" = 0 ]; then echo "npu-kernel-bitx: $n/$n prompt(s) bit-exact against the host integer arm"; exit 0; fi +echo "npu-kernel-bitx: $bad of $n prompt(s) differ from the host" >&2; exit 1 diff --git a/tests/audio_device_gemm_test.cpp b/tests/audio_device_gemm_test.cpp new file mode 100644 index 000000000..8579a03a1 --- /dev/null +++ b/tests/audio_device_gemm_test.cpp @@ -0,0 +1,109 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// tests/audio_device_gemm_test.cpp -- the decision layer around one encoder projection on the device (npu/forward/include/ +// audio_device_gemm.h, engine#1783), driven against a FAKE target: no device, no container, no kernels. +// +// The three things it pins are the three the real path gets wrong silently: the target must be asked for the STAGED width +// (N768 is named N1024), the caller must get its OWN width back (de-padded per row, not a staged stride), and every refusal +// (wrong output size, too many rows, no gemv, an unnamed projection, a target failure) must say why by name. +#include +#include +#include +#include "audio_device_gemm.h" + +using namespace onebit::audio; +namespace t = ::npu::target; + +static int failures = 0; +static void check(bool ok, const std::string& what) { + std::printf("%s %s\n", ok ? "ok " : "FAIL", what.c_str()); + if (!ok) ++failures; +} + +struct FakeTarget : t::NpuTarget { + t::Caps c; + bool supports_gemv = true; + bool fail = false; + size_t wrote = 0; // what the fake claims to have written; 0 = "rows x staged_n" + t::GemvArgs seen{}; + const char* name() const override { return "fake"; } + bool open(const onebit::npu::plan::Plan&, const std::string&, const char*, std::string&) override { return true; } + bool load_block(int, int, std::string&) override { return true; } + t::Caps caps() const override { return c; } + bool supports(t::Op op) const override { return op == t::Op::gemv && supports_gemv; } + bool gemv(const t::GemvArgs& a, std::vector& out, std::string& err) override { + seen = a; + if (fail) { err = "kernel not resident"; return false; } + const size_t n = wrote ? wrote : size_t(a.M) * size_t(a.N); + out.assign(n, 0.0f); + for (size_t i = 0; i < n; ++i) out[i] = float(i); // recognisable pattern + return true; + } +}; + +int main() { + FakeTarget ft; + ft.c.max_rows = 16; + ft.c.prefill_rows = 128; + std::vector x(128 * 768, 1.0f); + std::vector out; + std::string why; + + // 1. the de-pad: O at k768 n768 is named N1024, so the fake writes 128x1024 and the caller must get 128x768 + DeviceProjection p; + p.role = 1; p.layer = 3; p.rows = 128; p.k = 768; p.n = 768; p.x = x.data(); p.tag = "O"; + check(device_projection(ft, p, &out, &why), "a staged O projection runs"); + check(ft.seen.N == 1024 && ft.seen.K == 768, "the target is asked for the STAGED width (N1024), not the model's 768"); + check(ft.seen.M == 128 && ft.seen.rows, "a 128-row launch is the wide body"); + check(out.size() == size_t(128) * 768, "the caller gets its own width back (128 x 768), de-padded"); + check(out[1] == 1.0f && out[767] == 767.0f && out[768] == 1024.0f, + "the de-pad drops the padding column: row 1 starts at staged column 1024"); + + // 2. a target that writes the wrong size is refused, not repaired + ft.wrote = 128 * 768; + check(!device_projection(ft, p, &out, &why) && why.find("the target and the plan disagree") != std::string::npos, + "an output that is not rows x staged_n is refused by name"); + ft.wrote = 0; + + // 3. more rows than the target launches in one call is refused with both numbers + p.rows = 129; + check(!device_projection(ft, p, &out, &why) && why.find("prefill_rows=128") != std::string::npos, + "rows beyond the prefill launch is refused and names the limit"); + p.rows = 128; + + // 4. a target without gemv is refused, naming the target + ft.supports_gemv = false; + check(!device_projection(ft, p, &out, &why) && why.find("\"fake\"") != std::string::npos, + "a target without gemv is refused, naming it"); + ft.supports_gemv = true; + + // 5. a target failure keeps the target's own words + ft.fail = true; + check(!device_projection(ft, p, &out, &why) && why == "kernel not resident", "a target failure keeps its reason"); + ft.fail = false; + + // 6. an unnamed projection with no tag is refused (the log must be answerable) + p.tag = nullptr; p.k = 777; p.n = 5; + check(!device_projection(ft, p, &out, &why) && why.find("needs a tag") != std::string::npos, + "an unnamed shape with no tag is refused"); + + // 7. the decode tier: 16 rows is NOT the wide body + p.k = 768; p.n = 768; p.rows = 16; p.tag = "O"; + check(device_projection(ft, p, &out, &why) && !ft.seen.rows, "a 16-row launch is the per-row body"); + + std::printf(failures ? "audio device gemm: %d FAILED\n" : "audio device gemm: all cases ok\n", failures); + return failures ? 1 : 0; +} diff --git a/tests/audio_exec_test.cpp b/tests/audio_exec_test.cpp index 0b923244a..6d12cd7d1 100644 --- a/tests/audio_exec_test.cpp +++ b/tests/audio_exec_test.cpp @@ -26,6 +26,7 @@ #include #include "audio_exec.h" +#include "audio_pack.h" #include "onnx_graph.h" namespace { @@ -50,9 +51,26 @@ void expect_close(float got, double want, double tol, const std::string& what) { } // namespace int main(int argc, char** argv) { + // Two forms, like the tool: a pack directory (npu/audio_pack.h) or the two graph paths. The pack form is what the ctest row + // uses, so the container is exercised by the same run that checks the maths (engine#1783). + std::string pack_encoder, pack_decoder; const char* path = (argc > 1) ? argv[1] : std::getenv("ONEBIT_TEST_WHISPER"); + const char* dpath_arg = (argc > 2) ? argv[2] : nullptr; + if (path != nullptr && *path != '\0' && std::string(path).find(".onnx") == std::string::npos) { + onebit::audio::AudioPack pack; + std::string why; + if (!onebit::audio::discover_audio_pack(path, &pack, &why)) { + std::printf("FAIL pack: %s\n", why.c_str()); + return 1; + } + pack_encoder = pack.encoder; + pack_decoder = pack.decoder; + std::printf("pack: %s (mel bins %d from the %s)\n", path, pack.mel_bins, pack.mel_bins_source.c_str()); + path = pack_encoder.c_str(); + dpath_arg = pack_decoder.c_str(); + } if (path == nullptr || *path == '\0') { - std::printf("SKIP: pass the encoder .onnx, or set ONEBIT_TEST_WHISPER to it\n"); + std::printf("SKIP: pass a pack directory or the encoder .onnx, or set ONEBIT_TEST_WHISPER\n"); return 77; } using namespace onebit::npu::pack_import; @@ -127,7 +145,7 @@ int main(int argc, char** argv) { std::printf("audio exec: whole encoder on the host matches onnxruntime\n"); // ---- the decoder, on the same pack: 128-position token window x, the encoder output xa ---------------------------- - const char* dpath = (argc > 2) ? argv[2] : std::getenv("ONEBIT_TEST_WHISPER_DECODER"); + const char* dpath = (dpath_arg != nullptr) ? dpath_arg : std::getenv("ONEBIT_TEST_WHISPER_DECODER"); if (dpath == nullptr || *dpath == '\0') { std::printf("SKIP decoder: pass the decoder .onnx as the second argument\n"); return 0; diff --git a/tests/audio_kernel_plan.sh b/tests/audio_kernel_plan.sh new file mode 100755 index 000000000..99d824cf5 --- /dev/null +++ b/tests/audio_kernel_plan.sh @@ -0,0 +1,65 @@ +#!/bin/sh +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# tests/audio_kernel_plan.sh -- the launch plan a device route reads (engine#1783). +# +# The plan is the artifact the encoder's device arm consumes: one entry per projection, in the order the graph runs them, naming the +# ELF each one needs at the width the ENGINE stages it to. Three properties make it usable, and each has been wrong somewhere today: +# the list is not empty (an empty plan must not read as a servable pack), `order` is dense from 0 (a route indexes by it), and every +# staged width is on the engine's grid -- K a multiple of 256, N of 512 (`npu/forward/src/q4nx_pack.c`) -- because the forward asks +# for the staged name and a plan naming the model's own width sends a route after a file that is not there. +# +# Usage: audio_kernel_plan.sh +set -u +tool=${1:?audio_kernel_shapes binary} +pack=${2:-} +[ -n "$pack" ] || pack=${ONEBIT_TEST_WHISPER_PACK:-} +[ -n "$pack" ] || { echo "SKIP: no pack (pass one, or set ONEBIT_TEST_WHISPER_PACK)"; exit 77; } +[ -d "$pack" ] || { echo "SKIP: $pack is not a directory"; exit 77; } + +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +plan=$tmp/plan.json +if ! "$tool" "$pack" --write-plan "$plan" > "$tmp/out.txt" 2>&1; then + echo "FAIL the plan could not be written:"; sed 's/^/ /' "$tmp/out.txt"; exit 1 +fi +fail=0 +ok() { printf 'ok %s\n' "$1"; } +bad() { printf 'FAIL %s\n' "$1"; fail=1; } + +n=$(grep -c '"order":' "$plan" || true) +[ "$n" -gt 0 ] && ok "the plan lists $n projection(s)" || bad "the plan lists no projection -- nothing would launch" + +orders=$(grep -o '"order": [0-9]*' "$plan" | grep -o '[0-9]*' | tr '\n' ' ') +want=$(seq 0 $((n - 1)) | tr '\n' ' ') +[ "$orders" = "$want" ] && ok "...ordered 0..$((n - 1)) with no gaps" || bad "order is not dense from 0: $orders" + +offk=$(grep -o '"staged_k": [0-9]*' "$plan" | grep -o '[0-9]*' | awk '$1 % 256 != 0' | wc -l) +offn=$(grep -o '"staged_n": [0-9]*' "$plan" | grep -o '[0-9]*' | awk '$1 % 512 != 0' | wc -l) +[ "$offk" -eq 0 ] && ok "every staged K is a multiple of 256" || bad "$offk staged K value(s) off the grid" +[ "$offn" -eq 0 ] && ok "every staged N is a multiple of 512" || bad "$offn staged N value(s) off the grid" + +# The plan's ELF names are the family's, and they differ from the model's own widths exactly where staging bites. +bad_elf=$(grep -o '"elf_b8": "[^"]*"' "$plan" | grep -vc 'full_i8_[A-Z]*_K[0-9]*_N[0-9]*_b8\.elf' || true) +[ "$bad_elf" -eq 0 ] && ok "every elf_b8 is a full_i8__K_N_b8.elf name" \ + || bad "$bad_elf elf_b8 name(s) do not look like a family member" +if grep -q '"staged_n": 1024' "$plan"; then + grep -q '_N1024_b8.elf' "$plan" && ok "a staged N1024 is named N1024 (768 -> 1024), not the model's 768" \ + || bad "a launch is staged to 1024 but its ELF name is not" +fi + +[ "$fail" -eq 0 ] && { echo "audio kernel plan: ok"; exit 0; } +echo "audio kernel plan: FAILED"; exit 1 diff --git a/tests/audio_kernel_roles_test.cpp b/tests/audio_kernel_roles_test.cpp new file mode 100644 index 000000000..fd44c4ae0 --- /dev/null +++ b/tests/audio_kernel_roles_test.cpp @@ -0,0 +1,114 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// tests/audio_kernel_roles_test.cpp -- the shape-to-role rule of tools/audio_kernel_shapes.cpp, as a pure function +// (npu/forward/include/audio_kernel_roles.h, engine#1783). Hand-computed cases; no graph, no device. +// +// The rule was wrong twice before it was right, both times in ways a compiler cannot see: a name-based classifier reported +// `unknown` for every projection (packed exports name their weights `val_N`), and a version that classified BEFORE assigning +// the shape read (0, 0, hidden) and said `unknown` for everything. These arms are the ones that would have caught both. + +#include +#include + +#include "audio_kernel_roles.h" + +namespace { + +int failures = 0; + +void check(bool ok, const std::string& what) { + std::printf("%s %s\n", ok ? "ok " : "FAIL", what.c_str()); + if (!ok) ++failures; +} + +onebit::audio::ProjectionShape sh(int k, int n) { return onebit::audio::ProjectionShape{k, n}; } + +} // namespace + +using onebit::audio::elf_name_for; +using onebit::audio::q4nx_pad_k; +using onebit::audio::q4nx_pad_n; + +int main() { + using onebit::audio::elf_name_for; + using onebit::audio::hidden_from_square_widths; + using onebit::audio::role_of_shape; + + // 1. whisper-small's four shapes at h = 768 + check(std::string(role_of_shape(sh(768, 768), 768)) == "O", "h x h is O (q, k, v and o share it)"); + check(std::string(role_of_shape(sh(768, 3072), 768)) == "GU", "h -> 4h is GU"); + check(std::string(role_of_shape(sh(3072, 768), 768)) == "D", "4h -> h is D"); + check(std::string(role_of_shape(sh(768, 2304), 768)) == "QKV", "h -> 3h is fused QKV"); + + // 2. the ELF the family emits, both M tiers + // The emitted name is on the engine's staging grid (768 -> 1024), not the model's own width -- the defect the synth768 + // device run exposed: the forward asked for O_K768_N1024 while this said N768 and the set "covered". + check(elf_name_for(sh(768, 768), 768, false) == "full_i8_O_K768_N1024_b8.elf", "b8 ELF name, staged"); + check(elf_name_for(sh(768, 3072), 768, true) == "full_i8_GU_K768_N3072_b8_m16.elf", "m16 ELF name"); + + // 3. a shape that is none of the four is `unknown`, never a guess -- asking for the wrong ELF wastes a ticket + check(std::string(role_of_shape(sh(1000, 2000), 768)) == "unknown", "an unrecognised shape is unknown"); + check(std::string(role_of_shape(sh(768, 768), 0)) == "unknown", "no hidden width means no roles at all"); + + // 4. the hidden width comes from the most common SQUARE width, not the median of everything: embeddings and the 4h MLP + // would drag a median away from h, which is the bug the first version had. + { + const int widths[] = {768, 768, 768, 768, 3072, 3072, 51865, 1500, 128}; + check(hidden_from_square_widths(widths, 9) == 768, "the mode of the square widths is the hidden width"); + const int one[] = {1280}; + check(hidden_from_square_widths(one, 1) == 1280, "a single width is that width"); + // A tie is refused, not broken: two widths occurring equally often (here 768 and 896, four times each) mean the pack + // has two candidate hidden widths, and whichever one a loop happened to see first used to decide every role after it. + { + const int tied_widths[] = {768, 768, 768, 768, 896, 896, 896, 896}; + check(hidden_from_square_widths(tied_widths, 8) == 0, "an equal-mode tie is refused, not broken by order"); + const int clear[] = {768, 768, 768, 896, 896}; + check(hidden_from_square_widths(clear, 5) == 768, "a clear mode is still taken"); + check(hidden_from_square_widths(nullptr, 0) == 0, "no evidence is 0, not a guess"); + } + check(hidden_from_square_widths(nullptr, 0) == 0, "no widths means no hidden width, not a crash"); + } + + // 5. large-v3's shapes at h = 1280, so the rule is not tuned to one pack + check(std::string(role_of_shape(sh(1280, 1280), 1280)) == "O", "lv3: h x h is O"); + check(std::string(role_of_shape(sh(1280, 5120), 1280)) == "GU", "lv3: h -> 4h is GU"); + check(std::string(role_of_shape(sh(5120, 1280), 1280)) == "D", "lv3: 4h -> h is D"); + + // The engine stages every container on its own grid -- K a multiple of 256, N of 512 (npu/forward/src/q4nx_pack.c) -- and + // the forward asks for the ELF at THOSE widths. Measured on the synth768 device run (engine#1783): with the set emitted at + // the model's own widths the forward said "missing full ELF ... QKV_K768_N2560_b8" (2304 rounded up) and then "...O_K768_N1024" + // (768 rounded up), while this header was reporting the family covered. + check(q4nx_pad_n(768) == 1024, "N 768 stages to 1024"); + check(q4nx_pad_n(2304) == 2560, "N 2304 (a fused QKV at hidden 768) stages to 2560"); + check(q4nx_pad_n(3072) == 3072, "N 3072 is already on the 512 grid"); + check(q4nx_pad_k(768) == 768 && q4nx_pad_k(3000) == 3072, "K pads to 256: 768 in place, 3000 -> 3072"); + check(q4nx_pad_n(0) == 0 && q4nx_pad_k(-5) == -5, "a non-positive width is left alone, not rounded into a shape"); + check(elf_name_for({768, 2304}, 768, false) == "full_i8_QKV_K768_N2560_b8.elf", + "the QKV name the forward asks for at hidden 768 (2304 staged to 2560)"); + check(elf_name_for({768, 768}, 768, true) == "full_i8_O_K768_N1024_b8_m16.elf", + "the O name, m16 tier: the hidden width stages too (768 to 1024)"); + check(elf_name_for({3072, 768}, 768, false) == "full_i8_D_K3072_N1024_b8.elf", + "the D name: K already on its grid, N staged"); + check(elf_name_for({768, 3072}, 768, false) == "full_i8_GU_K768_N3072_b8.elf", + "the MLP name: 4h is on the 512 grid, so whisper-small's own width is the name"); + + if (failures != 0) { + std::printf("audio kernel roles: %d failure(s)\n", failures); + return 1; + } + std::printf("audio kernel roles: all cases ok\n"); + return 0; +} diff --git a/tests/audio_launch_plan_test.cpp b/tests/audio_launch_plan_test.cpp new file mode 100644 index 000000000..b389e6216 --- /dev/null +++ b/tests/audio_launch_plan_test.cpp @@ -0,0 +1,95 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// tests/audio_launch_plan_test.cpp -- the launch loop's routing and its ACCOUNTING (npu/forward/include/audio_launch_plan.h, +// engine#1783), driven against a fake target and a fake host: no device, no container, no kernels. +// +// The report is the measurement the row is scored on (`device_gemm= s + host_remainder= s`), so the arms here are about +// where a launch is COUNTED, not only whether it ran: a launch that falls back must land in the host column with the device's +// own reason beside it, and a launch both paths refuse must land in neither column -- counting it as host would quietly move an +// unexplained projection into the number the route is judged by. + +#include +#include +#include + +#include "audio_launch_plan.h" +using namespace onebit::audio; +namespace t = ::npu::target; +static int failures = 0; +static void check(bool ok, const std::string& what) { std::printf("%s %s\n", ok ? "ok " : "FAIL", what.c_str()); if (!ok) ++failures; } +struct FakeTarget : t::NpuTarget { + t::Caps c; int calls = 0; bool refuse_all = false; + const char* name() const override { return "fake"; } + bool open(const onebit::npu::plan::Plan&, const std::string&, const char*, std::string&) override { return true; } + bool load_block(int, int, std::string&) override { return true; } + t::Caps caps() const override { return c; } + bool supports(t::Op op) const override { return op == t::Op::gemv; } + bool gemv(const t::GemvArgs& a, std::vector& out, std::string& err) override { + ++calls; + if (refuse_all) { err = "no kernel for this shape"; return false; } + out.assign(size_t(a.M) * size_t(a.N), 1.0f); + return true; + } +}; +int main() { + std::vector x(128 * 768, 1.0f); + std::vector specs = { + {0, 1, 0, 768, 768, 128, "O"}, + {1, 0, 0, 768, 3072, 128, "GU"}, + {2, 2, 0, 3072, 768, 128, "D"}, + }; + auto input = [&](const LaunchSpec&) { return x.data(); }; + int host_calls = 0; + auto host = [&](const LaunchSpec&, std::vector*, std::string*) { ++host_calls; return true; }; + + FakeTarget ft; ft.c.max_rows = 16; ft.c.prefill_rows = 128; + auto r = run_launches(&ft, specs, input, host); + check(r.ok() && r.device_launches == 3 && r.host_launches == 0, "with a target, every launch goes to the device"); + check(ft.calls == 3 && host_calls == 0, "...the device was called three times and the host not at all"); + check(r.per_launch[0].device && r.per_launch[2].device, "...and each launch records which column it went in"); + + ft.refuse_all = true; host_calls = 0; + auto r2 = run_launches(&ft, specs, input, host); + check(r2.ok() && r2.host_launches == 3 && r2.device_launches == 0, "a target that refuses every shape falls back to the host"); + check(r2.per_launch[1].why == "no kernel for this shape", "...keeping the target's own reason per launch"); + check(host_calls == 3, "...the host ran all three"); + + ft.refuse_all = false; host_calls = 0; + auto r3 = run_launches(nullptr, specs, input, host); + check(r3.ok() && r3.host_launches == 3, "no target at all is the host path, counted as such"); + check(r3.per_launch[0].why == "no target", "...and says why it is not on the device"); + + auto bad_host = [&](const LaunchSpec&, std::vector*, std::string* why) { *why = "host GEMM refused"; return false; }; + auto r4 = run_launches(nullptr, specs, input, bad_host); + check(!r4.ok() && r4.failed == 3 && r4.device_launches == 0 && r4.host_launches == 0, + "a launch both paths refuse counts in NEITHER column"); + check(r4.failures.size() == 3 && r4.failures[0].find("launch 0 O") == 0 && r4.failures[0].find("host GEMM refused") != std::string::npos, + "...and the failure names the launch, its tag and both reasons"); + // The close condition's own line, built from a report rather than typed. A split with a silent third category (failed) is a + // split that adds up to less than the run, so the failure count is part of the string. + { + ft.refuse_all = true; // every launch falls back to the host + auto r5 = run_launches(&ft, specs, input, host); + const std::string line = format_split_run(r5, 163.0, 2050.0); + check(line == "device_gemm=0.163 s + host_remainder=2.050 s per window (0 device launch(es), 3 host, 0 failed)", + "the split line reads exactly as the close condition asks: " + line); + auto r6 = run_launches(nullptr, specs, input, bad_host); + const std::string bad = format_split_run(r6, 0.0, 0.0); + check(bad.find("3 failed") != std::string::npos, "...and a run whose launches all failed says so: " + bad); + } + std::printf(failures ? "audio launch plan: %d FAILED\n" : "audio launch plan: all cases ok\n", failures); + return failures ? 1 : 0; +} diff --git a/tests/audio_pack_test.cpp b/tests/audio_pack_test.cpp new file mode 100644 index 000000000..4d938a716 --- /dev/null +++ b/tests/audio_pack_test.cpp @@ -0,0 +1,155 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// tests/audio_pack_test.cpp -- audio pack discovery (npu/audio_pack.h, engine#1783). Fixture directories only: no model, no +// device, no network. The graph-derived arms need a real pack and skip (77) without ONEBIT_TEST_WHISPER. + +#include +#include +#include +#include +#include + +#include "audio_pack.h" + +namespace fs = std::filesystem; +namespace { + +int failures = 0; + +void check(bool ok, const std::string& what) { + if (ok) { + std::printf("ok %s\n", what.c_str()); + } else { + std::printf("FAIL %s\n", what.c_str()); + ++failures; + } +} + +void write(const fs::path& p, const std::string& body) { + fs::create_directories(p.parent_path()); + std::ofstream(p) << body; +} + +} // namespace + +int main() { + const fs::path root = fs::temp_directory_path() / "audio-pack-test"; + + // 7. a pack whose graphs are named after the model (large_v3_encoder.onnx): found by scan, and the choice is REPORTED -- + // and a `decoder_with_past` sibling is never picked, because it is a different graph with a different input list + { + const fs::path d = root / "model-named"; + write(d / "large_v3_encoder.onnx", "x"); + write(d / "large_v3_decoder_with_past.onnx", "x"); + write(d / "large_v3_decoder.onnx", "x"); + onebit::audio::AudioPack p; + std::string why; + // no bins anywhere: the graphs are not real, so discovery refuses on the bin count -- but the NAME choice must already + // have been made, which is what the two checks below read out of the pack struct. + onebit::audio::discover_audio_pack(d.string(), &p, &why); + check(p.decoder.find("large_v3_decoder.onnx") != std::string::npos, + "model-named: the plain decoder is chosen, not the with_past one"); + std::printf(" [debug] concern=<%s>\n", p.concern.c_str()); + check(p.concern.find("taken from its name") != std::string::npos, + "model-named: the name-based choice is reported, not silent"); + } + + fs::remove_all(root); + + // 1. a complete pack, whisper-* spelling, with a config.json that states the bins + { + const fs::path d = root / "complete"; + write(d / "encoder_model.onnx", "not really a graph"); + write(d / "decoder_model.onnx", "not really a graph"); + write(d / "config.json", R"({"num_mel_bins": 80, "sampling_rate": 16000, "n_fft": 400, "hop_length": 160})"); + onebit::audio::AudioPack p; + std::string why; + const bool ok = onebit::audio::discover_audio_pack(d.string(), &p, &why); + check(ok, "complete pack: discovered"); + check(p.encoder.find("encoder_model.onnx") != std::string::npos && + p.decoder.find("decoder_model.onnx") != std::string::npos, + "complete pack: both graphs found by their whisper names"); + check(p.mel_bins == 80 && p.mel_bins_source == "config.json", + "complete pack: the bins come from config.json when the graph cannot be read"); + check(!p.concern.empty(), "complete pack: the unreadable graph is reported as a concern, not a refusal"); + } + + // 2. our own spelling (encoder.onnx / decoder.onnx) is found too + { + const fs::path d = root / "own-names"; + write(d / "encoder.onnx", "x"); + write(d / "decoder.onnx", "x"); + write(d / "config.json", R"({"num_mel_bins": 128})"); + onebit::audio::AudioPack p; + std::string why; + check(onebit::audio::discover_audio_pack(d.string(), &p, &why), "own names: discovered"); + check(p.mel_bins == 128, "own names: 128 bins read from config.json"); + } + + // 3. a missing graph is refused, by name, and says which one + { + const fs::path d = root / "no-decoder"; + write(d / "encoder_model.onnx", "x"); + write(d / "config.json", R"({"num_mel_bins": 80})"); + onebit::audio::AudioPack p; + std::string why; + check(!onebit::audio::discover_audio_pack(d.string(), &p, &why), "no decoder: refused"); + check(p.missing.size() == 1 && p.missing[0].find("decoder") != std::string::npos, + "no decoder: the missing piece is named"); + check(why.find("missing") != std::string::npos, "no decoder: the reason says missing"); + } + + // 4. no mel-bin count anywhere: refused rather than guessed -- this is the run-that-answers-wrongly case + { + const fs::path d = root / "no-bins"; + write(d / "encoder_model.onnx", "x"); + write(d / "decoder_model.onnx", "x"); + onebit::audio::AudioPack p; + std::string why; + check(!onebit::audio::discover_audio_pack(d.string(), &p, &why), "no bin count: refused"); + check(why.find("guessing") != std::string::npos, "no bin count: the refusal says why guessing is not done"); + } + + // 5. not a directory at all + { + onebit::audio::AudioPack p; + std::string why; + check(!onebit::audio::discover_audio_pack((root / "nope").string(), &p, &why), "absent directory: refused"); + check(why.find("not a directory") != std::string::npos, "absent directory: said plainly"); + } + + // 6. a real pack, when one is named: the bins come from the encoder's own Conv + { + const char* pack = std::getenv("ONEBIT_TEST_WHISPER_PACK"); + if (pack == nullptr || *pack == '\0') { + std::printf("skip a real pack: set ONEBIT_TEST_WHISPER_PACK to a pack directory\n"); + } else { + onebit::audio::AudioPack p; + std::string why; + check(onebit::audio::discover_audio_pack(pack, &p, &why), std::string("real pack: discovered (") + why + ")"); + check(p.mel_bins > 0 && p.mel_bins_source == "encoder graph", + "real pack: the mel bins come from the encoder graph, not from a config field"); + } + } + + fs::remove_all(root); + if (failures != 0) { + std::printf("audio pack: %d failure(s)\n", failures); + return 1; + } + std::printf("audio pack: all cases ok\n"); + return 0; +} diff --git a/tests/data/whisper-jfk-ids.json b/tests/data/whisper-jfk-ids.json new file mode 100644 index 000000000..2a8554483 --- /dev/null +++ b/tests/data/whisper-jfk-ids.json @@ -0,0 +1,31 @@ +{ + "what": "the reference decode of jfk.wav through whisper-small (encoder_model.onnx + decoder_model.onnx)", + "source": "produced by tools/whisper_transcribe and checked against onnxruntime and the reference decode; see engine#1783", + "greedy_ids": [ + 400, + 370, + 452, + 7177, + 6280, + 11, + 1029, + 406, + 437, + 428, + 1941, + 393, + 360, + 337, + 291, + 11, + 1029, + 437, + 291, + 393, + 360, + 337, + 428, + 1941, + 13 + ] +} diff --git a/tests/whisper_ids_test.cpp b/tests/whisper_ids_test.cpp new file mode 100644 index 000000000..4fddfc898 --- /dev/null +++ b/tests/whisper_ids_test.cpp @@ -0,0 +1,74 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// tests/whisper_ids_test.cpp -- the special-token layout selection (npu/forward/include/whisper_ids.h, engine#1783). +// +// These are the two verdicts the tool got wrong before it read a pack: a hard-coded 50359/50363 decodes whisper-small and makes +// large-v3 answer <|endoftext|> at the first step. Both values here were measured against onnxruntime on the real packs. + +#include +#include + +#include "whisper_ids.h" + +namespace { + +int failures = 0; + +void check(bool ok, const std::string& what) { + std::printf("%s %s\n", ok ? "ok " : "FAIL", what.c_str()); + if (!ok) ++failures; +} + +} // namespace + +int main() { + using onebit::audio::whisper_ids_for_vocab; + using onebit::audio::WhisperIds; + + WhisperIds ids; + check(whisper_ids_for_vocab(51865, &ids), "51865 (v1/v2/small/medium): a layout exists"); + check(ids.sot == 50258 && ids.lang_en == 50259 && ids.eot == 50257, "51865: sot/en/eot are unchanged across families"); + check(ids.transcribe == 50359 && ids.no_timestamps == 50363, "51865: transcribe 50359, notimestamps 50363"); + + check(whisper_ids_for_vocab(51866, &ids), "51866 (large-v3): a layout exists"); + check(ids.transcribe == 50360 && ids.no_timestamps == 50364, "51866: both shift by one"); + + // A larger derivative of the v3 line keeps v3's layout; the whole point is that the ids are offsets into the vocabulary. + check(whisper_ids_for_vocab(51890, &ids) && ids.transcribe == 50360, "a wider v3 derivative keeps v3's layout"); + + // A `.en` model (51864) is refused: its prompt differs and nobody measured it here. Guessing one of the two known layouts is + // how a wrong prompt looks right on English audio. + WhisperIds untouched; + untouched.transcribe = -1; + check(!whisper_ids_for_vocab(51864, &untouched), "51864 (.en): refused rather than guessed"); + check(untouched.transcribe == -1, "... and the caller's struct is left untouched"); + check(!whisper_ids_for_vocab(0, &untouched), "vocabulary 0: refused"); + check(!whisper_ids_for_vocab(51865, nullptr), "a null output is refused, not dereferenced"); + + // The two families must not agree on transcribe: that difference is the defect this header exists to prevent. + WhisperIds small, v3; + whisper_ids_for_vocab(51865, &small); + whisper_ids_for_vocab(51866, &v3); + check(small.transcribe != v3.transcribe && small.no_timestamps != v3.no_timestamps, + "the two families differ in both ids (a shared constant is the bug)"); + + if (failures != 0) { + std::printf("whisper ids: %d failure(s)\n", failures); + return 1; + } + std::printf("whisper ids: all cases ok\n"); + return 0; +} diff --git a/tests/whisper_lv3_row.sh b/tests/whisper_lv3_row.sh new file mode 100755 index 000000000..f858becad --- /dev/null +++ b/tests/whisper_lv3_row.sh @@ -0,0 +1,38 @@ +#!/bin/sh +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# tests/whisper_lv3_row.sh -- the whisper-large-v3 half of the transcription row (engine#1783). +# +# Why a second pack and not just the first: every constant that fit whisper-small was wrong for large-v3 -- the token window +# (128 vs 448), the vocabulary width (51865 vs 51866) and the <|transcribe|>/<|notimestamps|> ids that follow from it +# (50359/50363 vs 50360/50364). Each one was invisible until the tool was pointed at the second pack, and each produced a +# plausible-looking wrong answer rather than a crash. So the row runs BOTH families and requires the same ids from both. +# +# Usage: whisper_lv3_row.sh +# ONEBIT_TEST_WHISPER_PACK_LV3 and ONEBIT_TEST_WHISPER_WAV_LV3 name the pack and the wav; unset means skip (77), because a +# pack is a multi-GB download and cannot live in the tree. +set -u +bin=${1:?whisper_transcribe binary} +ids=${2:?expected ids json} +[ -n "${ONEBIT_TEST_WHISPER_PACK_LV3:-}" ] || { + echo "SKIP: set ONEBIT_TEST_WHISPER_PACK_LV3 (and ONEBIT_TEST_WHISPER_WAV_LV3) to run the large-v3 arm" + exit 77 +} +[ -n "${ONEBIT_TEST_WHISPER_WAV_LV3:-}" ] || { + echo "SKIP: ONEBIT_TEST_WHISPER_PACK_LV3 is set but ONEBIT_TEST_WHISPER_WAV_LV3 is not" + exit 77 +} +exec "$bin" "$ONEBIT_TEST_WHISPER_PACK_LV3" "$ONEBIT_TEST_WHISPER_WAV_LV3" "$ids" diff --git a/tools/audio_kernel_shapes.cpp b/tools/audio_kernel_shapes.cpp new file mode 100644 index 000000000..100eaf4fc --- /dev/null +++ b/tools/audio_kernel_shapes.cpp @@ -0,0 +1,250 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// audio_kernel_shapes: which GEMM shapes an audio pack's encoder needs, and whether a kernel directory covers them (engine#1783). +// +// audio_kernel_shapes # the shapes the encoder's projections ask for +// audio_kernel_shapes --write-plan # ... as a launch plan (JSON): node order, staged shapes, ELF names +// audio_kernel_shapes --kernels # ... and whether each has a full ELF in that directory +// +// Why a tool and not a test: what it reports depends on which pack and which kernel directory are given, and the useful output is a +// list a lane reads before taking a device ticket. The shape derivation is the part worth trusting -- it reads the pack's own ONNX +// graph through the engine's reader, so a pack whose projections differ (768 for small, 1280 for large-v3) reports its own numbers +// instead of a table someone maintained. +// +// The role a projection plays is decided by the tensor it multiplies, not by node order: a packed export fuses q/k/v into one +// 3N-wide matrix, and the ELF it needs is the fused one (N = 3 x hidden). A name this tool does not recognise is reported as +// `unknown`, because a shape with no role is a shape nobody has built a kernel for. + +#include +#include "file_create.h" // onebit_fopen_private_wb: a dump must be 0600 and must not follow a symlink +#include +#include +#include +#include +#include +#include + +#include "audio_pack.h" +#include "audio_kernel_roles.h" +#include "onnx_graph.h" + +namespace fs = std::filesystem; +using onebit::npu::pack_import::OnnxGraphInfo; + +namespace { + +struct Need { + std::string role; // QKV | O | GU | D, or "unknown" + int kindex = 0; // the reduction (K) + int n = 0; // the output width (N) + std::string weight; // the tensor it multiplies, for the message +}; + +std::string role_of_unused(const std::string& weight) { + const auto has = [&](const char* s) { return weight.find(s) != std::string::npos; }; + if (has("qkv") || (has("q_proj") && has("k_proj"))) return "QKV"; + if (has("gate_up") || (has("gate_proj") && has("up_proj"))) return "GU"; + if (has("q_proj") || has("query")) return "QKV"; + if (has("k_proj") || has("key") || has("v_proj") || has("value")) return "QKV"; + if (has("out_proj") || has("o_proj") || has("dense")) return "O"; + if (has("down_proj")) return "D"; + if (has("fc1")) return "GU"; + if (has("fc2")) return "D"; + return "unknown"; +} + +// A packed export names its weights generically (`val_452`) and reaches them through a Transpose; the name that says what the +// projection IS (`blocks.0.attn.query`) sits on the Transpose's input. So the role comes from following the producer chain back to +// a named tensor -- without this every shape reports `unknown`, which is what the first run of this tool did. +const std::string* source_name(const std::map& producer, std::string name) { + for (int hop = 0; hop < 4; ++hop) { + const auto it = producer.find(name); + if (it == producer.end()) return nullptr; + name = it->second; + // a name that looks like a parameter path rather than a val_N intermediate is the one to report + if (name.find("val_") == std::string::npos && name.find("weight") != std::string::npos) { + static thread_local std::string keep; + keep = name; + return &keep; + } + } + static thread_local std::string keep; + keep = name; + return &keep; +} + +std::vector shapes_of(const OnnxGraphInfo& g, int hidden) { + std::map by_weight; // one entry per weight, so a shared weight is not counted twice + std::map> dims; + for (const auto& t : g.init_tensors) dims[t.name] = t.dims; + std::map producer; // output -> its single non-initializer input, for the role lookup + for (const auto& node : g.nodes) { + if (node.outs.empty() || node.ins.empty()) continue; + if (node.op != "Transpose" && node.op != "Reshape" && node.op != "Cast" && node.op != "Identity") continue; + producer[node.outs[0]] = node.ins[0]; + } + for (const auto& node : g.nodes) { + if (node.op != "MatMul" || node.ins.size() < 2) continue; + const auto it = dims.find(node.ins[1]); + if (it == dims.end() || it->second.size() != 2) continue; + Need need; + const std::string* named = source_name(producer, node.ins[1]); + need.weight = (named != nullptr && !named->empty()) ? *named : node.ins[1]; + need.kindex = int(it->second[0]); + need.n = int(it->second[1]); + need.role = onebit::audio::role_of_shape(onebit::audio::ProjectionShape{need.kindex, need.n}, hidden); // after the shape is set: this read 0, 0 on the first run + by_weight[need.weight] = need; + } + std::vector out; + for (auto& kv : by_weight) out.push_back(kv.second); + std::sort(out.begin(), out.end(), [](const Need& a, const Need& b) { + return a.role == b.role ? a.n < b.n : a.role < b.role; + }); + return out; +} + +// The ELF names the family generator produces: full_i8__K_N_b8[_m16].elf +std::string elf_name(const Need& n, bool m16) { + return onebit::audio::elf_name_for(onebit::audio::ProjectionShape{n.kindex, n.n}, 0, m16) == "" ? "" : + "full_i8_" + n.role + "_K" + std::to_string(onebit::audio::q4nx_pad_k(n.kindex)) + "_N" + + std::to_string(onebit::audio::q4nx_pad_n(n.n)) + "_b8" + + (m16 ? "_m16" : "") + ".elf"; +} + +} // namespace + +int main(int argc, char** argv) { + if (argc < 2) { + std::fprintf(stderr, "usage: %s [--kernels ]\n", argv[0]); + return 2; + } + onebit::audio::AudioPack pack; + std::string why; + if (!onebit::audio::discover_audio_pack(argv[1], &pack, &why)) { + std::fprintf(stderr, "pack: %s\n", why.c_str()); + return 1; + } + OnnxGraphInfo g; + try { + g = onebit::npu::pack_import::parse_onnx_graph(pack.encoder); + } catch (const std::exception& e) { + std::fprintf(stderr, "encoder: %s\n", e.what()); + return 1; + } + // The hidden width comes from the graph's own biggest projection: the widest N among the distinct shapes. + int hidden = 0; + { + // The hidden width is the most common SQUARE weight width: q/k/v/o are h x h in every layer, so the mode of that width is + // h, where a median over all weights is skewed by the embeddings and the 4h MLP. + // ONE implementation of the rule, in the header the tests drive: the mode of the square widths, and 0 when two widths + // tie. A second copy here is the drift this file exists to prevent -- and it had one: this loop took whichever + // width came first out of a std::map, i.e. the SMALLEST of a tie, silently. + std::vector square; + for (const auto& t : g.init_tensors) + if (t.dims.size() == 2 && t.dims[0] == t.dims[1] && t.dims[0] > 256) square.push_back(int(t.dims[0])); + hidden = onebit::audio::hidden_from_square_widths(square.data(), int(square.size())); + if (hidden <= 0) { + std::fprintf(stderr, + "audio_kernel_shapes: the hidden width is ambiguous or absent in %s (%zu square weight width(s), no " + "single most common one). Every role would be reported `unknown`, and a role this tool cannot name is " + "one whose kernel cannot be checked -- refusing rather than picking one.\n", + pack.encoder.c_str(), square.size()); + return 2; + } + } + const std::vector needs = shapes_of(g, hidden); + std::printf("pack %s: %zu projection shape(s) in %s (hidden %d)\n", argv[1], needs.size(), pack.encoder.c_str(), hidden); + std::map counts; + for (const auto& n : needs) { + ++counts[n.role + " K" + std::to_string(n.kindex) + " N" + std::to_string(n.n)]; + } + for (const auto& kv : counts) std::printf(" %-18s x%d\n", kv.first.c_str(), kv.second); + + std::string kernels, plan_path; + for (int i = 2; i + 1 < argc; ++i) { + if (std::strcmp(argv[i], "--kernels") == 0) kernels = argv[i + 1]; + else if (std::strcmp(argv[i], "--write-plan") == 0) plan_path = argv[i + 1]; + } + if (!plan_path.empty()) { + // The launch plan a device route would read: every projection IN TOPOLOGICAL ORDER (the graph's own order, because a + // route executes them in it), with the shape the model declares beside the shape the kernel is staged at and the ELF + // name that staging produces. It is written here rather than derived by the route so that the one place which knows the + // staging rule is the one place that names the kernel -- the defect this file exists to prevent (engine#1783). + std::FILE* f = onebit_fopen_private_wb(plan_path.c_str()); // 0600, no symlink (file_create.h) + if (f == nullptr) { + std::fprintf(stderr, "audio_kernel_shapes: cannot write %s\n", plan_path.c_str()); + return 1; + } + std::fprintf(f, "{\n \"pack\": \"%s\",\n \"encoder\": \"%s\",\n \"hidden\": %d,\n \"launches\": [\n", + argv[1], pack.encoder.c_str(), hidden); + int idx = 0; + std::vector dims; + std::vector mats; + for (const auto& t : g.init_tensors) + if (t.dims.size() == 2 && t.dims[1] > 1) mats.push_back(t.name); + for (const auto& node : g.nodes) { + if (node.op != "MatMul" || node.ins.size() < 2) continue; + const auto need = std::find_if(needs.begin(), needs.end(), [&](const Need& n) { + return n.weight == node.ins[1] || n.weight == node.ins[0]; + }); + if (need == needs.end()) continue; + const auto staged = onebit::audio::staged(onebit::audio::ProjectionShape{need->kindex, need->n}); + std::fprintf(f, + "%s {\"order\": %d, \"op\": \"MatMul\", \"weight\": \"%s\", \"role\": \"%s\", " + "\"k\": %d, \"n\": %d, \"staged_k\": %d, \"staged_n\": %d, \"elf_b8\": \"%s\", " + "\"elf_m16\": \"%s\"}", + idx == 0 ? "" : ",\n", idx, need->weight.c_str(), need->role.c_str(), need->kindex, need->n, + staged.k, staged.n, elf_name(*need, false).c_str(), elf_name(*need, true).c_str()); + ++idx; + } + std::fprintf(f, "\n ],\n \"projections\": %d,\n \"note\": \"staged_* are the widths the ELF is named at " + "(K%%256, N%%512); the forward asks for those names, not the model's\"\n}\n", idx); + std::fclose(f); + std::printf("plan: %d projection(s) in topological order -> %s\n", idx, plan_path.c_str()); + if (idx == 0) { + std::fprintf(stderr, "audio_kernel_shapes: the plan is empty -- the pack's graph has no 2-D MatMul on a " + "recognised weight, so nothing would launch\n"); + return 1; + } + } + + if (kernels.empty()) return 0; + + // One ELF per distinct (role, K, N), in both M tiers: the family the encoder needs is the same parameterised generator's output. + std::map distinct; + for (const auto& n : needs) distinct[n.role + ":" + std::to_string(n.kindex) + ":" + std::to_string(n.n)] = 1; + int missing = 0; + for (const auto& kv : distinct) { + const std::string key = kv.first; + const size_t a = key.find(':'), b = key.find(':', a + 1); + Need n; + n.role = key.substr(0, a); + n.kindex = std::stoi(key.substr(a + 1, b - a - 1)); + n.n = std::stoi(key.substr(b + 1)); + for (bool m16 : {false, true}) { + const std::string name = elf_name(n, m16); + const bool present = fs::is_regular_file(fs::path(kernels) / name); + if (!present) ++missing; + std::printf(" %-6s %5s %s\n", present ? "have" : "MISSING", m16 ? "m16" : "b8", name.c_str()); + } + } + std::printf("kernels %s: %d missing of %zu x 2\n", kernels.c_str(), missing, distinct.size()); + if (missing != 0) { + std::printf("note: a role this tool does not recognise is reported as `unknown` above, so an %s* ELF will always be MISSING\n", + "unknown"); + } + return missing == 0 ? 0 : 1; +} diff --git a/tools/whisper_transcribe.cpp b/tools/whisper_transcribe.cpp index 119a17131..1657023ee 100644 --- a/tools/whisper_transcribe.cpp +++ b/tools/whisper_transcribe.cpp @@ -15,8 +15,12 @@ // // whisper_transcribe.cpp -- the whole host pipeline: wav -> log-mel -> encoder -> greedy decode -> token ids. // +// ./whisper_transcribe [expected-ids-csv ...] // ./whisper_transcribe [expected-ids-csv ...] // +// The pack form reads both graphs, and the mel-bin count, from the directory (npu/audio_pack.h): 80 bins for whisper-small and 128 +// for large-v3, and passing the wrong one produces a graph that runs and answers wrongly, so the count comes from the encoder. +// // A tool rather than a test because one forward pass of each graph is ~26 s at these shapes and a sentence is 25 of them // (see PR #1931); `ctest` would time out for a reason that has nothing to do with correctness. The numbers it prints are // the ones a test would assert, so `expected-ids-csv` turns it into a check when run by hand. @@ -25,6 +29,7 @@ // stricter check than comparing text (two ids can share a detokenisation, and neither can share an id). #include +#include #include #include #include @@ -33,6 +38,9 @@ #include #include "audio_exec.h" +#include "audio_pack.h" +#include "whisper_ids.h" +#include "audio_parallel.h" #include "audio_mel.h" #include "onnx_graph.h" @@ -100,26 +108,132 @@ std::vector read_wav_16k_mono(const char* path, std::string* why) { } // namespace + +// The expected ids as a file (tests/data/whisper-jfk-ids.json), so a lane does not have to retype 25 numbers from a PR body. The +// parser is deliberately narrow: it finds "greedy_ids" and reads the integers that follow, and refuses a file with none. +static bool read_ids_file(const char* path, std::vector* out, std::string* why) { + FILE* f = std::fopen(path, "rb"); + if (f == nullptr) { + *why = std::string("cannot open ") + path; + return false; + } + std::string text; + char buf[4096]; + size_t n; + while ((n = std::fread(buf, 1, sizeof buf, f)) > 0) text.append(buf, n); + std::fclose(f); + const size_t key = text.find("\"greedy_ids\""); + if (key == std::string::npos) { + *why = std::string(path) + " has no \"greedy_ids\" list"; + return false; + } + // The scan stops at the array's closing bracket. Running to the END of the file (as this did) reads every digit after + // the key, including numbers in later keys and any mention of the key in a prose note. + size_t end = text.find('[', key); + if (end == std::string::npos) { + *why = std::string(path) + ": \"greedy_ids\" is not followed by a list"; + return false; + } + int depth = 0; + for (size_t i = end; i < text.size(); ++i) { + if (text[i] == '[') ++depth; + else if (text[i] == ']' && --depth == 0) { end = i; break; } + } + for (size_t i = key; i < end;) { + if (std::isdigit(static_cast(text[i])) || (text[i] == '-' && i + 1 < text.size())) { + const size_t start = i; + ++i; + while (i < text.size() && std::isdigit(static_cast(text[i]))) ++i; + out->emplace_back(text, start, i - start); + } else { + ++i; + } + } + if (out->empty()) { + *why = std::string(path) + ": the greedy_ids list is empty"; + return false; + } + return true; +} + int main(int argc, char** argv) { - if (argc < 4) { - std::fprintf(stderr, "usage: %s [expected-id ...]\n", argv[0]); + // ctest form: no arguments means "run the reference sentence from the environment", so the chain can be checked by CI or by + // a lane without remembering the 25 ids. Missing inputs skip with 77, the same convention as every other gated test here. + if (argc == 1) { + static std::vector from_env; + const char* pack = std::getenv("ONEBIT_TEST_WHISPER_PACK"); + const char* wav = std::getenv("ONEBIT_TEST_WHISPER_WAV"); + const char* ids = std::getenv("ONEBIT_TEST_WHISPER_IDS"); + const char* ids_file = std::getenv("ONEBIT_TEST_WHISPER_IDS_FILE"); + if (pack == nullptr || wav == nullptr || (ids == nullptr && ids_file == nullptr)) { + std::printf("SKIP: set ONEBIT_TEST_WHISPER_PACK and _WAV, and either _IDS or _IDS_FILE " + "(tests/data/whisper-jfk-ids.json)\n"); + return 77; + } + from_env.push_back(argv[0]); + from_env.push_back(pack); + from_env.push_back(wav); + if (ids == nullptr) { + std::string ids_why; + if (!read_ids_file(ids_file, &from_env, &ids_why)) { + std::fprintf(stderr, "ids: %s\n", ids_why.c_str()); + return 1; + } + ids = ""; + } + for (size_t i = 0; ids != nullptr && i < std::strlen(ids);) { // split on whitespace + while (i < std::strlen(ids) && std::isspace(static_cast(ids[i]))) ++i; + const size_t start = i; + while (i < std::strlen(ids) && !std::isspace(static_cast(ids[i]))) ++i; + if (i > start) from_env.emplace_back(ids + start, i - start); + } + std::vector argv_env; + for (auto& s : from_env) argv_env.push_back(const_cast(s.c_str())); + return main(static_cast(argv_env.size()), argv_env.data()); + } + if (argc < 3) { + std::fprintf(stderr, + "usage: %s [expected-id ...]\n" + " %s [expected-id ...]\n" + " The pack form reads the graphs, and the mel-bin count, from the directory (engine#1783).\n", + argv[0], argv[0]); return 2; } using namespace onebit::npu::pack_import; + std::string enc_path = (argc > 1) ? argv[1] : ""; + std::string dec_path = (argc > 2) ? argv[2] : ""; std::string why; - // The spans the last line reports, each named for what it contains and nothing else: wav (reading the file), mel - // (the log-mel front-end), load (parsing both graphs), encoder, decode. `wall` is their sum, from before the file - // is opened. - const auto t_begin = std::chrono::steady_clock::now(); - std::vector audio = read_wav_16k_mono(argv[3], &why); + // `1` means the two-graph form is in use and the wav is argv[3]; the pack form sets this to 2 after it discovers the graphs. + int audio_arg_index = 1; + onebit::audio::MelConfig cfg; + // The pack form is decided by the first argument: a directory (or anything without `.onnx`) is a pack, and it also carries the + // mel-bin count, which has to be known before the wav is turned into a spectrogram. + if (argc >= 3 && std::string(argv[1]).find(".onnx") == std::string::npos) audio_arg_index = 2; + if (audio_arg_index == 2) { + const std::string pack_dir = argv[1]; + onebit::audio::AudioPack pack; + if (!onebit::audio::discover_audio_pack(pack_dir, &pack, &why)) { + std::fprintf(stderr, "pack: %s\n", why.c_str()); + return 1; + } + if (!pack.concern.empty()) std::fprintf(stderr, "pack note: %s\n", pack.concern.c_str()); + std::fprintf(stderr, "pack: %s (mel bins %d from the %s)\n", pack_dir.c_str(), pack.mel_bins, + pack.mel_bins_source.c_str()); + cfg.bins = pack.mel_bins; + cfg.sample_rate = pack.sample_rate; + cfg.n_fft = pack.n_fft; + cfg.hop = pack.hop; + cfg.frames = pack.frames; + enc_path = pack.encoder; + dec_path = pack.decoder; + } + const size_t want = size_t(cfg.sample_rate) * 30; + std::vector audio = read_wav_16k_mono(argv[audio_arg_index], &why); if (audio.empty()) { std::fprintf(stderr, "wav: %s\n", why.c_str()); return 1; } - // Whisper's window is a fixed 30 s: pad with silence, do not scale. - onebit::audio::MelConfig cfg; - const size_t want = size_t(cfg.sample_rate) * size_t(cfg.frames) * size_t(cfg.hop) / size_t(cfg.frames); audio.resize(std::max(audio.size(), want), 0.0f); const auto t_start = std::chrono::steady_clock::now(); @@ -131,13 +245,12 @@ int main(int argc, char** argv) { const auto t_mel = std::chrono::steady_clock::now(); OnnxGraphInfo enc, dec; try { - enc = parse_onnx_graph(argv[1]); - dec = parse_onnx_graph(argv[2]); + enc = parse_onnx_graph(enc_path); + dec = parse_onnx_graph(dec_path); } catch (const std::exception& e) { std::fprintf(stderr, "graph: %s\n", e.what()); return 1; } - const auto t_load = std::chrono::steady_clock::now(); AudioRunResult xa; if (!run_audio_graph(enc, mel.data(), cfg.bins, cfg.frames, "", &xa, &why)) { std::fprintf(stderr, "encoder: %s\n", why.c_str()); @@ -150,8 +263,40 @@ int main(int argc, char** argv) { // <|startoftranscript|>, <|en|>, <|transcribe|>, <|notimestamps|> -- then greedy, reading the logits of the last real // prompt position (HF pads the 448/128-token window with <|endoftext|>, so the prompt is a prefix, not a suffix). - const int64_t kSot = 50258, kEn = 50259, kTranscribe = 50359, kNoTimestamps = 50363, kEot = 50257; - const int window = 128; + // The token window is the DECODER's declared input length, not a constant: whisper-small exports 128, large-v3 448, + // and a 128-wide window against a 448-wide graph builds a mask the scores cannot broadcast against (measured). + int window = 128; + if (!dec.input_dims.empty() && dec.input_dims[0].size() == 2 && dec.input_dims[0][1] > 0) { + window = int(dec.input_dims[0][1]); + std::printf("decoder window: %d positions (from the graph's input shape)\n", window); + } + + // Whisper's special ids are a property of the VOCABULARY, and the two packs here do not share one: whisper-small exports + // 51865 logits, large-v3 exports 51866 (v3 added a token), which shifts <|translate|>/<|transcribe|>/<|nospeech|>/ + // <|notimestamps|> by one. Hard-coding either set is what produced the first version of this tool: 50359/50363 decode + // whisper-small correctly *because English audio makes <|translate|> and <|transcribe|> agree*, and produce <|endoftext|> + // at the first step on large-v3 (measured: argmax 50257 with 50359/50363, 400 with 50360/50364, same pack and input). + // Neither pack ships a tokenizer or generation_config, so the vocabulary width is the only fact available -- and it is the + // right one, because the ids are offsets into it. + // The decoder's declared output shape is [1, window, vocab]; the vocabulary is its last dimension. (A scan over the MatMul + // weights cannot find it: the export applies the head through a Transpose, so the widest weight is the MLP's 4h.) + int64_t vocab = 0; + if (!dec.output_dims.empty() && !dec.output_dims[0].empty()) vocab = dec.output_dims[0].back(); + + // The ids come from the pack's own vocabulary width (npu/forward/include/whisper_ids.h), for the reason recorded there. + onebit::audio::WhisperIds slot; + if (!onebit::audio::whisper_ids_for_vocab(vocab, &slot)) { + std::fprintf(stderr, "special ids: no measured layout for a vocabulary of %lld\n", (long long)vocab); + return 1; + } + // Say what the split is: a rate quoted without it is unreadable, because the default follows the CPUs this process is + // ALLOWED (a thermal-run scope allows far fewer than the machine has) and the executor is only faster than serial + // when the threads actually fit (engine#1783). + std::printf("threads: %zu\n", onebit::npu::pack_import::audio_thread_count()); + std::printf("special ids: vocab %lld -> transcribe %lld, notimestamps %lld\n", (long long)vocab, + (long long)slot.transcribe, (long long)slot.no_timestamps); + const int64_t kEot = slot.eot, kSot = slot.sot, kEn = slot.lang_en; + const int64_t kTranscribe = slot.transcribe, kNoTimestamps = slot.no_timestamps; std::vector seq = {kSot, kEn, kTranscribe, kNoTimestamps}; std::vector ids; for (int step = 0; step < 60; ++step) { @@ -162,7 +307,8 @@ int main(int argc, char** argv) { std::fprintf(stderr, "decoder at step %d: %s\n", step, why.c_str()); return 1; } - const int64_t vocab = y.dims.empty() ? 0 : y.dims.back(); + const int64_t out_vocab = y.dims.empty() ? 0 : y.dims.back(); + const int64_t& vocab = out_vocab; const size_t row = (seq.size() - 1) * size_t(vocab); int64_t best = 0; for (int64_t v = 1; v < vocab; ++v) @@ -175,24 +321,43 @@ int main(int argc, char** argv) { const auto secs = [](std::chrono::steady_clock::time_point a, std::chrono::steady_clock::time_point b) { return std::chrono::duration(b - a).count(); }; - std::printf("time: wav %.3f s, mel %.3f s, load %.3f s, encoder %.3f s, decode %.3f s for %zu token(s) -> %.3f s/token, " - "wall %.3f s\n", - secs(t_begin, t_start), secs(t_start, t_mel), secs(t_mel, t_load), secs(t_load, t_enc), secs(t_enc, t_dec), - ids.size(), ids.empty() ? 0.0 : secs(t_enc, t_dec) / double(ids.size()), secs(t_begin, t_dec)); + std::printf("time: mel+load %.3f s, encoder %.3f s, decode %.3f s for %zu token(s) -> %.3f s/token, wall %.3f s\n", + secs(t_start, t_mel), secs(t_mel, t_enc), secs(t_enc, t_dec), ids.size(), + ids.empty() ? 0.0 : secs(t_enc, t_dec) / double(ids.size()), secs(t_start, t_dec)); + // The row's completion asks for "s/window AND time to first token", and TTFT for this chain is everything before the first + // decoded token: the mel front-end, the graph load, and the whole encoder pass. Named rather than left to be derived from the + // line above, because "which of those three is the TTFT" is exactly the question a reader gets wrong at 2 a.m. + std::printf("ttft: %.3f s (mel %.3f + encoder %.3f; the first token needs no extra forward)\n", + secs(t_start, t_enc), secs(t_start, t_mel), secs(t_mel, t_enc)); std::printf("ids:"); for (int64_t v : ids) std::printf(" %lld", (long long)v); std::printf("\n"); - if (argc > 4) { + // The expectation is either the ids themselves (one per argument) or, when a single .json path is given, that file's + // `greedy_ids`. A path used to be compared as a literal id, which reported "length: got 25 want 1" -- true, and useless: + // it said nothing about whether the file was readable or held the list. + std::vector want_ids; + if (argc > audio_arg_index + 1) { + const std::string first = argv[audio_arg_index + 1]; + if (argc == audio_arg_index + 2 && first.size() > 5 && first.compare(first.size() - 5, 5, ".json") == 0) { + std::string ids_why; + if (!read_ids_file(first.c_str(), &want_ids, &ids_why)) { + std::fprintf(stderr, "ids: %s\n", ids_why.c_str()); + return 1; + } + } else { + for (int i = audio_arg_index + 1; i < argc; ++i) want_ids.emplace_back(argv[i]); + } + } + if (!want_ids.empty()) { int bad = 0; - const int n = argc - 4; - if (int(ids.size()) != n) { - std::printf("FAIL length: got %zu want %d\n", ids.size(), n); + if (ids.size() != want_ids.size()) { + std::printf("FAIL length: got %zu want %zu\n", ids.size(), want_ids.size()); ++bad; } else { - for (int i = 0; i < n; ++i) - if (ids[size_t(i)] != std::atoll(argv[4 + i])) { - std::printf("FAIL id[%d]: got %lld want %s\n", i, (long long)ids[size_t(i)], argv[4 + i]); + for (size_t i = 0; i < want_ids.size(); ++i) + if (ids[i] != std::atoll(want_ids[i].c_str())) { + std::printf("FAIL id[%zu]: got %lld want %s\n", i, (long long)ids[i], want_ids[i].c_str()); ++bad; } }