From 672bbea43b433220525f3d3ded3d821e2df8bbfd Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Fri, 13 Nov 2026 07:22:59 -0400 Subject: [PATCH] npu: the whisper encoder's projections and conv stems on the NPU (engine#1783) The audio executor offers every product whose B is a weight (q, k, v, o, fc1, fc2, and both Conv stems as im2col GEMMs) to an AudioGemm device and runs the rest on the host, counting which side ran what and why. NpuAudioGemm is that device on an NpuTarget: - it routes by staged (K, N) to the design's role index; - it cuts rows into M-row launches; - it pads each weight once under its own residency key; - it strips the padding columns from the result. whisper_transcribe_npu (ONEBIT_WHISPER_NPU_KERNELS=) adopts one ELF per shape through open_prefill and prints the device/host split. Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 19 ++- npu/audio_exec.cpp | 92 ++++++++++++-- npu/audio_exec.h | 26 ++++ npu/forward/include/audio_npu_gemm.h | 175 ++++++++++++++++++++++++++ npu/forward/include/audio_ops.h | 27 ++++ tests/audio_exec_test.cpp | 72 +++++++++++ tests/audio_npu_gemm_test.cpp | 176 +++++++++++++++++++++++++++ tests/audio_ops_test.cpp | 31 ++++- tools/whisper_npu_device.h | 167 +++++++++++++++++++++++++ tools/whisper_transcribe.cpp | 37 +++++- 10 files changed, 811 insertions(+), 11 deletions(-) create mode 100644 npu/forward/include/audio_npu_gemm.h create mode 100644 tests/audio_npu_gemm_test.cpp create mode 100644 tools/whisper_npu_device.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 72b23a0da..8144e013e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1588,7 +1588,7 @@ target_link_libraries(audio_parallel_test PRIVATE onebit_npu Threads::Threads) # ci.yml's build list is not ours to edit without `workflow` scope, so these reach CI the way ops_dump_test does: as build # dependencies of a target it already builds (tools/ci/ci_targets_check.py follows this edge). # One line, deliberately: tools/ci/ci_targets_check.py follows `add_dependencies` line by line. -add_dependencies(onebit audio_ops_test audio_mel_test onnx_graph_test audio_exec_test audio_parallel_test audio_pack_test audio_kernel_roles_test whisper_ids_test audio_device_gemm_test audio_launch_plan_test audio_kernel_shapes whisper_transcribe) +add_dependencies(onebit audio_ops_test audio_mel_test onnx_graph_test audio_exec_test audio_parallel_test audio_pack_test audio_kernel_roles_test whisper_ids_test audio_device_gemm_test audio_npu_gemm_test audio_launch_plan_test audio_kernel_shapes whisper_transcribe) # A DFlash draft package as a model source (engine#1780): config + container -> a checked model whose tensors decode # one at a time. The loader is compiled here and not into onebit_npu because it needs the q4_1 tile decoder, which # lives with the forward (npu/forward/src/q4nx_dequant.c); it moves there with the draft's forward. Synthetic package, @@ -1986,6 +1986,11 @@ target_link_libraries(audio_kernel_shapes PRIVATE onebit_npu) add_executable(audio_device_gemm_test tests/audio_device_gemm_test.cpp) target_include_directories(audio_device_gemm_test PRIVATE npu/forward/include npu/target npu/plan) target_link_libraries(audio_device_gemm_test PRIVATE nlohmann_json::nlohmann_json) +# engine#1783: the encoder's products on an NpuTarget (routing by staged shape, M-row chunks, padded weight sent once, +# de-pad), against a fake target. +add_executable(audio_npu_gemm_test tests/audio_npu_gemm_test.cpp) +target_include_directories(audio_npu_gemm_test PRIVATE npu/forward/include npu/target npu/plan) +target_link_libraries(audio_npu_gemm_test PRIVATE nlohmann_json::nlohmann_json) add_executable(audio_launch_plan_test tests/audio_launch_plan_test.cpp) target_include_directories(audio_launch_plan_test PRIVATE npu/forward/include npu/target npu/plan) target_link_libraries(audio_launch_plan_test PRIVATE nlohmann_json::nlohmann_json) @@ -1993,6 +1998,7 @@ add_test(NAME whisper_ids COMMAND whisper_ids_test) add_test(NAME audio_kernel_roles COMMAND audio_kernel_roles_test) add_test(NAME audio_pack COMMAND audio_pack_test) add_test(NAME audio_device_gemm COMMAND audio_device_gemm_test) +add_test(NAME audio_npu_gemm COMMAND audio_npu_gemm_test) add_test(NAME audio_launch_plan COMMAND audio_launch_plan_test) add_test(NAME whisper_transcribe_jfk COMMAND whisper_transcribe) set_tests_properties(whisper_transcribe_jfk PROPERTIES @@ -2206,6 +2212,11 @@ endif() # The library body is onebit_npu_forward_library() (defined above the runtime options), so the CI build of the # same sources against the runtime stub (engine#1946) cannot drift from this one. onebit_npu_forward_library(onebit_npu_forward onebit_npu_rt) + # engine#1783: whisper_transcribe with the encoder's products on the NPU (ONEBIT_WHISPER_NPU_KERNELS=). + add_executable(whisper_transcribe_npu tools/whisper_transcribe.cpp) + target_compile_definitions(whisper_transcribe_npu PRIVATE ONEBIT_WHISPER_NPU) + target_include_directories(whisper_transcribe_npu PRIVATE npu/forward/include npu/target tools) + target_link_libraries(whisper_transcribe_npu PRIVATE onebit_npu_forward onebit_npu_rt) target_sources(onebit PRIVATE app/forward_serve.cpp app/stb_image_impl.cpp) # engine#2324: stb_image for pictures target_include_directories(onebit PRIVATE ${stb_SOURCE_DIR}) # engine#1781: forward-serve --spec-draft runs a DFlash draft package (npu/dflash_head.h) as its drafter. @@ -2330,6 +2341,12 @@ if(ONEBIT_NPU_HOST AND NOT ONEBIT_NPU AND NOT WIN32 AND Python3_Interpreter_FOUN target_link_libraries(npu_gemm_shape_guard_test PRIVATE onebit_npu_forward_ci) add_test(NAME npu_gemm_shape_guard COMMAND npu_gemm_shape_guard_test) add_dependencies(onebit npu_gemm_shape_guard_test) # CI builds `onebit` only; without this the test is Not Run + # engine#1783: the device build of whisper_transcribe, compiled here against the stub so CI sees its sources. + add_executable(whisper_transcribe_npu_ci tools/whisper_transcribe.cpp) + target_compile_definitions(whisper_transcribe_npu_ci PRIVATE ONEBIT_WHISPER_NPU) + target_include_directories(whisper_transcribe_npu_ci PRIVATE npu/forward/include npu/target tools) + target_link_libraries(whisper_transcribe_npu_ci PRIVATE onebit_npu_forward_ci) + add_dependencies(onebit whisper_transcribe_npu_ci) add_executable(batch_check_state_test tests/batch_check_state_test.cpp) target_link_libraries(batch_check_state_test PRIVATE onebit_npu_forward_ci) # direction B (tools/ci/ci_targets_check.py): ci.yml's --target list is a workflow file no lane can patch; the diff --git a/npu/audio_exec.cpp b/npu/audio_exec.cpp index a0d35168a..98aafa6e1 100644 --- a/npu/audio_exec.cpp +++ b/npu/audio_exec.cpp @@ -196,7 +196,7 @@ void load_init_i64(const OnnxGraphInfo& g, const OnnxInit& t, std::vector t, const std::string& out_name, AudioRunResult* out, - std::string* why) { + std::string* why, AudioGemm* dev = nullptr) { try { std::map inits; @@ -229,6 +229,37 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map t, con const char* ot = std::getenv("ONEBIT_AUDIO_OP_TIMES"); const bool op_times = ot != nullptr && ot[0] != '\0' && ot[0] != '0'; std::map> op_ms; // op -> (ms, calls) + + // engine#1783: offer one product to the device. True when the device ran it into `c`; false when the host must, with + // the reason counted. A device without a `gemm` is no device. + auto offer = [&](const std::string& key, const float* A, int64_t M, int64_t K, const float* B, int64_t N, + std::vector* c) -> bool { + if (dev == nullptr) return false; + std::string reason; + if (dev->gemm && dev->gemm(key, A, M, K, B, N, c, &reason)) { + if (c->size() != size_t(M) * size_t(N)) + fail("device gemm for '" + key + "' wrote " + std::to_string(c->size()) + " floats, not " + + std::to_string(M) + " x " + std::to_string(N)); + ++dev->device; + return true; + } + ++dev->host; + ++dev->host_why[reason.empty() ? std::string("declined without a reason") : reason]; + return false; + }; + + // Every node ends here, including the products the device ran (they leave the loop early), so ONEBIT_AUDIO_OP_TIMES + // counts a device-run MatMul or Conv under its op like a host one. + auto finish_node = [&](const std::string& op, const std::string& name, Tensor& res, + std::chrono::steady_clock::time_point t0) { + t[name] = std::move(res); + if (op_times) { + auto& e = op_ms[op]; + e.first += std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); + e.second++; + } + }; + int node_index = 0; for (const auto& n : g.nodes) { ++node_index; @@ -251,6 +282,22 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map t, con r.dims = bp.dims; r.dims.push_back(M); r.dims.push_back(N); + // A projection (B a rank-2 weight) is one GEMM over every row of A, whatever A's leading axes are. + if (dev != nullptr) { + if (rb == 2 && inits.count(n.ins[1]) != 0) { + std::vector c; + if (offer(n.ins[1], a.data.data(), int64_t(a.data.size() / size_t(K)), K, b.data.data(), N, &c)) { + r.dims = a.dims; + r.dims.back() = N; + r.data = std::move(c); + finish_node(n.op, oname, r, op_t0); + continue; + } + } else { + ++dev->host; + ++dev->host_why["MatMul with an activation as B (attention product): not offered"]; + } + } r.data.assign(nelem(r.dims), 0.0f); // Rows are independent, so the work splits over (batch, row) without touching any element's arithmetic. const size_t batches = nelem(bp.dims); @@ -392,6 +439,35 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map t, con const int64_t tout = audio::conv1d_out_len(int(T), int(k), stride, pad); r.dims = {a.dims[0], cout, tout}; r.data.resize(size_t(a.dims[0]) * size_t(cout) * size_t(tout)); + const auto ga = n.attrs.find("group"); + if (ga != n.attrs.end() && ga->second != 1) fail(where() + ": only group=1 is implemented"); + if (dev != nullptr) { + // The im2col GEMM: (tout, cin*k) x (cin*k, cout), then out[co][t] = bias[co] + C[t][co]. + const int64_t kk = cin * k; + std::vector bkn(size_t(kk) * size_t(cout)); + audio::conv1d_weight_as_kn(w.data.data(), int(cout), int(cin), int(k), bkn.data()); + std::vector acol(size_t(tout) * size_t(kk)); + bool all = true; + for (int64_t n0 = 0; n0 < a.dims[0] && all; ++n0) { + audio::conv1d_im2col(a.data.data() + size_t(n0) * size_t(cin) * size_t(T), int(cin), int(T), int(k), + stride, pad, acol.data()); + std::vector c; + if (!offer(n.ins[1] + "#conv", acol.data(), tout, kk, bkn.data(), cout, &c)) { + all = false; + break; + } + float* o = r.data.data() + size_t(n0) * size_t(cout) * size_t(tout); + for (int64_t co = 0; co < cout; ++co) { + const float bv = bias != nullptr ? bias->data[size_t(co)] : 0.0f; + for (int64_t tt = 0; tt < tout; ++tt) + o[size_t(co) * size_t(tout) + size_t(tt)] = bv + c[size_t(tt) * size_t(cout) + size_t(co)]; + } + } + if (all) { + finish_node(n.op, oname, r, op_t0); + continue; + } + } // engine#1783: one output channel per unit. conv1d with cout = 1 on that channel's weights, bias and output // row runs exactly the loop the full call runs for it, so the result is bit-identical; the mel stem's two // Convs were a serial 3.2 G multiply-adds per 30 s window. @@ -545,12 +621,7 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map t, con } else { fail(where() + " is not implemented; the pack needs an op this executor does not have"); } - t[oname] = std::move(r); - if (op_times) { - auto& e = op_ms[n.op]; - e.first += std::chrono::duration(std::chrono::steady_clock::now() - op_t0).count(); - e.second++; - } + finish_node(n.op, oname, r, op_t0); } if (op_times) { std::vector> order; @@ -580,6 +651,11 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map t, con bool run_audio_graph(const OnnxGraphInfo& g, const float* mel, int64_t mel_bins, int64_t frames, const std::string& out_name, AudioRunResult* out, std::string* why) { + return run_audio_graph(g, mel, mel_bins, frames, out_name, out, why, nullptr); +} + +bool run_audio_graph(const OnnxGraphInfo& g, const float* mel, int64_t mel_bins, int64_t frames, const std::string& out_name, + AudioRunResult* out, std::string* why, AudioGemm* dev) { std::map seed; Tensor in; in.dims = {1, mel_bins, frames}; @@ -589,7 +665,7 @@ bool run_audio_graph(const OnnxGraphInfo& g, const float* mel, int64_t mel_bins, return false; } seed[g.inputs[0]] = std::move(in); - return run_graph_impl(g, std::move(seed), out_name, out, why); + return run_graph_impl(g, std::move(seed), out_name, out, why, dev); } bool run_decoder_graph(const OnnxGraphInfo& g, const int64_t* tokens, int64_t n_tokens, const float* xa, diff --git a/npu/audio_exec.h b/npu/audio_exec.h index df7e099ce..d1789b38e 100644 --- a/npu/audio_exec.h +++ b/npu/audio_exec.h @@ -23,6 +23,8 @@ #pragma once +#include +#include #include #include @@ -30,6 +32,27 @@ namespace onebit::npu::pack_import { +// A device for the graph's matrix products (engine#1783). The executor stays host-only and device-free: it offers each +// product whose right operand is a CONSTANT weight to `gemm`, and runs it on the host itself when the device declines. +// +// Offered: every MatMul whose B is a rank-2 initializer (the projections q, k, v, o, fc1, fc2), with A's leading axes +// flattened into rows, and every Conv, as its im2col GEMM (A = (tout, cin*k), B = the kernel as (cin*k, cout)). +// Not offered: a MatMul whose B is an activation (attention's q.k^T and scores.v). Its operand changes every call, so it +// has no weight to stay resident, and the reason is recorded like any other host product. +// +// `key` names the weight and is stable across calls (the initializer's name; "#conv" for a Conv's transposed +// kernel), so an implementation may cache the packed weight by it. Return false with `why` to leave the product to +// the host. The executor then runs it, counts it under `host`, and keeps the reason in `host_why`, so a run whose +// products silently fell back to the host is visible in the counts. +struct AudioGemm { + std::function* C, std::string* why)> + gemm; + int device = 0; // products the device ran + int host = 0; // products the host ran (declined, or not offered) + std::map host_why; // reason -> count +}; + struct AudioRunResult { std::vector dims; // dims of the requested output std::vector data; // row-major f32 payload @@ -45,6 +68,9 @@ struct AudioRunResult { // Returns true and fills `out`; on failure returns false and sets `why` to the node and the reason. bool run_audio_graph(const OnnxGraphInfo& g, const float* mel, int64_t mel_bins, int64_t frames, const std::string& out_name, AudioRunResult* out, std::string* why); +// The same, with the matrix products offered to `dev` (null = host only, identical to the overload above). +bool run_audio_graph(const OnnxGraphInfo& g, const float* mel, int64_t mel_bins, int64_t frames, + const std::string& out_name, AudioRunResult* out, std::string* why, AudioGemm* dev); // The decoder: `tokens` is the int64 token window for the graph's first input; `xa` (the encoder output, row-major f32) is // bound to the graph's second input when the graph has one. `xa_dims` may be empty when there is no cross-attention input. diff --git a/npu/forward/include/audio_npu_gemm.h b/npu/forward/include/audio_npu_gemm.h new file mode 100644 index 000000000..e332e7e0e --- /dev/null +++ b/npu/forward/include/audio_npu_gemm.h @@ -0,0 +1,175 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// audio_npu_gemm.h -- the audio executor's matrix products on an NpuTarget (engine#1783, the encoder route). +// +// The executor (npu/audio_exec.h, AudioGemm) offers every product whose right operand is a weight: the projections and the +// two Conv stems as im2col GEMMs. This file is the device side of that offer, and it is device-free: it speaks to +// `NpuTarget::gemv`, so it is tested against a fake target (tests/audio_npu_gemm_test.cpp). +// +// What it decides, per product: +// * WHICH design. The caller passes the designs in the order it gave them to the target's open_prefill(); the target's +// role index IS that order (xdna_target.cpp: make_ctx(i)). A product goes to the design whose (K, N) is its STAGED +// shape -- K on the 256 grid, N on the 512 grid (audio_kernel_roles.h) -- so whisper-small's O (768 x 768) asks for +// K768 N1024, conv1 (240 x 768) for K256 N1024, conv2 (2304 x 768) for K2304 N1024. A shape no design covers is +// declined by name and runs on the host. Routing every launch to role 0 is the defect that crashed the first +// standalone harness (engine#1783, 08:48Z), so the index is never a default. +// * HOW MANY launches. One launch holds at most the design's M rows; more rows are cut into M-row chunks (conv1's 3000 +// frames at M=1536 are two launches). +// * WHAT the weight is. The target packs `Bmat` as K_staged x N_staged f32, so the model's K x N weight is zero-padded +// here once, under a stable residency key (layer 0, one wkey per weight). After the first launch the target serves +// the packed weight from its cache. The padded host copy is dropped then, unless the caller says the target does +// not cache. +// * WHAT comes back. The target writes rows x N_staged; the executor's tensor is rows x N, so the padding columns are +// dropped per row. +// +// It also keeps the device column of the issue's bar: the wall time inside gemv, the launches and the products, so the +// tool prints `device_gemm= s + host_remainder= s` from a run rather than from arithmetic. + +#pragma once + +#include +#include +#include +#include +#include + +#include "audio_kernel_roles.h" +#include "target.h" + +namespace onebit::audio { + +struct NpuAudioDesign { + int M = 0; // rows per launch (the ELF's MD) + int K = 0; // staged K + int N = 0; // staged N +}; + +class NpuAudioGemm { +public: + // `designs[i]` is role index i of `target` (the open_prefill() order). `target_caches` = the target was given a weight + // cache with at least one layer (set_device(..., weight_cache=true, n_layers>=1)); without it the padded weights stay + // on the host and are re-sent every launch. + NpuAudioGemm(::npu::target::NpuTarget& target, std::vector designs, bool target_caches = true) + : t_(target), designs_(std::move(designs)), caches_(target_caches) {} + + // The AudioGemm::gemm contract: C (M x N) = A (M x K) x B (K x N); false + why to leave it to the host. + bool gemm(const std::string& key, const float* A, int64_t M, int64_t K, const float* B, int64_t N, + std::vector* C, std::string* why) { + if (A == nullptr || B == nullptr || C == nullptr || M <= 0 || K <= 0 || N <= 0) { + if (why) *why = "empty product"; + return false; + } + const ProjectionShape st = staged(ProjectionShape{int(K), int(N)}); + const int role = role_for(st.k, st.n); + if (role < 0) { + if (why) + *why = "no design for staged K" + std::to_string(st.k) + " N" + std::to_string(st.n) + " (model " + + std::to_string(K) + " x " + std::to_string(N) + ")"; + return false; + } + const NpuAudioDesign& d = designs_[size_t(role)]; + if (d.M <= 0) { + if (why) *why = "design " + std::to_string(role) + " states no rows"; + return false; + } + + Weight& w = weight(key, B, K, N, st); + const auto t0 = std::chrono::steady_clock::now(); + C->assign(size_t(M) * size_t(N), 0.0f); + std::vector got; + for (int64_t r0 = 0; r0 < M; r0 += d.M) { + const int rows = int(std::min(d.M, M - r0)); + ::npu::target::GemvArgs a; + a.role = role; + a.layer = 0; + a.M = rows; + a.K = st.k; + a.N = st.n; + a.x = A + size_t(r0) * size_t(K); + a.a_k = int(K); + a.Bmat = w.padded.empty() ? nullptr : w.padded.data(); + a.wkey = caches_ ? w.wkey : -1; + a.rows = true; // the batched body, whatever the chunk's row count (target.h: M==1 with rows=true is batched) + a.tag = "audio"; + got.clear(); + std::string err; + if (!t_.gemv(a, got, err)) { + if (why) *why = "role " + std::to_string(role) + " (" + key + "): " + (err.empty() ? "gemv failed" : err); + return false; + } + if (got.size() != size_t(rows) * size_t(st.n)) { + if (why) + *why = "role " + std::to_string(role) + " wrote " + std::to_string(got.size()) + " floats for " + + std::to_string(rows) + " x " + std::to_string(st.n); + return false; + } + for (int r = 0; r < rows; ++r) + std::copy(got.begin() + size_t(r) * size_t(st.n), got.begin() + size_t(r) * size_t(st.n) + size_t(N), + C->begin() + size_t(r0 + r) * size_t(N)); + ++launches_; + ++per_role_[role]; + // The target now holds the packed weight under (layer 0, wkey): later chunks and later calls send none. + if (caches_ && !w.padded.empty()) w.padded = std::vector(); + } + device_ms_ += std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); + ++products_; + return true; + } + + double device_ms() const { return device_ms_; } + int launches() const { return launches_; } + int products() const { return products_; } + const std::map& launches_per_role() const { return per_role_; } + int weights() const { return int(w_.size()); } + +private: + struct Weight { + int wkey = -1; + std::vector padded; // K_staged x N_staged, until the target has it + }; + + int role_for(int k, int n) const { + for (size_t i = 0; i < designs_.size(); ++i) + if (designs_[i].K == k && designs_[i].N == n) return int(i); + return -1; + } + + Weight& weight(const std::string& key, const float* B, int64_t K, int64_t N, const ProjectionShape& st) { + // Padded once per key: either the copy is still here (not yet in the target, or the target does not cache), or + // the target holds it and the copy was dropped after its first launch. + auto it = w_.find(key); + if (it != w_.end()) return it->second; + Weight w; + w.wkey = next_wkey_++; + w.padded.assign(size_t(st.k) * size_t(st.n), 0.0f); + for (int64_t k = 0; k < K; ++k) + std::copy(B + size_t(k) * size_t(N), B + size_t(k) * size_t(N) + size_t(N), + w.padded.begin() + size_t(k) * size_t(st.n)); + return w_.emplace(key, std::move(w)).first->second; + } + + ::npu::target::NpuTarget& t_; + std::vector designs_; + bool caches_ = true; + std::map w_; + int next_wkey_ = 0; + double device_ms_ = 0.0; + int launches_ = 0; + int products_ = 0; + std::map per_role_; +}; + +} // namespace onebit::audio diff --git a/npu/forward/include/audio_ops.h b/npu/forward/include/audio_ops.h index 8fd3dfef6..d4f6229e8 100644 --- a/npu/forward/include/audio_ops.h +++ b/npu/forward/include/audio_ops.h @@ -56,6 +56,33 @@ inline void conv1d(const float* x, int cin, int T, const float* w, const float* } } +// The same convolution as a GEMM (engine#1783: so the encoder's two Conv stems can run on the NPU's GEMM family). +// A = im2col(x): (tout, cin*k), A[t][ci*k + j] = x[ci][t*stride + j - pad], 0 outside [0, T). +// B = w as (cin*k, cout): B[ci*k + j][co] = w[co][ci][j]. +// out[co][t] = b[co] + (A x B)[t][co] +// conv1d() above is the reference this is tested against (tests/audio_ops_test.cpp); the column order ci*k + j is the +// initializer's own (cout, cin, k) flattening, so B is a plain transpose of w viewed as (cout, cin*k). +inline void conv1d_im2col(const float* x, int cin, int T, int k, int stride, int pad, float* A) { + const int tout = conv1d_out_len(T, k, stride, pad); + const std::size_t kk = static_cast(cin) * static_cast(k); + for (int t = 0; t < tout; ++t) { + float* row = A + static_cast(t) * kk; + for (int ci = 0; ci < cin; ++ci) { + const float* xr = x + static_cast(ci) * T; + for (int j = 0; j < k; ++j) { + const int src = t * stride + j - pad; + row[static_cast(ci) * k + j] = (src >= 0 && src < T) ? xr[src] : 0.0f; + } + } + } +} + +inline void conv1d_weight_as_kn(const float* w, int cout, int cin, int k, float* B) { + const std::size_t kk = static_cast(cin) * static_cast(k); + for (int co = 0; co < cout; ++co) + for (std::size_t r = 0; r < kk; ++r) B[r * static_cast(cout) + co] = w[static_cast(co) * kk + r]; +} + // y = (x - mean) / sqrt(var + eps) * gamma + beta, biased variance (divide by n), one row. inline void layer_norm(const float* x, const float* gamma, const float* beta, int n, float eps, float* out) { double sum = 0.0; diff --git a/tests/audio_exec_test.cpp b/tests/audio_exec_test.cpp index 6d12cd7d1..516171b15 100644 --- a/tests/audio_exec_test.cpp +++ b/tests/audio_exec_test.cpp @@ -19,6 +19,7 @@ // The mel is mel[b,t] = sin(0.013*(b+1)*(t+1))*0.5 + 0.25 for 80 bins and 3000 frames, computed in double on both sides. // Expected values below are onnxruntime's, on whisper-small's encoder_model.onnx. Skips (77) without the pack. +#include #include #include #include @@ -26,6 +27,7 @@ #include #include "audio_exec.h" +#include "audio_parallel.h" #include "audio_pack.h" #include "onnx_graph.h" @@ -144,6 +146,76 @@ int main(int argc, char** argv) { } std::printf("audio exec: whole encoder on the host matches onnxruntime\n"); + // ---- engine#1783: the same encoder with its matrix products offered to a device ----------------------------------- + // The "device" here is an exact f32 GEMM on the host, so this checks the OFFER, not a kernel: that every projection + // and both Conv stems leave the executor as (A, B) GEMMs, that the result comes back in the right layout, and that + // only the attention products stay on the host. whisper-small: 12 layers x (q, k, v, o, fc1, fc2) + 2 convs = 74 + // offered, 12 x 2 attention products not offered. + { + AudioGemm dev; + dev.gemm = [](const std::string&, const float* A, int64_t M, int64_t K, const float* B, int64_t N, + std::vector* C, std::string*) { + C->assign(size_t(M) * size_t(N), 0.0f); + parallel_units(size_t(M), [&](size_t i) { + float* c = C->data() + i * size_t(N); + for (int64_t k = 0; k < K; ++k) { + const float a = A[i * size_t(K) + size_t(k)]; + const float* b = B + size_t(k) * size_t(N); + for (int64_t j = 0; j < N; ++j) c[j] += a * b[j]; + } + }); + return true; + }; + AudioRunResult rd; + if (!run_audio_graph(g, mel.data(), bins, frames, "", &rd, &why, &dev)) { + std::printf("FAIL run with a device: %s\n", why.c_str()); + return 1; + } + check(dev.device == 74, "74 products offered to the device (72 projections + 2 convs), got " + std::to_string(dev.device)); + check(dev.host == 24, "24 attention products stay on the host, got " + std::to_string(dev.host)); + check(rd.dims == r.dims, "the device run has the host run's dims"); + double maxd = 0.0; + for (size_t i = 0; i < r.data.size() && i < rd.data.size(); ++i) + maxd = std::max(maxd, std::fabs(double(r.data[i]) - double(rd.data[i]))); + std::printf("device-offer run: %d device, %d host, max |host - offered| %.3g\n", dev.device, dev.host, maxd); + // Not bit-identical: the im2col Conv adds its bias after the taps, conv1d() before them, and 12 layers carry that + // rounding to ~1.6e-3 at the output (std 1.68). So the offered run is held to the same onnxruntime values as the + // host run, plus a bound on its distance from the host run. + check(maxd < 5e-3, "the offered run is within 5e-3 of the host run"); + if (rd.data.size() == r.data.size()) { + double m2 = 0.0, v2 = 0.0; + for (float v : rd.data) m2 += v; + m2 /= double(rd.data.size()); + for (float v : rd.data) v2 += (double(v) - m2) * (double(v) - m2); + v2 /= double(rd.data.size()); + expect_close(float(m2), -0.009045244, 2e-4, "offered run: output mean"); + expect_close(float(std::sqrt(v2)), 1.678493023, 2e-4, "offered run: output std"); + auto at2 = [&](int64_t tt, int64_t h) { return rd.data[size_t(tt) * H + size_t(h)]; }; + for (int i = 0; i < 5; ++i) expect_close(at2(0, i), p0[i], 2e-3, "offered run: frame 0 col " + std::to_string(i)); + for (int i = 0; i < 4; ++i) expect_close(at2(750, i), p1[i], 2e-3, "offered run: frame 750 col " + std::to_string(i)); + for (int i = 0; i < 4; ++i) expect_close(at2(1499, i), p2[i], 2e-3, "offered run: frame 1499 col " + std::to_string(i)); + } + // A device that declines everything is the host run, counted as host. + AudioGemm none; + none.gemm = [](const std::string&, const float*, int64_t, int64_t, const float*, int64_t, std::vector*, + std::string* w) { + *w = "test: declined"; + return false; + }; + AudioRunResult rn; + if (!run_audio_graph(g, mel.data(), bins, frames, "", &rn, &why, &none)) { + std::printf("FAIL run with a declining device: %s\n", why.c_str()); + return 1; + } + check(none.device == 0 && none.host == 98 && none.host_why["test: declined"] == 74, + "a declining device runs all 98 products on the host and names the 74 refusals"); + check(rn.data == r.data, "a declining device is bit-identical to the host run"); + if (g_failures != 0) { + std::printf("audio exec: %d failures\n", g_failures); + return 1; + } + } + // ---- the decoder, on the same pack: 128-position token window x, the encoder output xa ---------------------------- const char* dpath = (dpath_arg != nullptr) ? dpath_arg : std::getenv("ONEBIT_TEST_WHISPER_DECODER"); if (dpath == nullptr || *dpath == '\0') { diff --git a/tests/audio_npu_gemm_test.cpp b/tests/audio_npu_gemm_test.cpp new file mode 100644 index 000000000..bc63814ea --- /dev/null +++ b/tests/audio_npu_gemm_test.cpp @@ -0,0 +1,176 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// tests/audio_npu_gemm_test.cpp -- the encoder's products on an NpuTarget (npu/forward/include/audio_npu_gemm.h, engine#1783), +// against a FAKE target that does the GEMM exactly from the staged Bmat and caches it by (layer, wkey) like XdnaTarget. +// +// Pinned: the role index is the design's position (never a default 0), each launch is at most the design's M rows, +// the weight is zero-padded to the staged K x N and sent once, the result is de-padded to the model's N, and an +// uncovered shape is declined by name. +#include +#include +#include +#include +#include + +#include "audio_npu_gemm.h" + +using namespace onebit::audio; +namespace t = ::npu::target; + +static int failures = 0; +static void check(bool ok, const std::string& what) { + std::printf("%s %s\n", ok ? "ok " : "FAIL", what.c_str()); + if (!ok) ++failures; +} + +struct FakeTarget : t::NpuTarget { + std::vector d; // what open_prefill adopted, by role index + std::map> cache; // wkey -> staged K x N weight + std::vector seen; + int with_bmat = 0; + const char* name() const override { return "fake"; } + bool open(const onebit::npu::plan::Plan&, const std::string&, const char*, std::string&) override { return true; } + bool load_block(int, int, std::string&) override { return true; } + t::Caps caps() const override { return t::Caps{}; } + bool gemv(const t::GemvArgs& a, std::vector& out, std::string& err) override { + seen.push_back(a); + if (a.role < 0 || a.role >= int(d.size())) { err = "no context for role"; return false; } + const NpuAudioDesign& g = d[size_t(a.role)]; + if (a.M > g.M || a.K != g.K || a.N != g.N) { err = "launch does not fit the role's context"; return false; } + const std::vector* w = nullptr; + if (a.Bmat != nullptr) { + ++with_bmat; + std::vector b(a.Bmat, a.Bmat + size_t(a.K) * size_t(a.N)); + if (a.wkey >= 0) w = &(cache[a.wkey] = std::move(b)); + else w = &(cache[-1000 - int(seen.size())] = std::move(b)); + } else { + const auto it = cache.find(a.wkey); + if (it == cache.end()) { err = "no Bmat and nothing cached"; return false; } + w = &it->second; + } + out.assign(size_t(a.M) * size_t(a.N), 0.0f); + for (int m = 0; m < a.M; ++m) + for (int k = 0; k < a.K; ++k) { + const float x = k < a.a_k ? a.x[size_t(m) * size_t(a.a_k) + size_t(k)] : 0.0f; + for (int n = 0; n < a.N; ++n) out[size_t(m) * size_t(a.N) + size_t(n)] += x * (*w)[size_t(k) * size_t(a.N) + size_t(n)]; + } + return true; + } +}; + +static void host_gemm(const std::vector& A, const std::vector& B, int M, int K, int N, std::vector* C) { + C->assign(size_t(M) * size_t(N), 0.0f); + for (int m = 0; m < M; ++m) + for (int k = 0; k < K; ++k) + for (int n = 0; n < N; ++n) (*C)[size_t(m) * N + n] += A[size_t(m) * K + k] * B[size_t(k) * N + n]; +} + +static double maxdiff(const std::vector& a, const std::vector& b) { + if (a.size() != b.size()) return 1e30; + double m = 0.0; + for (size_t i = 0; i < a.size(); ++i) m = std::fmax(m, std::fabs(double(a[i]) - double(b[i]))); + return m; +} + +int main() { + FakeTarget ft; + // Small stand-ins on the real grid: role 0 = "O" K256 N512, role 1 = "conv" K256 N1024, role 2 = "D" K512 N512. + ft.d = {{8, 256, 512}, {8, 256, 1024}, {8, 512, 512}}; + NpuAudioGemm g(ft, ft.d); + + auto fill = [](int n, float s) { + std::vector v(static_cast(n)); + for (int i = 0; i < n; ++i) v[size_t(i)] = std::sin(0.37f * float(i + 1) * s) * 0.5f; + return v; + }; + + // 1. a model-width projection (200 x 300 -> staged K256 N512) on role 0, 19 rows -> 3 launches (8, 8, 3) + { + const int M = 19, K = 200, N = 300; + const auto A = fill(M * K, 1.0f), B = fill(K * N, 0.7f); + std::vector C, R; + std::string why; + check(g.gemm("w.o", A.data(), M, K, B.data(), N, &C, &why), "a padded projection runs: " + why); + host_gemm(A, B, M, K, N, &R); + check(maxdiff(C, R) < 1e-4, "it equals the host GEMM after de-padding (max " + std::to_string(maxdiff(C, R)) + ")"); + check(ft.seen.size() == 3 && ft.seen[0].M == 8 && ft.seen[1].M == 8 && ft.seen[2].M == 3, + "19 rows at M=8 are three launches of 8, 8 and 3"); + check(ft.seen[0].role == 0 && ft.seen[0].K == 256 && ft.seen[0].N == 512 && ft.seen[0].a_k == 200, + "role 0 is asked for the staged K256 N512, with the activation's real width 200"); + check(ft.seen[0].rows && ft.seen[2].rows, "every chunk is the batched body"); + check(ft.with_bmat == 1, "the padded weight is sent once and the next launches use the cache"); + } + + // 2. a second shape routes by its staged (K, N), not by order of first use or a default + { + ft.seen.clear(); + const int M = 5, K = 240, N = 768; // conv1 in miniature: K240 -> K256, N768 -> N1024 + const auto A = fill(M * K, 1.3f), B = fill(K * N, 0.4f); + std::vector C, R; + std::string why; + check(g.gemm("conv1#conv", A.data(), M, K, B.data(), N, &C, &why), "a conv-stem shape runs: " + why); + host_gemm(A, B, M, K, N, &R); + check(ft.seen.size() == 1 && ft.seen[0].role == 1 && ft.seen[0].N == 1024, "K240 N768 goes to role 1 (K256 N1024)"); + check(maxdiff(C, R) < 1e-4, "and equals the host GEMM"); + check(ft.seen[0].wkey != 0, "a second weight gets its own residency key"); + } + + // 3. the same key a second time sends no weight + { + ft.seen.clear(); + const int before = ft.with_bmat; + const int M = 4, K = 200, N = 300; + const auto A = fill(M * K, 2.1f), B = fill(K * N, 0.7f); + std::vector C, R; + std::string why; + check(g.gemm("w.o", A.data(), M, K, B.data(), N, &C, &why), "a cached weight runs again"); + host_gemm(A, B, M, K, N, &R); + check(ft.with_bmat == before && ft.seen[0].Bmat == nullptr, "the second call sends no Bmat (the target holds it)"); + check(maxdiff(C, R) < 1e-4, "and still equals the host GEMM"); + } + + // 4. a shape no design covers is declined by name, and the target is not called + { + ft.seen.clear(); + std::vector C; + std::string why; + // K1000 stages to K1024: no design + const auto A2 = fill(1000, 1.0f), B2 = fill(1000 * 10, 1.0f); + check(!g.gemm("odd", A2.data(), 1, 1000, B2.data(), 10, &C, &why) && + why.find("no design for staged K1024 N512") != std::string::npos, + "an uncovered staged shape is declined by name: " + why); + check(ft.seen.empty(), "and the target is not called"); + } + + // 5. a target failure is passed through with the role and key + { + ft.d[2].M = 0; // the fake now refuses role 2 + NpuAudioGemm g2(ft, {{8, 256, 512}, {8, 256, 1024}, {8, 512, 512}}); + const auto A = fill(2 * 512, 1.0f), B = fill(512 * 512, 1.0f); + std::vector C; + std::string why; + check(!g2.gemm("w.d", A.data(), 2, 512, B.data(), 512, &C, &why) && why.find("role 2 (w.d)") != std::string::npos, + "a refused launch names its role and weight: " + why); + } + + check(g.launches() == 5 && g.products() == 3, "counts: 5 launches, 3 products"); + if (failures) { + std::printf("audio_npu_gemm: %d failure(s)\n", failures); + return 1; + } + std::printf("audio_npu_gemm: all pass\n"); + return 0; +} diff --git a/tests/audio_ops_test.cpp b/tests/audio_ops_test.cpp index 9107a8c53..cc915c6d5 100644 --- a/tests/audio_ops_test.cpp +++ b/tests/audio_ops_test.cpp @@ -78,6 +78,34 @@ void conv1d_arms() { expect_eq(onebit::audio::conv1d_out_len(5, 3, 1, 0), 3, "conv1d_out_len T5 k3 s1 p0"); } +// engine#1783: the Conv as a GEMM (what the device runs) must be the Conv. Same strided, padded case as above, so a +// wrong column order (j*cin + ci instead of ci*k + j) or a missed pad tap shows up as a value, not just a shape. +void conv1d_gemm_arm() { + const std::vector x = {0.5f, -1.0f, 0.25f, 2.0f, -0.75f, 1.5f, 0.0f, -0.5f, 1.0f, 0.25f}; + std::vector w(3 * 2 * 3); + for (int i = 0; i < 18; ++i) w[i] = static_cast((i + 1) * 0.0625); + const std::vector b = {0.125f, -0.25f, 0.5f}; + const int cin = 2, T = 5, cout = 3, k = 3, stride = 2, pad = 1; + const int tout = onebit::audio::conv1d_out_len(T, k, stride, pad); + std::vector ref(size_t(cout) * tout); + onebit::audio::conv1d(x.data(), cin, T, w.data(), b.data(), cout, k, stride, pad, ref.data()); + + std::vector A(size_t(tout) * cin * k), B(size_t(cin) * k * cout); + onebit::audio::conv1d_im2col(x.data(), cin, T, k, stride, pad, A.data()); + onebit::audio::conv1d_weight_as_kn(w.data(), cout, cin, k, B.data()); + expect_close(A[0], 0.0, 0.0, "im2col t0 tap 0 is the left pad"); + expect_close(A[1], 0.5, 0.0, "im2col t0 tap 1 is x[0][0]"); + expect_close(A[3], 0.0, 0.0, "im2col t0 channel 1 tap 0 is the left pad"); + expect_close(B[1 * cout + 2], w[2 * 6 + 1], 0.0, "B[ci0,j1][co2] = w[co2][ci0][j1]"); + for (int co = 0; co < cout; ++co) + for (int t = 0; t < tout; ++t) { + double acc = b[size_t(co)]; + for (int r = 0; r < cin * k; ++r) acc += double(A[size_t(t) * cin * k + r]) * B[size_t(r) * cout + co]; + expect_close(float(acc), ref[size_t(co) * tout + t], 1e-6, + "im2col GEMM = conv1d at co " + std::to_string(co) + " t " + std::to_string(t)); + } +} + // 2. Layer norm: biased variance, eps added under the root, gamma/beta applied after the normalise. void layernorm_arm() { const std::vector v = {0.25f, -1.5f, 2.0f, 0.75f}; @@ -136,6 +164,7 @@ void softmax_arm() { int main() { conv1d_arms(); + conv1d_gemm_arm(); layernorm_arm(); gelu_arm(); softmax_arm(); @@ -143,7 +172,7 @@ int main() { std::printf("audio ops: %d failures\n", g_failures); return 1; } - std::printf("audio ops: conv1d (strided + padded + edge), layer_norm (biased var, null affine), gelu (erf), softmax " + std::printf("audio ops: conv1d (strided + padded + edge), conv1d as im2col GEMM, layer_norm (biased var, null affine), gelu (erf), softmax " "(stable, shift-invariant) -- all pass\n"); return 0; } diff --git a/tools/whisper_npu_device.h b/tools/whisper_npu_device.h new file mode 100644 index 000000000..d2a88a14e --- /dev/null +++ b/tools/whisper_npu_device.h @@ -0,0 +1,167 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// whisper_npu_device.h -- open the NPU for whisper_transcribe's encoder (engine#1783, the encoder route). +// +// Compiled only into whisper_transcribe_npu (ONEBIT_WHISPER_NPU). It reads the design list from the ENCODER GRAPH, not from +// a hand-written plan: every MatMul whose B is a rank-2 weight, and every Conv as its im2col GEMM (K = cin*k, N = cout), staged on +// the engine's grid (audio_kernel_roles.h). For each distinct staged (K, N) it finds `full_i8__K_N_b8.elf` in the +// kernel directory, whatever the role prefix: cl-npurt's conv stems are D-named (`D K256 N1024`, `D K2304 N1024`), and the name +// is only how the generator files them. It then adopts them through open_prefill() in that order. The order is the role +// index NpuAudioGemm routes by. +// +// A staged shape with no ELF is left out and printed. Its products are declined by NpuAudioGemm and run on the host, +// counted, so a missing kernel shows in the split instead of as a crash or a silent zero. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "audio_exec.h" +#include "audio_kernel_roles.h" +#include "audio_npu_gemm.h" +#include "onnx_graph.h" +#include "q4nx_pack.h" +#include "target/xdna/xdna_target.h" + +namespace whisper_npu { + +namespace x = ::npu::target::xdna; + +struct Device { + std::unique_ptr dev; + std::unique_ptr target; + std::unique_ptr gemm; + onebit::npu::pack_import::AudioGemm hook; + std::vector missing; // staged shapes with no ELF, "K N" +}; + +// The staged (K, N) of every product the executor will offer, in graph order, without repeats. +inline std::vector> offered_shapes(const onebit::npu::pack_import::OnnxGraphInfo& g) { + std::vector> out; + auto add = [&](int64_t k, int64_t n) { + const auto s = onebit::audio::staged(onebit::audio::ProjectionShape{int(k), int(n)}); + for (const auto& p : out) + if (p.first == s.k && p.second == s.n) return; + out.emplace_back(s.k, s.n); + }; + auto init_dims = [&](const std::string& name) -> const std::vector* { + for (const auto& t : g.init_tensors) + if (t.name == name) return &t.dims; + return nullptr; + }; + for (const auto& n : g.nodes) { + if (n.ins.size() < 2) continue; + const std::vector* d = init_dims(n.ins[1]); + if (d == nullptr) continue; + if (n.op == "MatMul" && d->size() == 2) add((*d)[0], (*d)[1]); + if (n.op == "Conv" && d->size() == 3) add((*d)[1] * (*d)[2], (*d)[0]); + } + return out; +} + +// `full_i8__K_N_b8.elf` in `dir`, any role prefix; empty when none. Two matches is a refusal: which one the +// encoder ran would depend on directory order. +inline std::string find_elf(const std::string& dir, int k, int n, std::string* role, std::string* why) { + const std::string tail = "_K" + std::to_string(k) + "_N" + std::to_string(n) + "_b8.elf"; + std::string hit; + DIR* d = opendir(dir.c_str()); + if (d == nullptr) { + *why = "cannot open " + dir; + return {}; + } + while (dirent* e = readdir(d)) { + const std::string f = e->d_name; + if (f.rfind("full_i8_", 0) != 0 || f.size() <= tail.size() || f.compare(f.size() - tail.size(), tail.size(), tail) != 0) + continue; + if (!hit.empty()) { + *why = "two ELFs for K" + std::to_string(k) + " N" + std::to_string(n) + ": " + hit + " and " + f; + closedir(d); + return {}; + } + hit = f; + } + closedir(d); + if (!hit.empty()) *role = hit.substr(8, hit.size() - tail.size() - 8); + return hit; +} + +inline Q4nxDesign design_enum(const std::string& role) { + if (role == "QKV") return Q4NX_DESIGN_QKV; + if (role == "O") return Q4NX_DESIGN_O; + if (role == "GU") return Q4NX_DESIGN_GU; + if (role == "G") return Q4NX_DESIGN_G; + if (role == "U") return Q4NX_DESIGN_U; + return Q4NX_DESIGN_D; // the conv stems are filed as D; the enum only names the log line +} + +// Opens device 0, adopts one design per offered shape that has an ELF (M = `rows`, the ELFs' generated M), and wires the +// executor hook. False + why when the device or an ELF cannot be opened. +inline bool open(const onebit::npu::pack_import::OnnxGraphInfo& enc, const std::string& kdir, int rows, Device* out, + std::string* why) { + std::vector geoms; + std::vector designs; + for (const auto& s : offered_shapes(enc)) { + std::string role, err; + const std::string elf = find_elf(kdir, s.first, s.second, &role, &err); + if (!err.empty()) { + *why = err; + return false; + } + if (elf.empty()) { + out->missing.push_back("K" + std::to_string(s.first) + " N" + std::to_string(s.second)); + continue; + } + Q4nxDesignGeom g{}; + g.M = rows; + g.K = s.first; + g.N = s.second; + g.role = design_enum(role); + std::snprintf(g.elf_name, sizeof g.elf_name, "%s", elf.c_str()); + geoms.push_back(g); + designs.push_back(onebit::audio::NpuAudioDesign{rows, s.first, s.second}); + std::fprintf(stderr, "npu: role %zu = %s (staged K%d N%d, M=%d)\n", geoms.size() - 1, elf.c_str(), s.first, s.second, + rows); + } + if (geoms.empty()) { + *why = "no ELF in " + kdir + " covers any of the encoder's products"; + return false; + } + out->dev = std::make_unique(0); + out->target = std::make_unique(); + // One layer of weight cache: NpuAudioGemm keys every weight at layer 0 with its own wkey. + out->target->set_device(out->dev.get(), /*weight_layout_b8=*/true, /*weight_cache=*/true, /*n_layers=*/1); + x::XdnaTarget::Setup setup; + setup.kernels_dir = kdir; + setup.b8 = true; + out->target->configure(setup); + std::string err; + if (!out->target->open_prefill(geoms.data(), int(geoms.size()), err)) { + *why = "open_prefill: " + err; + return false; + } + out->gemm = std::make_unique(*out->target, designs, /*target_caches=*/true); + onebit::audio::NpuAudioGemm* gm = out->gemm.get(); + out->hook.gemm = [gm](const std::string& key, const float* A, int64_t M, int64_t K, const float* B, int64_t N, + std::vector* C, std::string* w) { return gm->gemm(key, A, M, K, B, N, C, w); }; + return true; +} + +} // namespace whisper_npu diff --git a/tools/whisper_transcribe.cpp b/tools/whisper_transcribe.cpp index 1657023ee..e7ed39a72 100644 --- a/tools/whisper_transcribe.cpp +++ b/tools/whisper_transcribe.cpp @@ -25,6 +25,11 @@ // (see PR #1931); `ctest` would time out for a reason that has nothing to do with correctness. The numbers it prints are // the ones a test would assert, so `expected-ids-csv` turns it into a check when run by hand. // +// whisper_transcribe_npu is this file built with ONEBIT_WHISPER_NPU (engine#1783, the encoder route). With +// ONEBIT_WHISPER_NPU_KERNELS= in the environment, the encoder's projections and Conv stems run on the NPU through +// NpuTarget::gemv, and the tool prints `device_gemm= s + host_remainder= s` for the encoder pass. Without the variable +// the binary is the host tool. The kernel ELFs are generated at one M; ONEBIT_WHISPER_NPU_ROWS names it (default 1536). +// // The pack on disk carries no tokenizer, so this stops at ids: `400 370 452 ...` is the sentence, and comparing ids is a // stricter check than comparing text (two ids can share a detokenisation, and neither can share an id). @@ -33,11 +38,15 @@ #include #include #include +#include #include #include #include #include "audio_exec.h" +#ifdef ONEBIT_WHISPER_NPU +#include "whisper_npu_device.h" +#endif #include "audio_pack.h" #include "whisper_ids.h" #include "audio_parallel.h" @@ -251,12 +260,38 @@ int main(int argc, char** argv) { std::fprintf(stderr, "graph: %s\n", e.what()); return 1; } + AudioGemm* dev_hook = nullptr; +#ifdef ONEBIT_WHISPER_NPU + whisper_npu::Device npu_dev; + if (const char* kdir = std::getenv("ONEBIT_WHISPER_NPU_KERNELS")) { + const char* rows_env = std::getenv("ONEBIT_WHISPER_NPU_ROWS"); + const int rows = rows_env != nullptr ? std::atoi(rows_env) : 1536; + if (!whisper_npu::open(enc, kdir, rows, &npu_dev, &why)) { + std::fprintf(stderr, "npu: %s\n", why.c_str()); + return 1; + } + for (const auto& m : npu_dev.missing) std::fprintf(stderr, "npu: no ELF for staged %s -- its products run on the host\n", m.c_str()); + dev_hook = &npu_dev.hook; + } +#endif AudioRunResult xa; - if (!run_audio_graph(enc, mel.data(), cfg.bins, cfg.frames, "", &xa, &why)) { + const auto t_enc0 = std::chrono::steady_clock::now(); + if (!run_audio_graph(enc, mel.data(), cfg.bins, cfg.frames, "", &xa, &why, dev_hook)) { std::fprintf(stderr, "encoder: %s\n", why.c_str()); return 1; } const auto t_enc = std::chrono::steady_clock::now(); +#ifdef ONEBIT_WHISPER_NPU + if (dev_hook != nullptr) { + // The split the issue is scored on, from this run: the wall inside gemv against everything else in the encoder pass. + const double enc_ms = std::chrono::duration(t_enc - t_enc0).count(); + const double dev_ms = npu_dev.gemm->device_ms(); + std::printf("encoder split: device_gemm=%.3f s + host_remainder=%.3f s per window (%d product(s) on the device in %d " + "launch(es), %d on the host)\n", + dev_ms / 1000.0, (enc_ms - dev_ms) / 1000.0, dev_hook->device, npu_dev.gemm->launches(), dev_hook->host); + for (const auto& kv : dev_hook->host_why) std::printf(" host: %d x %s\n", kv.second, kv.first.c_str()); + } +#endif std::printf("encoder: ["); for (int64_t d : xa.dims) std::printf(" %lld", (long long)d); std::printf(" ]\n");