diff --git a/CMakeLists.txt b/CMakeLists.txt index 72b23a0da..8144e013e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1588,7 +1588,7 @@ target_link_libraries(audio_parallel_test PRIVATE onebit_npu Threads::Threads) # ci.yml's build list is not ours to edit without `workflow` scope, so these reach CI the way ops_dump_test does: as build # dependencies of a target it already builds (tools/ci/ci_targets_check.py follows this edge). # One line, deliberately: tools/ci/ci_targets_check.py follows `add_dependencies` line by line. -add_dependencies(onebit audio_ops_test audio_mel_test onnx_graph_test audio_exec_test audio_parallel_test audio_pack_test audio_kernel_roles_test whisper_ids_test audio_device_gemm_test audio_launch_plan_test audio_kernel_shapes whisper_transcribe) +add_dependencies(onebit audio_ops_test audio_mel_test onnx_graph_test audio_exec_test audio_parallel_test audio_pack_test audio_kernel_roles_test whisper_ids_test audio_device_gemm_test audio_npu_gemm_test audio_launch_plan_test audio_kernel_shapes whisper_transcribe) # A DFlash draft package as a model source (engine#1780): config + container -> a checked model whose tensors decode # one at a time. The loader is compiled here and not into onebit_npu because it needs the q4_1 tile decoder, which # lives with the forward (npu/forward/src/q4nx_dequant.c); it moves there with the draft's forward. Synthetic package, @@ -1986,6 +1986,11 @@ target_link_libraries(audio_kernel_shapes PRIVATE onebit_npu) add_executable(audio_device_gemm_test tests/audio_device_gemm_test.cpp) target_include_directories(audio_device_gemm_test PRIVATE npu/forward/include npu/target npu/plan) target_link_libraries(audio_device_gemm_test PRIVATE nlohmann_json::nlohmann_json) +# engine#1783: the encoder's products on an NpuTarget (routing by staged shape, M-row chunks, padded weight sent once, +# de-pad), against a fake target. +add_executable(audio_npu_gemm_test tests/audio_npu_gemm_test.cpp) +target_include_directories(audio_npu_gemm_test PRIVATE npu/forward/include npu/target npu/plan) +target_link_libraries(audio_npu_gemm_test PRIVATE nlohmann_json::nlohmann_json) add_executable(audio_launch_plan_test tests/audio_launch_plan_test.cpp) target_include_directories(audio_launch_plan_test PRIVATE npu/forward/include npu/target npu/plan) target_link_libraries(audio_launch_plan_test PRIVATE nlohmann_json::nlohmann_json) @@ -1993,6 +1998,7 @@ add_test(NAME whisper_ids COMMAND whisper_ids_test) add_test(NAME audio_kernel_roles COMMAND audio_kernel_roles_test) add_test(NAME audio_pack COMMAND audio_pack_test) add_test(NAME audio_device_gemm COMMAND audio_device_gemm_test) +add_test(NAME audio_npu_gemm COMMAND audio_npu_gemm_test) add_test(NAME audio_launch_plan COMMAND audio_launch_plan_test) add_test(NAME whisper_transcribe_jfk COMMAND whisper_transcribe) set_tests_properties(whisper_transcribe_jfk PROPERTIES @@ -2206,6 +2212,11 @@ endif() # The library body is onebit_npu_forward_library() (defined above the runtime options), so the CI build of the # same sources against the runtime stub (engine#1946) cannot drift from this one. onebit_npu_forward_library(onebit_npu_forward onebit_npu_rt) + # engine#1783: whisper_transcribe with the encoder's products on the NPU (ONEBIT_WHISPER_NPU_KERNELS=). + add_executable(whisper_transcribe_npu tools/whisper_transcribe.cpp) + target_compile_definitions(whisper_transcribe_npu PRIVATE ONEBIT_WHISPER_NPU) + target_include_directories(whisper_transcribe_npu PRIVATE npu/forward/include npu/target tools) + target_link_libraries(whisper_transcribe_npu PRIVATE onebit_npu_forward onebit_npu_rt) target_sources(onebit PRIVATE app/forward_serve.cpp app/stb_image_impl.cpp) # engine#2324: stb_image for pictures target_include_directories(onebit PRIVATE ${stb_SOURCE_DIR}) # engine#1781: forward-serve --spec-draft runs a DFlash draft package (npu/dflash_head.h) as its drafter. @@ -2330,6 +2341,12 @@ if(ONEBIT_NPU_HOST AND NOT ONEBIT_NPU AND NOT WIN32 AND Python3_Interpreter_FOUN target_link_libraries(npu_gemm_shape_guard_test PRIVATE onebit_npu_forward_ci) add_test(NAME npu_gemm_shape_guard COMMAND npu_gemm_shape_guard_test) add_dependencies(onebit npu_gemm_shape_guard_test) # CI builds `onebit` only; without this the test is Not Run + # engine#1783: the device build of whisper_transcribe, compiled here against the stub so CI sees its sources. + add_executable(whisper_transcribe_npu_ci tools/whisper_transcribe.cpp) + target_compile_definitions(whisper_transcribe_npu_ci PRIVATE ONEBIT_WHISPER_NPU) + target_include_directories(whisper_transcribe_npu_ci PRIVATE npu/forward/include npu/target tools) + target_link_libraries(whisper_transcribe_npu_ci PRIVATE onebit_npu_forward_ci) + add_dependencies(onebit whisper_transcribe_npu_ci) add_executable(batch_check_state_test tests/batch_check_state_test.cpp) target_link_libraries(batch_check_state_test PRIVATE onebit_npu_forward_ci) # direction B (tools/ci/ci_targets_check.py): ci.yml's --target list is a workflow file no lane can patch; the diff --git a/npu/audio_exec.cpp b/npu/audio_exec.cpp index a0d35168a..98aafa6e1 100644 --- a/npu/audio_exec.cpp +++ b/npu/audio_exec.cpp @@ -196,7 +196,7 @@ void load_init_i64(const OnnxGraphInfo& g, const OnnxInit& t, std::vector t, const std::string& out_name, AudioRunResult* out, - std::string* why) { + std::string* why, AudioGemm* dev = nullptr) { try { std::map inits; @@ -229,6 +229,37 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map t, con const char* ot = std::getenv("ONEBIT_AUDIO_OP_TIMES"); const bool op_times = ot != nullptr && ot[0] != '\0' && ot[0] != '0'; std::map> op_ms; // op -> (ms, calls) + + // engine#1783: offer one product to the device. True when the device ran it into `c`; false when the host must, with + // the reason counted. A device without a `gemm` is no device. + auto offer = [&](const std::string& key, const float* A, int64_t M, int64_t K, const float* B, int64_t N, + std::vector* c) -> bool { + if (dev == nullptr) return false; + std::string reason; + if (dev->gemm && dev->gemm(key, A, M, K, B, N, c, &reason)) { + if (c->size() != size_t(M) * size_t(N)) + fail("device gemm for '" + key + "' wrote " + std::to_string(c->size()) + " floats, not " + + std::to_string(M) + " x " + std::to_string(N)); + ++dev->device; + return true; + } + ++dev->host; + ++dev->host_why[reason.empty() ? std::string("declined without a reason") : reason]; + return false; + }; + + // Every node ends here, including the products the device ran (they leave the loop early), so ONEBIT_AUDIO_OP_TIMES + // counts a device-run MatMul or Conv under its op like a host one. + auto finish_node = [&](const std::string& op, const std::string& name, Tensor& res, + std::chrono::steady_clock::time_point t0) { + t[name] = std::move(res); + if (op_times) { + auto& e = op_ms[op]; + e.first += std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); + e.second++; + } + }; + int node_index = 0; for (const auto& n : g.nodes) { ++node_index; @@ -251,6 +282,22 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map t, con r.dims = bp.dims; r.dims.push_back(M); r.dims.push_back(N); + // A projection (B a rank-2 weight) is one GEMM over every row of A, whatever A's leading axes are. + if (dev != nullptr) { + if (rb == 2 && inits.count(n.ins[1]) != 0) { + std::vector c; + if (offer(n.ins[1], a.data.data(), int64_t(a.data.size() / size_t(K)), K, b.data.data(), N, &c)) { + r.dims = a.dims; + r.dims.back() = N; + r.data = std::move(c); + finish_node(n.op, oname, r, op_t0); + continue; + } + } else { + ++dev->host; + ++dev->host_why["MatMul with an activation as B (attention product): not offered"]; + } + } r.data.assign(nelem(r.dims), 0.0f); // Rows are independent, so the work splits over (batch, row) without touching any element's arithmetic. const size_t batches = nelem(bp.dims); @@ -392,6 +439,35 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map t, con const int64_t tout = audio::conv1d_out_len(int(T), int(k), stride, pad); r.dims = {a.dims[0], cout, tout}; r.data.resize(size_t(a.dims[0]) * size_t(cout) * size_t(tout)); + const auto ga = n.attrs.find("group"); + if (ga != n.attrs.end() && ga->second != 1) fail(where() + ": only group=1 is implemented"); + if (dev != nullptr) { + // The im2col GEMM: (tout, cin*k) x (cin*k, cout), then out[co][t] = bias[co] + C[t][co]. + const int64_t kk = cin * k; + std::vector bkn(size_t(kk) * size_t(cout)); + audio::conv1d_weight_as_kn(w.data.data(), int(cout), int(cin), int(k), bkn.data()); + std::vector acol(size_t(tout) * size_t(kk)); + bool all = true; + for (int64_t n0 = 0; n0 < a.dims[0] && all; ++n0) { + audio::conv1d_im2col(a.data.data() + size_t(n0) * size_t(cin) * size_t(T), int(cin), int(T), int(k), + stride, pad, acol.data()); + std::vector c; + if (!offer(n.ins[1] + "#conv", acol.data(), tout, kk, bkn.data(), cout, &c)) { + all = false; + break; + } + float* o = r.data.data() + size_t(n0) * size_t(cout) * size_t(tout); + for (int64_t co = 0; co < cout; ++co) { + const float bv = bias != nullptr ? bias->data[size_t(co)] : 0.0f; + for (int64_t tt = 0; tt < tout; ++tt) + o[size_t(co) * size_t(tout) + size_t(tt)] = bv + c[size_t(tt) * size_t(cout) + size_t(co)]; + } + } + if (all) { + finish_node(n.op, oname, r, op_t0); + continue; + } + } // engine#1783: one output channel per unit. conv1d with cout = 1 on that channel's weights, bias and output // row runs exactly the loop the full call runs for it, so the result is bit-identical; the mel stem's two // Convs were a serial 3.2 G multiply-adds per 30 s window. @@ -545,12 +621,7 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map t, con } else { fail(where() + " is not implemented; the pack needs an op this executor does not have"); } - t[oname] = std::move(r); - if (op_times) { - auto& e = op_ms[n.op]; - e.first += std::chrono::duration(std::chrono::steady_clock::now() - op_t0).count(); - e.second++; - } + finish_node(n.op, oname, r, op_t0); } if (op_times) { std::vector> order; @@ -580,6 +651,11 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map t, con bool run_audio_graph(const OnnxGraphInfo& g, const float* mel, int64_t mel_bins, int64_t frames, const std::string& out_name, AudioRunResult* out, std::string* why) { + return run_audio_graph(g, mel, mel_bins, frames, out_name, out, why, nullptr); +} + +bool run_audio_graph(const OnnxGraphInfo& g, const float* mel, int64_t mel_bins, int64_t frames, const std::string& out_name, + AudioRunResult* out, std::string* why, AudioGemm* dev) { std::map seed; Tensor in; in.dims = {1, mel_bins, frames}; @@ -589,7 +665,7 @@ bool run_audio_graph(const OnnxGraphInfo& g, const float* mel, int64_t mel_bins, return false; } seed[g.inputs[0]] = std::move(in); - return run_graph_impl(g, std::move(seed), out_name, out, why); + return run_graph_impl(g, std::move(seed), out_name, out, why, dev); } bool run_decoder_graph(const OnnxGraphInfo& g, const int64_t* tokens, int64_t n_tokens, const float* xa, diff --git a/npu/audio_exec.h b/npu/audio_exec.h index df7e099ce..d1789b38e 100644 --- a/npu/audio_exec.h +++ b/npu/audio_exec.h @@ -23,6 +23,8 @@ #pragma once +#include +#include #include #include @@ -30,6 +32,27 @@ namespace onebit::npu::pack_import { +// A device for the graph's matrix products (engine#1783). The executor stays host-only and device-free: it offers each +// product whose right operand is a CONSTANT weight to `gemm`, and runs it on the host itself when the device declines. +// +// Offered: every MatMul whose B is a rank-2 initializer (the projections q, k, v, o, fc1, fc2), with A's leading axes +// flattened into rows, and every Conv, as its im2col GEMM (A = (tout, cin*k), B = the kernel as (cin*k, cout)). +// Not offered: a MatMul whose B is an activation (attention's q.k^T and scores.v). Its operand changes every call, so it +// has no weight to stay resident, and the reason is recorded like any other host product. +// +// `key` names the weight and is stable across calls (the initializer's name; "#conv" for a Conv's transposed +// kernel), so an implementation may cache the packed weight by it. Return false with `why` to leave the product to +// the host. The executor then runs it, counts it under `host`, and keeps the reason in `host_why`, so a run whose +// products silently fell back to the host is visible in the counts. +struct AudioGemm { + std::function* C, std::string* why)> + gemm; + int device = 0; // products the device ran + int host = 0; // products the host ran (declined, or not offered) + std::map host_why; // reason -> count +}; + struct AudioRunResult { std::vector dims; // dims of the requested output std::vector data; // row-major f32 payload @@ -45,6 +68,9 @@ struct AudioRunResult { // Returns true and fills `out`; on failure returns false and sets `why` to the node and the reason. bool run_audio_graph(const OnnxGraphInfo& g, const float* mel, int64_t mel_bins, int64_t frames, const std::string& out_name, AudioRunResult* out, std::string* why); +// The same, with the matrix products offered to `dev` (null = host only, identical to the overload above). +bool run_audio_graph(const OnnxGraphInfo& g, const float* mel, int64_t mel_bins, int64_t frames, + const std::string& out_name, AudioRunResult* out, std::string* why, AudioGemm* dev); // The decoder: `tokens` is the int64 token window for the graph's first input; `xa` (the encoder output, row-major f32) is // bound to the graph's second input when the graph has one. `xa_dims` may be empty when there is no cross-attention input. diff --git a/npu/forward/include/audio_npu_gemm.h b/npu/forward/include/audio_npu_gemm.h new file mode 100644 index 000000000..e332e7e0e --- /dev/null +++ b/npu/forward/include/audio_npu_gemm.h @@ -0,0 +1,175 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// audio_npu_gemm.h -- the audio executor's matrix products on an NpuTarget (engine#1783, the encoder route). +// +// The executor (npu/audio_exec.h, AudioGemm) offers every product whose right operand is a weight: the projections and the +// two Conv stems as im2col GEMMs. This file is the device side of that offer, and it is device-free: it speaks to +// `NpuTarget::gemv`, so it is tested against a fake target (tests/audio_npu_gemm_test.cpp). +// +// What it decides, per product: +// * WHICH design. The caller passes the designs in the order it gave them to the target's open_prefill(); the target's +// role index IS that order (xdna_target.cpp: make_ctx(i)). A product goes to the design whose (K, N) is its STAGED +// shape -- K on the 256 grid, N on the 512 grid (audio_kernel_roles.h) -- so whisper-small's O (768 x 768) asks for +// K768 N1024, conv1 (240 x 768) for K256 N1024, conv2 (2304 x 768) for K2304 N1024. A shape no design covers is +// declined by name and runs on the host. Routing every launch to role 0 is the defect that crashed the first +// standalone harness (engine#1783, 08:48Z), so the index is never a default. +// * HOW MANY launches. One launch holds at most the design's M rows; more rows are cut into M-row chunks (conv1's 3000 +// frames at M=1536 are two launches). +// * WHAT the weight is. The target packs `Bmat` as K_staged x N_staged f32, so the model's K x N weight is zero-padded +// here once, under a stable residency key (layer 0, one wkey per weight). After the first launch the target serves +// the packed weight from its cache. The padded host copy is dropped then, unless the caller says the target does +// not cache. +// * WHAT comes back. The target writes rows x N_staged; the executor's tensor is rows x N, so the padding columns are +// dropped per row. +// +// It also keeps the device column of the issue's bar: the wall time inside gemv, the launches and the products, so the +// tool prints `device_gemm= s + host_remainder= s` from a run rather than from arithmetic. + +#pragma once + +#include +#include +#include +#include +#include + +#include "audio_kernel_roles.h" +#include "target.h" + +namespace onebit::audio { + +struct NpuAudioDesign { + int M = 0; // rows per launch (the ELF's MD) + int K = 0; // staged K + int N = 0; // staged N +}; + +class NpuAudioGemm { +public: + // `designs[i]` is role index i of `target` (the open_prefill() order). `target_caches` = the target was given a weight + // cache with at least one layer (set_device(..., weight_cache=true, n_layers>=1)); without it the padded weights stay + // on the host and are re-sent every launch. + NpuAudioGemm(::npu::target::NpuTarget& target, std::vector designs, bool target_caches = true) + : t_(target), designs_(std::move(designs)), caches_(target_caches) {} + + // The AudioGemm::gemm contract: C (M x N) = A (M x K) x B (K x N); false + why to leave it to the host. + bool gemm(const std::string& key, const float* A, int64_t M, int64_t K, const float* B, int64_t N, + std::vector* C, std::string* why) { + if (A == nullptr || B == nullptr || C == nullptr || M <= 0 || K <= 0 || N <= 0) { + if (why) *why = "empty product"; + return false; + } + const ProjectionShape st = staged(ProjectionShape{int(K), int(N)}); + const int role = role_for(st.k, st.n); + if (role < 0) { + if (why) + *why = "no design for staged K" + std::to_string(st.k) + " N" + std::to_string(st.n) + " (model " + + std::to_string(K) + " x " + std::to_string(N) + ")"; + return false; + } + const NpuAudioDesign& d = designs_[size_t(role)]; + if (d.M <= 0) { + if (why) *why = "design " + std::to_string(role) + " states no rows"; + return false; + } + + Weight& w = weight(key, B, K, N, st); + const auto t0 = std::chrono::steady_clock::now(); + C->assign(size_t(M) * size_t(N), 0.0f); + std::vector got; + for (int64_t r0 = 0; r0 < M; r0 += d.M) { + const int rows = int(std::min(d.M, M - r0)); + ::npu::target::GemvArgs a; + a.role = role; + a.layer = 0; + a.M = rows; + a.K = st.k; + a.N = st.n; + a.x = A + size_t(r0) * size_t(K); + a.a_k = int(K); + a.Bmat = w.padded.empty() ? nullptr : w.padded.data(); + a.wkey = caches_ ? w.wkey : -1; + a.rows = true; // the batched body, whatever the chunk's row count (target.h: M==1 with rows=true is batched) + a.tag = "audio"; + got.clear(); + std::string err; + if (!t_.gemv(a, got, err)) { + if (why) *why = "role " + std::to_string(role) + " (" + key + "): " + (err.empty() ? "gemv failed" : err); + return false; + } + if (got.size() != size_t(rows) * size_t(st.n)) { + if (why) + *why = "role " + std::to_string(role) + " wrote " + std::to_string(got.size()) + " floats for " + + std::to_string(rows) + " x " + std::to_string(st.n); + return false; + } + for (int r = 0; r < rows; ++r) + std::copy(got.begin() + size_t(r) * size_t(st.n), got.begin() + size_t(r) * size_t(st.n) + size_t(N), + C->begin() + size_t(r0 + r) * size_t(N)); + ++launches_; + ++per_role_[role]; + // The target now holds the packed weight under (layer 0, wkey): later chunks and later calls send none. + if (caches_ && !w.padded.empty()) w.padded = std::vector(); + } + device_ms_ += std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); + ++products_; + return true; + } + + double device_ms() const { return device_ms_; } + int launches() const { return launches_; } + int products() const { return products_; } + const std::map& launches_per_role() const { return per_role_; } + int weights() const { return int(w_.size()); } + +private: + struct Weight { + int wkey = -1; + std::vector padded; // K_staged x N_staged, until the target has it + }; + + int role_for(int k, int n) const { + for (size_t i = 0; i < designs_.size(); ++i) + if (designs_[i].K == k && designs_[i].N == n) return int(i); + return -1; + } + + Weight& weight(const std::string& key, const float* B, int64_t K, int64_t N, const ProjectionShape& st) { + // Padded once per key: either the copy is still here (not yet in the target, or the target does not cache), or + // the target holds it and the copy was dropped after its first launch. + auto it = w_.find(key); + if (it != w_.end()) return it->second; + Weight w; + w.wkey = next_wkey_++; + w.padded.assign(size_t(st.k) * size_t(st.n), 0.0f); + for (int64_t k = 0; k < K; ++k) + std::copy(B + size_t(k) * size_t(N), B + size_t(k) * size_t(N) + size_t(N), + w.padded.begin() + size_t(k) * size_t(st.n)); + return w_.emplace(key, std::move(w)).first->second; + } + + ::npu::target::NpuTarget& t_; + std::vector designs_; + bool caches_ = true; + std::map w_; + int next_wkey_ = 0; + double device_ms_ = 0.0; + int launches_ = 0; + int products_ = 0; + std::map per_role_; +}; + +} // namespace onebit::audio diff --git a/npu/forward/include/audio_ops.h b/npu/forward/include/audio_ops.h index 8fd3dfef6..d4f6229e8 100644 --- a/npu/forward/include/audio_ops.h +++ b/npu/forward/include/audio_ops.h @@ -56,6 +56,33 @@ inline void conv1d(const float* x, int cin, int T, const float* w, const float* } } +// The same convolution as a GEMM (engine#1783: so the encoder's two Conv stems can run on the NPU's GEMM family). +// A = im2col(x): (tout, cin*k), A[t][ci*k + j] = x[ci][t*stride + j - pad], 0 outside [0, T). +// B = w as (cin*k, cout): B[ci*k + j][co] = w[co][ci][j]. +// out[co][t] = b[co] + (A x B)[t][co] +// conv1d() above is the reference this is tested against (tests/audio_ops_test.cpp); the column order ci*k + j is the +// initializer's own (cout, cin, k) flattening, so B is a plain transpose of w viewed as (cout, cin*k). +inline void conv1d_im2col(const float* x, int cin, int T, int k, int stride, int pad, float* A) { + const int tout = conv1d_out_len(T, k, stride, pad); + const std::size_t kk = static_cast(cin) * static_cast(k); + for (int t = 0; t < tout; ++t) { + float* row = A + static_cast(t) * kk; + for (int ci = 0; ci < cin; ++ci) { + const float* xr = x + static_cast(ci) * T; + for (int j = 0; j < k; ++j) { + const int src = t * stride + j - pad; + row[static_cast(ci) * k + j] = (src >= 0 && src < T) ? xr[src] : 0.0f; + } + } + } +} + +inline void conv1d_weight_as_kn(const float* w, int cout, int cin, int k, float* B) { + const std::size_t kk = static_cast(cin) * static_cast(k); + for (int co = 0; co < cout; ++co) + for (std::size_t r = 0; r < kk; ++r) B[r * static_cast(cout) + co] = w[static_cast(co) * kk + r]; +} + // y = (x - mean) / sqrt(var + eps) * gamma + beta, biased variance (divide by n), one row. inline void layer_norm(const float* x, const float* gamma, const float* beta, int n, float eps, float* out) { double sum = 0.0; diff --git a/tests/audio_exec_test.cpp b/tests/audio_exec_test.cpp index 6d12cd7d1..516171b15 100644 --- a/tests/audio_exec_test.cpp +++ b/tests/audio_exec_test.cpp @@ -19,6 +19,7 @@ // The mel is mel[b,t] = sin(0.013*(b+1)*(t+1))*0.5 + 0.25 for 80 bins and 3000 frames, computed in double on both sides. // Expected values below are onnxruntime's, on whisper-small's encoder_model.onnx. Skips (77) without the pack. +#include #include #include #include @@ -26,6 +27,7 @@ #include #include "audio_exec.h" +#include "audio_parallel.h" #include "audio_pack.h" #include "onnx_graph.h" @@ -144,6 +146,76 @@ int main(int argc, char** argv) { } std::printf("audio exec: whole encoder on the host matches onnxruntime\n"); + // ---- engine#1783: the same encoder with its matrix products offered to a device ----------------------------------- + // The "device" here is an exact f32 GEMM on the host, so this checks the OFFER, not a kernel: that every projection + // and both Conv stems leave the executor as (A, B) GEMMs, that the result comes back in the right layout, and that + // only the attention products stay on the host. whisper-small: 12 layers x (q, k, v, o, fc1, fc2) + 2 convs = 74 + // offered, 12 x 2 attention products not offered. + { + AudioGemm dev; + dev.gemm = [](const std::string&, const float* A, int64_t M, int64_t K, const float* B, int64_t N, + std::vector* C, std::string*) { + C->assign(size_t(M) * size_t(N), 0.0f); + parallel_units(size_t(M), [&](size_t i) { + float* c = C->data() + i * size_t(N); + for (int64_t k = 0; k < K; ++k) { + const float a = A[i * size_t(K) + size_t(k)]; + const float* b = B + size_t(k) * size_t(N); + for (int64_t j = 0; j < N; ++j) c[j] += a * b[j]; + } + }); + return true; + }; + AudioRunResult rd; + if (!run_audio_graph(g, mel.data(), bins, frames, "", &rd, &why, &dev)) { + std::printf("FAIL run with a device: %s\n", why.c_str()); + return 1; + } + check(dev.device == 74, "74 products offered to the device (72 projections + 2 convs), got " + std::to_string(dev.device)); + check(dev.host == 24, "24 attention products stay on the host, got " + std::to_string(dev.host)); + check(rd.dims == r.dims, "the device run has the host run's dims"); + double maxd = 0.0; + for (size_t i = 0; i < r.data.size() && i < rd.data.size(); ++i) + maxd = std::max(maxd, std::fabs(double(r.data[i]) - double(rd.data[i]))); + std::printf("device-offer run: %d device, %d host, max |host - offered| %.3g\n", dev.device, dev.host, maxd); + // Not bit-identical: the im2col Conv adds its bias after the taps, conv1d() before them, and 12 layers carry that + // rounding to ~1.6e-3 at the output (std 1.68). So the offered run is held to the same onnxruntime values as the + // host run, plus a bound on its distance from the host run. + check(maxd < 5e-3, "the offered run is within 5e-3 of the host run"); + if (rd.data.size() == r.data.size()) { + double m2 = 0.0, v2 = 0.0; + for (float v : rd.data) m2 += v; + m2 /= double(rd.data.size()); + for (float v : rd.data) v2 += (double(v) - m2) * (double(v) - m2); + v2 /= double(rd.data.size()); + expect_close(float(m2), -0.009045244, 2e-4, "offered run: output mean"); + expect_close(float(std::sqrt(v2)), 1.678493023, 2e-4, "offered run: output std"); + auto at2 = [&](int64_t tt, int64_t h) { return rd.data[size_t(tt) * H + size_t(h)]; }; + for (int i = 0; i < 5; ++i) expect_close(at2(0, i), p0[i], 2e-3, "offered run: frame 0 col " + std::to_string(i)); + for (int i = 0; i < 4; ++i) expect_close(at2(750, i), p1[i], 2e-3, "offered run: frame 750 col " + std::to_string(i)); + for (int i = 0; i < 4; ++i) expect_close(at2(1499, i), p2[i], 2e-3, "offered run: frame 1499 col " + std::to_string(i)); + } + // A device that declines everything is the host run, counted as host. + AudioGemm none; + none.gemm = [](const std::string&, const float*, int64_t, int64_t, const float*, int64_t, std::vector*, + std::string* w) { + *w = "test: declined"; + return false; + }; + AudioRunResult rn; + if (!run_audio_graph(g, mel.data(), bins, frames, "", &rn, &why, &none)) { + std::printf("FAIL run with a declining device: %s\n", why.c_str()); + return 1; + } + check(none.device == 0 && none.host == 98 && none.host_why["test: declined"] == 74, + "a declining device runs all 98 products on the host and names the 74 refusals"); + check(rn.data == r.data, "a declining device is bit-identical to the host run"); + if (g_failures != 0) { + std::printf("audio exec: %d failures\n", g_failures); + return 1; + } + } + // ---- the decoder, on the same pack: 128-position token window x, the encoder output xa ---------------------------- const char* dpath = (dpath_arg != nullptr) ? dpath_arg : std::getenv("ONEBIT_TEST_WHISPER_DECODER"); if (dpath == nullptr || *dpath == '\0') { diff --git a/tests/audio_npu_gemm_test.cpp b/tests/audio_npu_gemm_test.cpp new file mode 100644 index 000000000..bc63814ea --- /dev/null +++ b/tests/audio_npu_gemm_test.cpp @@ -0,0 +1,176 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// tests/audio_npu_gemm_test.cpp -- the encoder's products on an NpuTarget (npu/forward/include/audio_npu_gemm.h, engine#1783), +// against a FAKE target that does the GEMM exactly from the staged Bmat and caches it by (layer, wkey) like XdnaTarget. +// +// Pinned: the role index is the design's position (never a default 0), each launch is at most the design's M rows, +// the weight is zero-padded to the staged K x N and sent once, the result is de-padded to the model's N, and an +// uncovered shape is declined by name. +#include +#include +#include +#include +#include + +#include "audio_npu_gemm.h" + +using namespace onebit::audio; +namespace t = ::npu::target; + +static int failures = 0; +static void check(bool ok, const std::string& what) { + std::printf("%s %s\n", ok ? "ok " : "FAIL", what.c_str()); + if (!ok) ++failures; +} + +struct FakeTarget : t::NpuTarget { + std::vector d; // what open_prefill adopted, by role index + std::map> cache; // wkey -> staged K x N weight + std::vector seen; + int with_bmat = 0; + const char* name() const override { return "fake"; } + bool open(const onebit::npu::plan::Plan&, const std::string&, const char*, std::string&) override { return true; } + bool load_block(int, int, std::string&) override { return true; } + t::Caps caps() const override { return t::Caps{}; } + bool gemv(const t::GemvArgs& a, std::vector& out, std::string& err) override { + seen.push_back(a); + if (a.role < 0 || a.role >= int(d.size())) { err = "no context for role"; return false; } + const NpuAudioDesign& g = d[size_t(a.role)]; + if (a.M > g.M || a.K != g.K || a.N != g.N) { err = "launch does not fit the role's context"; return false; } + const std::vector* w = nullptr; + if (a.Bmat != nullptr) { + ++with_bmat; + std::vector b(a.Bmat, a.Bmat + size_t(a.K) * size_t(a.N)); + if (a.wkey >= 0) w = &(cache[a.wkey] = std::move(b)); + else w = &(cache[-1000 - int(seen.size())] = std::move(b)); + } else { + const auto it = cache.find(a.wkey); + if (it == cache.end()) { err = "no Bmat and nothing cached"; return false; } + w = &it->second; + } + out.assign(size_t(a.M) * size_t(a.N), 0.0f); + for (int m = 0; m < a.M; ++m) + for (int k = 0; k < a.K; ++k) { + const float x = k < a.a_k ? a.x[size_t(m) * size_t(a.a_k) + size_t(k)] : 0.0f; + for (int n = 0; n < a.N; ++n) out[size_t(m) * size_t(a.N) + size_t(n)] += x * (*w)[size_t(k) * size_t(a.N) + size_t(n)]; + } + return true; + } +}; + +static void host_gemm(const std::vector& A, const std::vector& B, int M, int K, int N, std::vector* C) { + C->assign(size_t(M) * size_t(N), 0.0f); + for (int m = 0; m < M; ++m) + for (int k = 0; k < K; ++k) + for (int n = 0; n < N; ++n) (*C)[size_t(m) * N + n] += A[size_t(m) * K + k] * B[size_t(k) * N + n]; +} + +static double maxdiff(const std::vector& a, const std::vector& b) { + if (a.size() != b.size()) return 1e30; + double m = 0.0; + for (size_t i = 0; i < a.size(); ++i) m = std::fmax(m, std::fabs(double(a[i]) - double(b[i]))); + return m; +} + +int main() { + FakeTarget ft; + // Small stand-ins on the real grid: role 0 = "O" K256 N512, role 1 = "conv" K256 N1024, role 2 = "D" K512 N512. + ft.d = {{8, 256, 512}, {8, 256, 1024}, {8, 512, 512}}; + NpuAudioGemm g(ft, ft.d); + + auto fill = [](int n, float s) { + std::vector v(static_cast(n)); + for (int i = 0; i < n; ++i) v[size_t(i)] = std::sin(0.37f * float(i + 1) * s) * 0.5f; + return v; + }; + + // 1. a model-width projection (200 x 300 -> staged K256 N512) on role 0, 19 rows -> 3 launches (8, 8, 3) + { + const int M = 19, K = 200, N = 300; + const auto A = fill(M * K, 1.0f), B = fill(K * N, 0.7f); + std::vector C, R; + std::string why; + check(g.gemm("w.o", A.data(), M, K, B.data(), N, &C, &why), "a padded projection runs: " + why); + host_gemm(A, B, M, K, N, &R); + check(maxdiff(C, R) < 1e-4, "it equals the host GEMM after de-padding (max " + std::to_string(maxdiff(C, R)) + ")"); + check(ft.seen.size() == 3 && ft.seen[0].M == 8 && ft.seen[1].M == 8 && ft.seen[2].M == 3, + "19 rows at M=8 are three launches of 8, 8 and 3"); + check(ft.seen[0].role == 0 && ft.seen[0].K == 256 && ft.seen[0].N == 512 && ft.seen[0].a_k == 200, + "role 0 is asked for the staged K256 N512, with the activation's real width 200"); + check(ft.seen[0].rows && ft.seen[2].rows, "every chunk is the batched body"); + check(ft.with_bmat == 1, "the padded weight is sent once and the next launches use the cache"); + } + + // 2. a second shape routes by its staged (K, N), not by order of first use or a default + { + ft.seen.clear(); + const int M = 5, K = 240, N = 768; // conv1 in miniature: K240 -> K256, N768 -> N1024 + const auto A = fill(M * K, 1.3f), B = fill(K * N, 0.4f); + std::vector C, R; + std::string why; + check(g.gemm("conv1#conv", A.data(), M, K, B.data(), N, &C, &why), "a conv-stem shape runs: " + why); + host_gemm(A, B, M, K, N, &R); + check(ft.seen.size() == 1 && ft.seen[0].role == 1 && ft.seen[0].N == 1024, "K240 N768 goes to role 1 (K256 N1024)"); + check(maxdiff(C, R) < 1e-4, "and equals the host GEMM"); + check(ft.seen[0].wkey != 0, "a second weight gets its own residency key"); + } + + // 3. the same key a second time sends no weight + { + ft.seen.clear(); + const int before = ft.with_bmat; + const int M = 4, K = 200, N = 300; + const auto A = fill(M * K, 2.1f), B = fill(K * N, 0.7f); + std::vector C, R; + std::string why; + check(g.gemm("w.o", A.data(), M, K, B.data(), N, &C, &why), "a cached weight runs again"); + host_gemm(A, B, M, K, N, &R); + check(ft.with_bmat == before && ft.seen[0].Bmat == nullptr, "the second call sends no Bmat (the target holds it)"); + check(maxdiff(C, R) < 1e-4, "and still equals the host GEMM"); + } + + // 4. a shape no design covers is declined by name, and the target is not called + { + ft.seen.clear(); + std::vector C; + std::string why; + // K1000 stages to K1024: no design + const auto A2 = fill(1000, 1.0f), B2 = fill(1000 * 10, 1.0f); + check(!g.gemm("odd", A2.data(), 1, 1000, B2.data(), 10, &C, &why) && + why.find("no design for staged K1024 N512") != std::string::npos, + "an uncovered staged shape is declined by name: " + why); + check(ft.seen.empty(), "and the target is not called"); + } + + // 5. a target failure is passed through with the role and key + { + ft.d[2].M = 0; // the fake now refuses role 2 + NpuAudioGemm g2(ft, {{8, 256, 512}, {8, 256, 1024}, {8, 512, 512}}); + const auto A = fill(2 * 512, 1.0f), B = fill(512 * 512, 1.0f); + std::vector C; + std::string why; + check(!g2.gemm("w.d", A.data(), 2, 512, B.data(), 512, &C, &why) && why.find("role 2 (w.d)") != std::string::npos, + "a refused launch names its role and weight: " + why); + } + + check(g.launches() == 5 && g.products() == 3, "counts: 5 launches, 3 products"); + if (failures) { + std::printf("audio_npu_gemm: %d failure(s)\n", failures); + return 1; + } + std::printf("audio_npu_gemm: all pass\n"); + return 0; +} diff --git a/tests/audio_ops_test.cpp b/tests/audio_ops_test.cpp index 9107a8c53..cc915c6d5 100644 --- a/tests/audio_ops_test.cpp +++ b/tests/audio_ops_test.cpp @@ -78,6 +78,34 @@ void conv1d_arms() { expect_eq(onebit::audio::conv1d_out_len(5, 3, 1, 0), 3, "conv1d_out_len T5 k3 s1 p0"); } +// engine#1783: the Conv as a GEMM (what the device runs) must be the Conv. Same strided, padded case as above, so a +// wrong column order (j*cin + ci instead of ci*k + j) or a missed pad tap shows up as a value, not just a shape. +void conv1d_gemm_arm() { + const std::vector x = {0.5f, -1.0f, 0.25f, 2.0f, -0.75f, 1.5f, 0.0f, -0.5f, 1.0f, 0.25f}; + std::vector w(3 * 2 * 3); + for (int i = 0; i < 18; ++i) w[i] = static_cast((i + 1) * 0.0625); + const std::vector b = {0.125f, -0.25f, 0.5f}; + const int cin = 2, T = 5, cout = 3, k = 3, stride = 2, pad = 1; + const int tout = onebit::audio::conv1d_out_len(T, k, stride, pad); + std::vector ref(size_t(cout) * tout); + onebit::audio::conv1d(x.data(), cin, T, w.data(), b.data(), cout, k, stride, pad, ref.data()); + + std::vector A(size_t(tout) * cin * k), B(size_t(cin) * k * cout); + onebit::audio::conv1d_im2col(x.data(), cin, T, k, stride, pad, A.data()); + onebit::audio::conv1d_weight_as_kn(w.data(), cout, cin, k, B.data()); + expect_close(A[0], 0.0, 0.0, "im2col t0 tap 0 is the left pad"); + expect_close(A[1], 0.5, 0.0, "im2col t0 tap 1 is x[0][0]"); + expect_close(A[3], 0.0, 0.0, "im2col t0 channel 1 tap 0 is the left pad"); + expect_close(B[1 * cout + 2], w[2 * 6 + 1], 0.0, "B[ci0,j1][co2] = w[co2][ci0][j1]"); + for (int co = 0; co < cout; ++co) + for (int t = 0; t < tout; ++t) { + double acc = b[size_t(co)]; + for (int r = 0; r < cin * k; ++r) acc += double(A[size_t(t) * cin * k + r]) * B[size_t(r) * cout + co]; + expect_close(float(acc), ref[size_t(co) * tout + t], 1e-6, + "im2col GEMM = conv1d at co " + std::to_string(co) + " t " + std::to_string(t)); + } +} + // 2. Layer norm: biased variance, eps added under the root, gamma/beta applied after the normalise. void layernorm_arm() { const std::vector v = {0.25f, -1.5f, 2.0f, 0.75f}; @@ -136,6 +164,7 @@ void softmax_arm() { int main() { conv1d_arms(); + conv1d_gemm_arm(); layernorm_arm(); gelu_arm(); softmax_arm(); @@ -143,7 +172,7 @@ int main() { std::printf("audio ops: %d failures\n", g_failures); return 1; } - std::printf("audio ops: conv1d (strided + padded + edge), layer_norm (biased var, null affine), gelu (erf), softmax " + std::printf("audio ops: conv1d (strided + padded + edge), conv1d as im2col GEMM, layer_norm (biased var, null affine), gelu (erf), softmax " "(stable, shift-invariant) -- all pass\n"); return 0; } diff --git a/tools/whisper_npu_device.h b/tools/whisper_npu_device.h new file mode 100644 index 000000000..d2a88a14e --- /dev/null +++ b/tools/whisper_npu_device.h @@ -0,0 +1,167 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// whisper_npu_device.h -- open the NPU for whisper_transcribe's encoder (engine#1783, the encoder route). +// +// Compiled only into whisper_transcribe_npu (ONEBIT_WHISPER_NPU). It reads the design list from the ENCODER GRAPH, not from +// a hand-written plan: every MatMul whose B is a rank-2 weight, and every Conv as its im2col GEMM (K = cin*k, N = cout), staged on +// the engine's grid (audio_kernel_roles.h). For each distinct staged (K, N) it finds `full_i8__K_N_b8.elf` in the +// kernel directory, whatever the role prefix: cl-npurt's conv stems are D-named (`D K256 N1024`, `D K2304 N1024`), and the name +// is only how the generator files them. It then adopts them through open_prefill() in that order. The order is the role +// index NpuAudioGemm routes by. +// +// A staged shape with no ELF is left out and printed. Its products are declined by NpuAudioGemm and run on the host, +// counted, so a missing kernel shows in the split instead of as a crash or a silent zero. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "audio_exec.h" +#include "audio_kernel_roles.h" +#include "audio_npu_gemm.h" +#include "onnx_graph.h" +#include "q4nx_pack.h" +#include "target/xdna/xdna_target.h" + +namespace whisper_npu { + +namespace x = ::npu::target::xdna; + +struct Device { + std::unique_ptr dev; + std::unique_ptr target; + std::unique_ptr gemm; + onebit::npu::pack_import::AudioGemm hook; + std::vector missing; // staged shapes with no ELF, "K N" +}; + +// The staged (K, N) of every product the executor will offer, in graph order, without repeats. +inline std::vector> offered_shapes(const onebit::npu::pack_import::OnnxGraphInfo& g) { + std::vector> out; + auto add = [&](int64_t k, int64_t n) { + const auto s = onebit::audio::staged(onebit::audio::ProjectionShape{int(k), int(n)}); + for (const auto& p : out) + if (p.first == s.k && p.second == s.n) return; + out.emplace_back(s.k, s.n); + }; + auto init_dims = [&](const std::string& name) -> const std::vector* { + for (const auto& t : g.init_tensors) + if (t.name == name) return &t.dims; + return nullptr; + }; + for (const auto& n : g.nodes) { + if (n.ins.size() < 2) continue; + const std::vector* d = init_dims(n.ins[1]); + if (d == nullptr) continue; + if (n.op == "MatMul" && d->size() == 2) add((*d)[0], (*d)[1]); + if (n.op == "Conv" && d->size() == 3) add((*d)[1] * (*d)[2], (*d)[0]); + } + return out; +} + +// `full_i8__K_N_b8.elf` in `dir`, any role prefix; empty when none. Two matches is a refusal: which one the +// encoder ran would depend on directory order. +inline std::string find_elf(const std::string& dir, int k, int n, std::string* role, std::string* why) { + const std::string tail = "_K" + std::to_string(k) + "_N" + std::to_string(n) + "_b8.elf"; + std::string hit; + DIR* d = opendir(dir.c_str()); + if (d == nullptr) { + *why = "cannot open " + dir; + return {}; + } + while (dirent* e = readdir(d)) { + const std::string f = e->d_name; + if (f.rfind("full_i8_", 0) != 0 || f.size() <= tail.size() || f.compare(f.size() - tail.size(), tail.size(), tail) != 0) + continue; + if (!hit.empty()) { + *why = "two ELFs for K" + std::to_string(k) + " N" + std::to_string(n) + ": " + hit + " and " + f; + closedir(d); + return {}; + } + hit = f; + } + closedir(d); + if (!hit.empty()) *role = hit.substr(8, hit.size() - tail.size() - 8); + return hit; +} + +inline Q4nxDesign design_enum(const std::string& role) { + if (role == "QKV") return Q4NX_DESIGN_QKV; + if (role == "O") return Q4NX_DESIGN_O; + if (role == "GU") return Q4NX_DESIGN_GU; + if (role == "G") return Q4NX_DESIGN_G; + if (role == "U") return Q4NX_DESIGN_U; + return Q4NX_DESIGN_D; // the conv stems are filed as D; the enum only names the log line +} + +// Opens device 0, adopts one design per offered shape that has an ELF (M = `rows`, the ELFs' generated M), and wires the +// executor hook. False + why when the device or an ELF cannot be opened. +inline bool open(const onebit::npu::pack_import::OnnxGraphInfo& enc, const std::string& kdir, int rows, Device* out, + std::string* why) { + std::vector geoms; + std::vector designs; + for (const auto& s : offered_shapes(enc)) { + std::string role, err; + const std::string elf = find_elf(kdir, s.first, s.second, &role, &err); + if (!err.empty()) { + *why = err; + return false; + } + if (elf.empty()) { + out->missing.push_back("K" + std::to_string(s.first) + " N" + std::to_string(s.second)); + continue; + } + Q4nxDesignGeom g{}; + g.M = rows; + g.K = s.first; + g.N = s.second; + g.role = design_enum(role); + std::snprintf(g.elf_name, sizeof g.elf_name, "%s", elf.c_str()); + geoms.push_back(g); + designs.push_back(onebit::audio::NpuAudioDesign{rows, s.first, s.second}); + std::fprintf(stderr, "npu: role %zu = %s (staged K%d N%d, M=%d)\n", geoms.size() - 1, elf.c_str(), s.first, s.second, + rows); + } + if (geoms.empty()) { + *why = "no ELF in " + kdir + " covers any of the encoder's products"; + return false; + } + out->dev = std::make_unique(0); + out->target = std::make_unique(); + // One layer of weight cache: NpuAudioGemm keys every weight at layer 0 with its own wkey. + out->target->set_device(out->dev.get(), /*weight_layout_b8=*/true, /*weight_cache=*/true, /*n_layers=*/1); + x::XdnaTarget::Setup setup; + setup.kernels_dir = kdir; + setup.b8 = true; + out->target->configure(setup); + std::string err; + if (!out->target->open_prefill(geoms.data(), int(geoms.size()), err)) { + *why = "open_prefill: " + err; + return false; + } + out->gemm = std::make_unique(*out->target, designs, /*target_caches=*/true); + onebit::audio::NpuAudioGemm* gm = out->gemm.get(); + out->hook.gemm = [gm](const std::string& key, const float* A, int64_t M, int64_t K, const float* B, int64_t N, + std::vector* C, std::string* w) { return gm->gemm(key, A, M, K, B, N, C, w); }; + return true; +} + +} // namespace whisper_npu diff --git a/tools/whisper_transcribe.cpp b/tools/whisper_transcribe.cpp index 1657023ee..e7ed39a72 100644 --- a/tools/whisper_transcribe.cpp +++ b/tools/whisper_transcribe.cpp @@ -25,6 +25,11 @@ // (see PR #1931); `ctest` would time out for a reason that has nothing to do with correctness. The numbers it prints are // the ones a test would assert, so `expected-ids-csv` turns it into a check when run by hand. // +// whisper_transcribe_npu is this file built with ONEBIT_WHISPER_NPU (engine#1783, the encoder route). With +// ONEBIT_WHISPER_NPU_KERNELS= in the environment, the encoder's projections and Conv stems run on the NPU through +// NpuTarget::gemv, and the tool prints `device_gemm= s + host_remainder= s` for the encoder pass. Without the variable +// the binary is the host tool. The kernel ELFs are generated at one M; ONEBIT_WHISPER_NPU_ROWS names it (default 1536). +// // The pack on disk carries no tokenizer, so this stops at ids: `400 370 452 ...` is the sentence, and comparing ids is a // stricter check than comparing text (two ids can share a detokenisation, and neither can share an id). @@ -33,11 +38,15 @@ #include #include #include +#include #include #include #include #include "audio_exec.h" +#ifdef ONEBIT_WHISPER_NPU +#include "whisper_npu_device.h" +#endif #include "audio_pack.h" #include "whisper_ids.h" #include "audio_parallel.h" @@ -251,12 +260,38 @@ int main(int argc, char** argv) { std::fprintf(stderr, "graph: %s\n", e.what()); return 1; } + AudioGemm* dev_hook = nullptr; +#ifdef ONEBIT_WHISPER_NPU + whisper_npu::Device npu_dev; + if (const char* kdir = std::getenv("ONEBIT_WHISPER_NPU_KERNELS")) { + const char* rows_env = std::getenv("ONEBIT_WHISPER_NPU_ROWS"); + const int rows = rows_env != nullptr ? std::atoi(rows_env) : 1536; + if (!whisper_npu::open(enc, kdir, rows, &npu_dev, &why)) { + std::fprintf(stderr, "npu: %s\n", why.c_str()); + return 1; + } + for (const auto& m : npu_dev.missing) std::fprintf(stderr, "npu: no ELF for staged %s -- its products run on the host\n", m.c_str()); + dev_hook = &npu_dev.hook; + } +#endif AudioRunResult xa; - if (!run_audio_graph(enc, mel.data(), cfg.bins, cfg.frames, "", &xa, &why)) { + const auto t_enc0 = std::chrono::steady_clock::now(); + if (!run_audio_graph(enc, mel.data(), cfg.bins, cfg.frames, "", &xa, &why, dev_hook)) { std::fprintf(stderr, "encoder: %s\n", why.c_str()); return 1; } const auto t_enc = std::chrono::steady_clock::now(); +#ifdef ONEBIT_WHISPER_NPU + if (dev_hook != nullptr) { + // The split the issue is scored on, from this run: the wall inside gemv against everything else in the encoder pass. + const double enc_ms = std::chrono::duration(t_enc - t_enc0).count(); + const double dev_ms = npu_dev.gemm->device_ms(); + std::printf("encoder split: device_gemm=%.3f s + host_remainder=%.3f s per window (%d product(s) on the device in %d " + "launch(es), %d on the host)\n", + dev_ms / 1000.0, (enc_ms - dev_ms) / 1000.0, dev_hook->device, npu_dev.gemm->launches(), dev_hook->host); + for (const auto& kv : dev_hook->host_why) std::printf(" host: %d x %s\n", kv.second, kv.first.c_str()); + } +#endif std::printf("encoder: ["); for (int64_t d : xa.dims) std::printf(" %lld", (long long)d); std::printf(" ]\n");