diff --git a/CMakeLists.txt b/CMakeLists.txt
index 72b23a0da..8144e013e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1588,7 +1588,7 @@ target_link_libraries(audio_parallel_test PRIVATE onebit_npu Threads::Threads)
# ci.yml's build list is not ours to edit without `workflow` scope, so these reach CI the way ops_dump_test does: as build
# dependencies of a target it already builds (tools/ci/ci_targets_check.py follows this edge).
# One line, deliberately: tools/ci/ci_targets_check.py follows `add_dependencies` line by line.
-add_dependencies(onebit audio_ops_test audio_mel_test onnx_graph_test audio_exec_test audio_parallel_test audio_pack_test audio_kernel_roles_test whisper_ids_test audio_device_gemm_test audio_launch_plan_test audio_kernel_shapes whisper_transcribe)
+add_dependencies(onebit audio_ops_test audio_mel_test onnx_graph_test audio_exec_test audio_parallel_test audio_pack_test audio_kernel_roles_test whisper_ids_test audio_device_gemm_test audio_npu_gemm_test audio_launch_plan_test audio_kernel_shapes whisper_transcribe)
# A DFlash draft package as a model source (engine#1780): config + container -> a checked model whose tensors decode
# one at a time. The loader is compiled here and not into onebit_npu because it needs the q4_1 tile decoder, which
# lives with the forward (npu/forward/src/q4nx_dequant.c); it moves there with the draft's forward. Synthetic package,
@@ -1986,6 +1986,11 @@ target_link_libraries(audio_kernel_shapes PRIVATE onebit_npu)
add_executable(audio_device_gemm_test tests/audio_device_gemm_test.cpp)
target_include_directories(audio_device_gemm_test PRIVATE npu/forward/include npu/target npu/plan)
target_link_libraries(audio_device_gemm_test PRIVATE nlohmann_json::nlohmann_json)
+# engine#1783: the encoder's products on an NpuTarget (routing by staged shape, M-row chunks, padded weight sent once,
+# de-pad), against a fake target.
+add_executable(audio_npu_gemm_test tests/audio_npu_gemm_test.cpp)
+target_include_directories(audio_npu_gemm_test PRIVATE npu/forward/include npu/target npu/plan)
+target_link_libraries(audio_npu_gemm_test PRIVATE nlohmann_json::nlohmann_json)
add_executable(audio_launch_plan_test tests/audio_launch_plan_test.cpp)
target_include_directories(audio_launch_plan_test PRIVATE npu/forward/include npu/target npu/plan)
target_link_libraries(audio_launch_plan_test PRIVATE nlohmann_json::nlohmann_json)
@@ -1993,6 +1998,7 @@ add_test(NAME whisper_ids COMMAND whisper_ids_test)
add_test(NAME audio_kernel_roles COMMAND audio_kernel_roles_test)
add_test(NAME audio_pack COMMAND audio_pack_test)
add_test(NAME audio_device_gemm COMMAND audio_device_gemm_test)
+add_test(NAME audio_npu_gemm COMMAND audio_npu_gemm_test)
add_test(NAME audio_launch_plan COMMAND audio_launch_plan_test)
add_test(NAME whisper_transcribe_jfk COMMAND whisper_transcribe)
set_tests_properties(whisper_transcribe_jfk PROPERTIES
@@ -2206,6 +2212,11 @@ endif()
# The library body is onebit_npu_forward_library() (defined above the runtime options), so the CI build of the
# same sources against the runtime stub (engine#1946) cannot drift from this one.
onebit_npu_forward_library(onebit_npu_forward onebit_npu_rt)
+ # engine#1783: whisper_transcribe with the encoder's products on the NPU (ONEBIT_WHISPER_NPU_KERNELS=
).
+ add_executable(whisper_transcribe_npu tools/whisper_transcribe.cpp)
+ target_compile_definitions(whisper_transcribe_npu PRIVATE ONEBIT_WHISPER_NPU)
+ target_include_directories(whisper_transcribe_npu PRIVATE npu/forward/include npu/target tools)
+ target_link_libraries(whisper_transcribe_npu PRIVATE onebit_npu_forward onebit_npu_rt)
target_sources(onebit PRIVATE app/forward_serve.cpp app/stb_image_impl.cpp) # engine#2324: stb_image for pictures
target_include_directories(onebit PRIVATE ${stb_SOURCE_DIR})
# engine#1781: forward-serve --spec-draft runs a DFlash draft package (npu/dflash_head.h) as its drafter.
@@ -2330,6 +2341,12 @@ if(ONEBIT_NPU_HOST AND NOT ONEBIT_NPU AND NOT WIN32 AND Python3_Interpreter_FOUN
target_link_libraries(npu_gemm_shape_guard_test PRIVATE onebit_npu_forward_ci)
add_test(NAME npu_gemm_shape_guard COMMAND npu_gemm_shape_guard_test)
add_dependencies(onebit npu_gemm_shape_guard_test) # CI builds `onebit` only; without this the test is Not Run
+ # engine#1783: the device build of whisper_transcribe, compiled here against the stub so CI sees its sources.
+ add_executable(whisper_transcribe_npu_ci tools/whisper_transcribe.cpp)
+ target_compile_definitions(whisper_transcribe_npu_ci PRIVATE ONEBIT_WHISPER_NPU)
+ target_include_directories(whisper_transcribe_npu_ci PRIVATE npu/forward/include npu/target tools)
+ target_link_libraries(whisper_transcribe_npu_ci PRIVATE onebit_npu_forward_ci)
+ add_dependencies(onebit whisper_transcribe_npu_ci)
add_executable(batch_check_state_test tests/batch_check_state_test.cpp)
target_link_libraries(batch_check_state_test PRIVATE onebit_npu_forward_ci)
# direction B (tools/ci/ci_targets_check.py): ci.yml's --target list is a workflow file no lane can patch; the
diff --git a/npu/audio_exec.cpp b/npu/audio_exec.cpp
index a0d35168a..98aafa6e1 100644
--- a/npu/audio_exec.cpp
+++ b/npu/audio_exec.cpp
@@ -196,7 +196,7 @@ void load_init_i64(const OnnxGraphInfo& g, const OnnxInit& t, std::vector t, const std::string& out_name, AudioRunResult* out,
- std::string* why) {
+ std::string* why, AudioGemm* dev = nullptr) {
try {
std::map inits;
@@ -229,6 +229,37 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map t, con
const char* ot = std::getenv("ONEBIT_AUDIO_OP_TIMES");
const bool op_times = ot != nullptr && ot[0] != '\0' && ot[0] != '0';
std::map> op_ms; // op -> (ms, calls)
+
+ // engine#1783: offer one product to the device. True when the device ran it into `c`; false when the host must, with
+ // the reason counted. A device without a `gemm` is no device.
+ auto offer = [&](const std::string& key, const float* A, int64_t M, int64_t K, const float* B, int64_t N,
+ std::vector* c) -> bool {
+ if (dev == nullptr) return false;
+ std::string reason;
+ if (dev->gemm && dev->gemm(key, A, M, K, B, N, c, &reason)) {
+ if (c->size() != size_t(M) * size_t(N))
+ fail("device gemm for '" + key + "' wrote " + std::to_string(c->size()) + " floats, not " +
+ std::to_string(M) + " x " + std::to_string(N));
+ ++dev->device;
+ return true;
+ }
+ ++dev->host;
+ ++dev->host_why[reason.empty() ? std::string("declined without a reason") : reason];
+ return false;
+ };
+
+ // Every node ends here, including the products the device ran (they leave the loop early), so ONEBIT_AUDIO_OP_TIMES
+ // counts a device-run MatMul or Conv under its op like a host one.
+ auto finish_node = [&](const std::string& op, const std::string& name, Tensor& res,
+ std::chrono::steady_clock::time_point t0) {
+ t[name] = std::move(res);
+ if (op_times) {
+ auto& e = op_ms[op];
+ e.first += std::chrono::duration(std::chrono::steady_clock::now() - t0).count();
+ e.second++;
+ }
+ };
+
int node_index = 0;
for (const auto& n : g.nodes) {
++node_index;
@@ -251,6 +282,22 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map t, con
r.dims = bp.dims;
r.dims.push_back(M);
r.dims.push_back(N);
+ // A projection (B a rank-2 weight) is one GEMM over every row of A, whatever A's leading axes are.
+ if (dev != nullptr) {
+ if (rb == 2 && inits.count(n.ins[1]) != 0) {
+ std::vector c;
+ if (offer(n.ins[1], a.data.data(), int64_t(a.data.size() / size_t(K)), K, b.data.data(), N, &c)) {
+ r.dims = a.dims;
+ r.dims.back() = N;
+ r.data = std::move(c);
+ finish_node(n.op, oname, r, op_t0);
+ continue;
+ }
+ } else {
+ ++dev->host;
+ ++dev->host_why["MatMul with an activation as B (attention product): not offered"];
+ }
+ }
r.data.assign(nelem(r.dims), 0.0f);
// Rows are independent, so the work splits over (batch, row) without touching any element's arithmetic.
const size_t batches = nelem(bp.dims);
@@ -392,6 +439,35 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map t, con
const int64_t tout = audio::conv1d_out_len(int(T), int(k), stride, pad);
r.dims = {a.dims[0], cout, tout};
r.data.resize(size_t(a.dims[0]) * size_t(cout) * size_t(tout));
+ const auto ga = n.attrs.find("group");
+ if (ga != n.attrs.end() && ga->second != 1) fail(where() + ": only group=1 is implemented");
+ if (dev != nullptr) {
+ // The im2col GEMM: (tout, cin*k) x (cin*k, cout), then out[co][t] = bias[co] + C[t][co].
+ const int64_t kk = cin * k;
+ std::vector bkn(size_t(kk) * size_t(cout));
+ audio::conv1d_weight_as_kn(w.data.data(), int(cout), int(cin), int(k), bkn.data());
+ std::vector acol(size_t(tout) * size_t(kk));
+ bool all = true;
+ for (int64_t n0 = 0; n0 < a.dims[0] && all; ++n0) {
+ audio::conv1d_im2col(a.data.data() + size_t(n0) * size_t(cin) * size_t(T), int(cin), int(T), int(k),
+ stride, pad, acol.data());
+ std::vector c;
+ if (!offer(n.ins[1] + "#conv", acol.data(), tout, kk, bkn.data(), cout, &c)) {
+ all = false;
+ break;
+ }
+ float* o = r.data.data() + size_t(n0) * size_t(cout) * size_t(tout);
+ for (int64_t co = 0; co < cout; ++co) {
+ const float bv = bias != nullptr ? bias->data[size_t(co)] : 0.0f;
+ for (int64_t tt = 0; tt < tout; ++tt)
+ o[size_t(co) * size_t(tout) + size_t(tt)] = bv + c[size_t(tt) * size_t(cout) + size_t(co)];
+ }
+ }
+ if (all) {
+ finish_node(n.op, oname, r, op_t0);
+ continue;
+ }
+ }
// engine#1783: one output channel per unit. conv1d with cout = 1 on that channel's weights, bias and output
// row runs exactly the loop the full call runs for it, so the result is bit-identical; the mel stem's two
// Convs were a serial 3.2 G multiply-adds per 30 s window.
@@ -545,12 +621,7 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map t, con
} else {
fail(where() + " is not implemented; the pack needs an op this executor does not have");
}
- t[oname] = std::move(r);
- if (op_times) {
- auto& e = op_ms[n.op];
- e.first += std::chrono::duration(std::chrono::steady_clock::now() - op_t0).count();
- e.second++;
- }
+ finish_node(n.op, oname, r, op_t0);
}
if (op_times) {
std::vector> order;
@@ -580,6 +651,11 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map t, con
bool run_audio_graph(const OnnxGraphInfo& g, const float* mel, int64_t mel_bins, int64_t frames, const std::string& out_name,
AudioRunResult* out, std::string* why) {
+ return run_audio_graph(g, mel, mel_bins, frames, out_name, out, why, nullptr);
+}
+
+bool run_audio_graph(const OnnxGraphInfo& g, const float* mel, int64_t mel_bins, int64_t frames, const std::string& out_name,
+ AudioRunResult* out, std::string* why, AudioGemm* dev) {
std::map seed;
Tensor in;
in.dims = {1, mel_bins, frames};
@@ -589,7 +665,7 @@ bool run_audio_graph(const OnnxGraphInfo& g, const float* mel, int64_t mel_bins,
return false;
}
seed[g.inputs[0]] = std::move(in);
- return run_graph_impl(g, std::move(seed), out_name, out, why);
+ return run_graph_impl(g, std::move(seed), out_name, out, why, dev);
}
bool run_decoder_graph(const OnnxGraphInfo& g, const int64_t* tokens, int64_t n_tokens, const float* xa,
diff --git a/npu/audio_exec.h b/npu/audio_exec.h
index df7e099ce..d1789b38e 100644
--- a/npu/audio_exec.h
+++ b/npu/audio_exec.h
@@ -23,6 +23,8 @@
#pragma once
+#include
+#include