From f60b9b6e430db0f819e97934d595e53911db844f Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Fri, 13 Nov 2026 06:30:49 -0400 Subject: [PATCH 1/3] audio_exec: ONEBIT_AUDIO_OP_TIMES=1 prints wall time per op type (engine#1783) The host split on #1783 (Conv, Softmax, attention products per 30 s window) was read from other tools. With the variable set, the executor prints one line per op type at the end of each graph run: calls and total ms, sorted, with the node count and thread count. Off by default; nothing else moves. Co-Authored-By: Claude Opus 5.5 --- npu/audio_exec.cpp | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/npu/audio_exec.cpp b/npu/audio_exec.cpp index edb394ad7..020cbdf61 100644 --- a/npu/audio_exec.cpp +++ b/npu/audio_exec.cpp @@ -19,6 +19,9 @@ #include "audio_parallel.h" #include +#include +#include +#include #include #include #include @@ -221,9 +224,15 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map t, con return t.count(name) != 0 ? need(name) : init_f32(name); }; + // ONEBIT_AUDIO_OP_TIMES=1 (engine#1783): wall time per op type over this graph, printed once at the end, so a + // host split ("Conv 3.74 s, Softmax 2.56 s") is read from the executor that runs the window rather than inferred. + const char* ot = std::getenv("ONEBIT_AUDIO_OP_TIMES"); + const bool op_times = ot != nullptr && ot[0] != '\0' && ot[0] != '0'; + std::map> op_ms; // op -> (ms, calls) int node_index = 0; for (const auto& n : g.nodes) { ++node_index; + const auto op_t0 = std::chrono::steady_clock::now(); const auto where = [&]() { return "node " + std::to_string(node_index) + " (" + n.op + ")"; }; if (n.outs.empty()) fail(where() + " has no output"); const std::string& oname = n.outs[0]; @@ -527,6 +536,20 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map t, con fail(where() + " is not implemented; the pack needs an op this executor does not have"); } t[oname] = std::move(r); + if (op_times) { + auto& e = op_ms[n.op]; + e.first += std::chrono::duration(std::chrono::steady_clock::now() - op_t0).count(); + e.second++; + } + } + if (op_times) { + std::vector> order; + double total = 0.0; + for (const auto& kv : op_ms) { order.push_back({kv.second.first, kv.first}); total += kv.second.first; } + std::sort(order.rbegin(), order.rend()); + std::fprintf(stderr, "[audio-op] %zu nodes, %.1f ms, %zu thread(s)\n", g.nodes.size(), total, audio_thread_count()); + for (const auto& o : order) + std::fprintf(stderr, "[audio-op] %-20s %6ld calls %10.1f ms\n", o.second.c_str(), op_ms[o.second].second, o.first); } const std::string want = out_name.empty() ? (g.outputs.empty() ? std::string() : g.outputs[0]) : out_name; From 7b0fb2f6e756e13cda238b12d34ce3ed52bee462 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Fri, 13 Nov 2026 06:30:49 -0400 Subject: [PATCH 2/3] audio_exec: Conv per output channel, Softmax and LayerNorm per row, over the audio threads (engine#1783) The whisper encoder's Conv stem (two 1-D convolutions, 3.2 G multiply-adds per 30 s window) and every Softmax and LayerNorm row ran on one thread, while MatMul already split its rows over parallel_units. Each unit now runs the same serial loop it ran before (conv1d with cout = 1 on that channel's weights, bias and output row; one softmax_row / layer_norm per row), so the result is bit-identical by construction. Co-Authored-By: Claude Opus 5.5 --- npu/audio_exec.cpp | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/npu/audio_exec.cpp b/npu/audio_exec.cpp index 020cbdf61..a0d35168a 100644 --- a/npu/audio_exec.cpp +++ b/npu/audio_exec.cpp @@ -311,19 +311,22 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map t, con if (int64_t(scale.data.size()) != nrm) fail(where() + ": scale is not the normalised width"); r.dims = a.dims; r.data.resize(a.data.size()); - for (size_t row = 0; row < a.data.size() / size_t(nrm); ++row) { + parallel_units(a.data.size() / size_t(nrm), [&](size_t row) { // per row, as above (engine#1783) audio::layer_norm(a.data.data() + row * size_t(nrm), scale.data.data(), bias != nullptr ? bias->data.data() : nullptr, int(nrm), eps, r.data.data() + row * size_t(nrm)); - } + }); } else if (n.op == "Softmax") { const Tensor& a = need(n.ins[0]); const int64_t axis = n.attrs.count("axis") ? n.attrs.at("axis") : -1; if (axis != -1 && axis != int64_t(a.dims.size()) - 1) fail(where() + ": only the last axis is implemented"); r = a; const int64_t nrm = a.dims.back(); - for (size_t row = 0; row < r.data.size() / size_t(nrm); ++row) + // engine#1783: rows are independent and each keeps its own serial arithmetic, so splitting them over + // threads moves no bit (attention's 12 x 1500 rows of 1500 per layer were the serial term). + parallel_units(r.data.size() / size_t(nrm), [&](size_t row) { audio::softmax_row(r.data.data() + row * size_t(nrm), int(nrm)); + }); } else if (n.op == "Transpose") { const Tensor& a = need(n.ins[0]); const auto it = n.attrs_list.find("perm"); @@ -389,10 +392,17 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map t, con const int64_t tout = audio::conv1d_out_len(int(T), int(k), stride, pad); r.dims = {a.dims[0], cout, tout}; r.data.resize(size_t(a.dims[0]) * size_t(cout) * size_t(tout)); - for (int64_t n0 = 0; n0 < a.dims[0]; ++n0) - audio::conv1d(a.data.data() + size_t(n0) * size_t(cin) * size_t(T), int(cin), int(T), w.data.data(), - bias != nullptr ? bias->data.data() : nullptr, int(cout), int(k), stride, pad, - r.data.data() + size_t(n0) * size_t(cout) * size_t(tout)); + // engine#1783: one output channel per unit. conv1d with cout = 1 on that channel's weights, bias and output + // row runs exactly the loop the full call runs for it, so the result is bit-identical; the mel stem's two + // Convs were a serial 3.2 G multiply-adds per 30 s window. + const size_t units = size_t(a.dims[0]) * size_t(cout); + parallel_units(units, [&](size_t u) { + const size_t n0 = u / size_t(cout), co = u % size_t(cout); + audio::conv1d(a.data.data() + n0 * size_t(cin) * size_t(T), int(cin), int(T), + w.data.data() + co * size_t(cin) * size_t(k), + bias != nullptr ? bias->data.data() + co : nullptr, 1, int(k), stride, pad, + r.data.data() + (n0 * size_t(cout) + co) * size_t(tout)); + }); } else if (n.op == "Gather") { // The token embedding: axis 0, indices are the int64 token window. const Tensor& a = any_t(n.ins[0]); From b77ef442ade1e8311134d0f01d3910b5a99599e7 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Fri, 13 Nov 2026 06:43:19 -0400 Subject: [PATCH 3/3] docs/npu.md: the Whisper host chain's thread and op-time variables (engine#1783) Co-Authored-By: Claude Opus 5.5 --- docs/npu.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/npu.md b/docs/npu.md index 64c1b8dc0..507a0fe3e 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -3957,3 +3957,23 @@ and `--device hybrid` splits a single request across them ([hybrid.md](hybrid.md The NPU needs the `amdxdna` kernel driver. The [1bit OS](os-v3.md) appliance ships a kernel built with it, and the [Linux kernel](kernel.md) page covers building your own. + +## The Whisper host chain: threads and per-op times (engine#1783) + +`tools/whisper_transcribe` runs a Whisper pack's two ONNX graphs on the host (`npu/audio_exec.cpp`). Two variables +control the executor: + +- `ONEBIT_AUDIO_THREADS=` sets the number of worker threads. The default is the process's CPU affinity count, capped + at 8. MatMul splits its rows over these threads; Conv splits its output channels; Softmax and LayerNormalization split + their rows. Each unit runs the same serial loop as one thread would, so the output does not depend on the count. +- `ONEBIT_AUDIO_OP_TIMES=1` prints, after each graph run, one line per op type with its calls and total wall time, + sorted, plus the node and thread counts: + +``` +[audio-op] 447 nodes, 5783.3 ms, 4 thread(s) +[audio-op] MatMul 96 calls 3491.8 ms +[audio-op] Softmax 12 calls 509.3 ms +``` + +On a loaded box, compare one op with MatMul in the same run rather than across runs: MatMul's share moves with load +much less than the wall total does.