Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions docs/npu.md
Original file line number Diff line number Diff line change
Expand Up @@ -3957,3 +3957,23 @@ and `--device hybrid` splits a single request across them ([hybrid.md](hybrid.md

The NPU needs the `amdxdna` kernel driver. The [1bit OS](os-v3.md) appliance ships a kernel built
with it, and the [Linux kernel](kernel.md) page covers building your own.

## The Whisper host chain: threads and per-op times (engine#1783)

`tools/whisper_transcribe` runs a Whisper pack's two ONNX graphs on the host (`npu/audio_exec.cpp`). Two variables
control the executor:

- `ONEBIT_AUDIO_THREADS=<n>` sets the number of worker threads. The default is the process's CPU affinity count, capped
at 8. MatMul splits its rows over these threads; Conv splits its output channels; Softmax and LayerNormalization split
their rows. Each unit runs the same serial loop as one thread would, so the output does not depend on the count.
- `ONEBIT_AUDIO_OP_TIMES=1` prints, after each graph run, one line per op type with its calls and total wall time,
sorted, plus the node and thread counts:

```
[audio-op] 447 nodes, 5783.3 ms, 4 thread(s)
[audio-op] MatMul 96 calls 3491.8 ms
[audio-op] Softmax 12 calls 509.3 ms
```

On a loaded box, compare one op with MatMul in the same run rather than across runs: MatMul's share moves with load
much less than the wall total does.
47 changes: 40 additions & 7 deletions npu/audio_exec.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@
#include "audio_parallel.h"

#include <cmath>
#include <algorithm>
#include <chrono>
#include <cstdio>
#include <cstring>
#include <map>
#include <set>
Expand Down Expand Up @@ -221,9 +224,15 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map<std::string, Tensor> t, con
return t.count(name) != 0 ? need(name) : init_f32(name);
};

// ONEBIT_AUDIO_OP_TIMES=1 (engine#1783): wall time per op type over this graph, printed once at the end, so a
// host split ("Conv 3.74 s, Softmax 2.56 s") is read from the executor that runs the window rather than inferred.
const char* ot = std::getenv("ONEBIT_AUDIO_OP_TIMES");
const bool op_times = ot != nullptr && ot[0] != '\0' && ot[0] != '0';
std::map<std::string, std::pair<double, long>> op_ms; // op -> (ms, calls)
int node_index = 0;
for (const auto& n : g.nodes) {
++node_index;
const auto op_t0 = std::chrono::steady_clock::now();
const auto where = [&]() { return "node " + std::to_string(node_index) + " (" + n.op + ")"; };
if (n.outs.empty()) fail(where() + " has no output");
const std::string& oname = n.outs[0];
Expand Down Expand Up @@ -302,19 +311,22 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map<std::string, Tensor> t, con
if (int64_t(scale.data.size()) != nrm) fail(where() + ": scale is not the normalised width");
r.dims = a.dims;
r.data.resize(a.data.size());
for (size_t row = 0; row < a.data.size() / size_t(nrm); ++row) {
parallel_units(a.data.size() / size_t(nrm), [&](size_t row) { // per row, as above (engine#1783)
audio::layer_norm(a.data.data() + row * size_t(nrm), scale.data.data(),
bias != nullptr ? bias->data.data() : nullptr, int(nrm), eps,
r.data.data() + row * size_t(nrm));
}
});
} else if (n.op == "Softmax") {
const Tensor& a = need(n.ins[0]);
const int64_t axis = n.attrs.count("axis") ? n.attrs.at("axis") : -1;
if (axis != -1 && axis != int64_t(a.dims.size()) - 1) fail(where() + ": only the last axis is implemented");
r = a;
const int64_t nrm = a.dims.back();
for (size_t row = 0; row < r.data.size() / size_t(nrm); ++row)
// engine#1783: rows are independent and each keeps its own serial arithmetic, so splitting them over
// threads moves no bit (attention's 12 x 1500 rows of 1500 per layer were the serial term).
parallel_units(r.data.size() / size_t(nrm), [&](size_t row) {
audio::softmax_row(r.data.data() + row * size_t(nrm), int(nrm));
});
} else if (n.op == "Transpose") {
const Tensor& a = need(n.ins[0]);
const auto it = n.attrs_list.find("perm");
Expand Down Expand Up @@ -380,10 +392,17 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map<std::string, Tensor> t, con
const int64_t tout = audio::conv1d_out_len(int(T), int(k), stride, pad);
r.dims = {a.dims[0], cout, tout};
r.data.resize(size_t(a.dims[0]) * size_t(cout) * size_t(tout));
for (int64_t n0 = 0; n0 < a.dims[0]; ++n0)
audio::conv1d(a.data.data() + size_t(n0) * size_t(cin) * size_t(T), int(cin), int(T), w.data.data(),
bias != nullptr ? bias->data.data() : nullptr, int(cout), int(k), stride, pad,
r.data.data() + size_t(n0) * size_t(cout) * size_t(tout));
// engine#1783: one output channel per unit. conv1d with cout = 1 on that channel's weights, bias and output
// row runs exactly the loop the full call runs for it, so the result is bit-identical; the mel stem's two
// Convs were a serial 3.2 G multiply-adds per 30 s window.
const size_t units = size_t(a.dims[0]) * size_t(cout);
parallel_units(units, [&](size_t u) {
const size_t n0 = u / size_t(cout), co = u % size_t(cout);
audio::conv1d(a.data.data() + n0 * size_t(cin) * size_t(T), int(cin), int(T),
w.data.data() + co * size_t(cin) * size_t(k),
bias != nullptr ? bias->data.data() + co : nullptr, 1, int(k), stride, pad,
r.data.data() + (n0 * size_t(cout) + co) * size_t(tout));
});
} else if (n.op == "Gather") {
// The token embedding: axis 0, indices are the int64 token window.
const Tensor& a = any_t(n.ins[0]);
Expand Down Expand Up @@ -527,6 +546,20 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map<std::string, Tensor> t, con
fail(where() + " is not implemented; the pack needs an op this executor does not have");
}
t[oname] = std::move(r);
if (op_times) {
auto& e = op_ms[n.op];
e.first += std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - op_t0).count();
e.second++;
}
}
if (op_times) {
std::vector<std::pair<double, std::string>> order;
double total = 0.0;
for (const auto& kv : op_ms) { order.push_back({kv.second.first, kv.first}); total += kv.second.first; }
std::sort(order.rbegin(), order.rend());
std::fprintf(stderr, "[audio-op] %zu nodes, %.1f ms, %zu thread(s)\n", g.nodes.size(), total, audio_thread_count());
for (const auto& o : order)
std::fprintf(stderr, "[audio-op] %-20s %6ld calls %10.1f ms\n", o.second.c_str(), op_ms[o.second].second, o.first);
}

const std::string want = out_name.empty() ? (g.outputs.empty() ? std::string() : g.outputs[0]) : out_name;
Expand Down
Loading