From ea5ea3b1cce60f76aeedfbc0ce4f2fbd8909f394 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Fri, 13 Nov 2026 19:22:43 -0400 Subject: [PATCH] audio_exec: split the elementwise ops and Transpose over the executor's threads, on top of #2383's flat paths (refs #2366) Rebuilt on main 6605b86, which carries #2383's flat paths in broadcast_binary. Those stay as the first two cases of broadcast_apply: equal dims and a scalar operand. They now run in kElemUnit ranges on parallel_units. Every other broadcast (bias rows, columns) takes a row loop: a row's source offsets are computed once, a stride-0 operand is read once per row, and row blocks are split over threads. Erf, Gelu and Transpose split their output into ranges. Each element is still the serial loop's one float operation on the same operands, so the output is bit-identical at any thread count. tests/audio_parallel_test.cpp: an Add (bias row), Mul (scalar), Sub (column on the left), Div (equal shape), Erf, Transpose chain at 301x193, memcmp against the plain serial arithmetic at 1-64 threads. Mutants of the equal-shape path, the stride-0 branch, the broadcast stride and the Transpose start index each fail it. Co-Authored-By: Claude Opus 5.5 --- docs/npu.md | 4 +- npu/audio_exec.cpp | 154 +++++++++++++++++++++++----------- tests/audio_parallel_test.cpp | 82 +++++++++++++++++- 3 files changed, 190 insertions(+), 50 deletions(-) diff --git a/docs/npu.md b/docs/npu.md index 6947d3c33..1ca04077c 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -3969,7 +3969,9 @@ control the executor: - `ONEBIT_AUDIO_THREADS=` sets the number of worker threads. The default is the process's CPU affinity count, capped at 8. MatMul splits its rows over these threads; Conv splits its output channels; Softmax and LayerNormalization split - their rows. Each unit runs the same serial loop as one thread would, so the output does not depend on the count. + their rows. The elementwise ops (Add, Sub, Mul and Div with their broadcasts, Erf, Gelu) and Transpose split their + output into ranges (engine#2366). Each unit runs the same serial loop as one thread would, so the output does not + depend on the count. `tests/audio_parallel_test.cpp` checks that byte for byte for MatMul and the elementwise chain. - `ONEBIT_AUDIO_OP_TIMES=1` prints, after each graph run, one line per op type with its calls and total wall time, sorted, plus the node and thread counts: diff --git a/npu/audio_exec.cpp b/npu/audio_exec.cpp index 44cda2422..2a518ed9c 100644 --- a/npu/audio_exec.cpp +++ b/npu/audio_exec.cpp @@ -97,44 +97,96 @@ size_t bcast_index(const std::vector& od, const std::vector& s return off; } +// Elements per parallel unit of the elementwise loops below: large enough that a unit is not dominated by the fork, small +// enough that a [1, 1500, 768] activation splits into ~70 units. +constexpr size_t kElemUnit = 16384; + +// engine#2366: one row (the output's last axis) at a time, the row's source offsets computed once from its index and then +// stepped by the last-axis strides, rows split over threads. Each element is still the one float operation of the serial +// loop on the same two operands, so the result is bit-identical at any thread count (tests/audio_parallel_test.cpp). +template +void broadcast_rows(const Tensor& a, const Tensor& b, const Broadcast& p, Tensor* out, Op op) { + const size_t rank = p.dims.size(); + const size_t width = rank == 0 ? 1 : size_t(p.dims[rank - 1]); + const size_t rows = width == 0 ? 0 : out->data.size() / width; + const size_t sa = rank == 0 ? 0 : p.stride_big[rank - 1], sb = rank == 0 ? 0 : p.stride_small[rank - 1]; + const size_t rows_per_unit = std::max(1, kElemUnit / std::max(1, width)); + const float* ad = a.data.data(); + const float* bd = b.data.data(); + float* od = out->data.data(); + parallel_units((rows + rows_per_unit - 1) / rows_per_unit, [&](size_t u) { + const size_t r0 = u * rows_per_unit, r1 = std::min(rows, r0 + rows_per_unit); + for (size_t row = r0; row < r1; ++row) { + size_t ia = 0, ib = 0, rem = row; + for (size_t d = rank > 0 ? rank - 1 : 0; d-- > 0;) { // the row's index on every axis but the last + const size_t i = rem % size_t(p.dims[d]); + rem /= size_t(p.dims[d]); + ia += i * p.stride_big[d]; + ib += i * p.stride_small[d]; + } + float* o = od + row * width; + // A last-axis stride of 0 (a scalar, a column) is one value for the whole row: read it once, which lets the + // loop vectorise (scalar Mul on 1x1500x3072, one thread: 1.39 -> 0.6 ms). Same operation per element. + if (sb == 0) { + const float y = bd[ib]; + for (size_t j = 0; j < width; ++j) o[j] = op(ad[ia + j * sa], y); + } else if (sa == 0) { + const float x = ad[ia]; + for (size_t j = 0; j < width; ++j) o[j] = op(x, bd[ib + j * sb]); + } else { + for (size_t j = 0; j < width; ++j) o[j] = op(ad[ia + j * sa], bd[ib + j * sb]); + } + } + }); +} + +// engine#2366: equal dims (residual adds, the GELU chain) and a scalar operand need no index arithmetic at all, so they +// take a flat loop (#2383: 66.76 -> 7.10 ms for Add/Mul/Div on 1x1500x3072). Every other broadcast takes the row loop +// above. Both split over the executor's threads in kElemUnit ranges or row blocks, and each element is the serial loop's +// one float operation on the same operands, so the output is bit-identical at any thread count. +template +void broadcast_apply(const Tensor& a, const Tensor& b, const Broadcast& p, Tensor* out, Op op) { + const size_t n = out->data.size(); + const float* ad = a.data.data(); + const float* bd = b.data.data(); + float* od = out->data.data(); + const bool equal = a.dims == p.dims && b.dims == p.dims; + if (equal || b.data.size() == 1) { + parallel_units((n + kElemUnit - 1) / kElemUnit, [&](size_t u) { + const size_t i1 = std::min(n, (u + 1) * kElemUnit); + if (equal) { + for (size_t i = u * kElemUnit; i < i1; ++i) od[i] = op(ad[i], bd[i]); + } else { + const float y = bd[0]; + for (size_t i = u * kElemUnit; i < i1; ++i) od[i] = op(ad[i], y); + } + }); + return; + } + broadcast_rows(a, b, p, out, op); +} + void broadcast_binary(const Tensor& a, const Tensor& b, const std::string& op, Tensor* out) { const Broadcast p = plan_broadcast(a.dims, b.dims); out->dims = p.dims; out->data.resize(nelem(p.dims)); - // engine#2366: 165 of the whisper encoder's nodes are one of these ops, 3.6 s of it, and the loop below does two dot - // products over the dims plus a carry step PER ELEMENT -- and compares the op name with std::string, twice, per - // element. For the two shapes that dominate the graph (equal dims: residual adds and the GELU chain; a scalar) there is - // no index arithmetic at all. Measured on the widest elementwise shape (1x1500x3072, 20 reps each, one process so load - // cancels): 66.76 ms -> 7.10 ms for Add/Mul/Div, **9.40x**, and the outputs are byte-identical because nothing about - // the arithmetic changes -- only where the indices come from. - const int kind = op == "Add" ? 0 : op == "Sub" ? 1 : op == "Mul" ? 2 : 3; - auto apply = [kind](float x, float y) { - return kind == 0 ? x + y : kind == 1 ? x - y : kind == 2 ? x * y : x / y; - }; - const size_t n = out->data.size(); - if (a.dims == p.dims && b.dims == p.dims) { - for (size_t o = 0; o < n; ++o) out->data[o] = apply(a.data[o], b.data[o]); - return; - } - if (b.data.size() == 1) { - const float y = b.data[0]; - for (size_t o = 0; o < n; ++o) out->data[o] = apply(a.data[o], y); - return; - } - std::vector idx(p.dims.size(), 0); - for (size_t o = 0; o < out->data.size(); ++o) { - size_t ia = 0, ib = 0; - for (size_t d = 0; d < idx.size(); ++d) { - ia += idx[d] * p.stride_big[d]; - ib += idx[d] * p.stride_small[d]; - } - const float x = a.data[ia], y = b.data[ib]; - out->data[o] = apply(x, y); - for (size_t d = idx.size(); d-- > 0;) { - if (++idx[d] < size_t(p.dims[d])) break; - idx[d] = 0; - } - } + if (op == "Add") broadcast_apply(a, b, p, out, [](float x, float y) { return x + y; }); + else if (op == "Sub") broadcast_apply(a, b, p, out, [](float x, float y) { return x - y; }); + else if (op == "Mul") broadcast_apply(a, b, p, out, [](float x, float y) { return x * y; }); + else broadcast_apply(a, b, p, out, [](float x, float y) { return x / y; }); +} + +// y[i] = f(x[i]) over kElemUnit-sized ranges on the executor's threads (engine#2366); per element the serial arithmetic. +template +void map_elements(const std::vector& x, std::vector* y, Fn f) { + y->resize(x.size()); + const float* xd = x.data(); + float* yd = y->data(); + const size_t n = x.size(); + parallel_units((n + kElemUnit - 1) / kElemUnit, [&](size_t u) { + const size_t i1 = std::min(n, (u + 1) * kElemUnit); + for (size_t i = u * kElemUnit; i < i1; ++i) yd[i] = f(xd[i]); + }); } std::vector dims_of(const OnnxInit& t) { return t.dims; } @@ -403,15 +455,13 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map t, con } else if (n.op == "Erf") { const Tensor& a = need(n.ins[0]); r.dims = a.dims; - r.data.resize(a.data.size()); - for (size_t i = 0; i < a.data.size(); ++i) r.data[i] = std::erf(a.data[i]); + map_elements(a.data, &r.data, [](float v) { return std::erf(v); }); } else if (n.op == "Gelu") { const Tensor& a = need(n.ins[0]); r.dims = a.dims; - r.data.resize(a.data.size()); const auto ap = n.attrs.find("approximate"); (void)ap; // ONNX spells it as a string; the export here is 'none', which is what audio_ops::gelu_erf is - for (size_t i = 0; i < a.data.size(); ++i) r.data[i] = audio::gelu_erf(a.data[i]); + map_elements(a.data, &r.data, [](float v) { return audio::gelu_erf(v); }); } else if (n.op == "LayerNormalization") { const Tensor& a = need(n.ins[0]); const Tensor& scale = init_f32(n.ins[1]); @@ -455,16 +505,26 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map t, con in_stride[i - 1] = in_stride[i] * size_t(a.dims[i]); out_stride[i - 1] = out_stride[i] * size_t(r.dims[i]); } - std::vector idx(perm.size(), 0); - for (size_t o = 0; o < r.data.size(); ++o) { - size_t src = 0; - for (size_t i = 0; i < idx.size(); ++i) src += idx[i] * in_stride[size_t(perm[i])]; - r.data[o] = a.data[src]; - for (size_t d = idx.size(); d-- > 0;) { - if (++idx[d] < size_t(r.dims[d])) break; - idx[d] = 0; + // engine#2366: kElemUnit-sized output ranges on the executor's threads, each starting its index walk from + // its first element. A copy, so any split is bit-identical. + const size_t total = r.data.size(); + parallel_units((total + kElemUnit - 1) / kElemUnit, [&](size_t u) { + const size_t o0 = u * kElemUnit, o1 = std::min(total, o0 + kElemUnit); + std::vector idx(perm.size(), 0); + for (size_t d = 0, rem = o0; d < idx.size(); ++d) { + idx[d] = rem / out_stride[d]; + rem %= out_stride[d]; } - } + for (size_t o = o0; o < o1; ++o) { + size_t src = 0; + for (size_t i = 0; i < idx.size(); ++i) src += idx[i] * in_stride[size_t(perm[i])]; + r.data[o] = a.data[src]; + for (size_t d = idx.size(); d-- > 0;) { + if (++idx[d] < size_t(r.dims[d])) break; + idx[d] = 0; + } + } + }); } else if (n.op == "Reshape") { const Tensor& a = need(n.ins[0]); std::vector want; diff --git a/tests/audio_parallel_test.cpp b/tests/audio_parallel_test.cpp index 15f2bc415..c8fab16c8 100644 --- a/tests/audio_parallel_test.cpp +++ b/tests/audio_parallel_test.cpp @@ -16,11 +16,12 @@ // audio_parallel_test.cpp -- the two claims npu/audio_parallel.h makes, as assertions (engine#1931 review): // // 1. the executor's MatMul is BIT-identical whatever ONEBIT_AUDIO_THREADS says. A tolerance cannot see this, and the -// pack-gated comparison against onnxruntime runs at one thread count, so here the same MatMul runs through +// pack-gated comparison against onnxruntime runs at one thread count, so here the same MatMul (and, for engine#2366, +// an Add/Mul/Sub/Div/Erf/Transpose chain) runs through // run_audio_graph() at 1, 2, 3, 7, 8 and 64 threads and unset, and the outputs are compared with memcmp; // 2. a unit that throws is an exception on the calling thread after every worker has been joined, not std::terminate. // -// The graph is built in memory (one MatMul against an inline initializer), so this needs no pack, no file and no device. +// The graphs are built in memory (against inline initializers), so this needs no pack, no file and no device. #include #include @@ -160,6 +161,82 @@ void test_bit_identity() { unsetenv("ONEBIT_AUDIO_THREADS"); } +// engine#2366: the elementwise families (Add/Sub/Mul/Div with each broadcast form the encoder uses, Erf) and Transpose +// split over threads too. The chain runs at several thread counts and is compared byte for byte with the plain serial +// arithmetic written out here. 301 x 193 spans several units, and the 193-wide rows do not divide a unit. +std::vector chain_at(const char* threads, int64_t M, int64_t K, const std::vector& x, + const std::vector& bias, const std::vector& col, const float& scale) { + if (threads != nullptr) setenv("ONEBIT_AUDIO_THREADS", threads, 1); + else unsetenv("ONEBIT_AUDIO_THREADS"); + OnnxGraphInfo g; + g.dir = "."; + g.inputs = {"x"}; + g.outputs = {"y"}; + auto init = [&](const char* name, std::vector dims, const float* data, size_t n) { + OnnxInit t; + t.name = name; + t.dtype = 1; + t.dims = std::move(dims); + t.raw = reinterpret_cast(data); + t.raw_len = n * sizeof(float); + g.initializers.push_back(name); + g.init_tensors.push_back(t); + }; + init("B", {K}, bias.data(), bias.size()); // a bias row: broadcast over the leading axes + init("S", {}, &scale, 1); // a scalar + init("C", {M, 1}, col.data(), col.size()); // a column: broadcast along the last axis + auto node = [&](const char* op, std::vector ins, const char* out) { + OnnxNodeInfo n; + n.op = op; + n.name = out; + n.ins = std::move(ins); + n.outs = {out}; + g.nodes.push_back(n); + return &g.nodes.back(); + }; + node("Add", {"x", "B"}, "a"); + node("Mul", {"a", "S"}, "m"); + node("Sub", {"C", "m"}, "s"); // the column first: its last-axis stride is 0 on the left + node("Div", {"s", "x"}, "d"); + node("Erf", {"d"}, "e"); + node("Transpose", {"e"}, "y")->attrs_list["perm"] = {0, 2, 1}; + AudioRunResult out; + std::string why; + if (!run_audio_graph(g, x.data(), M, K, "y", &out, &why)) { + std::printf("FAIL elementwise chain at ONEBIT_AUDIO_THREADS=%s refused: %s\n", threads ? threads : "(unset)", + why.c_str()); + ++g_failures; + return {}; + } + check(out.dims == std::vector({1, K, M}), "the chain's Transpose gives (1, K, M)"); + return out.data; +} + +void test_elementwise_bit_identity() { + const int64_t M = 301, K = 193; + std::vector x(size_t(M * K)), bias(static_cast(K)), col(static_cast(M)); + for (size_t i = 0; i < x.size(); ++i) x[i] = float(std::sin(0.29 * double(i + 1)) * 2.0 + 0.01); + for (size_t i = 0; i < bias.size(); ++i) bias[i] = float(std::cos(0.7 * double(i)) * 0.5); + for (size_t i = 0; i < col.size(); ++i) col[i] = float(0.001 * double(i) - 0.1); + const float scale = 0.125f; + + std::vector want(x.size()); + for (int64_t i = 0; i < M; ++i) + for (int64_t k = 0; k < K; ++k) { + const float xv = x[size_t(i * K + k)]; + const float d = (col[size_t(i)] - (xv + bias[size_t(k)]) * scale) / xv; + want[size_t(k * M + i)] = std::erf(d); + } + for (const char* threads : {"1", "2", "3", "7", "8", "64", static_cast(nullptr)}) { + const std::vector got = chain_at(threads, M, K, x, bias, col, scale); + const std::string name = threads ? threads : "(unset)"; + check(got.size() == want.size() && std::memcmp(got.data(), want.data(), want.size() * sizeof(float)) == 0, + "Add/Mul/Sub/Div/Erf/Transpose at ONEBIT_AUDIO_THREADS=" + name + + " is the serial per-element arithmetic, bit for bit"); + } + unsetenv("ONEBIT_AUDIO_THREADS"); +} + void test_every_unit_once() { for (size_t units : {size_t(0), size_t(1), size_t(2), size_t(7), size_t(37), size_t(1000)}) for (size_t want : {size_t(0), size_t(1), size_t(2), size_t(3), size_t(8), size_t(64), size_t(5000)}) { @@ -220,6 +297,7 @@ void test_a_throwing_unit() { int main() { test_bit_identity(); test_row_group_edges(); + test_elementwise_bit_identity(); test_every_unit_once(); test_a_throwing_unit(); if (g_failures == 0) std::printf("audio_parallel: ok\n");