diff --git a/docs/npu.md b/docs/npu.md index 6947d3c33..1ca04077c 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -3969,7 +3969,9 @@ control the executor: - `ONEBIT_AUDIO_THREADS=` sets the number of worker threads. The default is the process's CPU affinity count, capped at 8. MatMul splits its rows over these threads; Conv splits its output channels; Softmax and LayerNormalization split - their rows. Each unit runs the same serial loop as one thread would, so the output does not depend on the count. + their rows. The elementwise ops (Add, Sub, Mul and Div with their broadcasts, Erf, Gelu) and Transpose split their + output into ranges (engine#2366). Each unit runs the same serial loop as one thread would, so the output does not + depend on the count. `tests/audio_parallel_test.cpp` checks that byte for byte for MatMul and the elementwise chain. - `ONEBIT_AUDIO_OP_TIMES=1` prints, after each graph run, one line per op type with its calls and total wall time, sorted, plus the node and thread counts: diff --git a/npu/audio_exec.cpp b/npu/audio_exec.cpp index 44cda2422..2a518ed9c 100644 --- a/npu/audio_exec.cpp +++ b/npu/audio_exec.cpp @@ -97,44 +97,96 @@ size_t bcast_index(const std::vector& od, const std::vector& s return off; } +// Elements per parallel unit of the elementwise loops below: large enough that a unit is not dominated by the fork, small +// enough that a [1, 1500, 768] activation splits into ~70 units. +constexpr size_t kElemUnit = 16384; + +// engine#2366: one row (the output's last axis) at a time, the row's source offsets computed once from its index and then +// stepped by the last-axis strides, rows split over threads. Each element is still the one float operation of the serial +// loop on the same two operands, so the result is bit-identical at any thread count (tests/audio_parallel_test.cpp). +template +void broadcast_rows(const Tensor& a, const Tensor& b, const Broadcast& p, Tensor* out, Op op) { + const size_t rank = p.dims.size(); + const size_t width = rank == 0 ? 1 : size_t(p.dims[rank - 1]); + const size_t rows = width == 0 ? 0 : out->data.size() / width; + const size_t sa = rank == 0 ? 0 : p.stride_big[rank - 1], sb = rank == 0 ? 0 : p.stride_small[rank - 1]; + const size_t rows_per_unit = std::max(1, kElemUnit / std::max(1, width)); + const float* ad = a.data.data(); + const float* bd = b.data.data(); + float* od = out->data.data(); + parallel_units((rows + rows_per_unit - 1) / rows_per_unit, [&](size_t u) { + const size_t r0 = u * rows_per_unit, r1 = std::min(rows, r0 + rows_per_unit); + for (size_t row = r0; row < r1; ++row) { + size_t ia = 0, ib = 0, rem = row; + for (size_t d = rank > 0 ? rank - 1 : 0; d-- > 0;) { // the row's index on every axis but the last + const size_t i = rem % size_t(p.dims[d]); + rem /= size_t(p.dims[d]); + ia += i * p.stride_big[d]; + ib += i * p.stride_small[d]; + } + float* o = od + row * width; + // A last-axis stride of 0 (a scalar, a column) is one value for the whole row: read it once, which lets the + // loop vectorise (scalar Mul on 1x1500x3072, one thread: 1.39 -> 0.6 ms). Same operation per element. + if (sb == 0) { + const float y = bd[ib]; + for (size_t j = 0; j < width; ++j) o[j] = op(ad[ia + j * sa], y); + } else if (sa == 0) { + const float x = ad[ia]; + for (size_t j = 0; j < width; ++j) o[j] = op(x, bd[ib + j * sb]); + } else { + for (size_t j = 0; j < width; ++j) o[j] = op(ad[ia + j * sa], bd[ib + j * sb]); + } + } + }); +} + +// engine#2366: equal dims (residual adds, the GELU chain) and a scalar operand need no index arithmetic at all, so they +// take a flat loop (#2383: 66.76 -> 7.10 ms for Add/Mul/Div on 1x1500x3072). Every other broadcast takes the row loop +// above. Both split over the executor's threads in kElemUnit ranges or row blocks, and each element is the serial loop's +// one float operation on the same operands, so the output is bit-identical at any thread count. +template +void broadcast_apply(const Tensor& a, const Tensor& b, const Broadcast& p, Tensor* out, Op op) { + const size_t n = out->data.size(); + const float* ad = a.data.data(); + const float* bd = b.data.data(); + float* od = out->data.data(); + const bool equal = a.dims == p.dims && b.dims == p.dims; + if (equal || b.data.size() == 1) { + parallel_units((n + kElemUnit - 1) / kElemUnit, [&](size_t u) { + const size_t i1 = std::min(n, (u + 1) * kElemUnit); + if (equal) { + for (size_t i = u * kElemUnit; i < i1; ++i) od[i] = op(ad[i], bd[i]); + } else { + const float y = bd[0]; + for (size_t i = u * kElemUnit; i < i1; ++i) od[i] = op(ad[i], y); + } + }); + return; + } + broadcast_rows(a, b, p, out, op); +} + void broadcast_binary(const Tensor& a, const Tensor& b, const std::string& op, Tensor* out) { const Broadcast p = plan_broadcast(a.dims, b.dims); out->dims = p.dims; out->data.resize(nelem(p.dims)); - // engine#2366: 165 of the whisper encoder's nodes are one of these ops, 3.6 s of it, and the loop below does two dot - // products over the dims plus a carry step PER ELEMENT -- and compares the op name with std::string, twice, per - // element. For the two shapes that dominate the graph (equal dims: residual adds and the GELU chain; a scalar) there is - // no index arithmetic at all. Measured on the widest elementwise shape (1x1500x3072, 20 reps each, one process so load - // cancels): 66.76 ms -> 7.10 ms for Add/Mul/Div, **9.40x**, and the outputs are byte-identical because nothing about - // the arithmetic changes -- only where the indices come from. - const int kind = op == "Add" ? 0 : op == "Sub" ? 1 : op == "Mul" ? 2 : 3; - auto apply = [kind](float x, float y) { - return kind == 0 ? x + y : kind == 1 ? x - y : kind == 2 ? x * y : x / y; - }; - const size_t n = out->data.size(); - if (a.dims == p.dims && b.dims == p.dims) { - for (size_t o = 0; o < n; ++o) out->data[o] = apply(a.data[o], b.data[o]); - return; - } - if (b.data.size() == 1) { - const float y = b.data[0]; - for (size_t o = 0; o < n; ++o) out->data[o] = apply(a.data[o], y); - return; - } - std::vector idx(p.dims.size(), 0); - for (size_t o = 0; o < out->data.size(); ++o) { - size_t ia = 0, ib = 0; - for (size_t d = 0; d < idx.size(); ++d) { - ia += idx[d] * p.stride_big[d]; - ib += idx[d] * p.stride_small[d]; - } - const float x = a.data[ia], y = b.data[ib]; - out->data[o] = apply(x, y); - for (size_t d = idx.size(); d-- > 0;) { - if (++idx[d] < size_t(p.dims[d])) break; - idx[d] = 0; - } - } + if (op == "Add") broadcast_apply(a, b, p, out, [](float x, float y) { return x + y; }); + else if (op == "Sub") broadcast_apply(a, b, p, out, [](float x, float y) { return x - y; }); + else if (op == "Mul") broadcast_apply(a, b, p, out, [](float x, float y) { return x * y; }); + else broadcast_apply(a, b, p, out, [](float x, float y) { return x / y; }); +} + +// y[i] = f(x[i]) over kElemUnit-sized ranges on the executor's threads (engine#2366); per element the serial arithmetic. +template +void map_elements(const std::vector& x, std::vector* y, Fn f) { + y->resize(x.size()); + const float* xd = x.data(); + float* yd = y->data(); + const size_t n = x.size(); + parallel_units((n + kElemUnit - 1) / kElemUnit, [&](size_t u) { + const size_t i1 = std::min(n, (u + 1) * kElemUnit); + for (size_t i = u * kElemUnit; i < i1; ++i) yd[i] = f(xd[i]); + }); } std::vector dims_of(const OnnxInit& t) { return t.dims; } @@ -403,15 +455,13 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map t, con } else if (n.op == "Erf") { const Tensor& a = need(n.ins[0]); r.dims = a.dims; - r.data.resize(a.data.size()); - for (size_t i = 0; i < a.data.size(); ++i) r.data[i] = std::erf(a.data[i]); + map_elements(a.data, &r.data, [](float v) { return std::erf(v); }); } else if (n.op == "Gelu") { const Tensor& a = need(n.ins[0]); r.dims = a.dims; - r.data.resize(a.data.size()); const auto ap = n.attrs.find("approximate"); (void)ap; // ONNX spells it as a string; the export here is 'none', which is what audio_ops::gelu_erf is - for (size_t i = 0; i < a.data.size(); ++i) r.data[i] = audio::gelu_erf(a.data[i]); + map_elements(a.data, &r.data, [](float v) { return audio::gelu_erf(v); }); } else if (n.op == "LayerNormalization") { const Tensor& a = need(n.ins[0]); const Tensor& scale = init_f32(n.ins[1]); @@ -455,16 +505,26 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map t, con in_stride[i - 1] = in_stride[i] * size_t(a.dims[i]); out_stride[i - 1] = out_stride[i] * size_t(r.dims[i]); } - std::vector idx(perm.size(), 0); - for (size_t o = 0; o < r.data.size(); ++o) { - size_t src = 0; - for (size_t i = 0; i < idx.size(); ++i) src += idx[i] * in_stride[size_t(perm[i])]; - r.data[o] = a.data[src]; - for (size_t d = idx.size(); d-- > 0;) { - if (++idx[d] < size_t(r.dims[d])) break; - idx[d] = 0; + // engine#2366: kElemUnit-sized output ranges on the executor's threads, each starting its index walk from + // its first element. A copy, so any split is bit-identical. + const size_t total = r.data.size(); + parallel_units((total + kElemUnit - 1) / kElemUnit, [&](size_t u) { + const size_t o0 = u * kElemUnit, o1 = std::min(total, o0 + kElemUnit); + std::vector idx(perm.size(), 0); + for (size_t d = 0, rem = o0; d < idx.size(); ++d) { + idx[d] = rem / out_stride[d]; + rem %= out_stride[d]; } - } + for (size_t o = o0; o < o1; ++o) { + size_t src = 0; + for (size_t i = 0; i < idx.size(); ++i) src += idx[i] * in_stride[size_t(perm[i])]; + r.data[o] = a.data[src]; + for (size_t d = idx.size(); d-- > 0;) { + if (++idx[d] < size_t(r.dims[d])) break; + idx[d] = 0; + } + } + }); } else if (n.op == "Reshape") { const Tensor& a = need(n.ins[0]); std::vector want; diff --git a/tests/audio_parallel_test.cpp b/tests/audio_parallel_test.cpp index 15f2bc415..c8fab16c8 100644 --- a/tests/audio_parallel_test.cpp +++ b/tests/audio_parallel_test.cpp @@ -16,11 +16,12 @@ // audio_parallel_test.cpp -- the two claims npu/audio_parallel.h makes, as assertions (engine#1931 review): // // 1. the executor's MatMul is BIT-identical whatever ONEBIT_AUDIO_THREADS says. A tolerance cannot see this, and the -// pack-gated comparison against onnxruntime runs at one thread count, so here the same MatMul runs through +// pack-gated comparison against onnxruntime runs at one thread count, so here the same MatMul (and, for engine#2366, +// an Add/Mul/Sub/Div/Erf/Transpose chain) runs through // run_audio_graph() at 1, 2, 3, 7, 8 and 64 threads and unset, and the outputs are compared with memcmp; // 2. a unit that throws is an exception on the calling thread after every worker has been joined, not std::terminate. // -// The graph is built in memory (one MatMul against an inline initializer), so this needs no pack, no file and no device. +// The graphs are built in memory (against inline initializers), so this needs no pack, no file and no device. #include #include @@ -160,6 +161,82 @@ void test_bit_identity() { unsetenv("ONEBIT_AUDIO_THREADS"); } +// engine#2366: the elementwise families (Add/Sub/Mul/Div with each broadcast form the encoder uses, Erf) and Transpose +// split over threads too. The chain runs at several thread counts and is compared byte for byte with the plain serial +// arithmetic written out here. 301 x 193 spans several units, and the 193-wide rows do not divide a unit. +std::vector chain_at(const char* threads, int64_t M, int64_t K, const std::vector& x, + const std::vector& bias, const std::vector& col, const float& scale) { + if (threads != nullptr) setenv("ONEBIT_AUDIO_THREADS", threads, 1); + else unsetenv("ONEBIT_AUDIO_THREADS"); + OnnxGraphInfo g; + g.dir = "."; + g.inputs = {"x"}; + g.outputs = {"y"}; + auto init = [&](const char* name, std::vector dims, const float* data, size_t n) { + OnnxInit t; + t.name = name; + t.dtype = 1; + t.dims = std::move(dims); + t.raw = reinterpret_cast(data); + t.raw_len = n * sizeof(float); + g.initializers.push_back(name); + g.init_tensors.push_back(t); + }; + init("B", {K}, bias.data(), bias.size()); // a bias row: broadcast over the leading axes + init("S", {}, &scale, 1); // a scalar + init("C", {M, 1}, col.data(), col.size()); // a column: broadcast along the last axis + auto node = [&](const char* op, std::vector ins, const char* out) { + OnnxNodeInfo n; + n.op = op; + n.name = out; + n.ins = std::move(ins); + n.outs = {out}; + g.nodes.push_back(n); + return &g.nodes.back(); + }; + node("Add", {"x", "B"}, "a"); + node("Mul", {"a", "S"}, "m"); + node("Sub", {"C", "m"}, "s"); // the column first: its last-axis stride is 0 on the left + node("Div", {"s", "x"}, "d"); + node("Erf", {"d"}, "e"); + node("Transpose", {"e"}, "y")->attrs_list["perm"] = {0, 2, 1}; + AudioRunResult out; + std::string why; + if (!run_audio_graph(g, x.data(), M, K, "y", &out, &why)) { + std::printf("FAIL elementwise chain at ONEBIT_AUDIO_THREADS=%s refused: %s\n", threads ? threads : "(unset)", + why.c_str()); + ++g_failures; + return {}; + } + check(out.dims == std::vector({1, K, M}), "the chain's Transpose gives (1, K, M)"); + return out.data; +} + +void test_elementwise_bit_identity() { + const int64_t M = 301, K = 193; + std::vector x(size_t(M * K)), bias(static_cast(K)), col(static_cast(M)); + for (size_t i = 0; i < x.size(); ++i) x[i] = float(std::sin(0.29 * double(i + 1)) * 2.0 + 0.01); + for (size_t i = 0; i < bias.size(); ++i) bias[i] = float(std::cos(0.7 * double(i)) * 0.5); + for (size_t i = 0; i < col.size(); ++i) col[i] = float(0.001 * double(i) - 0.1); + const float scale = 0.125f; + + std::vector want(x.size()); + for (int64_t i = 0; i < M; ++i) + for (int64_t k = 0; k < K; ++k) { + const float xv = x[size_t(i * K + k)]; + const float d = (col[size_t(i)] - (xv + bias[size_t(k)]) * scale) / xv; + want[size_t(k * M + i)] = std::erf(d); + } + for (const char* threads : {"1", "2", "3", "7", "8", "64", static_cast(nullptr)}) { + const std::vector got = chain_at(threads, M, K, x, bias, col, scale); + const std::string name = threads ? threads : "(unset)"; + check(got.size() == want.size() && std::memcmp(got.data(), want.data(), want.size() * sizeof(float)) == 0, + "Add/Mul/Sub/Div/Erf/Transpose at ONEBIT_AUDIO_THREADS=" + name + + " is the serial per-element arithmetic, bit for bit"); + } + unsetenv("ONEBIT_AUDIO_THREADS"); +} + void test_every_unit_once() { for (size_t units : {size_t(0), size_t(1), size_t(2), size_t(7), size_t(37), size_t(1000)}) for (size_t want : {size_t(0), size_t(1), size_t(2), size_t(3), size_t(8), size_t(64), size_t(5000)}) { @@ -220,6 +297,7 @@ void test_a_throwing_unit() { int main() { test_bit_identity(); test_row_group_edges(); + test_elementwise_bit_identity(); test_every_unit_once(); test_a_throwing_unit(); if (g_failures == 0) std::printf("audio_parallel: ok\n");