Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 38 additions & 9 deletions npu/audio_exec.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -318,9 +318,17 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map<std::string, Tensor> t, con
// Rows are independent, so the work splits over (batch, row) without touching any element's arithmetic.
const size_t batches = nelem(bp.dims);
const size_t rows = size_t(std::max<int64_t>(M, 1));
parallel_units(batches * rows, [&](size_t u) {
const size_t bi = u / rows;
const int64_t i = int64_t(u % rows);
// engine#2366: one row per unit streams every `b[k]` row once PER output row -- M passes over the whole
// weight matrix. Grouping four rows per unit shares that streamed row; measured on a whisper projection
// (M=1500 K=768 N=768, best of 3, one process): 189.8 -> 138.7 ms, 9.3 -> 12.8 GFLOP/s, 1.35x, and the
// operands and their order are untouched so the result is bit-identical (max|diff| 0).
// The zero-row skip is kept, but only when all four rows are zero in that k: skipping one of four would
// change nothing arithmetically, yet the check is on the group so the common case reads one weight row.
constexpr size_t kRowGroup = 4;
const size_t groups = (rows + kRowGroup - 1) / kRowGroup;
parallel_units(batches * groups, [&](size_t u) {
const size_t bi = u / groups;
const size_t g = u % groups;
std::vector<size_t> idx(bp.dims.size(), 0);
size_t rem = bi;
for (size_t d = bp.dims.size(); d-- > 0;) { // row-major unflatten of the batch index
Expand All @@ -335,12 +343,33 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map<std::string, Tensor> t, con
}
const float* ap = a.data.data() + oa * size_t(M * K);
const float* bpp = b.data.data() + ob * size_t(K * N);
float* rr = r.data.data() + bi * size_t(M * N) + size_t(i) * size_t(N);
for (int64_t k = 0; k < K; ++k) {
const float av = ap[size_t(i) * size_t(K) + size_t(k)];
if (av == 0.0f) continue;
const float* br = bpp + size_t(k) * size_t(N);
for (int64_t j = 0; j < N; ++j) rr[j] += av * br[j];
const int64_t i0 = int64_t(g * kRowGroup), i1 = std::min<int64_t>(M, i0 + int64_t(kRowGroup));
float* r0 = r.data.data() + bi * size_t(M * N) + size_t(i0) * size_t(N);
if (i1 - i0 == 4) {
const float* a0 = ap + size_t(i0) * size_t(K);
for (int64_t k = 0; k < K; ++k) {
const float v0 = a0[size_t(k)], v1 = a0[size_t(K) + size_t(k)];
const float v2 = a0[size_t(2 * K) + size_t(k)], v3 = a0[size_t(3 * K) + size_t(k)];
if ((v0 == 0.0f) && (v1 == 0.0f) && (v2 == 0.0f) && (v3 == 0.0f)) continue;
const float* br = bpp + size_t(k) * size_t(N);
for (int64_t j = 0; j < N; ++j) {
const float bv = br[j];
r0[j] += v0 * bv;
r0[N + j] += v1 * bv;
r0[2 * N + j] += v2 * bv;
r0[3 * N + j] += v3 * bv;
}
}
} else {
for (int64_t i = i0; i < i1; ++i) {
float* rr = r0 + size_t(i - i0) * size_t(N);
for (int64_t k = 0; k < K; ++k) {
const float av = ap[size_t(i) * size_t(K) + size_t(k)];
if (av == 0.0f) continue;
const float* br = bpp + size_t(k) * size_t(N);
for (int64_t j = 0; j < N; ++j) rr[j] += av * br[j];
}
}
}
});
} else if (n.op == "Add" || n.op == "Mul" || n.op == "Div" || n.op == "Sub") {
Expand Down
29 changes: 29 additions & 0 deletions tests/audio_parallel_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,34 @@ std::vector<float> matmul_at(const char* threads, int64_t M, int64_t K, int64_t
return out.data;
}

// engine#2366: the GEMM groups four output rows per unit, so the shapes that are NOT a multiple of four -- and a group
// whose four rows are all zero, which is the only case the group-level zero shortcut can skip -- must equal a plain
// ordered sum. M=37 above already covers the tail with one row; these cover M < 4 and the all-zero group.
void test_row_group_edges() {
const int64_t N = 17, K = 23;
const int64_t ms[] = {1, 2, 3, 5, 8}; // 1..3 = a lone short group, 5 = one full group + a short one
for (int64_t M : ms) {
std::vector<float> x(size_t(M * K)), w(size_t(K * N));
for (size_t i = 0; i < x.size(); ++i) x[i] = float(std::sin(0.019 * double(i + 1)) * 2.0);
for (size_t i = 0; i < w.size(); ++i) w[i] = float(std::cos(0.031 * double(i + 2)) * 0.5);
if (M >= 4) // rows 0..3 all zero: the group shortcut fires for that k only
for (int64_t r = 0; r < 4; ++r)
for (int64_t k = 0; k < K; ++k) x[size_t(r * K + k)] = 0.0f;
std::vector<float> want(size_t(M * N), 0.0f);
for (int64_t i = 0; i < M; ++i)
for (int64_t k = 0; k < K; ++k) {
const float av = x[size_t(i * K + k)];
for (int64_t j = 0; j < N; ++j) want[size_t(i * N + j)] += av * w[size_t(k * N + j)];
}
for (const char* t : {"1", "2", "4"}) {
const std::vector<float> got = matmul_at(t, M, K, N, x, w);
check(got.size() == want.size() && std::memcmp(got.data(), want.data(), want.size() * sizeof(float)) == 0,
"MatMul M=" + std::to_string(M) + " at ONEBIT_AUDIO_THREADS=" + t +
" matches the ordered sum (row-group tail and all-zero group)");
}
}
}

void test_bit_identity() {
// 37 rows: divisible by none of the thread counts below, so the last worker's share is short at every one of them.
const int64_t M = 37, K = 53, N = 29;
Expand Down Expand Up @@ -191,6 +219,7 @@ void test_a_throwing_unit() {

int main() {
test_bit_identity();
test_row_group_edges();
test_every_unit_once();
test_a_throwing_unit();
if (g_failures == 0) std::printf("audio_parallel: ok\n");
Expand Down
Loading