Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 18 additions & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1588,7 +1588,7 @@ target_link_libraries(audio_parallel_test PRIVATE onebit_npu Threads::Threads)
# ci.yml's build list is not ours to edit without `workflow` scope, so these reach CI the way ops_dump_test does: as build
# dependencies of a target it already builds (tools/ci/ci_targets_check.py follows this edge).
# One line, deliberately: tools/ci/ci_targets_check.py follows `add_dependencies` line by line.
add_dependencies(onebit audio_ops_test audio_mel_test onnx_graph_test audio_exec_test audio_parallel_test audio_pack_test audio_kernel_roles_test whisper_ids_test audio_device_gemm_test audio_launch_plan_test audio_kernel_shapes whisper_transcribe)
add_dependencies(onebit audio_ops_test audio_mel_test onnx_graph_test audio_exec_test audio_parallel_test audio_pack_test audio_kernel_roles_test whisper_ids_test audio_device_gemm_test audio_npu_gemm_test audio_launch_plan_test audio_kernel_shapes whisper_transcribe)
# A DFlash draft package as a model source (engine#1780): config + container -> a checked model whose tensors decode
# one at a time. The loader is compiled here and not into onebit_npu because it needs the q4_1 tile decoder, which
# lives with the forward (npu/forward/src/q4nx_dequant.c); it moves there with the draft's forward. Synthetic package,
Expand Down Expand Up @@ -1986,13 +1986,19 @@ target_link_libraries(audio_kernel_shapes PRIVATE onebit_npu)
add_executable(audio_device_gemm_test tests/audio_device_gemm_test.cpp)
target_include_directories(audio_device_gemm_test PRIVATE npu/forward/include npu/target npu/plan)
target_link_libraries(audio_device_gemm_test PRIVATE nlohmann_json::nlohmann_json)
# engine#1783: the encoder's products on an NpuTarget (routing by staged shape, M-row chunks, padded weight sent once,
# de-pad), against a fake target.
add_executable(audio_npu_gemm_test tests/audio_npu_gemm_test.cpp)
target_include_directories(audio_npu_gemm_test PRIVATE npu/forward/include npu/target npu/plan)
target_link_libraries(audio_npu_gemm_test PRIVATE nlohmann_json::nlohmann_json)
add_executable(audio_launch_plan_test tests/audio_launch_plan_test.cpp)
target_include_directories(audio_launch_plan_test PRIVATE npu/forward/include npu/target npu/plan)
target_link_libraries(audio_launch_plan_test PRIVATE nlohmann_json::nlohmann_json)
add_test(NAME whisper_ids COMMAND whisper_ids_test)
add_test(NAME audio_kernel_roles COMMAND audio_kernel_roles_test)
add_test(NAME audio_pack COMMAND audio_pack_test)
add_test(NAME audio_device_gemm COMMAND audio_device_gemm_test)
add_test(NAME audio_npu_gemm COMMAND audio_npu_gemm_test)
add_test(NAME audio_launch_plan COMMAND audio_launch_plan_test)
add_test(NAME whisper_transcribe_jfk COMMAND whisper_transcribe)
set_tests_properties(whisper_transcribe_jfk PROPERTIES
Expand Down Expand Up @@ -2206,6 +2212,11 @@ endif()
# The library body is onebit_npu_forward_library() (defined above the runtime options), so the CI build of the
# same sources against the runtime stub (engine#1946) cannot drift from this one.
onebit_npu_forward_library(onebit_npu_forward onebit_npu_rt)
# engine#1783: whisper_transcribe with the encoder's products on the NPU (ONEBIT_WHISPER_NPU_KERNELS=<dir>).
add_executable(whisper_transcribe_npu tools/whisper_transcribe.cpp)
target_compile_definitions(whisper_transcribe_npu PRIVATE ONEBIT_WHISPER_NPU)
target_include_directories(whisper_transcribe_npu PRIVATE npu/forward/include npu/target tools)
target_link_libraries(whisper_transcribe_npu PRIVATE onebit_npu_forward onebit_npu_rt)
target_sources(onebit PRIVATE app/forward_serve.cpp app/stb_image_impl.cpp) # engine#2324: stb_image for pictures
target_include_directories(onebit PRIVATE ${stb_SOURCE_DIR})
# engine#1781: forward-serve --spec-draft runs a DFlash draft package (npu/dflash_head.h) as its drafter.
Expand Down Expand Up @@ -2330,6 +2341,12 @@ if(ONEBIT_NPU_HOST AND NOT ONEBIT_NPU AND NOT WIN32 AND Python3_Interpreter_FOUN
target_link_libraries(npu_gemm_shape_guard_test PRIVATE onebit_npu_forward_ci)
add_test(NAME npu_gemm_shape_guard COMMAND npu_gemm_shape_guard_test)
add_dependencies(onebit npu_gemm_shape_guard_test) # CI builds `onebit` only; without this the test is Not Run
# engine#1783: the device build of whisper_transcribe, compiled here against the stub so CI sees its sources.
add_executable(whisper_transcribe_npu_ci tools/whisper_transcribe.cpp)
target_compile_definitions(whisper_transcribe_npu_ci PRIVATE ONEBIT_WHISPER_NPU)
target_include_directories(whisper_transcribe_npu_ci PRIVATE npu/forward/include npu/target tools)
target_link_libraries(whisper_transcribe_npu_ci PRIVATE onebit_npu_forward_ci)
add_dependencies(onebit whisper_transcribe_npu_ci)
add_executable(batch_check_state_test tests/batch_check_state_test.cpp)
target_link_libraries(batch_check_state_test PRIVATE onebit_npu_forward_ci)
# direction B (tools/ci/ci_targets_check.py): ci.yml's --target list is a workflow file no lane can patch; the
Expand Down
92 changes: 84 additions & 8 deletions npu/audio_exec.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ void load_init_i64(const OnnxGraphInfo& g, const OnnxInit& t, std::vector<int64_
namespace {

bool run_graph_impl(const OnnxGraphInfo& g, std::map<std::string, Tensor> t, const std::string& out_name, AudioRunResult* out,
std::string* why) {
std::string* why, AudioGemm* dev = nullptr) {
try {

std::map<std::string, const OnnxInit*> inits;
Expand Down Expand Up @@ -229,6 +229,37 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map<std::string, Tensor> t, con
const char* ot = std::getenv("ONEBIT_AUDIO_OP_TIMES");
const bool op_times = ot != nullptr && ot[0] != '\0' && ot[0] != '0';
std::map<std::string, std::pair<double, long>> op_ms; // op -> (ms, calls)

// engine#1783: offer one product to the device. True when the device ran it into `c`; false when the host must, with
// the reason counted. A device without a `gemm` is no device.
auto offer = [&](const std::string& key, const float* A, int64_t M, int64_t K, const float* B, int64_t N,
std::vector<float>* c) -> bool {
if (dev == nullptr) return false;
std::string reason;
if (dev->gemm && dev->gemm(key, A, M, K, B, N, c, &reason)) {
if (c->size() != size_t(M) * size_t(N))
fail("device gemm for '" + key + "' wrote " + std::to_string(c->size()) + " floats, not " +
std::to_string(M) + " x " + std::to_string(N));
++dev->device;
return true;
}
++dev->host;
++dev->host_why[reason.empty() ? std::string("declined without a reason") : reason];
return false;
};

// Every node ends here, including the products the device ran (they leave the loop early), so ONEBIT_AUDIO_OP_TIMES
// counts a device-run MatMul or Conv under its op like a host one.
auto finish_node = [&](const std::string& op, const std::string& name, Tensor& res,
std::chrono::steady_clock::time_point t0) {
t[name] = std::move(res);
if (op_times) {
auto& e = op_ms[op];
e.first += std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count();
e.second++;
}
};

int node_index = 0;
for (const auto& n : g.nodes) {
++node_index;
Expand All @@ -251,6 +282,22 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map<std::string, Tensor> t, con
r.dims = bp.dims;
r.dims.push_back(M);
r.dims.push_back(N);
// A projection (B a rank-2 weight) is one GEMM over every row of A, whatever A's leading axes are.
if (dev != nullptr) {
if (rb == 2 && inits.count(n.ins[1]) != 0) {
std::vector<float> c;
if (offer(n.ins[1], a.data.data(), int64_t(a.data.size() / size_t(K)), K, b.data.data(), N, &c)) {
r.dims = a.dims;
r.dims.back() = N;
r.data = std::move(c);
finish_node(n.op, oname, r, op_t0);
continue;
}
} else {
++dev->host;
++dev->host_why["MatMul with an activation as B (attention product): not offered"];
}
}
r.data.assign(nelem(r.dims), 0.0f);
// Rows are independent, so the work splits over (batch, row) without touching any element's arithmetic.
const size_t batches = nelem(bp.dims);
Expand Down Expand Up @@ -392,6 +439,35 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map<std::string, Tensor> t, con
const int64_t tout = audio::conv1d_out_len(int(T), int(k), stride, pad);
r.dims = {a.dims[0], cout, tout};
r.data.resize(size_t(a.dims[0]) * size_t(cout) * size_t(tout));
const auto ga = n.attrs.find("group");
if (ga != n.attrs.end() && ga->second != 1) fail(where() + ": only group=1 is implemented");
if (dev != nullptr) {
// The im2col GEMM: (tout, cin*k) x (cin*k, cout), then out[co][t] = bias[co] + C[t][co].
const int64_t kk = cin * k;
std::vector<float> bkn(size_t(kk) * size_t(cout));
audio::conv1d_weight_as_kn(w.data.data(), int(cout), int(cin), int(k), bkn.data());
std::vector<float> acol(size_t(tout) * size_t(kk));
bool all = true;
for (int64_t n0 = 0; n0 < a.dims[0] && all; ++n0) {
audio::conv1d_im2col(a.data.data() + size_t(n0) * size_t(cin) * size_t(T), int(cin), int(T), int(k),
stride, pad, acol.data());
std::vector<float> c;
if (!offer(n.ins[1] + "#conv", acol.data(), tout, kk, bkn.data(), cout, &c)) {
all = false;
break;
}
float* o = r.data.data() + size_t(n0) * size_t(cout) * size_t(tout);
for (int64_t co = 0; co < cout; ++co) {
const float bv = bias != nullptr ? bias->data[size_t(co)] : 0.0f;
for (int64_t tt = 0; tt < tout; ++tt)
o[size_t(co) * size_t(tout) + size_t(tt)] = bv + c[size_t(tt) * size_t(cout) + size_t(co)];
}
}
if (all) {
finish_node(n.op, oname, r, op_t0);
continue;
}
}
// engine#1783: one output channel per unit. conv1d with cout = 1 on that channel's weights, bias and output
// row runs exactly the loop the full call runs for it, so the result is bit-identical; the mel stem's two
// Convs were a serial 3.2 G multiply-adds per 30 s window.
Expand Down Expand Up @@ -545,12 +621,7 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map<std::string, Tensor> t, con
} else {
fail(where() + " is not implemented; the pack needs an op this executor does not have");
}
t[oname] = std::move(r);
if (op_times) {
auto& e = op_ms[n.op];
e.first += std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - op_t0).count();
e.second++;
}
finish_node(n.op, oname, r, op_t0);
}
if (op_times) {
std::vector<std::pair<double, std::string>> order;
Expand Down Expand Up @@ -580,6 +651,11 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map<std::string, Tensor> t, con

bool run_audio_graph(const OnnxGraphInfo& g, const float* mel, int64_t mel_bins, int64_t frames, const std::string& out_name,
AudioRunResult* out, std::string* why) {
return run_audio_graph(g, mel, mel_bins, frames, out_name, out, why, nullptr);
}

bool run_audio_graph(const OnnxGraphInfo& g, const float* mel, int64_t mel_bins, int64_t frames, const std::string& out_name,
AudioRunResult* out, std::string* why, AudioGemm* dev) {
std::map<std::string, Tensor> seed;
Tensor in;
in.dims = {1, mel_bins, frames};
Expand All @@ -589,7 +665,7 @@ bool run_audio_graph(const OnnxGraphInfo& g, const float* mel, int64_t mel_bins,
return false;
}
seed[g.inputs[0]] = std::move(in);
return run_graph_impl(g, std::move(seed), out_name, out, why);
return run_graph_impl(g, std::move(seed), out_name, out, why, dev);
}

bool run_decoder_graph(const OnnxGraphInfo& g, const int64_t* tokens, int64_t n_tokens, const float* xa,
Expand Down
26 changes: 26 additions & 0 deletions npu/audio_exec.h
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,36 @@

#pragma once

#include <functional>
#include <map>
#include <string>
#include <vector>

#include "onnx_graph.h"

namespace onebit::npu::pack_import {

// A device for the graph's matrix products (engine#1783). The executor stays host-only and device-free: it offers each
// product whose right operand is a CONSTANT weight to `gemm`, and runs it on the host itself when the device declines.
//
// Offered: every MatMul whose B is a rank-2 initializer (the projections q, k, v, o, fc1, fc2), with A's leading axes
// flattened into rows, and every Conv, as its im2col GEMM (A = (tout, cin*k), B = the kernel as (cin*k, cout)).
// Not offered: a MatMul whose B is an activation (attention's q.k^T and scores.v). Its operand changes every call, so it
// has no weight to stay resident, and the reason is recorded like any other host product.
//
// `key` names the weight and is stable across calls (the initializer's name; "<name>#conv" for a Conv's transposed
// kernel), so an implementation may cache the packed weight by it. Return false with `why` to leave the product to
// the host. The executor then runs it, counts it under `host`, and keeps the reason in `host_why`, so a run whose
// products silently fell back to the host is visible in the counts.
struct AudioGemm {
std::function<bool(const std::string& key, const float* A, int64_t M, int64_t K, const float* B, int64_t N,
std::vector<float>* C, std::string* why)>
gemm;
int device = 0; // products the device ran
int host = 0; // products the host ran (declined, or not offered)
std::map<std::string, int> host_why; // reason -> count
};

struct AudioRunResult {
std::vector<int64_t> dims; // dims of the requested output
std::vector<float> data; // row-major f32 payload
Expand All @@ -45,6 +68,9 @@ struct AudioRunResult {
// Returns true and fills `out`; on failure returns false and sets `why` to the node and the reason.
bool run_audio_graph(const OnnxGraphInfo& g, const float* mel, int64_t mel_bins, int64_t frames,
const std::string& out_name, AudioRunResult* out, std::string* why);
// The same, with the matrix products offered to `dev` (null = host only, identical to the overload above).
bool run_audio_graph(const OnnxGraphInfo& g, const float* mel, int64_t mel_bins, int64_t frames,
const std::string& out_name, AudioRunResult* out, std::string* why, AudioGemm* dev);

// The decoder: `tokens` is the int64 token window for the graph's first input; `xa` (the encoder output, row-major f32) is
// bound to the graph's second input when the graph has one. `xa_dims` may be empty when there is no cross-attention input.
Expand Down
Loading
Loading