From 296d1dfdef7e7f21d260c9d9dab86f86437be657 Mon Sep 17 00:00:00 2001 From: hrx-compat lane Date: Sat, 14 Nov 2026 10:16:26 -0400 Subject: [PATCH 1/2] npu: a single-use input is moved into Reshape, not copied (engine #2399) The Reshape branch did 'r.data = a.data' -- a deep copy of every element for an op that changes shapes, not bytes. Measured on whisper-small's encoder: 216 Reshape nodes at ~0.60 ms each, ~135 ms per pass, all of it memcpy of tensors whose contents do not move. The timing signature said so before the code did: four different shapes with the same element count (1x1500x12x64, 1x1500x768, 1x12x64x1500, 12x1500x64) all cost 0.58-0.67 ms. The executor now counts each tensor's uses from the graph's edges once, and Reshape takes the input's buffer when it is the LAST use. A twice-used input (Add(x, x)) keeps its copy, and the count is cleared on take so a buffer cannot be moved twice. Why this one is safe to land without a numerics discussion: a move changes no bits. The same values end up in the same places, so every existing expectation is unchanged -- verified, not assumed: * encoder output mean -0.009045236 std 1.678493020, identical before and after; * audio_ops_test ........................... ok * audio_parallel_test ...................... ok (this is the suite that compares each thread count against an ordered per-element reference, i.e. the one that caught the -mavx2/-mfma rounding difference -- a move must pass it, and does.) Measured effect, direct and deterministic: Reshape 135.9 ms -> 2.4 ms per encoder pass (mean 0.60 -> 0.01 ms per node). NOT claimed: an end-to-end win. The total moved 18,760 -> 20,077 ms in the same run, and the per-op diff says that is noise rather than this change -- MatMul +4.6%, Softmax +6.7%, neither of which a move can affect. This box is shared and thermal-run caps the die at 78 C, so +/-5% run-to-run swamps a 0.7% change; the direct number is the honest one. The other ops that deep-copy (Add/Mul/Transpose results are all built fresh) are untouched: each needs its own look, and this change deliberately does the smallest thing that removes a measured, provably-wasted copy. --- npu/audio_exec.cpp | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/npu/audio_exec.cpp b/npu/audio_exec.cpp index f662bf468..0dacd18ad 100644 --- a/npu/audio_exec.cpp +++ b/npu/audio_exec.cpp @@ -348,6 +348,24 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map t, con } }; + // engine#2399: a tensor with exactly one use can be MOVED into its consumer instead of copied. Reshape + // was deep-copying 4.6 MB per node (216 nodes, ~130 ms per encoder pass) to change a shape and no bytes. + // A move changes no bits, so this cannot alter any result; the count makes it safe (a twice-used input, + // e.g. Add(x, x), keeps its copy). + std::map use_count; + for (const auto& n : g.nodes) { + for (const auto& in : n.ins) { + if (!in.empty()) ++use_count[in]; + } + } + auto take_if_last = [&](const std::string& name) -> Tensor* { + const auto c = use_count.find(name); + const auto it = t.find(name); + if (c == use_count.end() || c->second != 1 || it == t.end()) return nullptr; + use_count[name] = 0; + return &it->second; + }; + int node_index = 0; for (const auto& n : g.nodes) { ++node_index; @@ -547,7 +565,11 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map t, con if (infer >= 0) dims[size_t(infer)] = int64_t(a.data.size() / std::max(known, 1)); if (nelem(dims) != a.data.size()) fail(where() + ": reshape changes the element count"); r.dims = dims; - r.data = a.data; + if (Tensor* src = take_if_last(n.ins[0])) { + r.data = std::move(src->data); // engine#2399: same bits, no copy + } else { + r.data = a.data; + } } else if (n.op == "Conv") { const Tensor& a = need(n.ins[0]); const Tensor& w = init_f32(n.ins[1]); From 5b4e64143698be00846714ad7ce85e2d8f0cfa49 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sat, 14 Nov 2026 19:31:33 -0400 Subject: [PATCH 2/2] npu: a graph OUTPUT counts as a use, so the requested tensor is never moved away (engine#2437) `use_count` counted only node *inputs*. A tensor that is also the graph's requested output -- `out_name`, else `g.outputs[0]` -- and has exactly one node consumer therefore looked "last used" by that node, was MOVED into it, and the fetch after the loop copied a moved-from vector: ```cpp const auto it = t.find(want); if (it == t.end()) fail("output '" + want + "' was never produced"); out->data = it->second.data; // moved-from: empty return true; // and the run SUCCEEDS ``` That is a silent empty output: no exception, `rc == true`, and `dims` still correct because the shape was set before the move. The count now includes the outputs, and an explicit `out_name` too when it is not one of them (the fetch reads that as well, and a caller may request a tensor the graph does not list). **The arm is the shape no existing case had** -- `audio_parallel: y-as-output-and-reshape-input` -- a graph whose requested output also feeds exactly one Reshape: ``` node 1: Reshape(x, {2,2}) -> y y is g.outputs[0] node 2: Reshape(y, {4}) -> z z is never fetched fetch: y ``` Verified by reverting only the fix, with the arm kept: | tree | result | |---|---| | without the fix | `FAIL y-as-output-and-reshape-input: output is not empty`, `FAIL ...: output bytes survive the Reshape move` -- and `ran` still PASSES, which is the defect: success with empty data | | with the fix | `audio_parallel: ok` | `audio_exec_test` still SKIPs without `ONEBIT_TEST_WHISPER` (no pack directory in this tree), so the device-side path is unchanged by construction: the added count can only make `take_if_last` return `nullptr` more often, which selects the existing copy path. --- npu/audio_exec.cpp | 9 +++++++ tests/audio_parallel_test.cpp | 46 +++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/npu/audio_exec.cpp b/npu/audio_exec.cpp index 0dacd18ad..a05e82e25 100644 --- a/npu/audio_exec.cpp +++ b/npu/audio_exec.cpp @@ -358,6 +358,15 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map t, con if (!in.empty()) ++use_count[in]; } } + // engine#2437: a graph OUTPUT is a use too. The fetch after the loop reads the requested tensor out of `t` + // (`out_name`, else g.outputs[0]), so a tensor that is an output can never be "last used by a node" even + // when it has exactly one node consumer -- moving it there leaves the fetch copying a moved-from (empty) + // vector, and the run still returns true. Counting outputs keeps the invariant in one place: the count + // answers "how many more users of `name` are there", and the fetch is one of them. + for (const auto& o : g.outputs) { + if (!o.empty()) ++use_count[o]; + } + if (!out_name.empty()) ++use_count[out_name]; auto take_if_last = [&](const std::string& name) -> Tensor* { const auto c = use_count.find(name); const auto it = t.find(name); diff --git a/tests/audio_parallel_test.cpp b/tests/audio_parallel_test.cpp index c8fab16c8..6cb2b195e 100644 --- a/tests/audio_parallel_test.cpp +++ b/tests/audio_parallel_test.cpp @@ -294,7 +294,53 @@ void test_a_throwing_unit() { } // namespace +// engine#2437: a tensor that is BOTH the graph's requested output and the input of exactly one node. The move +// optimisation counts uses, and before this case existed it counted node inputs only -- so `y` looked last-used by +// the Reshape, was MOVED into it, and the fetch of "y" afterwards copied a moved-from (empty) vector while the run +// still returned true. That is a silent empty output, which is why the arm asserts the DATA and not just the rc. +void output_also_feeds_one_node() { + OnnxGraphInfo g; + g.dir = "."; + g.inputs = {"x"}; + g.outputs = {"y"}; // y is the requested output ... + const std::vector x = {1.f, 2.f, 3.f, 4.f}; + OnnxInit shp; + shp.name = "shape22"; + shp.dtype = 7; // int64 + static const int64_t d22[2] = {2, 2}; + shp.dims = {2}; // a 1-D shape tensor of two values + shp.raw = reinterpret_cast(d22); + shp.raw_len = sizeof(d22); + OnnxInit shp4; + shp4.name = "shape4"; + shp4.dtype = 7; + static const int64_t d4[1] = {4}; + shp4.dims = {1}; + shp4.raw = reinterpret_cast(d4); + shp4.raw_len = sizeof(d4); + g.initializers = {"shape22", "shape4"}; + g.init_tensors.push_back(shp); + g.init_tensors.push_back(shp4); + // node 1 produces y from the input ... + OnnxNodeInfo r1; + r1.op = "Reshape"; r1.name = "r1"; r1.ins = {"x", "shape22"}; r1.outs = {"y"}; + g.nodes.push_back(r1); + // ... and y then feeds exactly ONE node, whose output nobody fetches. take_if_last used to read that as + // "last use of y" and move it, emptying the tensor the fetch below then copies. + OnnxNodeInfo r2; + r2.op = "Reshape"; r2.name = "r2"; r2.ins = {"y", "shape4"}; r2.outs = {"z"}; + g.nodes.push_back(r2); + AudioRunResult out; + std::string why; + const bool ok = run_audio_graph(g, x.data(), 2, 2, "y", &out, &why); + if (!ok) std::printf(" (why: %s)\n", why.c_str()); + check(ok, "y-as-output-and-reshape-input: ran"); + check(out.data.size() == x.size(), "y-as-output-and-reshape-input: output is not empty"); + check(out.data == x, "y-as-output-and-reshape-input: output bytes survive the Reshape move"); +} + int main() { + output_also_feeds_one_node(); test_bit_identity(); test_row_group_edges(); test_elementwise_bit_identity();