Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -586,6 +586,12 @@ add_executable(bf16_matrix_test tests/bf16_matrix_test.cpp)
target_include_directories(bf16_matrix_test PRIVATE npu/forward/include)
add_test(NAME bf16_matrix COMMAND bf16_matrix_test)
add_dependencies(onebit bf16_matrix_test) # ci.yml builds onebit: this is how the arm reaches CI's ctest
# engine#1783: the NPU_PROFILE accumulator keeps a weight-cache fill out of `pack` (npu/target/xdna/prof_acc.h). The
# target that uses it needs the private runtime, so this header-only arm is what CI compiles of the rule.
add_executable(prof_acc_test tests/prof_acc_test.cpp)
target_include_directories(prof_acc_test PRIVATE npu/target/xdna)
add_test(NAME prof_acc COMMAND prof_acc_test)
add_dependencies(onebit prof_acc_test)
add_executable(npu_sampling_test tests/npu_sampling_test.cpp)
# The shape rule for a 1.8 external-data initializer (npu/pack_initializer_shape.h, engine#1503's graph-initializer
# reader). Those packs zero every `dims`, so a byte length has to become a shape -- and a shape inferred wrongly imports
Expand Down
26 changes: 25 additions & 1 deletion docs/npu.md
Original file line number Diff line number Diff line change
Expand Up @@ -885,7 +885,31 @@ blocks per arm, two arms per binary run A B B A in one ticket):
| of which GDN recurrence | 30.5-33.4 ms | 27.0-30.0 ms | |

So prefill on this route is bound by host work outside the launches and by the M=16 launch geometry above, in that
order, and not by packing. "After" is one change: the per-(row, head) alpha/beta dot products of a GDN layer ran on one
order, and not by packing.

**The `pack` column no longer carries the weight-cache fill (#1783).** The fill moved before the ready line, but the
warm-up pass is itself a profiled prefill, so a short request still printed a `[prof] pre` table where half the
launches were fills: `pack` read 2.7-3.5 ms per launch on the 768 family and 2.9-7.2 ms on the 1280 family, against
0.00 in the decode rows of the same process. An encoder window at those numbers was extrapolated as twelve launches per
projection per layer, each paying the one-time fill again. `NPU_PROFILE=1` now times each fill on its own and leaves
it out of `pack`. `pack` is what every launch pays, and a separate line gives the fills. The same request on the same
768 container and kernel set (2026-11-13, one `npu-hold` ticket, `amdxdna` error count 632 before and after, same
greedy text):

| `[prof] pre`, M=128, 24 launches per role | QKV | O | GU | D |
|---|---:|---:|---:|---:|
| `pack` before (ms/launch) | 3.10 | 3.09 | 3.49 | 2.73 |
| `pack` after (ms/launch) | 0.00 | 0.00 | 0.00 | 0.00 |
| `ms/launch` after | 1.269 | 1.359 | 1.935 | 1.595 |

```
[prof] pre weight-cache fills (one-time, not in pack): 48, 310.4 ms, 6.466 ms/fill
```

The 48 fills are 12 layers x 4 roles, all in the warm-up pass, and they account for the old `pack` totals (24 x 12.41
= 298 ms).

An uncached launch (`NPU_NO_WCACHE=1`) packs its weight on every launch, and that stays in `pack`. "After" is one change: the per-(row, head) alpha/beta dot products of a GDN layer ran on one
thread inside the row loop (0.099 ms per row and layer, 48 GDN layers), and they now run as a second worksharing loop
inside the parallel region that layer already opens for the recurrence. No region is added, because the region count is
what a loaded box pays for. Each dot product is the same sequential loop as before and the build has no `-ffast-math`,
Expand Down
45 changes: 45 additions & 0 deletions npu/target/xdna/prof_acc.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// npu/target/xdna/prof_acc.h -- the NPU_PROFILE per-role accumulator and its one rule, header-only so a test with no
// device and no private runtime can arm it (xdna_target.cpp includes the private runtime header, so no CI job compiles
// it).
//
// engine#1783: a launch that missed the weight cache spent the fill inside the span the `pack` column times. The fill
// is a one-time cost per (layer, weight), so it is counted apart and taken out of `pack`, clamped at zero so a clock
// that reads the fill a little longer than the span can never print a negative cost.
#ifndef ONEBIT_NPU_TARGET_XDNA_PROF_ACC_H
#define ONEBIT_NPU_TARGET_XDNA_PROF_ACC_H

struct ProfAcc {
double pack = 0, quant = 0, wait = 0, rb = 0;
long n = 0;
double fill = 0; // weight-cache fills: one-time, never in `pack`
long nfill = 0;
};

// What a launch paid for `pack` once its weight-cache fill (0 on a cached launch) is taken out.
inline double prof_pack_without_fill(double pack, double fill) {
if (!(fill > 0.0)) return pack;
return pack > fill ? pack - fill : 0.0;
}

inline void prof_acc_add(ProfAcc& a, double pack, double quant, double wait, double rb, double fill) {
a.pack += prof_pack_without_fill(pack, fill);
a.quant += quant; a.wait += wait; a.rb += rb;
a.n++;
if (fill > 0.0) { a.fill += fill; a.nfill++; }
}

#endif
45 changes: 34 additions & 11 deletions npu/target/xdna/xdna_target.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -528,7 +528,14 @@ int64_t XdnaTarget::m16_check_gemm_rows(I8Ctx& ref, I8Ctx& fast, const float* A,
}

void XdnaTarget::prof_add(ProfAcc* acc, int design, double pack, double quant,
double wait, double rb) {
double wait, double rb, double fill) {
// engine#1783: `pack` (tp->tl) also spanned the weight-cache fill on a launch that missed. The warm-up pass and a
// short request each launch every (layer, role) once or twice, so half the prefill launches in such a window were
// fills, and `pack` read ~3 ms/launch on the 768 family where a cached launch's host side is ~0. An encoder
// window (12 launches per projection per layer) then extrapolated a one-time cost twelve times. The fill now
// is printed on its own line after the table, and `pack` is what every launch pays.
// The g_ph totals below take the same rule as the per-role accumulator (prof_acc.h).
const double pack_paid = prof_pack_without_fill(pack, fill);
// g_ph is what the [bs] (batch) and [ph] (decode step) lines print, and both read it. Only the DECODE
// accumulator used to be counted, so a 219-row prefill -- which launches through prof_pre_ -- reported
// "n=0 launches, host=93113 ms" for 224 launches and 93 s of device wait: the wait fell into the line's
Expand All @@ -544,14 +551,12 @@ void XdnaTarget::prof_add(ProfAcc* acc, int design, double pack, double quant,
// (it can go negative) on exactly the multi-row passes this line exists for. The head keeps its own accounting.
if (q4nx_lh::phprof_on() && acc != &prof_head_ &&
(acc == prof_dec_ || (q4nx_lh::g_in_batch && acc == prof_pre_)) && !q4nx_lh::g_in_head) {
q4nx_lh::g_ph.g_pack += pack; q4nx_lh::g_ph.g_ql += quant;
q4nx_lh::g_ph.g_pack += pack_paid; q4nx_lh::g_ph.g_ql += quant;
q4nx_lh::g_ph.g_wait += wait; q4nx_lh::g_ph.g_rb += rb;
q4nx_lh::g_ph.launches++;
}
if (design < 0 || design >= Q4NX_MAX_DESIGNS) return;
ProfAcc& a = acc[design];
a.pack += pack; a.quant += quant; a.wait += wait; a.rb += rb;
a.n++;
prof_acc_add(acc[design], pack, quant, wait, rb, fill);
}

void XdnaTarget::prof_dump() {
Expand All @@ -571,6 +576,14 @@ void XdnaTarget::prof_dump() {
(a.pack + a.quant + a.wait + a.rb) / n);
}
}
for (int ph = 0; ph < 2; ph++) {
ProfAcc* acc = ph ? prof_pre_ : prof_dec_;
long nf = 0; double fill = 0;
for (int i = 0; i < ndesigns_; i++) { nf += acc[i].nfill; fill += acc[i].fill; }
if (nf)
fprintf(stderr, "[prof] %-5s weight-cache fills (one-time, not in pack): %ld, %.1f ms, %.3f ms/fill\n",
ph ? "pre" : "dec", nf, fill, fill / (double)nf);
}
if (prof_head_.n) {
const double n = (double)prof_head_.n;
fprintf(stderr, "[prof] %-5s %-5s %2d %8ld %7.2f %9.2f %7.3f %9.3f %9.3f\n", "dec",
Expand Down Expand Up @@ -908,6 +921,7 @@ void XdnaTarget::npu_gemm(const GemvArgs& ar, std::vector<float>& out, std::stri
I8Ctx& kctx = *kctx1;
const bool prof = prof_;
const auto tp = prof ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{};
double fill_ms = 0.0; // engine#1783: weight-cache fill time inside tp->tl, reported apart from `pack`

// pb16i: the weight BO holds the sidecar image unchanged, so a cache miss is a
// file read (no dequant, no quantise, no pack) and the result is dequantised with
Expand All @@ -922,6 +936,7 @@ void XdnaTarget::npu_gemm(const GemvArgs& ar, std::vector<float>& out, std::stri
auto& m = wcache_[size_t(ar.layer)];
auto it = m.find(wkey);
if (it == m.end()) {
const auto f0 = std::chrono::steady_clock::now(); // a weight-cache fill: one-time, not per launch
q4nx_lh::LhScope _tlh_pk(LH_PACK_UPLOAD);
PackedWeight pw;
pw.bo = kctx.make_weight_bo(*dev_, /*zero=*/false);
Expand All @@ -934,6 +949,7 @@ void XdnaTarget::npu_gemm(const GemvArgs& ar, std::vector<float>& out, std::stri
}
pw.eref = std::move(eref);
pw.pb16i_mode = mode;
fill_ms = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - f0).count();
it = m.emplace(wkey, std::move(pw)).first;
}
slot = &it->second;
Expand Down Expand Up @@ -1021,7 +1037,7 @@ void XdnaTarget::npu_gemm(const GemvArgs& ar, std::vector<float>& out, std::stri
const double dql = std::chrono::duration<double, std::milli>(tq - tl).count();
const double dwt = std::chrono::duration<double, std::milli>(tw - tq).count();
const double drb = std::chrono::duration<double, std::milli>(tr - tw).count();
prof_add(prof_dec_, design_index(&kctx), dpk, dql, dwt, drb);
prof_add(prof_dec_, design_index(&kctx), dpk, dql, dwt, drb, fill_ms);
if (++prof_dec_n_ % 100 == 0) prof_dump();
}
return;
Expand All @@ -1037,6 +1053,7 @@ void XdnaTarget::npu_gemm(const GemvArgs& ar, std::vector<float>& out, std::stri
auto& m = wcache_[size_t(ar.layer)];
auto it = m.find(wkey);
if (it == m.end()) {
const auto f0 = std::chrono::steady_clock::now(); // a weight-cache fill: one-time, not per launch
// Cache miss only: this is the per-request re-pack/re-upload the
// `prefill` row would hide. It must be zero for a warm process.
q4nx_lh::LhScope _tlh_pk(LH_PACK_UPLOAD);
Expand All @@ -1054,6 +1071,7 @@ void XdnaTarget::npu_gemm(const GemvArgs& ar, std::vector<float>& out, std::stri
float sout = 1.0f;
kctx.packB_into(*pw.bo, Bmat, K, N, sout, pw.scales);
}
fill_ms = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - f0).count();
it = m.emplace(wkey, std::move(pw)).first;
}
slot = &it->second;
Expand Down Expand Up @@ -1112,7 +1130,7 @@ void XdnaTarget::npu_gemm(const GemvArgs& ar, std::vector<float>& out, std::stri
const double dql = std::chrono::duration<double, std::milli>(tq - tl).count();
const double dwt = std::chrono::duration<double, std::milli>(tw - tq).count();
const double drb = std::chrono::duration<double, std::milli>(tr - tw).count();
prof_add(prof_dec_, design_index(&kctx), dpk, dql, dwt, drb);
prof_add(prof_dec_, design_index(&kctx), dpk, dql, dwt, drb, fill_ms);
if (++prof_dec_n_ % 100 == 0) prof_dump();
}
}
Expand Down Expand Up @@ -1192,6 +1210,7 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector<float>& out, std:
I8Ctx& kctx = *kctxp;
const bool prof = prof_;
const auto tp = prof ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{};
double fill_ms = 0.0; // engine#1783: weight-cache fill time inside tp->tl, reported apart from `pack`
if (pb16i_ && kctx.pb16i) {
out.assign((size_t)M * (size_t)N, 0.0f); // sized before every refusal path below
if (M > kctx.MD) {
Expand All @@ -1206,6 +1225,7 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector<float>& out, std:
auto& m = wcache_[size_t(ar.layer)];
auto it = m.find(wkey);
if (it == m.end()) {
const auto f0 = std::chrono::steady_clock::now(); // a weight-cache fill: one-time, not per launch
q4nx_lh::LhScope _tlh_pk(LH_PACK_UPLOAD);
PackedWeight pw;
pw.bo = kctx.make_weight_bo(*dev_, /*zero=*/false);
Expand All @@ -1218,6 +1238,7 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector<float>& out, std:
}
pw.eref = std::move(eref);
pw.pb16i_mode = mode;
fill_ms = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - f0).count();
it = m.emplace(wkey, std::move(pw)).first;
}
slot = &it->second;
Expand Down Expand Up @@ -1292,10 +1313,10 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector<float>& out, std:
const double drb = std::chrono::duration<double, std::milli>(tr - tw).count();
const int di = design_index(&kctx);
if (M == 1) {
prof_add(prof_dec_, di, dpk, dql, dwt, drb);
prof_add(prof_dec_, di, dpk, dql, dwt, drb, fill_ms);
prof_dec_n_++;
} else {
prof_add(prof_pre_, di, dpk, dql, dwt, drb);
prof_add(prof_pre_, di, dpk, dql, dwt, drb, fill_ms);
prof_pre_n_++;
}
// Dump on either family's count. The `prof_dec_n_ &&` guard meant a PREFILL-ONLY process never dumped at
Expand All @@ -1314,6 +1335,7 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector<float>& out, std:
auto& m = wcache_[size_t(ar.layer)];
auto it = m.find(wkey);
if (it == m.end()) {
const auto f0 = std::chrono::steady_clock::now(); // a weight-cache fill: one-time, not per launch
q4nx_lh::LhScope _tlh_pk(LH_PACK_UPLOAD);
PackedWeight pw;
pw.bo = kctx.make_weight_bo(*dev_);
Expand All @@ -1326,6 +1348,7 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector<float>& out, std:
if (std::getenv("NPU_PACK_SPLIT") && wmiss_n_ % 20 == 0)
fprintf(stderr, "[wm] n=%ld cache-miss repacks, avg packB_into=%.3f ms\n",
wmiss_n_, wmiss_ms_ / wmiss_n_);
fill_ms = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - f0).count();
it = m.emplace(wkey, std::move(pw)).first;
}
slot = &it->second;
Expand Down Expand Up @@ -1392,10 +1415,10 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector<float>& out, std:
const double drb = std::chrono::duration<double, std::milli>(tr - tw).count();
const int di = design_index(&kctx);
if (M == 1) {
prof_add(prof_dec_, di, dpk, dql, dwt, drb);
prof_add(prof_dec_, di, dpk, dql, dwt, drb, fill_ms);
prof_dec_n_++;
} else {
prof_add(prof_pre_, di, dpk, dql, dwt, drb);
prof_add(prof_pre_, di, dpk, dql, dwt, drb, fill_ms);
prof_pre_n_++;
}
if ((prof_dec_n_ + prof_pre_n_) % 100 == 0) prof_dump(); // same reason as above: prefill-only runs
Expand Down
8 changes: 3 additions & 5 deletions npu/target/xdna/xdna_target.h
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@

#include "target/target.h"
#include "plan/plan.h" // onebit::npu::plan::Plan (engine #799 P1)
#include "target/xdna/prof_acc.h" // ProfAcc and the fill rule (engine#1783), header-only so CI tests it
#include "q4nx_pack.h" // Q4nxDesignGeom, Q4NX_MAX_DESIGNS
#include "rt.h" // xrt::device, xrt::bo

Expand All @@ -52,10 +53,6 @@ struct PackedWeight {
};

// Per-role dispatch accounting for NPU_PROFILE=1.
struct ProfAcc {
double pack = 0, quant = 0, wait = 0, rb = 0;
long n = 0;
};

class XdnaTarget final : public NpuTarget {
public:
Expand Down Expand Up @@ -221,7 +218,8 @@ class XdnaTarget final : public NpuTarget {
private:
I8Ctx* ctx_for(I8Ctx* prefill, int rows, bool decode_twin);
int design_index(I8Ctx* c) const;
void prof_add(ProfAcc* acc, int design, double pack, double quant, double wait, double rb);
void prof_add(ProfAcc* acc, int design, double pack, double quant, double wait, double rb,
double fill = 0.0);
void prof_dump();
void npu_gemm(const GemvArgs& a, std::vector<float>& out, std::string& err);
void npu_gemm_rows(const GemvArgs& a, std::vector<float>& out, std::string& err);
Expand Down
51 changes: 51 additions & 0 deletions tests/prof_acc_test.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// engine#1783: npu/target/xdna/prof_acc.h -- the NPU_PROFILE accumulator keeps a weight-cache fill out of `pack`.
// Four cases of the rule, then the accumulator over a window shaped like the warm-up pass (half fills, half hits).
#include "prof_acc.h"

#include <cmath>
#include <cstdio>

static int failures = 0;
static void check(bool ok, const char* what) {
if (!ok) { std::printf("FAIL %s\n", what); failures++; }
}
static bool near(double a, double b) { return std::fabs(a - b) < 1e-9; }

int main() {
check(near(prof_pack_without_fill(0.25, 0.0), 0.25), "fill == 0: pack unchanged (a cached launch)");
check(near(prof_pack_without_fill(6.5, 6.25), 0.25), "0 < fill < pack: pack - fill");
check(near(prof_pack_without_fill(6.5, 6.5), 0.0), "fill == pack: 0");
check(near(prof_pack_without_fill(6.5, 7.0), 0.0), "fill > pack: 0, not negative");
check(near(prof_pack_without_fill(0.25, -1.0), 0.25), "a negative fill is no fill");

// 12 layers: each launches once with a fill (6.25 ms of a 6.5 ms span) and once cached (0.25 ms).
ProfAcc a;
for (int l = 0; l < 12; l++) {
prof_acc_add(a, 6.5, 0.1, 1.2, 0.002, 6.25);
prof_acc_add(a, 0.25, 0.1, 1.2, 0.002, 0.0);
}
check(a.n == 24, "every launch is counted");
check(a.nfill == 12, "only the fills are counted as fills");
check(near(a.fill, 12 * 6.25), "the fill total is the fills' own time");
check(near(a.pack / a.n, 0.25), "pack per launch is what a cached launch pays");
check(near(a.wait / a.n, 1.2) && near(a.quant / a.n, 0.1), "wait and quant are untouched");

if (failures) { std::printf("prof_acc: %d failure(s)\n", failures); return 1; }
std::printf("prof_acc: ok\n");
return 0;
}
Loading