diff --git a/CMakeLists.txt b/CMakeLists.txt index 7f360bc35..405e9e998 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -586,6 +586,12 @@ add_executable(bf16_matrix_test tests/bf16_matrix_test.cpp) target_include_directories(bf16_matrix_test PRIVATE npu/forward/include) add_test(NAME bf16_matrix COMMAND bf16_matrix_test) add_dependencies(onebit bf16_matrix_test) # ci.yml builds onebit: this is how the arm reaches CI's ctest +# engine#1783: the NPU_PROFILE accumulator keeps a weight-cache fill out of `pack` (npu/target/xdna/prof_acc.h). The +# target that uses it needs the private runtime, so this header-only arm is what CI compiles of the rule. +add_executable(prof_acc_test tests/prof_acc_test.cpp) +target_include_directories(prof_acc_test PRIVATE npu/target/xdna) +add_test(NAME prof_acc COMMAND prof_acc_test) +add_dependencies(onebit prof_acc_test) add_executable(npu_sampling_test tests/npu_sampling_test.cpp) # The shape rule for a 1.8 external-data initializer (npu/pack_initializer_shape.h, engine#1503's graph-initializer # reader). Those packs zero every `dims`, so a byte length has to become a shape -- and a shape inferred wrongly imports diff --git a/docs/npu.md b/docs/npu.md index efca2dd34..db1ada36d 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -885,7 +885,31 @@ blocks per arm, two arms per binary run A B B A in one ticket): | of which GDN recurrence | 30.5-33.4 ms | 27.0-30.0 ms | | So prefill on this route is bound by host work outside the launches and by the M=16 launch geometry above, in that -order, and not by packing. "After" is one change: the per-(row, head) alpha/beta dot products of a GDN layer ran on one +order, and not by packing. + +**The `pack` column no longer carries the weight-cache fill (#1783).** The fill moved before the ready line, but the +warm-up pass is itself a profiled prefill, so a short request still printed a `[prof] pre` table where half the +launches were fills: `pack` read 2.7-3.5 ms per launch on the 768 family and 2.9-7.2 ms on the 1280 family, against +0.00 in the decode rows of the same process. An encoder window at those numbers was extrapolated as twelve launches per +projection per layer, each paying the one-time fill again. `NPU_PROFILE=1` now times each fill on its own and leaves +it out of `pack`. `pack` is what every launch pays, and a separate line gives the fills. The same request on the same +768 container and kernel set (2026-11-13, one `npu-hold` ticket, `amdxdna` error count 632 before and after, same +greedy text): + +| `[prof] pre`, M=128, 24 launches per role | QKV | O | GU | D | +|---|---:|---:|---:|---:| +| `pack` before (ms/launch) | 3.10 | 3.09 | 3.49 | 2.73 | +| `pack` after (ms/launch) | 0.00 | 0.00 | 0.00 | 0.00 | +| `ms/launch` after | 1.269 | 1.359 | 1.935 | 1.595 | + +``` +[prof] pre weight-cache fills (one-time, not in pack): 48, 310.4 ms, 6.466 ms/fill +``` + +The 48 fills are 12 layers x 4 roles, all in the warm-up pass, and they account for the old `pack` totals (24 x 12.41 += 298 ms). + +An uncached launch (`NPU_NO_WCACHE=1`) packs its weight on every launch, and that stays in `pack`. "After" is one change: the per-(row, head) alpha/beta dot products of a GDN layer ran on one thread inside the row loop (0.099 ms per row and layer, 48 GDN layers), and they now run as a second worksharing loop inside the parallel region that layer already opens for the recurrence. No region is added, because the region count is what a loaded box pays for. Each dot product is the same sequential loop as before and the build has no `-ffast-math`, diff --git a/npu/target/xdna/prof_acc.h b/npu/target/xdna/prof_acc.h new file mode 100644 index 000000000..d1a28c67a --- /dev/null +++ b/npu/target/xdna/prof_acc.h @@ -0,0 +1,45 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// npu/target/xdna/prof_acc.h -- the NPU_PROFILE per-role accumulator and its one rule, header-only so a test with no +// device and no private runtime can arm it (xdna_target.cpp includes the private runtime header, so no CI job compiles +// it). +// +// engine#1783: a launch that missed the weight cache spent the fill inside the span the `pack` column times. The fill +// is a one-time cost per (layer, weight), so it is counted apart and taken out of `pack`, clamped at zero so a clock +// that reads the fill a little longer than the span can never print a negative cost. +#ifndef ONEBIT_NPU_TARGET_XDNA_PROF_ACC_H +#define ONEBIT_NPU_TARGET_XDNA_PROF_ACC_H + +struct ProfAcc { + double pack = 0, quant = 0, wait = 0, rb = 0; + long n = 0; + double fill = 0; // weight-cache fills: one-time, never in `pack` + long nfill = 0; +}; + +// What a launch paid for `pack` once its weight-cache fill (0 on a cached launch) is taken out. +inline double prof_pack_without_fill(double pack, double fill) { + if (!(fill > 0.0)) return pack; + return pack > fill ? pack - fill : 0.0; +} + +inline void prof_acc_add(ProfAcc& a, double pack, double quant, double wait, double rb, double fill) { + a.pack += prof_pack_without_fill(pack, fill); + a.quant += quant; a.wait += wait; a.rb += rb; + a.n++; + if (fill > 0.0) { a.fill += fill; a.nfill++; } +} + +#endif diff --git a/npu/target/xdna/xdna_target.cpp b/npu/target/xdna/xdna_target.cpp index cf753062a..6d59fc846 100644 --- a/npu/target/xdna/xdna_target.cpp +++ b/npu/target/xdna/xdna_target.cpp @@ -528,7 +528,14 @@ int64_t XdnaTarget::m16_check_gemm_rows(I8Ctx& ref, I8Ctx& fast, const float* A, } void XdnaTarget::prof_add(ProfAcc* acc, int design, double pack, double quant, - double wait, double rb) { + double wait, double rb, double fill) { + // engine#1783: `pack` (tp->tl) also spanned the weight-cache fill on a launch that missed. The warm-up pass and a + // short request each launch every (layer, role) once or twice, so half the prefill launches in such a window were + // fills, and `pack` read ~3 ms/launch on the 768 family where a cached launch's host side is ~0. An encoder + // window (12 launches per projection per layer) then extrapolated a one-time cost twelve times. The fill now + // is printed on its own line after the table, and `pack` is what every launch pays. + // The g_ph totals below take the same rule as the per-role accumulator (prof_acc.h). + const double pack_paid = prof_pack_without_fill(pack, fill); // g_ph is what the [bs] (batch) and [ph] (decode step) lines print, and both read it. Only the DECODE // accumulator used to be counted, so a 219-row prefill -- which launches through prof_pre_ -- reported // "n=0 launches, host=93113 ms" for 224 launches and 93 s of device wait: the wait fell into the line's @@ -544,14 +551,12 @@ void XdnaTarget::prof_add(ProfAcc* acc, int design, double pack, double quant, // (it can go negative) on exactly the multi-row passes this line exists for. The head keeps its own accounting. if (q4nx_lh::phprof_on() && acc != &prof_head_ && (acc == prof_dec_ || (q4nx_lh::g_in_batch && acc == prof_pre_)) && !q4nx_lh::g_in_head) { - q4nx_lh::g_ph.g_pack += pack; q4nx_lh::g_ph.g_ql += quant; + q4nx_lh::g_ph.g_pack += pack_paid; q4nx_lh::g_ph.g_ql += quant; q4nx_lh::g_ph.g_wait += wait; q4nx_lh::g_ph.g_rb += rb; q4nx_lh::g_ph.launches++; } if (design < 0 || design >= Q4NX_MAX_DESIGNS) return; - ProfAcc& a = acc[design]; - a.pack += pack; a.quant += quant; a.wait += wait; a.rb += rb; - a.n++; + prof_acc_add(acc[design], pack, quant, wait, rb, fill); } void XdnaTarget::prof_dump() { @@ -571,6 +576,14 @@ void XdnaTarget::prof_dump() { (a.pack + a.quant + a.wait + a.rb) / n); } } + for (int ph = 0; ph < 2; ph++) { + ProfAcc* acc = ph ? prof_pre_ : prof_dec_; + long nf = 0; double fill = 0; + for (int i = 0; i < ndesigns_; i++) { nf += acc[i].nfill; fill += acc[i].fill; } + if (nf) + fprintf(stderr, "[prof] %-5s weight-cache fills (one-time, not in pack): %ld, %.1f ms, %.3f ms/fill\n", + ph ? "pre" : "dec", nf, fill, fill / (double)nf); + } if (prof_head_.n) { const double n = (double)prof_head_.n; fprintf(stderr, "[prof] %-5s %-5s %2d %8ld %7.2f %9.2f %7.3f %9.3f %9.3f\n", "dec", @@ -908,6 +921,7 @@ void XdnaTarget::npu_gemm(const GemvArgs& ar, std::vector& out, std::stri I8Ctx& kctx = *kctx1; const bool prof = prof_; const auto tp = prof ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + double fill_ms = 0.0; // engine#1783: weight-cache fill time inside tp->tl, reported apart from `pack` // pb16i: the weight BO holds the sidecar image unchanged, so a cache miss is a // file read (no dequant, no quantise, no pack) and the result is dequantised with @@ -922,6 +936,7 @@ void XdnaTarget::npu_gemm(const GemvArgs& ar, std::vector& out, std::stri auto& m = wcache_[size_t(ar.layer)]; auto it = m.find(wkey); if (it == m.end()) { + const auto f0 = std::chrono::steady_clock::now(); // a weight-cache fill: one-time, not per launch q4nx_lh::LhScope _tlh_pk(LH_PACK_UPLOAD); PackedWeight pw; pw.bo = kctx.make_weight_bo(*dev_, /*zero=*/false); @@ -934,6 +949,7 @@ void XdnaTarget::npu_gemm(const GemvArgs& ar, std::vector& out, std::stri } pw.eref = std::move(eref); pw.pb16i_mode = mode; + fill_ms = std::chrono::duration(std::chrono::steady_clock::now() - f0).count(); it = m.emplace(wkey, std::move(pw)).first; } slot = &it->second; @@ -1021,7 +1037,7 @@ void XdnaTarget::npu_gemm(const GemvArgs& ar, std::vector& out, std::stri const double dql = std::chrono::duration(tq - tl).count(); const double dwt = std::chrono::duration(tw - tq).count(); const double drb = std::chrono::duration(tr - tw).count(); - prof_add(prof_dec_, design_index(&kctx), dpk, dql, dwt, drb); + prof_add(prof_dec_, design_index(&kctx), dpk, dql, dwt, drb, fill_ms); if (++prof_dec_n_ % 100 == 0) prof_dump(); } return; @@ -1037,6 +1053,7 @@ void XdnaTarget::npu_gemm(const GemvArgs& ar, std::vector& out, std::stri auto& m = wcache_[size_t(ar.layer)]; auto it = m.find(wkey); if (it == m.end()) { + const auto f0 = std::chrono::steady_clock::now(); // a weight-cache fill: one-time, not per launch // Cache miss only: this is the per-request re-pack/re-upload the // `prefill` row would hide. It must be zero for a warm process. q4nx_lh::LhScope _tlh_pk(LH_PACK_UPLOAD); @@ -1054,6 +1071,7 @@ void XdnaTarget::npu_gemm(const GemvArgs& ar, std::vector& out, std::stri float sout = 1.0f; kctx.packB_into(*pw.bo, Bmat, K, N, sout, pw.scales); } + fill_ms = std::chrono::duration(std::chrono::steady_clock::now() - f0).count(); it = m.emplace(wkey, std::move(pw)).first; } slot = &it->second; @@ -1112,7 +1130,7 @@ void XdnaTarget::npu_gemm(const GemvArgs& ar, std::vector& out, std::stri const double dql = std::chrono::duration(tq - tl).count(); const double dwt = std::chrono::duration(tw - tq).count(); const double drb = std::chrono::duration(tr - tw).count(); - prof_add(prof_dec_, design_index(&kctx), dpk, dql, dwt, drb); + prof_add(prof_dec_, design_index(&kctx), dpk, dql, dwt, drb, fill_ms); if (++prof_dec_n_ % 100 == 0) prof_dump(); } } @@ -1192,6 +1210,7 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector& out, std: I8Ctx& kctx = *kctxp; const bool prof = prof_; const auto tp = prof ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + double fill_ms = 0.0; // engine#1783: weight-cache fill time inside tp->tl, reported apart from `pack` if (pb16i_ && kctx.pb16i) { out.assign((size_t)M * (size_t)N, 0.0f); // sized before every refusal path below if (M > kctx.MD) { @@ -1206,6 +1225,7 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector& out, std: auto& m = wcache_[size_t(ar.layer)]; auto it = m.find(wkey); if (it == m.end()) { + const auto f0 = std::chrono::steady_clock::now(); // a weight-cache fill: one-time, not per launch q4nx_lh::LhScope _tlh_pk(LH_PACK_UPLOAD); PackedWeight pw; pw.bo = kctx.make_weight_bo(*dev_, /*zero=*/false); @@ -1218,6 +1238,7 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector& out, std: } pw.eref = std::move(eref); pw.pb16i_mode = mode; + fill_ms = std::chrono::duration(std::chrono::steady_clock::now() - f0).count(); it = m.emplace(wkey, std::move(pw)).first; } slot = &it->second; @@ -1292,10 +1313,10 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector& out, std: const double drb = std::chrono::duration(tr - tw).count(); const int di = design_index(&kctx); if (M == 1) { - prof_add(prof_dec_, di, dpk, dql, dwt, drb); + prof_add(prof_dec_, di, dpk, dql, dwt, drb, fill_ms); prof_dec_n_++; } else { - prof_add(prof_pre_, di, dpk, dql, dwt, drb); + prof_add(prof_pre_, di, dpk, dql, dwt, drb, fill_ms); prof_pre_n_++; } // Dump on either family's count. The `prof_dec_n_ &&` guard meant a PREFILL-ONLY process never dumped at @@ -1314,6 +1335,7 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector& out, std: auto& m = wcache_[size_t(ar.layer)]; auto it = m.find(wkey); if (it == m.end()) { + const auto f0 = std::chrono::steady_clock::now(); // a weight-cache fill: one-time, not per launch q4nx_lh::LhScope _tlh_pk(LH_PACK_UPLOAD); PackedWeight pw; pw.bo = kctx.make_weight_bo(*dev_); @@ -1326,6 +1348,7 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector& out, std: if (std::getenv("NPU_PACK_SPLIT") && wmiss_n_ % 20 == 0) fprintf(stderr, "[wm] n=%ld cache-miss repacks, avg packB_into=%.3f ms\n", wmiss_n_, wmiss_ms_ / wmiss_n_); + fill_ms = std::chrono::duration(std::chrono::steady_clock::now() - f0).count(); it = m.emplace(wkey, std::move(pw)).first; } slot = &it->second; @@ -1392,10 +1415,10 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector& out, std: const double drb = std::chrono::duration(tr - tw).count(); const int di = design_index(&kctx); if (M == 1) { - prof_add(prof_dec_, di, dpk, dql, dwt, drb); + prof_add(prof_dec_, di, dpk, dql, dwt, drb, fill_ms); prof_dec_n_++; } else { - prof_add(prof_pre_, di, dpk, dql, dwt, drb); + prof_add(prof_pre_, di, dpk, dql, dwt, drb, fill_ms); prof_pre_n_++; } if ((prof_dec_n_ + prof_pre_n_) % 100 == 0) prof_dump(); // same reason as above: prefill-only runs diff --git a/npu/target/xdna/xdna_target.h b/npu/target/xdna/xdna_target.h index cc3767933..347ec43cd 100644 --- a/npu/target/xdna/xdna_target.h +++ b/npu/target/xdna/xdna_target.h @@ -34,6 +34,7 @@ #include "target/target.h" #include "plan/plan.h" // onebit::npu::plan::Plan (engine #799 P1) +#include "target/xdna/prof_acc.h" // ProfAcc and the fill rule (engine#1783), header-only so CI tests it #include "q4nx_pack.h" // Q4nxDesignGeom, Q4NX_MAX_DESIGNS #include "rt.h" // xrt::device, xrt::bo @@ -52,10 +53,6 @@ struct PackedWeight { }; // Per-role dispatch accounting for NPU_PROFILE=1. -struct ProfAcc { - double pack = 0, quant = 0, wait = 0, rb = 0; - long n = 0; -}; class XdnaTarget final : public NpuTarget { public: @@ -221,7 +218,8 @@ class XdnaTarget final : public NpuTarget { private: I8Ctx* ctx_for(I8Ctx* prefill, int rows, bool decode_twin); int design_index(I8Ctx* c) const; - void prof_add(ProfAcc* acc, int design, double pack, double quant, double wait, double rb); + void prof_add(ProfAcc* acc, int design, double pack, double quant, double wait, double rb, + double fill = 0.0); void prof_dump(); void npu_gemm(const GemvArgs& a, std::vector& out, std::string& err); void npu_gemm_rows(const GemvArgs& a, std::vector& out, std::string& err); diff --git a/tests/prof_acc_test.cpp b/tests/prof_acc_test.cpp new file mode 100644 index 000000000..49d4d8252 --- /dev/null +++ b/tests/prof_acc_test.cpp @@ -0,0 +1,51 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// engine#1783: npu/target/xdna/prof_acc.h -- the NPU_PROFILE accumulator keeps a weight-cache fill out of `pack`. +// Four cases of the rule, then the accumulator over a window shaped like the warm-up pass (half fills, half hits). +#include "prof_acc.h" + +#include +#include + +static int failures = 0; +static void check(bool ok, const char* what) { + if (!ok) { std::printf("FAIL %s\n", what); failures++; } +} +static bool near(double a, double b) { return std::fabs(a - b) < 1e-9; } + +int main() { + check(near(prof_pack_without_fill(0.25, 0.0), 0.25), "fill == 0: pack unchanged (a cached launch)"); + check(near(prof_pack_without_fill(6.5, 6.25), 0.25), "0 < fill < pack: pack - fill"); + check(near(prof_pack_without_fill(6.5, 6.5), 0.0), "fill == pack: 0"); + check(near(prof_pack_without_fill(6.5, 7.0), 0.0), "fill > pack: 0, not negative"); + check(near(prof_pack_without_fill(0.25, -1.0), 0.25), "a negative fill is no fill"); + + // 12 layers: each launches once with a fill (6.25 ms of a 6.5 ms span) and once cached (0.25 ms). + ProfAcc a; + for (int l = 0; l < 12; l++) { + prof_acc_add(a, 6.5, 0.1, 1.2, 0.002, 6.25); + prof_acc_add(a, 0.25, 0.1, 1.2, 0.002, 0.0); + } + check(a.n == 24, "every launch is counted"); + check(a.nfill == 12, "only the fills are counted as fills"); + check(near(a.fill, 12 * 6.25), "the fill total is the fills' own time"); + check(near(a.pack / a.n, 0.25), "pack per launch is what a cached launch pays"); + check(near(a.wait / a.n, 1.2) && near(a.quant / a.n, 0.1), "wait and quant are untouched"); + + if (failures) { std::printf("prof_acc: %d failure(s)\n", failures); return 1; } + std::printf("prof_acc: ok\n"); + return 0; +}