From ae542ac0b39192124c91e2f225437dd62a43a28a Mon Sep 17 00:00:00 2001 From: agent Date: Sat, 7 Nov 2026 06:47:20 -0400 Subject: [PATCH] npu: a plan that asks for an int8 KV cache is refused by name, not served as f16 Follow-up to #806 (plan P7 part A) now that P4 has landed. #1273 put `kv {key, value, hadamard}` in the plan and promised, in docs/npu-plan.md, that "a target without the capability must refuse `int8` by name, never serve f16 in its place". That was prose. #808 (P4, A1) landed npu/target/target.h with `Caps::int8_kv`, and the xdna target reports it -- and nothing read it. So a plan asking for an int8 KV cache loaded clean and opened clean, and would have been answered out of an f16 cache: the same text from a different cache contract, which is the failure this layer exists to remove and the one that cannot be noticed from outside. Closing it before a writer exists, because afterwards the fallback would be born masking a real bug. - target.h: check_kv_caps(key, value, caps, err), the rule in one place, taking the plan's two dtype strings so the header stays free of nlohmann/json. A function rather than a per-target check because xdna is not the last target (P5 adds Hexagon) and a rule re-remembered per target is one that will be missing from a target. - target.h also gains for uint8_t in TileFn, which it needed all along and only got when an includer happened to pull it in first. Found by compiling a TU that includes this header alone. - xdna_target.cpp: open() calls it before the device checks, so the answer does not depend on whether a device is attached. - tests/npu_target_caps_test.cpp: device-free, and end to end from a plan file to the refusal. Pins both directions -- too little (int8 K, int8 V and int8/int8 each refused, message naming the plan fields and the missing capability) and too much (a plan with no kv block, i.e. every container shipped so far, is accepted with no error text; refusing a working model would be the worse regression) -- plus that a target reporting int8_kv = true accepts all three. - docs/npu-plan.md: the schema row says where the enforcement lives, the refusal list carries the target-side rule (today every target reports int8_kv = false, so every int8 plan is refused, because the writer does not exist yet), P7's remaining list drops this item, and P4's stale "in PR #808 (open)" line is corrected to A1 landed / A2 remaining. Checks: ctest -R 'npu_target_caps|npu_kv_int8|npu_plan' 5/5 pass through CMake in a fresh tree (including npu_plan_generate), the ci.yml add_dependencies binding verified in the generated build graph, tools/copyright.py --check rc 0. Stated limitation: xdna_target.cpp needs XRT headers and this host has none, so that TU is not compiled here. The line added there is fully qualified, the function it calls is proven by the new test, and the exact call-site namespace context was probed by a throwaway TU -- but I cannot rule a compile error out locally, so it is flagged rather than assumed. No device: 0 s GPU, 0 s NPU, no ticket taken. --- CMakeLists.txt | 8 ++ docs/npu-plan.md | 17 +++- npu/target/target.h | 18 ++++ npu/target/xdna/xdna_target.cpp | 4 + tests/npu_target_caps_test.cpp | 150 ++++++++++++++++++++++++++++++++ 5 files changed, 193 insertions(+), 4 deletions(-) create mode 100644 tests/npu_target_caps_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index d77166dd1..7898dd271 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -835,6 +835,14 @@ add_test(NAME npu_plan COMMAND npu_plan_test) # Same reason as the other device-free tests: ci.yml builds a named list of targets, so a test whose # target is not built reports "Not Run" (#635). This lane's token cannot edit the workflow. add_dependencies(npu_sampling_test npu_plan_test) +# The plan-vs-target rule (npu/target/target.h, check_kv_caps): a plan that asks for a cache the target +# cannot store is a refusal by name, never a silent f16 store. Device-free: the refusal happens before any +# kernel is touched, and it is the same answer for a target that does not exist yet (engine#806 P7a). +add_executable(npu_target_caps_test tests/npu_target_caps_test.cpp) +target_include_directories(npu_target_caps_test PRIVATE ${CMAKE_SOURCE_DIR}/npu) +target_link_libraries(npu_target_caps_test PRIVATE nlohmann_json::nlohmann_json) +add_test(NAME npu_target_caps COMMAND npu_target_caps_test) +add_dependencies(npu_sampling_test npu_target_caps_test) # The generator behind that loader: a synthetic model dir, so the fused/split decision and --check are # tested without a model or a kernel build. diff --git a/docs/npu-plan.md b/docs/npu-plan.md index 8388ce183..30e5a9494 100644 --- a/docs/npu-plan.md +++ b/docs/npu-plan.md @@ -61,7 +61,7 @@ tools/npu_plan.py [--kernels DIR] [--out PATH] [--check] [--print] | `decode.sealed_elf` | pb16i only: the sealed ELF the decode roles run | | `kernels.tag` | artifact-name suffix: `""`, `_b8`, `_pb16i`, `_t2s` or `_t2s2`; must agree with the container layout and `container.pb16i` | | `attention.mode` | `causal` (default) or `noncausal_full_attention`. The latter is a prompt-encoding mode: every full-attention layer sees the whole prompt, and the forward refuses to generate with it | -| `kv.key`, `kv.value` | what the KV cache is stored as, per side: `f16` (the meaning of an absent `kv` block, and of every plan written before P7) or `int8`. `int8` is int8 with **one fp32 scale per (head, token)**, taken *after* a per-head Hadamard rotation — the rotation is what makes one scale affordable, because it collapses the per-channel magnitude spread the scale would otherwise have to cover. `key` is the side that costs no un-rotation (`(QH)(KH)ᵀ = QKᵀ`), so storing K rotated and rotating Q needs no inverse transform; `value` is a separate choice because an int8 V needs one on the output, which is why K-only is the cheap variant to prove the path with. This is a description: a target without the capability must refuse `int8` **by name**, never serve f16 in its place | +| `kv.key`, `kv.value` | what the KV cache is stored as, per side: `f16` (the meaning of an absent `kv` block, and of every plan written before P7) or `int8`. `int8` is int8 with **one fp32 scale per (head, token)**, taken *after* a per-head Hadamard rotation — the rotation is what makes one scale affordable, because it collapses the per-channel magnitude spread the scale would otherwise have to cover. `key` is the side that costs no un-rotation (`(QH)(KH)ᵀ = QKᵀ`), so storing K rotated and rotating Q needs no inverse transform; `value` is a separate choice because an int8 V needs one on the output, which is why K-only is the cheap variant to prove the path with. This is a description until a target opens it: a target without the capability refuses `int8` **by name**, never serves f16 in its place. That check is `check_kv_caps` in `npu/target/target.h`, called from every target's `open()`, and it is gated device-free (`ctest -R npu_target_caps`) | | `kv.hadamard.block`, `kv.hadamard.signs` | the rotation itself: `block` is `0` (no rotation) or a power of two in `[2, 65536]` — the head width the rotation runs over — and `signs` is an optional **bare file name** (no directory part, under the model dir) for per-channel signs. A `signs` file with `block: 0` is refused rather than ignored | | `moe.batched_expert_gu` | `{enabled, K, N}`; when enabled, K and N must be named | | `layers[]` | one entry per layer, in order: `index`, `role` (`attn`/`gdn`/`moe`/`mla`), and projections `qkv {K, N, fused}`, `attn_gate {K, N}`, `gate_up {K, N}`, `down {K, N}`. The count must equal `model.layers`. The projection shapes are recorded for P4’s `NpuModel`; the current forward reads only the count (for its [plan] log line) and derives designs from the container | @@ -86,6 +86,11 @@ The loader never substitutes a default for something it cannot honour; every ref - `kv.key`/`kv.value` outside `f16`/`int8`; a `kv.hadamard.block` that is not `0` or a power of two in `[2, 65536]`; a `kv.hadamard.signs` that is a path rather than a bare name, or that names a file while `kv.hadamard.block` is `0`; +- `kv.key`/`kv.value` asking for `int8` on a target whose caps say `int8_kv` is false — this one is the + **target's** refusal, not the loader's (the loader accepts the plan; the target refuses to run it), and it + names both the plan fields and the missing capability. Today every target reports `int8_kv = false`, so + every `int8` plan is refused: the cache writer does not exist yet (see P7 below), and the refusal is what + keeps a request for it from being answered out of an f16 cache; - `moe.batched_expert_gu.enabled` without positive K and N; - `layers` missing/out of order, a role outside the four, a projection naming only one of K and N, or a layer count that disagrees with `model.layers`; @@ -166,12 +171,16 @@ layers[0] {"index": 0, "role": "gdn", "qkv": {"K": 5120, "N": 10240, "fused" ## Not yet -- **P4** (#803): `NpuTarget`/`NpuModel` extraction is in PR #808 (open); the plan stays the description - until that interface lands. +- **P4** (#803): **landed** in #808 (`0a2cbb9`) as **A1** — the `NpuTarget` interface (`npu/target/target.h`), + the XDNA target behind it, and `open()` consuming the plan instead of duplicating the derivation. **A2** is what + remains: the design derivation moves out of the forward and into the target, so `open()` becomes the source rather + than the check (and `load_block`, which is a named no-op today). - **P5** (#804): Hexagon target (t2 gemv on HVX/HMX, QEMU v73 oracle) has no engine rows yet. - **P6** (#805): the shared drafter protocol is on main; tau=0.6 and Hexagon acceptance rows are still missing. - **P7** (#806): the `kv` contract above is in the plan and its arithmetic is gated in CI (`ctest -R npu_kv_int8`, - which needs no device: see below). Still open, in the order it can be taken: the cache **writer/reader** in the + which needs no device: see below), and the **target-side refusal** is enforced as well (`npu_target_caps`, so an + `int8` plan cannot be run as f16 by any target, including P5's Hexagon). Still open, in the order it can be taken: + the cache **writer/reader** in the forward (an int8 K store plus its scale sidecar, the prefix-cache snapshot's byte accounting with it), the **KLD gate on real KV from a model on the device** — the CI test bounds the mechanism on synthetic vectors, not the model — and **part B**, prefill in 1024/2048-token chunks with block-fp expansion, which waits on P3's on-core diff --git a/npu/target/target.h b/npu/target/target.h index 4ec87faad..3bd956ca9 100644 --- a/npu/target/target.h +++ b/npu/target/target.h @@ -26,6 +26,7 @@ #ifndef ONEBIT_NPU_TARGET_TARGET_H #define ONEBIT_NPU_TARGET_TARGET_H +#include // uint8_t: TileFn's tile buffer. Listed rather than relied on from an includer. #include #include @@ -125,6 +126,23 @@ class NpuTarget { virtual Caps caps() const = 0; }; +// The plan describes what the model directory asks for; a target's caps describe what it can actually run. +// A plan that asks for something the caps do not cover is a refusal BY NAME, never a silent downgrade -- +// for the KV cache the downgrade would be a model that answers the same text out of a different contract, +// which is the failure this whole layer exists to remove. +// +// One function rather than a check inside each target: the xdna target is not the last one (P5 adds +// Hexagon), and a rule that has to be re-remembered per target is a rule that will be missing from one. +// Takes the plan's two dtype strings rather than the Plan so this header stays free of nlohmann/json. +inline bool check_kv_caps(const std::string& key, const std::string& value, const Caps& caps, std::string& err) { + if (key != "int8" && value != "int8") return true; + if (caps.int8_kv) return true; + err = "plan asks for an int8 KV cache (kv.key=\"" + key + "\", kv.value=\"" + value + + "\") but this target has no int8 KV path (caps().int8_kv is false): refusing rather than " + "storing f16 in its place"; + return false; +} + } // namespace npu::target #endif // ONEBIT_NPU_TARGET_TARGET_H diff --git a/npu/target/xdna/xdna_target.cpp b/npu/target/xdna/xdna_target.cpp index 5b4fc7182..4ca6dae6c 100644 --- a/npu/target/xdna/xdna_target.cpp +++ b/npu/target/xdna/xdna_target.cpp @@ -93,6 +93,10 @@ bool XdnaTarget::open(const onebit::npu::plan::Plan& plan, const std::string& ke // A2 (#803 §3) moves the derivation in here: then this becomes the source rather than the check. kernels_dir_ = kernels_dir; plan_ = plan; + // Before the device checks: a plan that asks for a cache this target cannot store is refused whatever + // the device is doing, and the refusal names the field. Without this an int8 KV plan loads and is + // quietly served an f16 cache -- the plan layer's own documented trap. + if (!::npu::target::check_kv_caps(plan_.kv_key, plan_.kv_value, caps(), err)) return false; if (!dev_) { err = "xdna: no device attached (host-GEMM mode has no target)"; return false; } if (ndesigns_ == 0) { err = "xdna: no design adopted yet (A2: the plan supplies it)"; return false; } if (!plan_.loaded()) return true; diff --git a/tests/npu_target_caps_test.cpp b/tests/npu_target_caps_test.cpp new file mode 100644 index 000000000..1c34f85f3 --- /dev/null +++ b/tests/npu_target_caps_test.cpp @@ -0,0 +1,150 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// npu_target_caps_test: the rule that a plan asking for a cache its target cannot store is refused BY NAME +// (npu/target/target.h, check_kv_caps) -- engine#806 P7a's promise, enforced instead of documented. +// +// Why this is its own test rather than an arm of a device run: the refusal has to happen before any kernel +// or ELF is touched, and it has to be the same answer on every target, including the ones that do not exist +// yet (P5's Hexagon). A rule only a device can exercise is a rule that regresses silently, so this runs on +// any host with no model and no NPU. +// +// The two ways it can be wrong are both checked here: refusing too little (an int8 plan loads and is served +// an f16 cache -- a different model answering the same text, which is the trap the plan layer exists to +// remove) and refusing too much (a shipped f16 plan fails to load, which is a worse regression than the bug +// the check prevents). +#include "plan/plan.h" +#include "target/target.h" + +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using onebit::npu::plan::Plan; +using namespace npu::target; + +namespace { + +int failures = 0; +void check(bool ok, const std::string& what) { + if (ok) return; + ++failures; + std::printf("FAIL %s\n", what.c_str()); +} + +bool has(const std::string& hay, const std::string& needle) { + return hay.find(needle) != std::string::npos; +} + +// The smallest plan that loads, so the kv block is the only variable. +nlohmann::json base_plan() { + return nlohmann::json{ + {"format", "1bit-npu-plan/1"}, + {"model", {{"hidden", 1024}, {"layers", 1}, {"arch", "qwen3"}}}, + {"container", {{"layout", "rm"}, {"chunk_kind", "mode0"}, {"pb16i", false}}}, + {"fusion", {{"gate_up", "fused"}}}, + {"kernels", {{"tag", ""}}}, + {"layers", nlohmann::json::array({nlohmann::json{ + {"index", 0}, {"role", "attn"}, + {"qkv", {{"K", 1024}, {"N", 4096}, {"fused", true}}}, + {"gate_up", {{"K", 1024}, {"N", 6144}}}, + {"down", {{"K", 3072}, {"N", 1024}}}}})}, + }; +} + +std::string write_plan(const std::string& name, const nlohmann::json& body) { + const fs::path p = fs::temp_directory_path() / ("npu_target_caps_test_" + name + ".json"); + std::ofstream f(p); + f << body.dump(2) << "\n"; + f.close(); + return p.string(); +} + +// What today's targets report: the xdna target sets this false (P4), and nothing in the tree can store an +// int8 cache yet, which is precisely why every int8 plan must be refused today. +Caps no_int8_kv() { + Caps c; + c.max_rows = 16; + c.prefill_rows = 128; + c.t2_expansion = false; + c.int8_kv = false; + return c; +} + +void test_refuses_int8_without_caps() { + auto j = base_plan(); + j["kv"] = {{"key", "int8"}, {"value", "f16"}, {"hadamard", {{"block", 256}}}}; + const Plan p = Plan::load(write_plan("i8k", j)); + check(p.kv_int8(), "the plan reports itself as int8 before the target is asked"); + + std::string err; + const bool ok = check_kv_caps(p.kv_key, p.kv_value, no_int8_kv(), err); + check(!ok, "an int8 K plan is refused by a target with no int8 KV path"); + check(has(err, "int8") && has(err, "int8_kv"), + "the refusal names what was asked for and what is missing: " + err); + check(has(err, "kv.key") && has(err, "kv.value"), + "the refusal names the plan fields, so a user can find the line to edit"); + check(has(err, "refusing") && has(err, "f16"), + "the refusal says it is refusing rather than serving f16 (no silent downgrade)"); + // Printed, not just asserted: what a user actually reads is part of the contract, and a wording change that + // keeps every substring above intact should still be visible to whoever runs this by hand. + std::printf(" a refusal reads: %s\n", err.c_str()); +} + +void test_refuses_int8_value_without_caps() { + // K-only is the cheap variant; V is a separate choice, and asking for V int8 must be caught the same way + // rather than only being spotted when the writer lands. + std::string err; + const bool ok = check_kv_caps("f16", "int8", no_int8_kv(), err); + check(!ok && has(err, "kv.value"), + "an int8 V plan is refused by a target with no int8 KV path"); +} + +void test_accepts_when_caps_allow() { + Caps c = no_int8_kv(); + c.int8_kv = true; + for (const auto& kinds : {std::pair{"int8", "f16"}, + std::pair{"f16", "int8"}, + std::pair{"int8", "int8"}}) { + std::string err; + check(check_kv_caps(kinds.first, kinds.second, c, err), + "a target that reports int8 KV accepts " + kinds.first + "/" + kinds.second); + check(err.empty(), "and sets no error text when it accepts"); + } +} + +void test_does_not_refuse_shipped_plans() { + // The regression this guards: every container shipped so far is f16 KV, and refusing one of those would + // be worse than the bug the check prevents. Built from a real generator-shaped plan, not a stub. + const Plan p = Plan::load(write_plan("f16", base_plan())); + std::string err; + check(check_kv_caps(p.kv_key, p.kv_value, no_int8_kv(), err), + "a plan with no kv block (f16, what every shipped container means) is accepted, with no device"); + check(err.empty(), "and sets no error text"); +} + +} // namespace + +int main() { + test_refuses_int8_without_caps(); + test_refuses_int8_value_without_caps(); + test_accepts_when_caps_allow(); + test_does_not_refuse_shipped_plans(); + std::printf("%s\n", failures ? "FAILED" : "npu_target_caps: ok"); + return failures ? 1 : 0; +}