diff --git a/CMakeLists.txt b/CMakeLists.txt index 8b0fe5cc3..5469522c5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -835,6 +835,14 @@ add_test(NAME npu_plan COMMAND npu_plan_test) # Same reason as the other device-free tests: ci.yml builds a named list of targets, so a test whose # target is not built reports "Not Run" (#635). This lane's token cannot edit the workflow. add_dependencies(npu_sampling_test npu_plan_test) +# The plan-vs-target rule (npu/target/target.h, check_kv_caps): a plan that asks for a cache the target +# cannot store is a refusal by name, never a silent f16 store. Device-free: the refusal happens before any +# kernel is touched, and it is the same answer for a target that does not exist yet (engine#806 P7a). +add_executable(npu_target_caps_test tests/npu_target_caps_test.cpp) +target_include_directories(npu_target_caps_test PRIVATE ${CMAKE_SOURCE_DIR}/npu) +target_link_libraries(npu_target_caps_test PRIVATE nlohmann_json::nlohmann_json) +add_test(NAME npu_target_caps COMMAND npu_target_caps_test) +add_dependencies(npu_sampling_test npu_target_caps_test) # The generator behind that loader: a synthetic model dir, so the fused/split decision and --check are # tested without a model or a kernel build. diff --git a/docs/npu-plan.md b/docs/npu-plan.md index 8388ce183..30e5a9494 100644 --- a/docs/npu-plan.md +++ b/docs/npu-plan.md @@ -61,7 +61,7 @@ tools/npu_plan.py [--kernels DIR] [--out PATH] [--check] [--print] | `decode.sealed_elf` | pb16i only: the sealed ELF the decode roles run | | `kernels.tag` | artifact-name suffix: `""`, `_b8`, `_pb16i`, `_t2s` or `_t2s2`; must agree with the container layout and `container.pb16i` | | `attention.mode` | `causal` (default) or `noncausal_full_attention`. The latter is a prompt-encoding mode: every full-attention layer sees the whole prompt, and the forward refuses to generate with it | -| `kv.key`, `kv.value` | what the KV cache is stored as, per side: `f16` (the meaning of an absent `kv` block, and of every plan written before P7) or `int8`. `int8` is int8 with **one fp32 scale per (head, token)**, taken *after* a per-head Hadamard rotation — the rotation is what makes one scale affordable, because it collapses the per-channel magnitude spread the scale would otherwise have to cover. `key` is the side that costs no un-rotation (`(QH)(KH)ᵀ = QKᵀ`), so storing K rotated and rotating Q needs no inverse transform; `value` is a separate choice because an int8 V needs one on the output, which is why K-only is the cheap variant to prove the path with. This is a description: a target without the capability must refuse `int8` **by name**, never serve f16 in its place | +| `kv.key`, `kv.value` | what the KV cache is stored as, per side: `f16` (the meaning of an absent `kv` block, and of every plan written before P7) or `int8`. `int8` is int8 with **one fp32 scale per (head, token)**, taken *after* a per-head Hadamard rotation — the rotation is what makes one scale affordable, because it collapses the per-channel magnitude spread the scale would otherwise have to cover. `key` is the side that costs no un-rotation (`(QH)(KH)ᵀ = QKᵀ`), so storing K rotated and rotating Q needs no inverse transform; `value` is a separate choice because an int8 V needs one on the output, which is why K-only is the cheap variant to prove the path with. This is a description until a target opens it: a target without the capability refuses `int8` **by name**, never serves f16 in its place. That check is `check_kv_caps` in `npu/target/target.h`, called from every target's `open()`, and it is gated device-free (`ctest -R npu_target_caps`) | | `kv.hadamard.block`, `kv.hadamard.signs` | the rotation itself: `block` is `0` (no rotation) or a power of two in `[2, 65536]` — the head width the rotation runs over — and `signs` is an optional **bare file name** (no directory part, under the model dir) for per-channel signs. A `signs` file with `block: 0` is refused rather than ignored | | `moe.batched_expert_gu` | `{enabled, K, N}`; when enabled, K and N must be named | | `layers[]` | one entry per layer, in order: `index`, `role` (`attn`/`gdn`/`moe`/`mla`), and projections `qkv {K, N, fused}`, `attn_gate {K, N}`, `gate_up {K, N}`, `down {K, N}`. The count must equal `model.layers`. The projection shapes are recorded for P4’s `NpuModel`; the current forward reads only the count (for its [plan] log line) and derives designs from the container | @@ -86,6 +86,11 @@ The loader never substitutes a default for something it cannot honour; every ref - `kv.key`/`kv.value` outside `f16`/`int8`; a `kv.hadamard.block` that is not `0` or a power of two in `[2, 65536]`; a `kv.hadamard.signs` that is a path rather than a bare name, or that names a file while `kv.hadamard.block` is `0`; +- `kv.key`/`kv.value` asking for `int8` on a target whose caps say `int8_kv` is false — this one is the + **target's** refusal, not the loader's (the loader accepts the plan; the target refuses to run it), and it + names both the plan fields and the missing capability. Today every target reports `int8_kv = false`, so + every `int8` plan is refused: the cache writer does not exist yet (see P7 below), and the refusal is what + keeps a request for it from being answered out of an f16 cache; - `moe.batched_expert_gu.enabled` without positive K and N; - `layers` missing/out of order, a role outside the four, a projection naming only one of K and N, or a layer count that disagrees with `model.layers`; @@ -166,12 +171,16 @@ layers[0] {"index": 0, "role": "gdn", "qkv": {"K": 5120, "N": 10240, "fused" ## Not yet -- **P4** (#803): `NpuTarget`/`NpuModel` extraction is in PR #808 (open); the plan stays the description - until that interface lands. +- **P4** (#803): **landed** in #808 (`0a2cbb9`) as **A1** — the `NpuTarget` interface (`npu/target/target.h`), + the XDNA target behind it, and `open()` consuming the plan instead of duplicating the derivation. **A2** is what + remains: the design derivation moves out of the forward and into the target, so `open()` becomes the source rather + than the check (and `load_block`, which is a named no-op today). - **P5** (#804): Hexagon target (t2 gemv on HVX/HMX, QEMU v73 oracle) has no engine rows yet. - **P6** (#805): the shared drafter protocol is on main; tau=0.6 and Hexagon acceptance rows are still missing. - **P7** (#806): the `kv` contract above is in the plan and its arithmetic is gated in CI (`ctest -R npu_kv_int8`, - which needs no device: see below). Still open, in the order it can be taken: the cache **writer/reader** in the + which needs no device: see below), and the **target-side refusal** is enforced as well (`npu_target_caps`, so an + `int8` plan cannot be run as f16 by any target, including P5's Hexagon). Still open, in the order it can be taken: + the cache **writer/reader** in the forward (an int8 K store plus its scale sidecar, the prefix-cache snapshot's byte accounting with it), the **KLD gate on real KV from a model on the device** — the CI test bounds the mechanism on synthetic vectors, not the model — and **part B**, prefill in 1024/2048-token chunks with block-fp expansion, which waits on P3's on-core diff --git a/npu/target/target.h b/npu/target/target.h index 4ec87faad..3bd956ca9 100644 --- a/npu/target/target.h +++ b/npu/target/target.h @@ -26,6 +26,7 @@ #ifndef ONEBIT_NPU_TARGET_TARGET_H #define ONEBIT_NPU_TARGET_TARGET_H +#include // uint8_t: TileFn's tile buffer. Listed rather than relied on from an includer. #include #include @@ -125,6 +126,23 @@ class NpuTarget { virtual Caps caps() const = 0; }; +// The plan describes what the model directory asks for; a target's caps describe what it can actually run. +// A plan that asks for something the caps do not cover is a refusal BY NAME, never a silent downgrade -- +// for the KV cache the downgrade would be a model that answers the same text out of a different contract, +// which is the failure this whole layer exists to remove. +// +// One function rather than a check inside each target: the xdna target is not the last one (P5 adds +// Hexagon), and a rule that has to be re-remembered per target is a rule that will be missing from one. +// Takes the plan's two dtype strings rather than the Plan so this header stays free of nlohmann/json. +inline bool check_kv_caps(const std::string& key, const std::string& value, const Caps& caps, std::string& err) { + if (key != "int8" && value != "int8") return true; + if (caps.int8_kv) return true; + err = "plan asks for an int8 KV cache (kv.key=\"" + key + "\", kv.value=\"" + value + + "\") but this target has no int8 KV path (caps().int8_kv is false): refusing rather than " + "storing f16 in its place"; + return false; +} + } // namespace npu::target #endif // ONEBIT_NPU_TARGET_TARGET_H diff --git a/npu/target/xdna/xdna_target.cpp b/npu/target/xdna/xdna_target.cpp index 5b4fc7182..4ca6dae6c 100644 --- a/npu/target/xdna/xdna_target.cpp +++ b/npu/target/xdna/xdna_target.cpp @@ -93,6 +93,10 @@ bool XdnaTarget::open(const onebit::npu::plan::Plan& plan, const std::string& ke // A2 (#803 §3) moves the derivation in here: then this becomes the source rather than the check. kernels_dir_ = kernels_dir; plan_ = plan; + // Before the device checks: a plan that asks for a cache this target cannot store is refused whatever + // the device is doing, and the refusal names the field. Without this an int8 KV plan loads and is + // quietly served an f16 cache -- the plan layer's own documented trap. + if (!::npu::target::check_kv_caps(plan_.kv_key, plan_.kv_value, caps(), err)) return false; if (!dev_) { err = "xdna: no device attached (host-GEMM mode has no target)"; return false; } if (ndesigns_ == 0) { err = "xdna: no design adopted yet (A2: the plan supplies it)"; return false; } if (!plan_.loaded()) return true; diff --git a/tests/npu_target_caps_test.cpp b/tests/npu_target_caps_test.cpp new file mode 100644 index 000000000..1c34f85f3 --- /dev/null +++ b/tests/npu_target_caps_test.cpp @@ -0,0 +1,150 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// npu_target_caps_test: the rule that a plan asking for a cache its target cannot store is refused BY NAME +// (npu/target/target.h, check_kv_caps) -- engine#806 P7a's promise, enforced instead of documented. +// +// Why this is its own test rather than an arm of a device run: the refusal has to happen before any kernel +// or ELF is touched, and it has to be the same answer on every target, including the ones that do not exist +// yet (P5's Hexagon). A rule only a device can exercise is a rule that regresses silently, so this runs on +// any host with no model and no NPU. +// +// The two ways it can be wrong are both checked here: refusing too little (an int8 plan loads and is served +// an f16 cache -- a different model answering the same text, which is the trap the plan layer exists to +// remove) and refusing too much (a shipped f16 plan fails to load, which is a worse regression than the bug +// the check prevents). +#include "plan/plan.h" +#include "target/target.h" + +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using onebit::npu::plan::Plan; +using namespace npu::target; + +namespace { + +int failures = 0; +void check(bool ok, const std::string& what) { + if (ok) return; + ++failures; + std::printf("FAIL %s\n", what.c_str()); +} + +bool has(const std::string& hay, const std::string& needle) { + return hay.find(needle) != std::string::npos; +} + +// The smallest plan that loads, so the kv block is the only variable. +nlohmann::json base_plan() { + return nlohmann::json{ + {"format", "1bit-npu-plan/1"}, + {"model", {{"hidden", 1024}, {"layers", 1}, {"arch", "qwen3"}}}, + {"container", {{"layout", "rm"}, {"chunk_kind", "mode0"}, {"pb16i", false}}}, + {"fusion", {{"gate_up", "fused"}}}, + {"kernels", {{"tag", ""}}}, + {"layers", nlohmann::json::array({nlohmann::json{ + {"index", 0}, {"role", "attn"}, + {"qkv", {{"K", 1024}, {"N", 4096}, {"fused", true}}}, + {"gate_up", {{"K", 1024}, {"N", 6144}}}, + {"down", {{"K", 3072}, {"N", 1024}}}}})}, + }; +} + +std::string write_plan(const std::string& name, const nlohmann::json& body) { + const fs::path p = fs::temp_directory_path() / ("npu_target_caps_test_" + name + ".json"); + std::ofstream f(p); + f << body.dump(2) << "\n"; + f.close(); + return p.string(); +} + +// What today's targets report: the xdna target sets this false (P4), and nothing in the tree can store an +// int8 cache yet, which is precisely why every int8 plan must be refused today. +Caps no_int8_kv() { + Caps c; + c.max_rows = 16; + c.prefill_rows = 128; + c.t2_expansion = false; + c.int8_kv = false; + return c; +} + +void test_refuses_int8_without_caps() { + auto j = base_plan(); + j["kv"] = {{"key", "int8"}, {"value", "f16"}, {"hadamard", {{"block", 256}}}}; + const Plan p = Plan::load(write_plan("i8k", j)); + check(p.kv_int8(), "the plan reports itself as int8 before the target is asked"); + + std::string err; + const bool ok = check_kv_caps(p.kv_key, p.kv_value, no_int8_kv(), err); + check(!ok, "an int8 K plan is refused by a target with no int8 KV path"); + check(has(err, "int8") && has(err, "int8_kv"), + "the refusal names what was asked for and what is missing: " + err); + check(has(err, "kv.key") && has(err, "kv.value"), + "the refusal names the plan fields, so a user can find the line to edit"); + check(has(err, "refusing") && has(err, "f16"), + "the refusal says it is refusing rather than serving f16 (no silent downgrade)"); + // Printed, not just asserted: what a user actually reads is part of the contract, and a wording change that + // keeps every substring above intact should still be visible to whoever runs this by hand. + std::printf(" a refusal reads: %s\n", err.c_str()); +} + +void test_refuses_int8_value_without_caps() { + // K-only is the cheap variant; V is a separate choice, and asking for V int8 must be caught the same way + // rather than only being spotted when the writer lands. + std::string err; + const bool ok = check_kv_caps("f16", "int8", no_int8_kv(), err); + check(!ok && has(err, "kv.value"), + "an int8 V plan is refused by a target with no int8 KV path"); +} + +void test_accepts_when_caps_allow() { + Caps c = no_int8_kv(); + c.int8_kv = true; + for (const auto& kinds : {std::pair{"int8", "f16"}, + std::pair{"f16", "int8"}, + std::pair{"int8", "int8"}}) { + std::string err; + check(check_kv_caps(kinds.first, kinds.second, c, err), + "a target that reports int8 KV accepts " + kinds.first + "/" + kinds.second); + check(err.empty(), "and sets no error text when it accepts"); + } +} + +void test_does_not_refuse_shipped_plans() { + // The regression this guards: every container shipped so far is f16 KV, and refusing one of those would + // be worse than the bug the check prevents. Built from a real generator-shaped plan, not a stub. + const Plan p = Plan::load(write_plan("f16", base_plan())); + std::string err; + check(check_kv_caps(p.kv_key, p.kv_value, no_int8_kv(), err), + "a plan with no kv block (f16, what every shipped container means) is accepted, with no device"); + check(err.empty(), "and sets no error text"); +} + +} // namespace + +int main() { + test_refuses_int8_without_caps(); + test_refuses_int8_value_without_caps(); + test_accepts_when_caps_allow(); + test_does_not_refuse_shipped_plans(); + std::printf("%s\n", failures ? "FAILED" : "npu_target_caps: ok"); + return failures ? 1 : 0; +}