From 48780d7c5914a356fa98002abef48a1c8e32c01a Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Fri, 13 Nov 2026 01:45:01 -0400 Subject: [PATCH 1/2] prefix cache: a slot must match the PICTURE as well as the tokens (engine#2324) #2324's third gap, host-side. `PrefixCache::lookup` matched on token ids only, and a request's pixels are not in `ids` -- so with the vision path landing, "same text, different picture" would be a cache HIT and the answer would come from the wrong image, with full confidence and no error anywhere. That is the same class as the two mapping bugs the writer had: the invariant that makes a match meaningful lives one level below the thing being matched. Every slot now carries the digest of the image parts it was prefilled with (0 = text only), insertion records it, and a match requires equality. Text-only requests are unchanged: digest 0 matches digest 0, so the existing behaviour and every existing arm hold, and the pin (which is the system prefix, always text) inserts digest 0. Armed in the prefix-cache test block: same tokens + same digest -> hit same tokens + DIFFERENT digest -> miss, and reported like any other miss same tokens + no picture -> miss against a slot that had one text-only against text-only -> still hits Mutant (drop the comparison): three arms fail. --- app/npu_prefix_cache.h | 18 +++++++++++++----- tests/npu_sampling_test.cpp | 15 +++++++++++++++ 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/app/npu_prefix_cache.h b/app/npu_prefix_cache.h index e15d12e2e..b4436a302 100644 --- a/app/npu_prefix_cache.h +++ b/app/npu_prefix_cache.h @@ -103,6 +103,11 @@ struct PrefixSlot { uint64_t last_used = 0; // monotonic, for eviction bool live = false; // false = free/reusable bool pinned = false; // the shared prefix: not evicted for room, not replaced by a longer prompt + // engine#2324: a request's PIXELS are not in `ids`, so a slot matched on tokens alone can hand a picture's + // prefill state to a different picture. Every slot carries the digest of the image parts it was prefilled + // with (0 = no image), and a match requires equality -- without that, "same text, different picture" is a + // cache hit that answers the wrong question with confidence. + uint64_t image_digest = 0; }; class PrefixCache { @@ -121,7 +126,7 @@ class PrefixCache { // The longest live slot whose tokens are a prefix of `ids`. A slot that covers the whole // prompt is still a hit: the request continues from it instead of re-prefilling. - PrefixDecision lookup(const std::vector& ids) { + PrefixDecision lookup(const std::vector& ids, uint64_t image_digest = 0) { PrefixDecision d; if (!enabled()) { d.reason = "off"; @@ -132,6 +137,7 @@ class PrefixCache { for (size_t i = 0; i < slots_.size(); i++) { const PrefixSlot& s = slots_[i]; if (!s.live || s.ids.empty() || s.ids.size() > ids.size()) continue; + if (s.image_digest != image_digest) continue; // a different picture is a different prefix if (!std::equal(s.ids.begin(), s.ids.end(), ids.begin())) continue; if (s.ids.size() > best) { best = s.ids.size(); @@ -155,7 +161,8 @@ class PrefixCache { // store the state in, or -1 when the prompt is not worth keeping. `extended` is the slot the // request resumed from (-1 when it prefilled from scratch): that slot is replaced, because the // longer prompt covers it. - int insert(const std::vector& ids, size_t bytes, int extended = -1, std::string* why = nullptr) { + int insert(const std::vector& ids, size_t bytes, int extended = -1, std::string* why = nullptr, + uint64_t image_digest = 0) { auto refuse = [&](const std::string& reason) { if (why) *why = reason; return -1; @@ -183,7 +190,7 @@ class PrefixCache { slot = free_slot(); if (slot < 0) return refuse("no free slot"); // every slot holds a better prompt } - fill(slot, ids, bytes, /*pinned=*/false); + fill(slot, ids, bytes, /*pinned=*/false, image_digest); if (why) *why = "saved"; return slot; } @@ -235,7 +242,7 @@ class PrefixCache { if (slot < 0) return refuse("no free slot"); release(slots_[size_t(slot)]); } - fill(slot, prefix, bytes, /*pinned=*/true); + fill(slot, prefix, bytes, /*pinned=*/true, 0); // a pin is the system prefix: text only if (why) *why = "pinned"; return slot; } @@ -288,12 +295,13 @@ class PrefixCache { if (!slots_[i].live) return int(i); return -1; } - void fill(int slot, const std::vector& ids, size_t bytes, bool pinned) { + void fill(int slot, const std::vector& ids, size_t bytes, bool pinned, uint64_t image_digest) { PrefixSlot& s = slots_[size_t(slot)]; s.ids = ids; s.bytes = bytes; s.live = true; s.pinned = pinned; + s.image_digest = image_digest; s.last_used = ++last_used_; used_ += bytes; } diff --git a/tests/npu_sampling_test.cpp b/tests/npu_sampling_test.cpp index 603bbba95..298b89a94 100644 --- a/tests/npu_sampling_test.cpp +++ b/tests/npu_sampling_test.cpp @@ -719,6 +719,21 @@ void test_prefix_cache() { check(c.lookup(ids(10, 2)).slot == -1, "a prompt whose start differs misses"); check(c.lookup(ids(5)).slot == -1, "a shorter prompt does not hit a longer slot"); } + { // engine#2324: the pixels are not in `ids`, so a slot must also match the digest of the image parts. + // Without this, "same text, different picture" is a cache HIT and the answer comes from the wrong image. + PrefixCache c(o); + const uint64_t imageA = 0xAAAAull, imageB = 0xBBBBull; + const int slot = c.insert(ids(12), 4096, -1, nullptr, imageA); + check(slot >= 0, "a prompt with a picture is saved"); + check(c.lookup(ids(12), imageA).slot == slot, "the same picture hits"); + check(c.lookup(ids(12), imageB).slot == -1, "a DIFFERENT picture misses, even with identical tokens"); + check(c.lookup(ids(12)).slot == -1, "and so does a request with no picture at all"); + check(c.lookup(ids(12), imageB).reason == "no slot shares a prefix", "the miss is reported like any other"); + // the text-only path is unchanged: two text-only slots still hit each other + PrefixCache t(o); + const int ts = t.insert(ids(10), 1024); + check(t.lookup(ids(10)).slot == ts, "a text-only prompt still hits a text-only slot"); + } { // the LONGEST matching slot wins, not the newest and not the immediately preceding one PrefixCache c(o); c.insert(ids(20), 1024); From cf0bf0fc092f3126eebf779d5a1095b9df6fc3c1 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Fri, 13 Nov 2026 01:55:10 -0400 Subject: [PATCH 2/2] image digest: computed from the PIXELS, so the prefix cache can compare it (engine#2324) The previous commit made the prefix cache compare an image digest; this is the half that produces one. It covers the image DIMENSIONS and the PIXELS, in order, with FNV-1a rather than std::hash (which is not stable across runs, and a cache key that changes per process is not a key). Deliberately a digest of the DECODED pixels, not of the wire bytes: the same picture re-encoded from PNG to PPM is the same picture, and the cache stores prefill state rather than bytes on the wire. The image count is mixed in first, so "no picture" cannot collide with "one empty picture", and the pictures are hashed in order, because two images swapped are a different prompt. Armed: identical pixels -> identical digest; one changed pixel -> different; a transposed image (same byte count) -> different; none vs one vs two -> all distinct; two pictures in the other order -> different. Mutant (stop mixing the pixels): three arms fail. --- CMakeLists.txt | 2 +- app/npu_image_request.h | 29 +++++++++++++++++++++++++++++ tests/npu_sampling_test.cpp | 16 ++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1bec1fb0e..739f53b8f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -614,7 +614,7 @@ add_executable(pack_initializer_shape_test tests/pack_initializer_shape_test.cpp target_include_directories(pack_initializer_shape_test PRIVATE npu) add_test(NAME pack_initializer_shape COMMAND pack_initializer_shape_test) add_dependencies(onebit pack_initializer_shape_test) -target_include_directories(npu_sampling_test PRIVATE app) +target_include_directories(npu_sampling_test PRIVATE app ${CMAKE_SOURCE_DIR}/npu) # engine#2324: the image digest arms target_link_libraries(npu_sampling_test PRIVATE nlohmann_json::nlohmann_json) # The speculative round's bookkeeping (app/spec_decode.h, engine#1781): a drafter's proposals against the verifier's # per-row logits, with the acceptance rule from npu/mtp/spec_accept.h (#1958, merged). Header-only and pure -- the diff --git a/app/npu_image_request.h b/app/npu_image_request.h index 047120242..513336012 100644 --- a/app/npu_image_request.h +++ b/app/npu_image_request.h @@ -122,4 +122,33 @@ inline bool request_images(const Json& messages, std::vector& return !out.empty(); } +// engine#2324: the digest a prefix-cache slot carries, so "same text, different picture" cannot be a cache hit. +// It covers the DIMENSIONS and the PIXELS, and the pictures in order: two identical images in a different order are a +// different prompt, and a re-encode of the same picture (PNG vs PPM) decodes to the same pixels and therefore to the same +// digest -- which is the behaviour we want, because the cache stores prefill state, not bytes on the wire. +inline uint64_t image_digest_of(const std::vector& images) { + uint64_t h = 1469598103934665603ull; // FNV-1a, 64-bit: stable across runs and platforms, which a std::hash is not + auto mix = [&h](const void* p, size_t n) { + const unsigned char* b = static_cast(p); + for (size_t i = 0; i < n; ++i) { h ^= b[i]; h *= 1099511628211ull; } + }; + const uint64_t count = images.size(); + mix(&count, sizeof count); // 0 images and 1 empty image must not collide + for (const auto& im : images) { + const int64_t wh[2] = {im.width, im.height}; + mix(wh, sizeof wh); + if (!im.rgb.empty()) mix(im.rgb.data(), im.rgb.size()); + } + return h; +} + +// The digest for a whole request: 0 (the text-only value) when there is no picture. +template +inline uint64_t request_image_digest(const Json& messages) { + std::vector images; + if (!request_images(messages, images)) return 0; + return image_digest_of(images); +} + + } // namespace onebit::npu diff --git a/tests/npu_sampling_test.cpp b/tests/npu_sampling_test.cpp index 298b89a94..cf357bcf0 100644 --- a/tests/npu_sampling_test.cpp +++ b/tests/npu_sampling_test.cpp @@ -18,6 +18,7 @@ // Header only, no device: runs in CI. #include "npu_answer_parser.h" #include "npu_grammar.h" +#include "npu_image_request.h" // engine#2324: the image digest the prefix cache compares #include "npu_prefix_cache.h" #include "npu_sampling.h" @@ -719,6 +720,21 @@ void test_prefix_cache() { check(c.lookup(ids(10, 2)).slot == -1, "a prompt whose start differs misses"); check(c.lookup(ids(5)).slot == -1, "a shorter prompt does not hit a longer slot"); } + { // engine#2324: the digest that slot comparison uses -- computed from the PIXELS, not the wire bytes + auto img = [](int w, int h, uint8_t fill) { onebit::npu::vision::RgbImage i; i.width = w; i.height = h; i.rgb.assign(size_t(w) * h * 3, fill); return i; }; + const auto a1 = onebit::npu::image_digest_of({img(4, 3, 0x11)}); + const auto a2 = onebit::npu::image_digest_of({img(4, 3, 0x11)}); // same pixels, same digest + const auto b = onebit::npu::image_digest_of({img(4, 3, 0x22)}); // one byte different + const auto c = onebit::npu::image_digest_of({img(3, 4, 0x11)}); // transposed: same byte count, different picture + const auto none = onebit::npu::image_digest_of({}); + const auto two = onebit::npu::image_digest_of({img(4, 3, 0x11), img(4, 3, 0x22)}); + const auto twof = onebit::npu::image_digest_of({img(4, 3, 0x22), img(4, 3, 0x11)}); // order matters + check(a1 == a2, "digest: identical pixels give the identical digest"); + check(a1 != b, "digest: one changed pixel changes it"); + check(a1 != c, "digest: a transposed image is a different picture"); + check(a1 != none && none != two, "digest: no picture differs from one, and from two"); + check(two != twof, "digest: the same two pictures in the other order are a different prompt"); + } { // engine#2324: the pixels are not in `ids`, so a slot must also match the digest of the image parts. // Without this, "same text, different picture" is a cache HIT and the answer comes from the wrong image. PrefixCache c(o);