diff --git a/CMakeLists.txt b/CMakeLists.txt index 1bec1fb0e..739f53b8f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -614,7 +614,7 @@ add_executable(pack_initializer_shape_test tests/pack_initializer_shape_test.cpp target_include_directories(pack_initializer_shape_test PRIVATE npu) add_test(NAME pack_initializer_shape COMMAND pack_initializer_shape_test) add_dependencies(onebit pack_initializer_shape_test) -target_include_directories(npu_sampling_test PRIVATE app) +target_include_directories(npu_sampling_test PRIVATE app ${CMAKE_SOURCE_DIR}/npu) # engine#2324: the image digest arms target_link_libraries(npu_sampling_test PRIVATE nlohmann_json::nlohmann_json) # The speculative round's bookkeeping (app/spec_decode.h, engine#1781): a drafter's proposals against the verifier's # per-row logits, with the acceptance rule from npu/mtp/spec_accept.h (#1958, merged). Header-only and pure -- the diff --git a/app/npu_image_request.h b/app/npu_image_request.h index 047120242..513336012 100644 --- a/app/npu_image_request.h +++ b/app/npu_image_request.h @@ -122,4 +122,33 @@ inline bool request_images(const Json& messages, std::vector& return !out.empty(); } +// engine#2324: the digest a prefix-cache slot carries, so "same text, different picture" cannot be a cache hit. +// It covers the DIMENSIONS and the PIXELS, and the pictures in order: two identical images in a different order are a +// different prompt, and a re-encode of the same picture (PNG vs PPM) decodes to the same pixels and therefore to the same +// digest -- which is the behaviour we want, because the cache stores prefill state, not bytes on the wire. +inline uint64_t image_digest_of(const std::vector& images) { + uint64_t h = 1469598103934665603ull; // FNV-1a, 64-bit: stable across runs and platforms, which a std::hash is not + auto mix = [&h](const void* p, size_t n) { + const unsigned char* b = static_cast(p); + for (size_t i = 0; i < n; ++i) { h ^= b[i]; h *= 1099511628211ull; } + }; + const uint64_t count = images.size(); + mix(&count, sizeof count); // 0 images and 1 empty image must not collide + for (const auto& im : images) { + const int64_t wh[2] = {im.width, im.height}; + mix(wh, sizeof wh); + if (!im.rgb.empty()) mix(im.rgb.data(), im.rgb.size()); + } + return h; +} + +// The digest for a whole request: 0 (the text-only value) when there is no picture. +template +inline uint64_t request_image_digest(const Json& messages) { + std::vector images; + if (!request_images(messages, images)) return 0; + return image_digest_of(images); +} + + } // namespace onebit::npu diff --git a/app/npu_prefix_cache.h b/app/npu_prefix_cache.h index e15d12e2e..b4436a302 100644 --- a/app/npu_prefix_cache.h +++ b/app/npu_prefix_cache.h @@ -103,6 +103,11 @@ struct PrefixSlot { uint64_t last_used = 0; // monotonic, for eviction bool live = false; // false = free/reusable bool pinned = false; // the shared prefix: not evicted for room, not replaced by a longer prompt + // engine#2324: a request's PIXELS are not in `ids`, so a slot matched on tokens alone can hand a picture's + // prefill state to a different picture. Every slot carries the digest of the image parts it was prefilled + // with (0 = no image), and a match requires equality -- without that, "same text, different picture" is a + // cache hit that answers the wrong question with confidence. + uint64_t image_digest = 0; }; class PrefixCache { @@ -121,7 +126,7 @@ class PrefixCache { // The longest live slot whose tokens are a prefix of `ids`. A slot that covers the whole // prompt is still a hit: the request continues from it instead of re-prefilling. - PrefixDecision lookup(const std::vector& ids) { + PrefixDecision lookup(const std::vector& ids, uint64_t image_digest = 0) { PrefixDecision d; if (!enabled()) { d.reason = "off"; @@ -132,6 +137,7 @@ class PrefixCache { for (size_t i = 0; i < slots_.size(); i++) { const PrefixSlot& s = slots_[i]; if (!s.live || s.ids.empty() || s.ids.size() > ids.size()) continue; + if (s.image_digest != image_digest) continue; // a different picture is a different prefix if (!std::equal(s.ids.begin(), s.ids.end(), ids.begin())) continue; if (s.ids.size() > best) { best = s.ids.size(); @@ -155,7 +161,8 @@ class PrefixCache { // store the state in, or -1 when the prompt is not worth keeping. `extended` is the slot the // request resumed from (-1 when it prefilled from scratch): that slot is replaced, because the // longer prompt covers it. - int insert(const std::vector& ids, size_t bytes, int extended = -1, std::string* why = nullptr) { + int insert(const std::vector& ids, size_t bytes, int extended = -1, std::string* why = nullptr, + uint64_t image_digest = 0) { auto refuse = [&](const std::string& reason) { if (why) *why = reason; return -1; @@ -183,7 +190,7 @@ class PrefixCache { slot = free_slot(); if (slot < 0) return refuse("no free slot"); // every slot holds a better prompt } - fill(slot, ids, bytes, /*pinned=*/false); + fill(slot, ids, bytes, /*pinned=*/false, image_digest); if (why) *why = "saved"; return slot; } @@ -235,7 +242,7 @@ class PrefixCache { if (slot < 0) return refuse("no free slot"); release(slots_[size_t(slot)]); } - fill(slot, prefix, bytes, /*pinned=*/true); + fill(slot, prefix, bytes, /*pinned=*/true, 0); // a pin is the system prefix: text only if (why) *why = "pinned"; return slot; } @@ -288,12 +295,13 @@ class PrefixCache { if (!slots_[i].live) return int(i); return -1; } - void fill(int slot, const std::vector& ids, size_t bytes, bool pinned) { + void fill(int slot, const std::vector& ids, size_t bytes, bool pinned, uint64_t image_digest) { PrefixSlot& s = slots_[size_t(slot)]; s.ids = ids; s.bytes = bytes; s.live = true; s.pinned = pinned; + s.image_digest = image_digest; s.last_used = ++last_used_; used_ += bytes; } diff --git a/tests/npu_sampling_test.cpp b/tests/npu_sampling_test.cpp index 603bbba95..cf357bcf0 100644 --- a/tests/npu_sampling_test.cpp +++ b/tests/npu_sampling_test.cpp @@ -18,6 +18,7 @@ // Header only, no device: runs in CI. #include "npu_answer_parser.h" #include "npu_grammar.h" +#include "npu_image_request.h" // engine#2324: the image digest the prefix cache compares #include "npu_prefix_cache.h" #include "npu_sampling.h" @@ -719,6 +720,36 @@ void test_prefix_cache() { check(c.lookup(ids(10, 2)).slot == -1, "a prompt whose start differs misses"); check(c.lookup(ids(5)).slot == -1, "a shorter prompt does not hit a longer slot"); } + { // engine#2324: the digest that slot comparison uses -- computed from the PIXELS, not the wire bytes + auto img = [](int w, int h, uint8_t fill) { onebit::npu::vision::RgbImage i; i.width = w; i.height = h; i.rgb.assign(size_t(w) * h * 3, fill); return i; }; + const auto a1 = onebit::npu::image_digest_of({img(4, 3, 0x11)}); + const auto a2 = onebit::npu::image_digest_of({img(4, 3, 0x11)}); // same pixels, same digest + const auto b = onebit::npu::image_digest_of({img(4, 3, 0x22)}); // one byte different + const auto c = onebit::npu::image_digest_of({img(3, 4, 0x11)}); // transposed: same byte count, different picture + const auto none = onebit::npu::image_digest_of({}); + const auto two = onebit::npu::image_digest_of({img(4, 3, 0x11), img(4, 3, 0x22)}); + const auto twof = onebit::npu::image_digest_of({img(4, 3, 0x22), img(4, 3, 0x11)}); // order matters + check(a1 == a2, "digest: identical pixels give the identical digest"); + check(a1 != b, "digest: one changed pixel changes it"); + check(a1 != c, "digest: a transposed image is a different picture"); + check(a1 != none && none != two, "digest: no picture differs from one, and from two"); + check(two != twof, "digest: the same two pictures in the other order are a different prompt"); + } + { // engine#2324: the pixels are not in `ids`, so a slot must also match the digest of the image parts. + // Without this, "same text, different picture" is a cache HIT and the answer comes from the wrong image. + PrefixCache c(o); + const uint64_t imageA = 0xAAAAull, imageB = 0xBBBBull; + const int slot = c.insert(ids(12), 4096, -1, nullptr, imageA); + check(slot >= 0, "a prompt with a picture is saved"); + check(c.lookup(ids(12), imageA).slot == slot, "the same picture hits"); + check(c.lookup(ids(12), imageB).slot == -1, "a DIFFERENT picture misses, even with identical tokens"); + check(c.lookup(ids(12)).slot == -1, "and so does a request with no picture at all"); + check(c.lookup(ids(12), imageB).reason == "no slot shares a prefix", "the miss is reported like any other"); + // the text-only path is unchanged: two text-only slots still hit each other + PrefixCache t(o); + const int ts = t.insert(ids(10), 1024); + check(t.lookup(ids(10)).slot == ts, "a text-only prompt still hits a text-only slot"); + } { // the LONGEST matching slot wins, not the newest and not the immediately preceding one PrefixCache c(o); c.insert(ids(20), 1024);