Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -614,7 +614,7 @@ add_executable(pack_initializer_shape_test tests/pack_initializer_shape_test.cpp
target_include_directories(pack_initializer_shape_test PRIVATE npu)
add_test(NAME pack_initializer_shape COMMAND pack_initializer_shape_test)
add_dependencies(onebit pack_initializer_shape_test)
target_include_directories(npu_sampling_test PRIVATE app)
target_include_directories(npu_sampling_test PRIVATE app ${CMAKE_SOURCE_DIR}/npu) # engine#2324: the image digest arms
target_link_libraries(npu_sampling_test PRIVATE nlohmann_json::nlohmann_json)
# The speculative round's bookkeeping (app/spec_decode.h, engine#1781): a drafter's proposals against the verifier's
# per-row logits, with the acceptance rule from npu/mtp/spec_accept.h (#1958, merged). Header-only and pure -- the
Expand Down
29 changes: 29 additions & 0 deletions app/npu_image_request.h
Original file line number Diff line number Diff line change
Expand Up @@ -122,4 +122,33 @@ inline bool request_images(const Json& messages, std::vector<vision::RgbImage>&
return !out.empty();
}

// engine#2324: the digest a prefix-cache slot carries, so "same text, different picture" cannot be a cache hit.
// It covers the DIMENSIONS and the PIXELS, and the pictures in order: two identical images in a different order are a
// different prompt, and a re-encode of the same picture (PNG vs PPM) decodes to the same pixels and therefore to the same
// digest -- which is the behaviour we want, because the cache stores prefill state, not bytes on the wire.
inline uint64_t image_digest_of(const std::vector<vision::RgbImage>& images) {
uint64_t h = 1469598103934665603ull; // FNV-1a, 64-bit: stable across runs and platforms, which a std::hash is not
auto mix = [&h](const void* p, size_t n) {
const unsigned char* b = static_cast<const unsigned char*>(p);
for (size_t i = 0; i < n; ++i) { h ^= b[i]; h *= 1099511628211ull; }
};
const uint64_t count = images.size();
mix(&count, sizeof count); // 0 images and 1 empty image must not collide
for (const auto& im : images) {
const int64_t wh[2] = {im.width, im.height};
mix(wh, sizeof wh);
if (!im.rgb.empty()) mix(im.rgb.data(), im.rgb.size());
}
return h;
}

// The digest for a whole request: 0 (the text-only value) when there is no picture.
template <class Json>
inline uint64_t request_image_digest(const Json& messages) {
std::vector<vision::RgbImage> images;
if (!request_images(messages, images)) return 0;
return image_digest_of(images);
}


} // namespace onebit::npu
18 changes: 13 additions & 5 deletions app/npu_prefix_cache.h
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,11 @@ struct PrefixSlot {
uint64_t last_used = 0; // monotonic, for eviction
bool live = false; // false = free/reusable
bool pinned = false; // the shared prefix: not evicted for room, not replaced by a longer prompt
// engine#2324: a request's PIXELS are not in `ids`, so a slot matched on tokens alone can hand a picture's
// prefill state to a different picture. Every slot carries the digest of the image parts it was prefilled
// with (0 = no image), and a match requires equality -- without that, "same text, different picture" is a
// cache hit that answers the wrong question with confidence.
uint64_t image_digest = 0;
};

class PrefixCache {
Expand All @@ -121,7 +126,7 @@ class PrefixCache {

// The longest live slot whose tokens are a prefix of `ids`. A slot that covers the whole
// prompt is still a hit: the request continues from it instead of re-prefilling.
PrefixDecision lookup(const std::vector<int>& ids) {
PrefixDecision lookup(const std::vector<int>& ids, uint64_t image_digest = 0) {
PrefixDecision d;
if (!enabled()) {
d.reason = "off";
Expand All @@ -132,6 +137,7 @@ class PrefixCache {
for (size_t i = 0; i < slots_.size(); i++) {
const PrefixSlot& s = slots_[i];
if (!s.live || s.ids.empty() || s.ids.size() > ids.size()) continue;
if (s.image_digest != image_digest) continue; // a different picture is a different prefix
if (!std::equal(s.ids.begin(), s.ids.end(), ids.begin())) continue;
if (s.ids.size() > best) {
best = s.ids.size();
Expand All @@ -155,7 +161,8 @@ class PrefixCache {
// store the state in, or -1 when the prompt is not worth keeping. `extended` is the slot the
// request resumed from (-1 when it prefilled from scratch): that slot is replaced, because the
// longer prompt covers it.
int insert(const std::vector<int>& ids, size_t bytes, int extended = -1, std::string* why = nullptr) {
int insert(const std::vector<int>& ids, size_t bytes, int extended = -1, std::string* why = nullptr,
uint64_t image_digest = 0) {
auto refuse = [&](const std::string& reason) {
if (why) *why = reason;
return -1;
Expand Down Expand Up @@ -183,7 +190,7 @@ class PrefixCache {
slot = free_slot();
if (slot < 0) return refuse("no free slot"); // every slot holds a better prompt
}
fill(slot, ids, bytes, /*pinned=*/false);
fill(slot, ids, bytes, /*pinned=*/false, image_digest);
if (why) *why = "saved";
return slot;
}
Expand Down Expand Up @@ -235,7 +242,7 @@ class PrefixCache {
if (slot < 0) return refuse("no free slot");
release(slots_[size_t(slot)]);
}
fill(slot, prefix, bytes, /*pinned=*/true);
fill(slot, prefix, bytes, /*pinned=*/true, 0); // a pin is the system prefix: text only
if (why) *why = "pinned";
return slot;
}
Expand Down Expand Up @@ -288,12 +295,13 @@ class PrefixCache {
if (!slots_[i].live) return int(i);
return -1;
}
void fill(int slot, const std::vector<int>& ids, size_t bytes, bool pinned) {
void fill(int slot, const std::vector<int>& ids, size_t bytes, bool pinned, uint64_t image_digest) {
PrefixSlot& s = slots_[size_t(slot)];
s.ids = ids;
s.bytes = bytes;
s.live = true;
s.pinned = pinned;
s.image_digest = image_digest;
s.last_used = ++last_used_;
used_ += bytes;
}
Expand Down
31 changes: 31 additions & 0 deletions tests/npu_sampling_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
// Header only, no device: runs in CI.
#include "npu_answer_parser.h"
#include "npu_grammar.h"
#include "npu_image_request.h" // engine#2324: the image digest the prefix cache compares
#include "npu_prefix_cache.h"
#include "npu_sampling.h"

Expand Down Expand Up @@ -719,6 +720,36 @@ void test_prefix_cache() {
check(c.lookup(ids(10, 2)).slot == -1, "a prompt whose start differs misses");
check(c.lookup(ids(5)).slot == -1, "a shorter prompt does not hit a longer slot");
}
{ // engine#2324: the digest that slot comparison uses -- computed from the PIXELS, not the wire bytes
auto img = [](int w, int h, uint8_t fill) { onebit::npu::vision::RgbImage i; i.width = w; i.height = h; i.rgb.assign(size_t(w) * h * 3, fill); return i; };
const auto a1 = onebit::npu::image_digest_of({img(4, 3, 0x11)});
const auto a2 = onebit::npu::image_digest_of({img(4, 3, 0x11)}); // same pixels, same digest
const auto b = onebit::npu::image_digest_of({img(4, 3, 0x22)}); // one byte different
const auto c = onebit::npu::image_digest_of({img(3, 4, 0x11)}); // transposed: same byte count, different picture
const auto none = onebit::npu::image_digest_of({});
const auto two = onebit::npu::image_digest_of({img(4, 3, 0x11), img(4, 3, 0x22)});
const auto twof = onebit::npu::image_digest_of({img(4, 3, 0x22), img(4, 3, 0x11)}); // order matters
check(a1 == a2, "digest: identical pixels give the identical digest");
check(a1 != b, "digest: one changed pixel changes it");
check(a1 != c, "digest: a transposed image is a different picture");
check(a1 != none && none != two, "digest: no picture differs from one, and from two");
check(two != twof, "digest: the same two pictures in the other order are a different prompt");
}
{ // engine#2324: the pixels are not in `ids`, so a slot must also match the digest of the image parts.
// Without this, "same text, different picture" is a cache HIT and the answer comes from the wrong image.
PrefixCache c(o);
const uint64_t imageA = 0xAAAAull, imageB = 0xBBBBull;
const int slot = c.insert(ids(12), 4096, -1, nullptr, imageA);
check(slot >= 0, "a prompt with a picture is saved");
check(c.lookup(ids(12), imageA).slot == slot, "the same picture hits");
check(c.lookup(ids(12), imageB).slot == -1, "a DIFFERENT picture misses, even with identical tokens");
check(c.lookup(ids(12)).slot == -1, "and so does a request with no picture at all");
check(c.lookup(ids(12), imageB).reason == "no slot shares a prefix", "the miss is reported like any other");
// the text-only path is unchanged: two text-only slots still hit each other
PrefixCache t(o);
const int ts = t.insert(ids(10), 1024);
check(t.lookup(ids(10)).slot == ts, "a text-only prompt still hits a text-only slot");
}
{ // the LONGEST matching slot wins, not the newest and not the immediately preceding one
PrefixCache c(o);
c.insert(ids(20), 1024);
Expand Down
Loading