From c88b339d6b9e313fa5a08804a8ce8260fe63d9ee Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 10:51:22 -0400 Subject: [PATCH] npu: a pinned slot for the prefix single-turn requests share (engine#1785, F5) A slot holds a whole prompt, so two questions under one system prompt reuse nothing from each other. The slot table gains pin_point() (the tokens a request shares with the previous one, when worth a slot) and pin() (one pinned slot, kept through eviction and replacement). Table and tests only: forward-serve does not call them yet. Docs in docs/npu.md. Co-Authored-By: Claude Fable 5.2 --- app/npu_prefix_cache.h | 150 ++++++++++++++++++++++++++++-------- docs/npu.md | 22 ++++++ tests/npu_sampling_test.cpp | 100 ++++++++++++++++++++++++ 3 files changed, 242 insertions(+), 30 deletions(-) diff --git a/app/npu_prefix_cache.h b/app/npu_prefix_cache.h index c76368ed7..b66562b1c 100644 --- a/app/npu_prefix_cache.h +++ b/app/npu_prefix_cache.h @@ -30,6 +30,16 @@ // saved prompt that is already a prefix of it (FLM default 128) // Whole tokens only: the forward's step is token-at-a-time, so a partial-chunk match would buy // nothing and risk a wrong position. +// +// The pinned prefix (engine#1785, the single-turn "flash" shape). A slot holds a whole prompt, and a hit +// needs the slot to be a prefix of the request. Single-turn requests that share a system prompt and differ +// in the user turn therefore never hit: `system + question 1` is not a prefix of `system + question 2`. +// What they share is the part before the first difference, so that part gets a slot of its own: +// pin_point(prev, ids) how many leading tokens this request shares with the previous one, when that +// is worth a slot and no slot covers it already +// pin(prefix, bytes) keep the state at that point. One pinned slot at a time; the budget's eviction +// and a longer prompt's replacement both leave it alone, a new pin() replaces it. +// The rule reads token ids only, so it needs no knowledge of the template or of what a system prompt is. #pragma once #include @@ -55,8 +65,17 @@ struct PrefixDecision { int slot = -1; // slot the request resumes from, -1 = prefill from scratch int cached_tokens = 0; // tokens of this prompt the slot already covers std::string reason; // "hit", "off", "no slot shares a prefix", "slot has no state", ... + bool pinned = false; // the slot is the pinned shared prefix, not a whole saved prompt }; +// How many tokens two prompts share from position 0. +inline size_t shared_prefix_len(const std::vector& a, const std::vector& b) { + const size_t n = std::min(a.size(), b.size()); + size_t i = 0; + while (i < n && a[i] == b[i]) i++; + return i; +} + // Where a hit may resume. A slot's snapshot is taken one token short of the prompt it was saved for // (forward-serve splits the last prefill chunk and saves before it, because the answer is drawn from the last // position's logits and a snapshot holds state, not logits), so the positions it can restore are `cached - 1`, @@ -74,6 +93,7 @@ struct PrefixSlot { size_t bytes = 0; // what its snapshot costs in the budget uint64_t last_used = 0; // monotonic, for eviction bool live = false; // false = free/reusable + bool pinned = false; // the shared prefix: not evicted for room, not replaced by a longer prompt }; class PrefixCache { @@ -116,6 +136,7 @@ class PrefixCache { d.slot = best_slot; d.cached_tokens = int(best); d.reason = "hit"; + d.pinned = slots_[size_t(best_slot)].pinned; last_used_++; slots_[best_slot].last_used = last_used_; return d; @@ -139,55 +160,124 @@ class PrefixCache { if (gain < size_t(opt_.min_gain)) return refuse("gain below prefix-min-gain"); if (bytes > opt_.cache_mb * 1024u * 1024u) return refuse("larger than prefix-cache-mb"); - // The slot extended is replaced: its tokens are a prefix of the new ones. + // The slot extended is replaced: its tokens are a prefix of the new ones. A pinned slot is not: + // the next single-turn request shares the pin, not this prompt, so the prompt takes a slot of its own. int slot = extended; + if (slot >= 0 && size_t(slot) < slots_.size() && slots_[size_t(slot)].pinned) slot = -1; if (slot >= 0 && size_t(slot) < slots_.size() && slots_[size_t(slot)].live) { used_ -= slots_[size_t(slot)].bytes; slots_[size_t(slot)].live = false; } - // Fit the budget by dropping the least recently used slot. - while (used_ + bytes > opt_.cache_mb * 1024u * 1024u) { - int victim = -1; - uint64_t oldest = UINT64_MAX; - for (size_t i = 0; i < slots_.size(); i++) { - if (!slots_[i].live) continue; - if (slots_[i].last_used < oldest) { - oldest = slots_[i].last_used; - victim = int(i); - } - } - if (victim < 0) break; // nothing left to evict - used_ -= slots_[size_t(victim)].bytes; - slots_[size_t(victim)].live = false; - } - if (used_ + bytes > opt_.cache_mb * 1024u * 1024u) return refuse("no room under prefix-cache-mb"); + // Fit the budget by dropping the least recently used slot (never the pinned one). + if (!make_room(bytes)) return refuse("no room under prefix-cache-mb"); if (slot < 0 || size_t(slot) >= slots_.size() || slots_[size_t(slot)].live) { - slot = -1; - for (size_t i = 0; i < slots_.size(); i++) - if (!slots_[i].live) { - slot = int(i); - break; - } + slot = free_slot(); if (slot < 0) return refuse("no free slot"); // every slot holds a better prompt } - PrefixSlot& s = slots_[size_t(slot)]; - s.ids = ids; - s.bytes = bytes; - s.live = true; - s.last_used = ++last_used_; - used_ += bytes; + fill(slot, ids, bytes, /*pinned=*/false); if (why) *why = "saved"; return slot; } + // The number of leading tokens `ids` shares with `prev` (the previous request's prompt) when the state at + // that point is worth pinning, else 0. It is 0 when one prompt is a prefix of the other (a repeat or a + // continued conversation: insert() already keeps those), when the shared part adds fewer than + // prefix-min-gain tokens over the slot that already covers this request, and when the cache is off. + size_t pin_point(const std::vector& prev, const std::vector& ids) const { + if (!enabled()) return 0; + const size_t shared = shared_prefix_len(prev, ids); + if (shared == prev.size() || shared == ids.size()) return 0; + size_t covered = 0; // the longest live slot that is a prefix of this request + for (const PrefixSlot& s : slots_) { + if (!s.live || s.ids.size() > ids.size() || s.ids.size() <= covered) continue; + if (std::equal(s.ids.begin(), s.ids.end(), ids.begin())) covered = s.ids.size(); + } + if (shared <= covered || shared - covered < size_t(opt_.min_gain)) return 0; + return shared; + } + + // Keep `prefix` (the first pin_point() tokens of a request) as the pinned slot. Returns the slot the + // caller should store the state in, or -1. The previous pin, if any, is released first: there is one + // shared prefix at a time, and the newest pair of requests is the evidence for what it is. + int pin(const std::vector& prefix, size_t bytes, std::string* why = nullptr) { + auto refuse = [&](const std::string& reason) { + if (why) *why = reason; + return -1; + }; + if (!enabled() || prefix.empty()) return refuse("off"); + if (bytes > opt_.cache_mb * 1024u * 1024u) return refuse("larger than prefix-cache-mb"); + for (PrefixSlot& s : slots_) + if (s.live && s.pinned) release(s); + if (!make_room(bytes)) return refuse("no room under prefix-cache-mb"); + int slot = free_slot(); + if (slot < 0) { + slot = least_recently_used(); // a whole prompt gives way to the prefix every request shares + if (slot < 0) return refuse("no free slot"); + release(slots_[size_t(slot)]); + } + fill(slot, prefix, bytes, /*pinned=*/true); + if (why) *why = "pinned"; + return slot; + } + + // The pinned slot, or -1. + int pinned_slot() const { + for (size_t i = 0; i < slots_.size(); i++) + if (slots_[i].live && slots_[i].pinned) return int(i); + return -1; + } + const PrefixSlot& slot(int i) const { return slots_.at(size_t(i)); } size_t slot_count() const { return slots_.size(); } // A cancelled request, or a cleared context, must not throw away other conversations' prompts // (FLM v1.0.11's own fix), so there is deliberately no clear() here: entries age out by the - // budget and are replaced only by prompts that cover more tokens. + // budget and are replaced only by prompts that cover more tokens. The pinned slot does neither. private: + void release(PrefixSlot& s) { + used_ -= s.bytes; + s.live = false; + s.pinned = false; + } + // The least recently used live slot that is not pinned, or -1. + int least_recently_used() const { + int victim = -1; + uint64_t oldest = UINT64_MAX; + for (size_t i = 0; i < slots_.size(); i++) { + if (!slots_[i].live || slots_[i].pinned) continue; + if (slots_[i].last_used < oldest) { + oldest = slots_[i].last_used; + victim = int(i); + } + } + return victim; + } + // Evict until `bytes` more fit under the budget; false when they cannot. + bool make_room(size_t bytes) { + const size_t budget = opt_.cache_mb * 1024u * 1024u; + while (used_ + bytes > budget) { + const int victim = least_recently_used(); + if (victim < 0) break; // nothing left to evict + release(slots_[size_t(victim)]); + } + return used_ + bytes <= budget; + } + int free_slot() const { + for (size_t i = 0; i < slots_.size(); i++) + if (!slots_[i].live) return int(i); + return -1; + } + void fill(int slot, const std::vector& ids, size_t bytes, bool pinned) { + PrefixSlot& s = slots_[size_t(slot)]; + s.ids = ids; + s.bytes = bytes; + s.live = true; + s.pinned = pinned; + s.last_used = ++last_used_; + used_ += bytes; + } + PrefixOptions opt_; std::vector slots_; size_t used_ = 0; diff --git a/docs/npu.md b/docs/npu.md index 6862549d4..66f1b18a6 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1353,6 +1353,28 @@ again to get the logits the answer is drawn from. Restoring `cached - 1` positio version of this fix, the cache hit (`[prefix] slot=0 cached=179 of 180 reason=hit`, TTFT 5.3 s against 15.6 s) and answered the same prompt with different, fragmented text. +### The prefix cache's pinned slot: single-turn requests under one system prompt (engine#1785) + +A slot holds a whole prompt, and a request hits a slot only when the slot is a prefix of it. Requests that +share a system prompt and differ in the user turn therefore reuse nothing from each other: `system + question 1` +is not a prefix of `system + question 2`. `tests/npu_sampling_test.cpp` (`test_prefix_pin`) states that as its +first row: a 900-token system prompt, a second question, `cached_tokens: 0`. + +The slot table (`app/npu_prefix_cache.h`) has a slot kind for the part such requests do share: + +| call | what it decides | +|---|---| +| `pin_point(prev, ids)` | how many leading tokens this request shares with the previous one. 0 when one prompt is a prefix of the other (a repeat or a continued conversation, which the ordinary slots already keep), when the shared part adds fewer than `--prefix-min-gain` tokens over a slot that already covers the request, or when the cache is off | +| `pin(prefix, bytes)` | keep the state at that point. One pinned slot at a time: a new pin replaces it; eviction for room and a longer prompt's replacement both leave it alone | +| `lookup(ids)` | unchanged, and reports `pinned` when the hit is the pinned slot | + +It reads token ids only, so it does not depend on the template or on which message is the system prompt. + +**State of this, plainly:** the table and its tests are in; `forward-serve` does not call `pin_point()`/`pin()` +yet, so no request is faster because of it today. The forward half is one more cut in the prefill loop (stop one +token short of the pin point, `snapshot_state`, `pin`), and it needs a device run to be claimed. + + ### Sampling and structured output `1bit serve -m --device npu` serves through `1bit forward-serve` diff --git a/tests/npu_sampling_test.cpp b/tests/npu_sampling_test.cpp index 28bc24fac..91a054cbe 100644 --- a/tests/npu_sampling_test.cpp +++ b/tests/npu_sampling_test.cpp @@ -449,6 +449,105 @@ void test_logprob_options() { "top_logprobs without logprobs: true is refused rather than ignored"); } +// The pinned shared prefix (engine#1785): single-turn requests that share a system prompt and differ in the +// user turn. `sys(n)` + `turn(k)` stand for the two parts; the table sees token ids only. +void test_prefix_pin() { + using onebit::npu::shared_prefix_len; + auto prompt = [](int sys, int turn_start, int turn) { + std::vector v; + for (int i = 0; i < sys; i++) v.push_back(1 + i); + for (int i = 0; i < turn; i++) v.push_back(turn_start + i); + return v; + }; + PrefixOptions o; + o.slots = 4; + o.cache_mb = 1; + o.min_gain = 128; + const std::vector q1 = prompt(900, 5000, 40), q2 = prompt(900, 6000, 30), q3 = prompt(900, 7000, 50); + + { // the gap, on the table as it was: a whole saved prompt is no use to the next single-turn request + PrefixCache c(o); + check(c.insert(q1, 1000) >= 0, "the first single-turn prompt is saved whole"); + const auto d = c.lookup(q2); + check(d.slot == -1 && d.cached_tokens == 0, + "a second question under the same 900-token system prompt reuses nothing from the first"); + } + { // the pin: request 2 finds the shared part, request 3 resumes from it + PrefixCache c(o); + check(c.pin_point({}, q1) == 0, "the first request has nothing to compare with"); + c.insert(q1, 1000); + check(shared_prefix_len(q1, q2) == 900, "the two prompts share exactly the system part"); + const size_t at = c.pin_point(q1, q2); + check(at == 900, "the second request's pin point is the end of what it shares with the first"); + std::string why; + const int pin = c.pin(std::vector(q2.begin(), q2.begin() + long(at)), 2000, &why); + check(pin >= 0 && why == "pinned" && c.pinned_slot() == pin, "the shared part is pinned"); + const auto d = c.lookup(q3); + check(d.slot == pin && d.cached_tokens == 900 && d.pinned && d.reason == "hit", + "a third question resumes from the pin: 900 of its 950 tokens are already there"); + check(onebit::npu::resume_point(d, q3.size()) == 899, "and resumes one token short of it, as every hit does"); + check(c.pin_point(q2, q3) == 0, "a prefix the pin already covers is not pinned again"); + check(!c.lookup(q1).pinned && c.lookup(q1).cached_tokens == 940, + "a whole-prompt slot is still preferred when it is the longer match, and says it is not the pin"); + } + { // what is not a pin point + PrefixCache c(o); + check(c.pin_point(q1, q1) == 0, "a repeated prompt is insert()'s business, not a pin"); + std::vector longer = q1; + for (int i = 0; i < 200; i++) longer.push_back(9000 + i); + check(c.pin_point(q1, longer) == 0, "a continued conversation is insert()'s business too"); + check(c.pin_point(longer, q1) == 0, "and so is a prompt the previous one continued"); + check(c.pin_point(prompt(100, 5000, 40), prompt(100, 6000, 40)) == 0, + "100 shared tokens do not beat prefix-min-gain 128"); + check(c.pin_point(prompt(900, 5000, 40), prompt(0, 6000, 40)) == 0, "prompts that share nothing have no pin point"); + PrefixOptions off = o; + off.slots = 0; + check(PrefixCache(off).pin_point(q1, q2) == 0, "an off cache pins nothing"); + check(PrefixCache(off).pin(q1, 10) == -1, "and refuses a pin"); + } + { // the pin outlives what evicts and replaces everything else + PrefixOptions b = o; + b.slots = 4; // one more than the three snapshots the budget holds, so a slot is always free + b.min_gain = 1; + PrefixCache c(b); // 1 MB + const std::vector sys(q1.begin(), q1.begin() + 900); + const int pin = c.pin(sys, 300000); + const int a = c.insert(prompt(0, 100, 10), 300000); + const int bb = c.insert(prompt(0, 200, 10), 300000); + check(pin >= 0 && a >= 0 && bb >= 0 && c.used_bytes() == 900000, "a pin and two prompts fit under 1 MB"); + // The pin is the least recently used of the three, and 300000 more does not fit: something goes. + const int cc = c.insert(prompt(0, 300, 10), 300000); + check(cc == a, "room is made from the oldest whole prompt"); + check(c.pinned_slot() == pin && c.lookup(q2).slot == pin, "not from the pin, though it is the oldest slot"); + // A prompt that extends the pin does not replace it. + std::string why; + const int whole = c.insert(q2, 1000, pin, &why); + check(whole >= 0 && whole != pin && c.pinned_slot() == pin, + "a prompt that resumed from the pin is saved beside it, not over it"); + check(c.lookup(q3).slot == pin && c.lookup(q3).cached_tokens == 900, "so the next question still finds the pin"); + // A new pin replaces the old one: one shared prefix at a time. + const std::vector other = prompt(0, 40000, 500); + const int pin2 = c.pin(other, 1000); + check(pin2 >= 0 && c.pinned_slot() == pin2, "a new pin takes over"); + check(c.lookup(q3).slot != pin || !c.lookup(q3).pinned, "and the old one is no longer pinned"); + check(c.pin(other, 2u * 1024u * 1024u, &why) == -1 && why == "larger than prefix-cache-mb" && + c.pinned_slot() == pin2, + "a pin over the budget is refused and the standing pin is kept"); + } + { // every slot holds a whole prompt: the pin takes the oldest + PrefixOptions b = o; + b.slots = 2; + b.min_gain = 1; + PrefixCache c(b); + const int s0 = c.insert(prompt(0, 100, 10), 100); + c.insert(prompt(0, 200, 10), 100); + const int pin = c.pin(prompt(900, 0, 0), 100); + check(pin == s0, "with no free slot the pin takes the least recently used prompt's"); + check(c.insert(prompt(0, 300, 10), 100) != pin, "and a later prompt cannot take it back"); + check(c.pinned_slot() == pin, "the pin is still there"); + } +} + void test_prefix_cache() { auto ids = [](int n, int start = 1) { std::vector v; @@ -974,6 +1073,7 @@ int main() { test_logprobs(); test_logprob_options(); test_prefix_cache(); + test_prefix_pin(); test_json_schema(); test_json_schema_refusals(); test_answer_parser();