Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
150 changes: 120 additions & 30 deletions app/npu_prefix_cache.h
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,16 @@
// saved prompt that is already a prefix of it (FLM default 128)
// Whole tokens only: the forward's step is token-at-a-time, so a partial-chunk match would buy
// nothing and risk a wrong position.
//
// The pinned prefix (engine#1785, the single-turn "flash" shape). A slot holds a whole prompt, and a hit
// needs the slot to be a prefix of the request. Single-turn requests that share a system prompt and differ
// in the user turn therefore never hit: `system + question 1` is not a prefix of `system + question 2`.
// What they share is the part before the first difference, so that part gets a slot of its own:
// pin_point(prev, ids) how many leading tokens this request shares with the previous one, when that
// is worth a slot and no slot covers it already
// pin(prefix, bytes) keep the state at that point. One pinned slot at a time; the budget's eviction
// and a longer prompt's replacement both leave it alone, a new pin() replaces it.
// The rule reads token ids only, so it needs no knowledge of the template or of what a system prompt is.
#pragma once

#include <algorithm>
Expand All @@ -55,8 +65,17 @@ struct PrefixDecision {
int slot = -1; // slot the request resumes from, -1 = prefill from scratch
int cached_tokens = 0; // tokens of this prompt the slot already covers
std::string reason; // "hit", "off", "no slot shares a prefix", "slot has no state", ...
bool pinned = false; // the slot is the pinned shared prefix, not a whole saved prompt
};

// How many tokens two prompts share from position 0.
inline size_t shared_prefix_len(const std::vector<int>& a, const std::vector<int>& b) {
const size_t n = std::min(a.size(), b.size());
size_t i = 0;
while (i < n && a[i] == b[i]) i++;
return i;
}

// Where a hit may resume. A slot's snapshot is taken one token short of the prompt it was saved for
// (forward-serve splits the last prefill chunk and saves before it, because the answer is drawn from the last
// position's logits and a snapshot holds state, not logits), so the positions it can restore are `cached - 1`,
Expand All @@ -74,6 +93,7 @@ struct PrefixSlot {
size_t bytes = 0; // what its snapshot costs in the budget
uint64_t last_used = 0; // monotonic, for eviction
bool live = false; // false = free/reusable
bool pinned = false; // the shared prefix: not evicted for room, not replaced by a longer prompt
};

class PrefixCache {
Expand Down Expand Up @@ -116,6 +136,7 @@ class PrefixCache {
d.slot = best_slot;
d.cached_tokens = int(best);
d.reason = "hit";
d.pinned = slots_[size_t(best_slot)].pinned;
last_used_++;
slots_[best_slot].last_used = last_used_;
return d;
Expand All @@ -139,55 +160,124 @@ class PrefixCache {
if (gain < size_t(opt_.min_gain)) return refuse("gain below prefix-min-gain");
if (bytes > opt_.cache_mb * 1024u * 1024u) return refuse("larger than prefix-cache-mb");

// The slot extended is replaced: its tokens are a prefix of the new ones.
// The slot extended is replaced: its tokens are a prefix of the new ones. A pinned slot is not:
// the next single-turn request shares the pin, not this prompt, so the prompt takes a slot of its own.
int slot = extended;
if (slot >= 0 && size_t(slot) < slots_.size() && slots_[size_t(slot)].pinned) slot = -1;
if (slot >= 0 && size_t(slot) < slots_.size() && slots_[size_t(slot)].live) {
used_ -= slots_[size_t(slot)].bytes;
slots_[size_t(slot)].live = false;
}
// Fit the budget by dropping the least recently used slot.
while (used_ + bytes > opt_.cache_mb * 1024u * 1024u) {
int victim = -1;
uint64_t oldest = UINT64_MAX;
for (size_t i = 0; i < slots_.size(); i++) {
if (!slots_[i].live) continue;
if (slots_[i].last_used < oldest) {
oldest = slots_[i].last_used;
victim = int(i);
}
}
if (victim < 0) break; // nothing left to evict
used_ -= slots_[size_t(victim)].bytes;
slots_[size_t(victim)].live = false;
}
if (used_ + bytes > opt_.cache_mb * 1024u * 1024u) return refuse("no room under prefix-cache-mb");
// Fit the budget by dropping the least recently used slot (never the pinned one).
if (!make_room(bytes)) return refuse("no room under prefix-cache-mb");
if (slot < 0 || size_t(slot) >= slots_.size() || slots_[size_t(slot)].live) {
slot = -1;
for (size_t i = 0; i < slots_.size(); i++)
if (!slots_[i].live) {
slot = int(i);
break;
}
slot = free_slot();
if (slot < 0) return refuse("no free slot"); // every slot holds a better prompt
}
PrefixSlot& s = slots_[size_t(slot)];
s.ids = ids;
s.bytes = bytes;
s.live = true;
s.last_used = ++last_used_;
used_ += bytes;
fill(slot, ids, bytes, /*pinned=*/false);
if (why) *why = "saved";
return slot;
}

// The number of leading tokens `ids` shares with `prev` (the previous request's prompt) when the state at
// that point is worth pinning, else 0. It is 0 when one prompt is a prefix of the other (a repeat or a
// continued conversation: insert() already keeps those), when the shared part adds fewer than
// prefix-min-gain tokens over the slot that already covers this request, and when the cache is off.
size_t pin_point(const std::vector<int>& prev, const std::vector<int>& ids) const {
if (!enabled()) return 0;
const size_t shared = shared_prefix_len(prev, ids);
if (shared == prev.size() || shared == ids.size()) return 0;
size_t covered = 0; // the longest live slot that is a prefix of this request
for (const PrefixSlot& s : slots_) {
if (!s.live || s.ids.size() > ids.size() || s.ids.size() <= covered) continue;
if (std::equal(s.ids.begin(), s.ids.end(), ids.begin())) covered = s.ids.size();
}
if (shared <= covered || shared - covered < size_t(opt_.min_gain)) return 0;
return shared;
}

// Keep `prefix` (the first pin_point() tokens of a request) as the pinned slot. Returns the slot the
// caller should store the state in, or -1. The previous pin, if any, is released first: there is one
// shared prefix at a time, and the newest pair of requests is the evidence for what it is.
int pin(const std::vector<int>& prefix, size_t bytes, std::string* why = nullptr) {
auto refuse = [&](const std::string& reason) {
if (why) *why = reason;
return -1;
};
if (!enabled() || prefix.empty()) return refuse("off");
if (bytes > opt_.cache_mb * 1024u * 1024u) return refuse("larger than prefix-cache-mb");
for (PrefixSlot& s : slots_)
if (s.live && s.pinned) release(s);
if (!make_room(bytes)) return refuse("no room under prefix-cache-mb");
int slot = free_slot();
if (slot < 0) {
slot = least_recently_used(); // a whole prompt gives way to the prefix every request shares
if (slot < 0) return refuse("no free slot");
release(slots_[size_t(slot)]);
}
fill(slot, prefix, bytes, /*pinned=*/true);
if (why) *why = "pinned";
return slot;
}

// The pinned slot, or -1.
int pinned_slot() const {
for (size_t i = 0; i < slots_.size(); i++)
if (slots_[i].live && slots_[i].pinned) return int(i);
return -1;
}

const PrefixSlot& slot(int i) const { return slots_.at(size_t(i)); }
size_t slot_count() const { return slots_.size(); }

// A cancelled request, or a cleared context, must not throw away other conversations' prompts
// (FLM v1.0.11's own fix), so there is deliberately no clear() here: entries age out by the
// budget and are replaced only by prompts that cover more tokens.
// budget and are replaced only by prompts that cover more tokens. The pinned slot does neither.

private:
void release(PrefixSlot& s) {
used_ -= s.bytes;
s.live = false;
s.pinned = false;
}
// The least recently used live slot that is not pinned, or -1.
int least_recently_used() const {
int victim = -1;
uint64_t oldest = UINT64_MAX;
for (size_t i = 0; i < slots_.size(); i++) {
if (!slots_[i].live || slots_[i].pinned) continue;
if (slots_[i].last_used < oldest) {
oldest = slots_[i].last_used;
victim = int(i);
}
}
return victim;
}
// Evict until `bytes` more fit under the budget; false when they cannot.
bool make_room(size_t bytes) {
const size_t budget = opt_.cache_mb * 1024u * 1024u;
while (used_ + bytes > budget) {
const int victim = least_recently_used();
if (victim < 0) break; // nothing left to evict
release(slots_[size_t(victim)]);
}
return used_ + bytes <= budget;
}
int free_slot() const {
for (size_t i = 0; i < slots_.size(); i++)
if (!slots_[i].live) return int(i);
return -1;
}
void fill(int slot, const std::vector<int>& ids, size_t bytes, bool pinned) {
PrefixSlot& s = slots_[size_t(slot)];
s.ids = ids;
s.bytes = bytes;
s.live = true;
s.pinned = pinned;
s.last_used = ++last_used_;
used_ += bytes;
}

PrefixOptions opt_;
std::vector<PrefixSlot> slots_;
size_t used_ = 0;
Expand Down
22 changes: 22 additions & 0 deletions docs/npu.md
Original file line number Diff line number Diff line change
Expand Up @@ -1353,6 +1353,28 @@ again to get the logits the answer is drawn from. Restoring `cached - 1` positio
version of this fix, the cache hit (`[prefix] slot=0 cached=179 of 180 reason=hit`, TTFT 5.3 s against 15.6 s) and
answered the same prompt with different, fragmented text.

### The prefix cache's pinned slot: single-turn requests under one system prompt (engine#1785)

A slot holds a whole prompt, and a request hits a slot only when the slot is a prefix of it. Requests that
share a system prompt and differ in the user turn therefore reuse nothing from each other: `system + question 1`
is not a prefix of `system + question 2`. `tests/npu_sampling_test.cpp` (`test_prefix_pin`) states that as its
first row: a 900-token system prompt, a second question, `cached_tokens: 0`.

The slot table (`app/npu_prefix_cache.h`) has a slot kind for the part such requests do share:

| call | what it decides |
|---|---|
| `pin_point(prev, ids)` | how many leading tokens this request shares with the previous one. 0 when one prompt is a prefix of the other (a repeat or a continued conversation, which the ordinary slots already keep), when the shared part adds fewer than `--prefix-min-gain` tokens over a slot that already covers the request, or when the cache is off |
| `pin(prefix, bytes)` | keep the state at that point. One pinned slot at a time: a new pin replaces it; eviction for room and a longer prompt's replacement both leave it alone |
| `lookup(ids)` | unchanged, and reports `pinned` when the hit is the pinned slot |

It reads token ids only, so it does not depend on the template or on which message is the system prompt.

**State of this, plainly:** the table and its tests are in; `forward-serve` does not call `pin_point()`/`pin()`
yet, so no request is faster because of it today. The forward half is one more cut in the prefill loop (stop one
token short of the pin point, `snapshot_state`, `pin`), and it needs a device run to be claimed.


### Sampling and structured output

`1bit serve -m <NPU model dir> --device npu` serves through `1bit forward-serve`
Expand Down
100 changes: 100 additions & 0 deletions tests/npu_sampling_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -449,6 +449,105 @@ void test_logprob_options() {
"top_logprobs without logprobs: true is refused rather than ignored");
}

// The pinned shared prefix (engine#1785): single-turn requests that share a system prompt and differ in the
// user turn. `sys(n)` + `turn(k)` stand for the two parts; the table sees token ids only.
void test_prefix_pin() {
using onebit::npu::shared_prefix_len;
auto prompt = [](int sys, int turn_start, int turn) {
std::vector<int> v;
for (int i = 0; i < sys; i++) v.push_back(1 + i);
for (int i = 0; i < turn; i++) v.push_back(turn_start + i);
return v;
};
PrefixOptions o;
o.slots = 4;
o.cache_mb = 1;
o.min_gain = 128;
const std::vector<int> q1 = prompt(900, 5000, 40), q2 = prompt(900, 6000, 30), q3 = prompt(900, 7000, 50);

{ // the gap, on the table as it was: a whole saved prompt is no use to the next single-turn request
PrefixCache c(o);
check(c.insert(q1, 1000) >= 0, "the first single-turn prompt is saved whole");
const auto d = c.lookup(q2);
check(d.slot == -1 && d.cached_tokens == 0,
"a second question under the same 900-token system prompt reuses nothing from the first");
}
{ // the pin: request 2 finds the shared part, request 3 resumes from it
PrefixCache c(o);
check(c.pin_point({}, q1) == 0, "the first request has nothing to compare with");
c.insert(q1, 1000);
check(shared_prefix_len(q1, q2) == 900, "the two prompts share exactly the system part");
const size_t at = c.pin_point(q1, q2);
check(at == 900, "the second request's pin point is the end of what it shares with the first");
std::string why;
const int pin = c.pin(std::vector<int>(q2.begin(), q2.begin() + long(at)), 2000, &why);
check(pin >= 0 && why == "pinned" && c.pinned_slot() == pin, "the shared part is pinned");
const auto d = c.lookup(q3);
check(d.slot == pin && d.cached_tokens == 900 && d.pinned && d.reason == "hit",
"a third question resumes from the pin: 900 of its 950 tokens are already there");
check(onebit::npu::resume_point(d, q3.size()) == 899, "and resumes one token short of it, as every hit does");
check(c.pin_point(q2, q3) == 0, "a prefix the pin already covers is not pinned again");
check(!c.lookup(q1).pinned && c.lookup(q1).cached_tokens == 940,
"a whole-prompt slot is still preferred when it is the longer match, and says it is not the pin");
}
{ // what is not a pin point
PrefixCache c(o);
check(c.pin_point(q1, q1) == 0, "a repeated prompt is insert()'s business, not a pin");
std::vector<int> longer = q1;
for (int i = 0; i < 200; i++) longer.push_back(9000 + i);
check(c.pin_point(q1, longer) == 0, "a continued conversation is insert()'s business too");
check(c.pin_point(longer, q1) == 0, "and so is a prompt the previous one continued");
check(c.pin_point(prompt(100, 5000, 40), prompt(100, 6000, 40)) == 0,
"100 shared tokens do not beat prefix-min-gain 128");
check(c.pin_point(prompt(900, 5000, 40), prompt(0, 6000, 40)) == 0, "prompts that share nothing have no pin point");
PrefixOptions off = o;
off.slots = 0;
check(PrefixCache(off).pin_point(q1, q2) == 0, "an off cache pins nothing");
check(PrefixCache(off).pin(q1, 10) == -1, "and refuses a pin");
}
{ // the pin outlives what evicts and replaces everything else
PrefixOptions b = o;
b.slots = 4; // one more than the three snapshots the budget holds, so a slot is always free
b.min_gain = 1;
PrefixCache c(b); // 1 MB
const std::vector<int> sys(q1.begin(), q1.begin() + 900);
const int pin = c.pin(sys, 300000);
const int a = c.insert(prompt(0, 100, 10), 300000);
const int bb = c.insert(prompt(0, 200, 10), 300000);
check(pin >= 0 && a >= 0 && bb >= 0 && c.used_bytes() == 900000, "a pin and two prompts fit under 1 MB");
// The pin is the least recently used of the three, and 300000 more does not fit: something goes.
const int cc = c.insert(prompt(0, 300, 10), 300000);
check(cc == a, "room is made from the oldest whole prompt");
check(c.pinned_slot() == pin && c.lookup(q2).slot == pin, "not from the pin, though it is the oldest slot");
// A prompt that extends the pin does not replace it.
std::string why;
const int whole = c.insert(q2, 1000, pin, &why);
check(whole >= 0 && whole != pin && c.pinned_slot() == pin,
"a prompt that resumed from the pin is saved beside it, not over it");
check(c.lookup(q3).slot == pin && c.lookup(q3).cached_tokens == 900, "so the next question still finds the pin");
// A new pin replaces the old one: one shared prefix at a time.
const std::vector<int> other = prompt(0, 40000, 500);
const int pin2 = c.pin(other, 1000);
check(pin2 >= 0 && c.pinned_slot() == pin2, "a new pin takes over");
check(c.lookup(q3).slot != pin || !c.lookup(q3).pinned, "and the old one is no longer pinned");
check(c.pin(other, 2u * 1024u * 1024u, &why) == -1 && why == "larger than prefix-cache-mb" &&
c.pinned_slot() == pin2,
"a pin over the budget is refused and the standing pin is kept");
}
{ // every slot holds a whole prompt: the pin takes the oldest
PrefixOptions b = o;
b.slots = 2;
b.min_gain = 1;
PrefixCache c(b);
const int s0 = c.insert(prompt(0, 100, 10), 100);
c.insert(prompt(0, 200, 10), 100);
const int pin = c.pin(prompt(900, 0, 0), 100);
check(pin == s0, "with no free slot the pin takes the least recently used prompt's");
check(c.insert(prompt(0, 300, 10), 100) != pin, "and a later prompt cannot take it back");
check(c.pinned_slot() == pin, "the pin is still there");
}
}

void test_prefix_cache() {
auto ids = [](int n, int start = 1) {
std::vector<int> v;
Expand Down Expand Up @@ -974,6 +1073,7 @@ int main() {
test_logprobs();
test_logprob_options();
test_prefix_cache();
test_prefix_pin();
test_json_schema();
test_json_schema_refusals();
test_answer_parser();
Expand Down
Loading