Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 34 additions & 2 deletions app/forward_serve.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -216,6 +216,12 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib
// from `ids`. Do not log `prompt` as the request's text without handling this case first.
std::vector<int> prompt_ids;
bool prompt_is_ids = false;
// --prefix-pin: the leading system messages and a probe user message (npu::pin_probe_messages), rendered. Whatever
// of that the full prompt starts with is the part every question under this system prompt shares, so it can be pinned on
// the FIRST request (PrefixCache::pin_at) -- comparing with the previous request only finds it on the second,
// and can run a few tokens into the question when two questions open with the same words. Empty = the
// request has no system message, or the route is not pinning.
std::string system_prompt;
try {
if (chat) {
if (!e.chat_ok) throw std::runtime_error(e.chat_refusal);
Expand All @@ -232,8 +238,25 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib
extra["enable_thinking"] = npu::thinking_from_body(body);
prompt = npu::render_chat_template(e.tmpl, ordered.at("messages"), /*add_generation_prompt=*/true,
extra, e.bos_text, e.eos_text);
if (e.prefix_pin) {
// A template that cannot render the probe is no reason to refuse the request: it only means
// there is no early pin, and the log says so.
try {
const nlohmann::ordered_json probe = npu::pin_probe_messages(ordered.at("messages"));
if (!probe.empty())
system_prompt = npu::render_chat_template(e.tmpl, probe, /*add_generation_prompt=*/false, extra,
e.bos_text, e.eos_text);
} catch (const std::exception& ex) {
system_prompt.clear();
std::fprintf(stderr, "[prefix] no early pin: the template did not render the system turn (%s)\n", ex.what());
}
}
} else {
prompt = npu::chatml(body.at("messages"), npu::thinking_from_body(body));
if (e.prefix_pin) {
const json probe = npu::pin_probe_messages(body.at("messages"));
if (!probe.empty()) system_prompt = npu::chatml(probe, npu::thinking_from_body(body));
}
}
} else {
prompt_is_ids = npu::prompt_ids_from_body(body, e.tok->size(), prompt_ids);
Expand Down Expand Up @@ -291,7 +314,7 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib
// Returns the finish_reason the request earned: "stop" for an end-of-sequence token,
// a stop sequence or a grammar that allows nothing more, "length" when the token
// budget ran out.
auto run = [&e, &req, prompt, prompt_ids, prompt_is_ids, max_tokens, chat, &opts, gram, &stats, dump_req, bridge](
auto run = [&e, &req, prompt, system_prompt, prompt_ids, prompt_is_ids, max_tokens, chat, &opts, gram, &stats, dump_req, bridge](
const std::function<bool(const std::string&, const std::vector<Emitted>&)>& on_text) {
std::string finish = "stop";
// The tokenizer is const and keeps no per-call state, so it runs before the lock. A caller
Expand Down Expand Up @@ -358,7 +381,16 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib
int pin_cut = -1;
size_t pin_tokens = 0;
if (e.prefix_pin && e.prefix && e.prefix->enabled()) {
pin_tokens = e.prefix->pin_point(e.prev_ids, ids);
// The end of the system turn first: it is known on the first request and it is the right place. Only
// a request with no system message falls back to what it shares with the previous request.
if (!system_prompt.empty()) {
std::vector<int> lead = e.tok->encode(system_prompt);
if (chat && e.add_bos && e.bos_id >= 0 && (lead.empty() || lead.front() != e.bos_id))
lead.insert(lead.begin(), e.bos_id);
pin_tokens = e.prefix->pin_at(ids, npu::shared_prefix_len(lead, ids));
} else {
pin_tokens = e.prefix->pin_point(e.prev_ids, ids);
}
if (pin_tokens > 0 && int(pin_tokens) - 1 >= resume) pin_cut = int(pin_tokens) - 1;
}
{
Expand Down
25 changes: 25 additions & 0 deletions app/npu_chatml.h
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,31 @@ inline std::string chatml(const nlohmann::json& messages, bool thinking = true)
return p;
}

// The messages a chat route renders to find where a request's system turn ends (--prefix-pin, engine#1785): the
// request's leading system messages, then one user message whose text no question starts with. The rendering of
// these and the rendering of the real request agree up to the first token of the user's text and differ there, so
// their common prefix is the part every question under this system prompt shares.
//
// The user message is there because a template may refuse to render without one. Measured: the Qwen3.5 template
// rendered alone on its system message throws, and the early pin was silently not made on that container.
//
// Empty array when the request does not open with a system message: there is nothing to pin early.
template <typename Json>
Json pin_probe_messages(const Json& messages) {
Json lead = Json::array();
if (!messages.is_array()) return lead;
for (const auto& m : messages) {
if (!m.is_object() || !m.contains("role") || !m["role"].is_string() || m["role"].template get<std::string>() != "system") break;
lead.push_back(m);
}
if (lead.empty()) return lead;
Json probe = Json::object();
probe["role"] = "user";
probe["content"] = "\xC2\xA4\xC2\xA4 pin probe \xC2\xA4\xC2\xA4";
lead.push_back(probe);
return lead;
}

// Whether the container's tokenizer can represent the prompt above. `chatml()` is rendered for
// every container (#832), and a tokenizer without the anchors spells `<|im_start|>` as ordinary
// pieces, so the model is prompted with text its own template never produces (#963: on
Expand Down
18 changes: 15 additions & 3 deletions app/npu_prefix_cache.h
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
// What they share is the part before the first difference, so that part gets a slot of its own:
// pin_point(prev, ids) how many leading tokens this request shares with the previous one, when that
// is worth a slot and no slot covers it already
// pin_at(ids, n) the same decision for a point the caller knows already (the end of the system turn)
// pin(prefix, bytes) keep the state at that point. One pinned slot at a time; the budget's eviction
// and a longer prompt's replacement both leave it alone, a new pin() replaces it.
// The rule reads token ids only, so it needs no knowledge of the template or of what a system prompt is.
Expand Down Expand Up @@ -186,14 +187,25 @@ class PrefixCache {
size_t pin_point(const std::vector<int>& prev, const std::vector<int>& ids) const {
if (!enabled()) return 0;
const size_t shared = shared_prefix_len(prev, ids);
if (shared == prev.size() || shared == ids.size()) return 0;
if (shared == prev.size()) return 0;
return pin_at(ids, shared);
}

// The same decision for a point the caller already knows: `n` leading tokens of `ids` that every request of
// this kind will share -- the end of the system turn, which a chat route can find on the FIRST request by
// rendering the system messages alone, instead of waiting for a second request to compare with. Returns `n`
// when the state there is worth pinning, else 0: 0 when `n` is the whole prompt or more (nothing follows it,
// so insert() keeps it), when it adds fewer than prefix-min-gain tokens over the slot that already covers
// this request (the pin itself, on every request after the one that made it), and when the cache is off.
size_t pin_at(const std::vector<int>& ids, size_t n) const {
if (!enabled() || n == 0 || n >= ids.size()) return 0;
size_t covered = 0; // the longest live slot that is a prefix of this request
for (const PrefixSlot& s : slots_) {
if (!s.live || s.ids.size() > ids.size() || s.ids.size() <= covered) continue;
if (std::equal(s.ids.begin(), s.ids.end(), ids.begin())) covered = s.ids.size();
}
if (shared <= covered || shared - covered < size_t(opt_.min_gain)) return 0;
return shared;
if (n <= covered || n - covered < size_t(opt_.min_gain)) return 0;
return n;
}

// Keep `prefix` (the first pin_point() tokens of a request) as the pinned slot. Returns the slot the
Expand Down
25 changes: 24 additions & 1 deletion docs/npu.md
Original file line number Diff line number Diff line change
Expand Up @@ -1746,10 +1746,12 @@ The slot table (`app/npu_prefix_cache.h`) has a slot kind for the part such requ
| call | what it decides |
|---|---|
| `pin_point(prev, ids)` | how many leading tokens this request shares with the previous one. 0 when one prompt is a prefix of the other (a repeat or a continued conversation, which the ordinary slots already keep), when the shared part adds fewer than `--prefix-min-gain` tokens over a slot that already covers the request, or when the cache is off |
| `pin_at(ids, n)` | the same decision for a point the caller already knows: `n` leading tokens every such request shares. 0 when `n` is the whole prompt, when a slot already covers it, or when it is under `--prefix-min-gain` |
| `pin(prefix, bytes)` | keep the state at that point. One pinned slot at a time: a new pin replaces it; eviction for room and a longer prompt's replacement both leave it alone |
| `lookup(ids)` | unchanged, and reports `pinned` when the hit is the pinned slot |

It reads token ids only, so it does not depend on the template or on which message is the system prompt.
The table itself reads token ids only, so it does not depend on the template. Where the pin goes is the caller's
choice between two rules, below.

**`--prefix-pin`** (on `1bit serve --device npu` and on `forward-serve`; off by default) turns it on. The
forward keeps the previous request's prompt ids, and when `pin_point()` is non-zero it stops the prefill one
Expand All @@ -1774,6 +1776,27 @@ The default-cache arm's times are not in the table: the ticket ended during its
three cache decisions (from the server log) are known. A request's time here is the whole HTTP call, prefill
plus 16 generated tokens, not TTFT.

**Pinned on the first request (engine#1785).** That run needed two requests to find the shared part, so the
third was the first warm one, and when two questions opened with the same words the pin ran a few tokens into the
question, which a question opening differently could not use. A chat request with a leading system message now
gives the point directly: the route renders the leading system messages followed by a probe user message (the
container's template, or ChatML) and pins where the full prompt stops agreeing with that, on the first request
(`pin_at`). The probe user message is there because a template may refuse to render a system turn alone (the
Qwen3.5 template does); if a template cannot render even that, the request has no early pin and the log says so
(`[prefix] no early pin: …`). Only a request with no
system message still uses the comparison with the previous request (`pin_point`). On FastFlowLM's
`Qwen3.5-2B-NPU2` container under `--flash`, a 1,016-token prompt, time to first token (two server loads):

| request | before | after |
|---|---|---|
| first question | 8.8 s | 10.6 / 11.1 s (busier box; makes the pin: 1,003 tokens) |
| **second question** | 9.1–9.5 s (made the pin: 1,009 tokens) | **1.39 / 1.38 s** (1,002 of 1,017 cached) |
| third question | 1.06 s | 1.40 / 1.35 s |

The "before" column was taken with two other changes that make decode about a third faster on this model, the
"after" column without them and at twice the box load, so compare the rows' shape (which request is the first warm
one), not the third row's two figures.

**`--flash`** (`1bit serve --device npu --flash`) is the single-turn profile as one word: `--ctx-size 1024`
plus `--prefix-pin`. It refuses what contradicts it (`--ctx-size` other than 1024, `--prefix-slots 0`, a device
other than `npu`) instead of quietly winning or losing.
Expand Down
28 changes: 28 additions & 0 deletions tests/npu_chatml_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,34 @@ int main(int argc, char** argv) {
check(!without.encode(chatml(msgs, true)).empty(),
"the refusal is not about whether the prompt tokenizes at all -- it does, into the wrong ids");

// pin_probe_messages (engine#1785): what the route renders to find the end of the system turn
{
using onebit::npu::pin_probe_messages;
const nlohmann::json req = nlohmann::json::parse(
R"([{"role":"system","content":"Be brief."},{"role":"user","content":"What is item 3?"}])");
const nlohmann::json probe = pin_probe_messages(req);
check(probe.size() == 2 && probe[0] == req[0] && probe[1]["role"] == "user",
"the probe is the system message and one user message (a template may refuse to render without one)");
check(probe[1]["content"] != req[1]["content"], "whose text is not the question");
// the two renderings agree through the user turn's opening and part at the first character of its text
const std::string full = chatml(req, true), lead = chatml(probe, true);
size_t same = 0;
while (same < full.size() && same < lead.size() && full[same] == lead[same]) ++same;
const std::string expect = "<|im_start|>system\nBe brief.<|im_end|>\n<|im_start|>user\n";
check(full.compare(0, same, expect) == 0 && same == expect.size(),
"the common prefix of the two ChatML renderings is the system turn plus the user turn's opening");
const nlohmann::json two = nlohmann::json::parse(
R"([{"role":"system","content":"A"},{"role":"system","content":"B"},{"role":"user","content":"q"},{"role":"system","content":"late"}])");
check(pin_probe_messages(two).size() == 3, "every LEADING system message is kept, a later one is not");
check(pin_probe_messages(nlohmann::json::parse(R"([{"role":"user","content":"q"}])")).empty(),
"a request that does not open with a system message has no probe");
check(pin_probe_messages(nlohmann::json::parse(R"([{"role":7},{"content":"x"}])")).empty() &&
pin_probe_messages(nlohmann::json::parse("{}")).empty(),
"malformed messages give no probe instead of throwing");
const nlohmann::ordered_json oreq = nlohmann::ordered_json::parse(R"([{"role":"system","content":"S"},{"role":"user","content":"q"}])");
check(pin_probe_messages(oreq).size() == 2, "the same for the ordered JSON the template path uses");
}

if (failures == 0) std::printf("npu_chatml: ok\n");
return failures == 0 ? 0 : 1;
}
33 changes: 33 additions & 0 deletions tests/npu_sampling_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -529,6 +529,39 @@ void test_prefix_pin() {
check(PrefixCache(off).pin_point(q1, q2) == 0, "an off cache pins nothing");
check(PrefixCache(off).pin(q1, 10) == -1, "and refuses a pin");
}
{ // pin_at: the end of the system turn is known on the FIRST request, so the second one is already warm
PrefixCache c(o);
check(c.pin_at(q1, 900) == 900, "the first request under a 900-token system prompt pins its 900 tokens");
check(c.pin_point({}, q1) == 0, "where comparing with a previous request has nothing to offer yet");
std::string why;
const int pin = c.pin(std::vector<int>(q1.begin(), q1.begin() + 900), 2000, &why);
c.insert(q1, 1000);
const auto d = c.lookup(q2);
check(pin >= 0 && d.slot == pin && d.cached_tokens == 900 && d.pinned,
"so the SECOND question resumes from the pin: 900 of its 930 tokens");
check(c.pin_at(q2, 900) == 0 && c.pin_at(q3, 900) == 0, "and later requests do not pin the same point again");
// what is not worth a pin, or is not one
check(c.pin_at(q1, 0) == 0, "no system turn, no pin");
check(c.pin_at(q1, q1.size()) == 0 && c.pin_at(q1, q1.size() + 5) == 0, "a point at or past the end of the prompt is insert()'s");
PrefixCache fresh(o);
check(fresh.pin_at(q1, 127) == 0 && fresh.pin_at(q1, 128) == 128, "a system turn shorter than prefix-min-gain is not pinned");
PrefixOptions off = o;
off.slots = 0;
check(PrefixCache(off).pin_at(q1, 900) == 0, "an off cache pins nothing");
// a pin that ran into the question (two questions opening alike) does not cover a question that opens
// differently, and the system turn's own end then replaces it
PrefixCache over(o);
std::vector<int> a1 = prompt(900, 5000, 40), a2 = prompt(900, 5000, 40), b1 = prompt(900, 8000, 40);
a2[905] = 4242; // same first five question tokens, then different
const size_t ran_over = over.pin_point(a1, a2);
check(ran_over == 905, "comparing two alike-opening questions pins five tokens into the question");
over.pin(std::vector<int>(a2.begin(), a2.begin() + long(ran_over)), 2000);
check(over.lookup(b1).cached_tokens == 0, "which a differently opening question cannot use");
check(over.pin_at(b1, 900) == 900, "the system turn's end is still worth pinning for it");
const int repin = over.pin(std::vector<int>(b1.begin(), b1.begin() + 900), 2000);
check(repin >= 0 && over.lookup(b1).cached_tokens == 900 && over.lookup(a1).cached_tokens == 900,
"and once pinned there, every question under that system prompt resumes from it");
}
{ // the pin outlives what evicts and replaces everything else
PrefixOptions b = o;
b.slots = 4; // one more than the three snapshots the budget holds, so a slot is always free
Expand Down
Loading