diff --git a/app/forward_serve.cpp b/app/forward_serve.cpp index 5acc1c741..4a43ab585 100644 --- a/app/forward_serve.cpp +++ b/app/forward_serve.cpp @@ -216,6 +216,12 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib // from `ids`. Do not log `prompt` as the request's text without handling this case first. std::vector prompt_ids; bool prompt_is_ids = false; + // --prefix-pin: the leading system messages and a probe user message (npu::pin_probe_messages), rendered. Whatever + // of that the full prompt starts with is the part every question under this system prompt shares, so it can be pinned on + // the FIRST request (PrefixCache::pin_at) -- comparing with the previous request only finds it on the second, + // and can run a few tokens into the question when two questions open with the same words. Empty = the + // request has no system message, or the route is not pinning. + std::string system_prompt; try { if (chat) { if (!e.chat_ok) throw std::runtime_error(e.chat_refusal); @@ -232,8 +238,25 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib extra["enable_thinking"] = npu::thinking_from_body(body); prompt = npu::render_chat_template(e.tmpl, ordered.at("messages"), /*add_generation_prompt=*/true, extra, e.bos_text, e.eos_text); + if (e.prefix_pin) { + // A template that cannot render the probe is no reason to refuse the request: it only means + // there is no early pin, and the log says so. + try { + const nlohmann::ordered_json probe = npu::pin_probe_messages(ordered.at("messages")); + if (!probe.empty()) + system_prompt = npu::render_chat_template(e.tmpl, probe, /*add_generation_prompt=*/false, extra, + e.bos_text, e.eos_text); + } catch (const std::exception& ex) { + system_prompt.clear(); + std::fprintf(stderr, "[prefix] no early pin: the template did not render the system turn (%s)\n", ex.what()); + } + } } else { prompt = npu::chatml(body.at("messages"), npu::thinking_from_body(body)); + if (e.prefix_pin) { + const json probe = npu::pin_probe_messages(body.at("messages")); + if (!probe.empty()) system_prompt = npu::chatml(probe, npu::thinking_from_body(body)); + } } } else { prompt_is_ids = npu::prompt_ids_from_body(body, e.tok->size(), prompt_ids); @@ -291,7 +314,7 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib // Returns the finish_reason the request earned: "stop" for an end-of-sequence token, // a stop sequence or a grammar that allows nothing more, "length" when the token // budget ran out. - auto run = [&e, &req, prompt, prompt_ids, prompt_is_ids, max_tokens, chat, &opts, gram, &stats, dump_req, bridge]( + auto run = [&e, &req, prompt, system_prompt, prompt_ids, prompt_is_ids, max_tokens, chat, &opts, gram, &stats, dump_req, bridge]( const std::function&)>& on_text) { std::string finish = "stop"; // The tokenizer is const and keeps no per-call state, so it runs before the lock. A caller @@ -358,7 +381,16 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib int pin_cut = -1; size_t pin_tokens = 0; if (e.prefix_pin && e.prefix && e.prefix->enabled()) { - pin_tokens = e.prefix->pin_point(e.prev_ids, ids); + // The end of the system turn first: it is known on the first request and it is the right place. Only + // a request with no system message falls back to what it shares with the previous request. + if (!system_prompt.empty()) { + std::vector lead = e.tok->encode(system_prompt); + if (chat && e.add_bos && e.bos_id >= 0 && (lead.empty() || lead.front() != e.bos_id)) + lead.insert(lead.begin(), e.bos_id); + pin_tokens = e.prefix->pin_at(ids, npu::shared_prefix_len(lead, ids)); + } else { + pin_tokens = e.prefix->pin_point(e.prev_ids, ids); + } if (pin_tokens > 0 && int(pin_tokens) - 1 >= resume) pin_cut = int(pin_tokens) - 1; } { diff --git a/app/npu_chatml.h b/app/npu_chatml.h index ed8094357..7ec367fa1 100644 --- a/app/npu_chatml.h +++ b/app/npu_chatml.h @@ -61,6 +61,31 @@ inline std::string chatml(const nlohmann::json& messages, bool thinking = true) return p; } +// The messages a chat route renders to find where a request's system turn ends (--prefix-pin, engine#1785): the +// request's leading system messages, then one user message whose text no question starts with. The rendering of +// these and the rendering of the real request agree up to the first token of the user's text and differ there, so +// their common prefix is the part every question under this system prompt shares. +// +// The user message is there because a template may refuse to render without one. Measured: the Qwen3.5 template +// rendered alone on its system message throws, and the early pin was silently not made on that container. +// +// Empty array when the request does not open with a system message: there is nothing to pin early. +template +Json pin_probe_messages(const Json& messages) { + Json lead = Json::array(); + if (!messages.is_array()) return lead; + for (const auto& m : messages) { + if (!m.is_object() || !m.contains("role") || !m["role"].is_string() || m["role"].template get() != "system") break; + lead.push_back(m); + } + if (lead.empty()) return lead; + Json probe = Json::object(); + probe["role"] = "user"; + probe["content"] = "\xC2\xA4\xC2\xA4 pin probe \xC2\xA4\xC2\xA4"; + lead.push_back(probe); + return lead; +} + // Whether the container's tokenizer can represent the prompt above. `chatml()` is rendered for // every container (#832), and a tokenizer without the anchors spells `<|im_start|>` as ordinary // pieces, so the model is prompted with text its own template never produces (#963: on diff --git a/app/npu_prefix_cache.h b/app/npu_prefix_cache.h index b66562b1c..0412a676d 100644 --- a/app/npu_prefix_cache.h +++ b/app/npu_prefix_cache.h @@ -37,6 +37,7 @@ // What they share is the part before the first difference, so that part gets a slot of its own: // pin_point(prev, ids) how many leading tokens this request shares with the previous one, when that // is worth a slot and no slot covers it already +// pin_at(ids, n) the same decision for a point the caller knows already (the end of the system turn) // pin(prefix, bytes) keep the state at that point. One pinned slot at a time; the budget's eviction // and a longer prompt's replacement both leave it alone, a new pin() replaces it. // The rule reads token ids only, so it needs no knowledge of the template or of what a system prompt is. @@ -186,14 +187,25 @@ class PrefixCache { size_t pin_point(const std::vector& prev, const std::vector& ids) const { if (!enabled()) return 0; const size_t shared = shared_prefix_len(prev, ids); - if (shared == prev.size() || shared == ids.size()) return 0; + if (shared == prev.size()) return 0; + return pin_at(ids, shared); + } + + // The same decision for a point the caller already knows: `n` leading tokens of `ids` that every request of + // this kind will share -- the end of the system turn, which a chat route can find on the FIRST request by + // rendering the system messages alone, instead of waiting for a second request to compare with. Returns `n` + // when the state there is worth pinning, else 0: 0 when `n` is the whole prompt or more (nothing follows it, + // so insert() keeps it), when it adds fewer than prefix-min-gain tokens over the slot that already covers + // this request (the pin itself, on every request after the one that made it), and when the cache is off. + size_t pin_at(const std::vector& ids, size_t n) const { + if (!enabled() || n == 0 || n >= ids.size()) return 0; size_t covered = 0; // the longest live slot that is a prefix of this request for (const PrefixSlot& s : slots_) { if (!s.live || s.ids.size() > ids.size() || s.ids.size() <= covered) continue; if (std::equal(s.ids.begin(), s.ids.end(), ids.begin())) covered = s.ids.size(); } - if (shared <= covered || shared - covered < size_t(opt_.min_gain)) return 0; - return shared; + if (n <= covered || n - covered < size_t(opt_.min_gain)) return 0; + return n; } // Keep `prefix` (the first pin_point() tokens of a request) as the pinned slot. Returns the slot the diff --git a/docs/npu.md b/docs/npu.md index 1c4396014..189b69e74 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1746,10 +1746,12 @@ The slot table (`app/npu_prefix_cache.h`) has a slot kind for the part such requ | call | what it decides | |---|---| | `pin_point(prev, ids)` | how many leading tokens this request shares with the previous one. 0 when one prompt is a prefix of the other (a repeat or a continued conversation, which the ordinary slots already keep), when the shared part adds fewer than `--prefix-min-gain` tokens over a slot that already covers the request, or when the cache is off | +| `pin_at(ids, n)` | the same decision for a point the caller already knows: `n` leading tokens every such request shares. 0 when `n` is the whole prompt, when a slot already covers it, or when it is under `--prefix-min-gain` | | `pin(prefix, bytes)` | keep the state at that point. One pinned slot at a time: a new pin replaces it; eviction for room and a longer prompt's replacement both leave it alone | | `lookup(ids)` | unchanged, and reports `pinned` when the hit is the pinned slot | -It reads token ids only, so it does not depend on the template or on which message is the system prompt. +The table itself reads token ids only, so it does not depend on the template. Where the pin goes is the caller's +choice between two rules, below. **`--prefix-pin`** (on `1bit serve --device npu` and on `forward-serve`; off by default) turns it on. The forward keeps the previous request's prompt ids, and when `pin_point()` is non-zero it stops the prefill one @@ -1774,6 +1776,27 @@ The default-cache arm's times are not in the table: the ticket ended during its three cache decisions (from the server log) are known. A request's time here is the whole HTTP call, prefill plus 16 generated tokens, not TTFT. +**Pinned on the first request (engine#1785).** That run needed two requests to find the shared part, so the +third was the first warm one, and when two questions opened with the same words the pin ran a few tokens into the +question, which a question opening differently could not use. A chat request with a leading system message now +gives the point directly: the route renders the leading system messages followed by a probe user message (the +container's template, or ChatML) and pins where the full prompt stops agreeing with that, on the first request +(`pin_at`). The probe user message is there because a template may refuse to render a system turn alone (the +Qwen3.5 template does); if a template cannot render even that, the request has no early pin and the log says so +(`[prefix] no early pin: …`). Only a request with no +system message still uses the comparison with the previous request (`pin_point`). On FastFlowLM's +`Qwen3.5-2B-NPU2` container under `--flash`, a 1,016-token prompt, time to first token (two server loads): + +| request | before | after | +|---|---|---| +| first question | 8.8 s | 10.6 / 11.1 s (busier box; makes the pin: 1,003 tokens) | +| **second question** | 9.1–9.5 s (made the pin: 1,009 tokens) | **1.39 / 1.38 s** (1,002 of 1,017 cached) | +| third question | 1.06 s | 1.40 / 1.35 s | + +The "before" column was taken with two other changes that make decode about a third faster on this model, the +"after" column without them and at twice the box load, so compare the rows' shape (which request is the first warm +one), not the third row's two figures. + **`--flash`** (`1bit serve --device npu --flash`) is the single-turn profile as one word: `--ctx-size 1024` plus `--prefix-pin`. It refuses what contradicts it (`--ctx-size` other than 1024, `--prefix-slots 0`, a device other than `npu`) instead of quietly winning or losing. diff --git a/tests/npu_chatml_test.cpp b/tests/npu_chatml_test.cpp index 16b1526f3..d43909570 100644 --- a/tests/npu_chatml_test.cpp +++ b/tests/npu_chatml_test.cpp @@ -126,6 +126,34 @@ int main(int argc, char** argv) { check(!without.encode(chatml(msgs, true)).empty(), "the refusal is not about whether the prompt tokenizes at all -- it does, into the wrong ids"); + // pin_probe_messages (engine#1785): what the route renders to find the end of the system turn + { + using onebit::npu::pin_probe_messages; + const nlohmann::json req = nlohmann::json::parse( + R"([{"role":"system","content":"Be brief."},{"role":"user","content":"What is item 3?"}])"); + const nlohmann::json probe = pin_probe_messages(req); + check(probe.size() == 2 && probe[0] == req[0] && probe[1]["role"] == "user", + "the probe is the system message and one user message (a template may refuse to render without one)"); + check(probe[1]["content"] != req[1]["content"], "whose text is not the question"); + // the two renderings agree through the user turn's opening and part at the first character of its text + const std::string full = chatml(req, true), lead = chatml(probe, true); + size_t same = 0; + while (same < full.size() && same < lead.size() && full[same] == lead[same]) ++same; + const std::string expect = "<|im_start|>system\nBe brief.<|im_end|>\n<|im_start|>user\n"; + check(full.compare(0, same, expect) == 0 && same == expect.size(), + "the common prefix of the two ChatML renderings is the system turn plus the user turn's opening"); + const nlohmann::json two = nlohmann::json::parse( + R"([{"role":"system","content":"A"},{"role":"system","content":"B"},{"role":"user","content":"q"},{"role":"system","content":"late"}])"); + check(pin_probe_messages(two).size() == 3, "every LEADING system message is kept, a later one is not"); + check(pin_probe_messages(nlohmann::json::parse(R"([{"role":"user","content":"q"}])")).empty(), + "a request that does not open with a system message has no probe"); + check(pin_probe_messages(nlohmann::json::parse(R"([{"role":7},{"content":"x"}])")).empty() && + pin_probe_messages(nlohmann::json::parse("{}")).empty(), + "malformed messages give no probe instead of throwing"); + const nlohmann::ordered_json oreq = nlohmann::ordered_json::parse(R"([{"role":"system","content":"S"},{"role":"user","content":"q"}])"); + check(pin_probe_messages(oreq).size() == 2, "the same for the ordered JSON the template path uses"); + } + if (failures == 0) std::printf("npu_chatml: ok\n"); return failures == 0 ? 0 : 1; } diff --git a/tests/npu_sampling_test.cpp b/tests/npu_sampling_test.cpp index 45932a92b..e791743f6 100644 --- a/tests/npu_sampling_test.cpp +++ b/tests/npu_sampling_test.cpp @@ -529,6 +529,39 @@ void test_prefix_pin() { check(PrefixCache(off).pin_point(q1, q2) == 0, "an off cache pins nothing"); check(PrefixCache(off).pin(q1, 10) == -1, "and refuses a pin"); } + { // pin_at: the end of the system turn is known on the FIRST request, so the second one is already warm + PrefixCache c(o); + check(c.pin_at(q1, 900) == 900, "the first request under a 900-token system prompt pins its 900 tokens"); + check(c.pin_point({}, q1) == 0, "where comparing with a previous request has nothing to offer yet"); + std::string why; + const int pin = c.pin(std::vector(q1.begin(), q1.begin() + 900), 2000, &why); + c.insert(q1, 1000); + const auto d = c.lookup(q2); + check(pin >= 0 && d.slot == pin && d.cached_tokens == 900 && d.pinned, + "so the SECOND question resumes from the pin: 900 of its 930 tokens"); + check(c.pin_at(q2, 900) == 0 && c.pin_at(q3, 900) == 0, "and later requests do not pin the same point again"); + // what is not worth a pin, or is not one + check(c.pin_at(q1, 0) == 0, "no system turn, no pin"); + check(c.pin_at(q1, q1.size()) == 0 && c.pin_at(q1, q1.size() + 5) == 0, "a point at or past the end of the prompt is insert()'s"); + PrefixCache fresh(o); + check(fresh.pin_at(q1, 127) == 0 && fresh.pin_at(q1, 128) == 128, "a system turn shorter than prefix-min-gain is not pinned"); + PrefixOptions off = o; + off.slots = 0; + check(PrefixCache(off).pin_at(q1, 900) == 0, "an off cache pins nothing"); + // a pin that ran into the question (two questions opening alike) does not cover a question that opens + // differently, and the system turn's own end then replaces it + PrefixCache over(o); + std::vector a1 = prompt(900, 5000, 40), a2 = prompt(900, 5000, 40), b1 = prompt(900, 8000, 40); + a2[905] = 4242; // same first five question tokens, then different + const size_t ran_over = over.pin_point(a1, a2); + check(ran_over == 905, "comparing two alike-opening questions pins five tokens into the question"); + over.pin(std::vector(a2.begin(), a2.begin() + long(ran_over)), 2000); + check(over.lookup(b1).cached_tokens == 0, "which a differently opening question cannot use"); + check(over.pin_at(b1, 900) == 900, "the system turn's end is still worth pinning for it"); + const int repin = over.pin(std::vector(b1.begin(), b1.begin() + 900), 2000); + check(repin >= 0 && over.lookup(b1).cached_tokens == 900 && over.lookup(a1).cached_tokens == 900, + "and once pinned there, every question under that system prompt resumes from it"); + } { // the pin outlives what evicts and replaces everything else PrefixOptions b = o; b.slots = 4; // one more than the three snapshots the budget holds, so a slot is always free