Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -278,6 +278,9 @@ add_test(NAME serve_port COMMAND bash ${CMAKE_SOURCE_DIR}/tests/serve_port_test.
# device. Deliberately OUTSIDE if(ONEBIT_SERVE_TEST_GGUF): that row is device-gated, and the whole point is that the
# classification is checked in every configure, including the ones that never load a model.
add_test(NAME serve_context_limit_classify COMMAND bash ${CMAKE_SOURCE_DIR}/tests/serve_context_limit_test.sh --self-test)
# engine#1785: `serve --flash` names two settings (--ctx-size 1024, --prefix-pin on --device npu) and refuses what
# contradicts them. Every case ends in the option check, so the row needs no model, no device and no port.
add_test(NAME serve_flash_refusals COMMAND bash ${CMAKE_SOURCE_DIR}/tests/serve_flash_test.sh $<TARGET_FILE:onebit>)
# engine#1594: every target ci.yml's `cmake --build --target …` list names must exist in the configuration ci.yml
# creates. #1583 added one name for a target defined inside if(ONEBIT_NPU), which CI does not configure -- and ninja
# rejects the whole command line before building anything, so `onebit` stopped being compiled for every PR. Reads the
Expand Down
31 changes: 30 additions & 1 deletion app/forward_serve.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,16 @@ namespace onebit {

namespace {

// A prompt that leaves no room for an answer inside the context (the model's, or --ctx-size).
struct ContextOverflow : std::runtime_error {
size_t prompt_tokens;
int context;
ContextOverflow(size_t n, int ctx)
: std::runtime_error("request (" + std::to_string(n) + " tokens) exceeds the available context size (" +
std::to_string(ctx) + " tokens)"),
prompt_tokens(n), context(ctx) {}
};

using json = nlohmann::json;

struct ForwardEngine {
Expand Down Expand Up @@ -297,7 +307,7 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res
// The KV cache grows with every position: the prompt and the answer together stay
// inside the model's context.
const int room = e.max_context - int(ids.size());
if (room < 1) throw std::runtime_error("prompt longer than the context");
if (room < 1) throw ContextOverflow(ids.size(), e.max_context);
const int limit = std::clamp(max_tokens, 0, room);
std::lock_guard<std::mutex> lock(e.mu);

Expand Down Expand Up @@ -729,6 +739,13 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res
return true;
});
}
} catch (const ContextOverflow& ex) {
// The client's request, not a server fault: 400, with the type and the wording the GGUF route's
// backend uses for the same refusal, so one client-side check covers both routes.
res.status = 400;
res.set_content(json{{"error", {{"message", ex.what()}, {"type", "exceed_context_size_error"},
{"n_prompt_tokens", ex.prompt_tokens}, {"n_ctx", ex.context}}}}.dump(),
"application/json");
} catch (const std::exception& ex) {
res.status = 500;
res.set_content(json{{"error", {{"message", ex.what()}}}}.dump(), "application/json");
Expand Down Expand Up @@ -906,6 +923,7 @@ int run_forward_serve(int argc, char** argv) {
std::string npu_target = ::npu::target::xdna::XdnaTarget::kName;
npu::PrefixOptions prefix_opts; // --prefix-slots / --prefix-cache-mb / --prefix-min-gain (#650)
bool prefix_pin = false; // --prefix-pin (engine#1785)
int ctx_size = 0; // --ctx-size N: prompt plus answer, at most the model's own (0 = the model's)
for (int i = 0; i < argc; i++) {
const std::string a = argv[i];
auto next = [&]() -> std::string {
Expand All @@ -921,6 +939,7 @@ int run_forward_serve(int argc, char** argv) {
else if (a == "--prefix-cache-mb") prefix_opts.cache_mb = std::stoul(next());
else if (a == "--prefix-min-gain") prefix_opts.min_gain = std::stoi(next());
else if (a == "--prefix-pin") prefix_pin = true;
else if (a == "-c" || a == "--ctx-size") ctx_size = std::stoi(next());
else if (a == "--npu-streams") streams = std::stoi(next());
else if (a == "--npu-target") npu_target = next();
else if (a == "--dump-logits") dump_logits = next();
Expand Down Expand Up @@ -963,6 +982,16 @@ int run_forward_serve(int argc, char** argv) {
// q4nx_config_int finds the key at any depth (a text_config's too).
e.max_context = q4nx_config_int(cfg.c_str(), "max_position_embeddings", 0);
if (e.max_context <= 0) e.max_context = 32768;
// --ctx-size lowers the limit and never raises it: the model's positions are what they are. The same
// guard that refuses a prompt longer than the model's context then refuses one longer than this.
if (ctx_size < 0) throw std::runtime_error("--ctx-size: must be zero or more");
if (ctx_size > 0) {
if (ctx_size > e.max_context)
std::fprintf(stderr, "1bit forward-serve: --ctx-size %d is above the model's context %d; using %d\n", ctx_size,
e.max_context, e.max_context);
e.max_context = std::min(e.max_context, ctx_size);
}
std::fprintf(stderr, "1bit forward-serve: context %d tokens (prompt plus answer)\n", e.max_context);

// #963: the chat route's prompt was ChatML for every container, and a tokenizer with no
// <|im_start|>/<|im_end|> added token cannot represent it -- the marker becomes ordinary pieces and
Expand Down
16 changes: 16 additions & 0 deletions app/serve.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,7 @@ struct Options {
int prefix_cache_mb = -1; // --prefix-cache-mb N
int prefix_min_gain = -1; // --prefix-min-gain N: tokens a prompt must add to be worth keeping
bool prefix_pin = false; // --prefix-pin: keep the part single-turn requests share (docs/npu.md)
bool flash = false; // --flash: the single-turn NPU profile = --ctx-size 1024 + --prefix-pin (engine#1785)
std::string npu_target; // --npu-target NAME: which NPU target serves the per-op route (empty = the build's default)
std::string prefill_provider; // --prefill-provider PATH (or ONEBIT_HYBRID_PREFILL): --device hybrid's prefill (docs/hybrid.md)
std::string laya_model; // --laya-model DIR: route each request by the Laya scorer (docs/laya.md)
Expand Down Expand Up @@ -3173,6 +3174,7 @@ void usage(FILE* out) {
" [--prefix-slots N] [--prefix-cache-mb N] [--prefix-min-gain N] the prompt prefix cache\n"
" (docs/npu.md; 0 slots turns it off). Unset keeps the forward's defaults: 8 / 4096 / 128\n"
" [--prefix-pin] keep the part a request shares with the previous one (one system prompt, many questions)\n"
" [--flash] with --device npu: the single-turn profile, --ctx-size 1024 plus --prefix-pin\n"
" (default 1, the device exclusive; docs/npu.md)\n"
" [--load-queue-ms N] a load refused because another client holds the NPU array\n"
" waits at most N ms for it to free before the driver is called dead\n"
Expand Down Expand Up @@ -3289,6 +3291,7 @@ int run_serve(int argc, char** argv) {
else if (a == "--prefix-cache-mb") o.prefix_cache_mb = std::stoi(next());
else if (a == "--prefix-min-gain") o.prefix_min_gain = std::stoi(next());
else if (a == "--prefix-pin") o.prefix_pin = true;
else if (a == "--flash") o.flash = true;
else if (a == "--npu-target") o.npu_target = next();
else if (a == "--prefill-provider") o.prefill_provider = next();
else if (a == "--laya-model") o.laya_model = next();
Expand Down Expand Up @@ -3325,6 +3328,16 @@ int run_serve(int argc, char** argv) {
else if (a == "-h" || a == "--help") { usage(stdout); return 0; }
else throw std::runtime_error("unknown option " + a);
}
// --flash is a name for two settings, so that the profile is one word and its parts stay visible: a 1024-token
// context and the pinned shared prefix. It refuses what contradicts it instead of quietly winning or losing.
if (o.flash) {
if (o.ctx_size != 0 && o.ctx_size != 1024)
throw std::runtime_error("--flash fixes the context at 1024 tokens; drop --ctx-size " + std::to_string(o.ctx_size));
if (o.prefix_slots == 0) throw std::runtime_error("--flash needs the prefix cache; drop --prefix-slots 0");
if (o.device != "npu") throw std::runtime_error("--flash is a profile of --device npu");
o.ctx_size = 1024;
o.prefix_pin = true;
}
if (!o.laya_models.empty()) {
#if defined(ONEBIT_LAYA) && !defined(_WIN32)
if (o.laya_model.empty() && !o.laya_auto) throw std::runtime_error("--laya-models needs --laya or --laya-model DIR");
Expand Down Expand Up @@ -3456,6 +3469,9 @@ int run_serve(int argc, char** argv) {
if (o.prefix_cache_mb >= 0) { fargs.push_back("--prefix-cache-mb"); fargs.push_back(std::to_string(o.prefix_cache_mb)); }
if (o.prefix_min_gain >= 0) { fargs.push_back("--prefix-min-gain"); fargs.push_back(std::to_string(o.prefix_min_gain)); }
if (o.prefix_pin) fargs.push_back("--prefix-pin");
// The context limit reaches this route too: it used to stop here, so `serve --ctx-size N` on a
// model-generic container was accepted and the forward kept the model's own limit.
if (o.ctx_size > 0) { fargs.push_back("--ctx-size"); fargs.push_back(std::to_string(o.ctx_size)); }
std::vector<char*> fav;
for (auto& a : fargs) fav.push_back(a.data());
return run_forward_serve(int(fav.size()), fav.data());
Expand Down
26 changes: 25 additions & 1 deletion docs/npu.md
Original file line number Diff line number Diff line change
Expand Up @@ -1622,7 +1622,7 @@ The cost is one more launch on the request that makes the pin. Every decision is
(`[prefix] pin slot=1 tokens=342 state_at=341 pinned`), and `usage.flm_prefix_cache.pinned` says when a hit
came from the pinned slot.

Measured 2026-11-10 on `bonsai2-t2s2-perrole876` with its `kernels-m16`, `--ctx-size 1024`, one `npu-hold`
Measured 2026-11-10 on `bonsai2-t2s2-perrole876` with its `kernels-m16`, one `npu-hold`
ticket, one server per arm, four requests: the same 14-rule system prompt, then three different questions and
the second one again. 16 tokens generated, temperature 0.

Expand All @@ -1638,6 +1638,30 @@ The default-cache arm's times are not in the table: the ticket ended during its
three cache decisions (from the server log) are known. A request's time here is the whole HTTP call, prefill
plus 16 generated tokens, not TTFT.

**`--flash`** (`1bit serve --device npu --flash`) is the single-turn profile as one word: `--ctx-size 1024`
plus `--prefix-pin`. It refuses what contradicts it (`--ctx-size` other than 1024, `--prefix-slots 0`, a device
other than `npu`) instead of quietly winning or losing.

`--ctx-size N` now reaches this route. Before engine#1785's flash slice, `serve --ctx-size N` on a
model-generic container was accepted and dropped at the hand-off to `forward-serve`, which kept the model's
own `max_position_embeddings` (so the run in the table above was passed `--ctx-size 1024` and it had no effect;
its prompts were about 355 tokens, so nothing in the table depends on it). The limit is prompt plus answer,
it can lower the model's context and never raise it, and the load line says what is in force:
`1bit forward-serve: context 1024 tokens (prompt plus answer)`.

A request that does not fit is the client's error, so it is a 400 with the type the GGUF route's backend uses
for the same refusal (it was a 500 with a bare message):

```
HTTP 400 {"error": {"message": "request (1597 tokens) exceeds the available context size (1024 tokens)",
"type": "exceed_context_size_error", "n_prompt_tokens": 1597, "n_ctx": 1024}}
```

Measured 2026-11-10 with `--flash`, same pack and kernels, one `npu-hold` ticket, `max_tokens 8`: the
1597-token request above refused in 0.02 s; then three questions under one system prompt: request 1
`cached 0` (56.2 s), request 2 `cached 0` (48.9 s, makes the pin at 342 tokens), request 3
`cached 341 of 355, pinned` (13.9 s). Whole HTTP calls, one run.

### Sampling and structured output

`1bit serve -m <NPU model dir> --device npu` serves through `1bit forward-serve`
Expand Down
76 changes: 76 additions & 0 deletions tests/serve_flash_test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
#!/bin/bash
# Copyright 2026 bong-water-water-bong
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# `1bit serve --flash` (engine #1785) is a name for two settings: --ctx-size 1024 and --prefix-pin on --device npu.
# What contradicts the profile must be refused by name, before any model is opened or any device is touched, instead
# of one side quietly winning.
#
# No model, no device, no port: every case here ends in the option check. The model path does not exist on purpose;
# a case that got past the check would fail on it with a different message, and the pattern would not match.
#
# serve_flash_test.sh <1bit binary>
set -uo pipefail

bin=${1:?usage: serve_flash_test.sh <1bit binary>}
[ -x "$bin" ] || { echo "serve_flash_test: $bin is not executable"; exit 1; }

fail=0
missing=/nonexistent/serve-flash-test-model
# refused NAME PATTERN ARGS...: the command must exit non-zero within the limit and name the reason.
refused() {
local name=$1 pattern=$2 out rc
shift 2
out=$(timeout 20 "$bin" serve -m "$missing" "$@" 2>&1)
rc=$?
if [ "$rc" -ne 0 ] && [ "$rc" -ne 124 ] && printf '%s' "$out" | grep -qF -- "$pattern"; then
echo "ok $name"
else
echo "FAIL $name (rc=$rc): $out"
fail=$(( fail + 1 ))
fi
}
# not_refused_by_flash NAME ARGS...: the command may fail (the model is missing), but not on a --flash rule and
# not in the option parser.
not_refused_by_flash() {
local name=$1 out rc
shift
out=$(timeout 20 "$bin" serve -m "$missing" "$@" 2>&1)
rc=$?
if [ "$rc" -ne 124 ] && ! printf '%s' "$out" | grep -qF -e "--flash" -e "unknown option"; then
echo "ok $name"
else
echo "FAIL $name (rc=$rc): $out"
fail=$(( fail + 1 ))
fi
}

refused "another context size is refused" "--flash fixes the context at 1024 tokens; drop --ctx-size 2048" \
--device npu --flash --ctx-size 2048
refused "and so is one given before --flash" "--flash fixes the context at 1024 tokens; drop --ctx-size 512" \
--device npu --ctx-size 512 --flash
refused "a cache turned off is refused" "--flash needs the prefix cache; drop --prefix-slots 0" \
--device npu --flash --prefix-slots 0
refused "another device is refused" "--flash is a profile of --device npu" \
--device cpu --flash
refused "no device at all is refused too" "--flash is a profile of --device npu" \
--flash

not_refused_by_flash "its own context size, spelled out, is accepted" --device npu --flash --ctx-size 1024
not_refused_by_flash "--prefix-pin next to it is accepted" --device npu --flash --prefix-pin
not_refused_by_flash "without --flash none of the rules apply" --device cpu --ctx-size 2048 --prefix-slots 0

[ "$fail" -eq 0 ] && echo "serve_flash: ok" || echo "serve_flash: $fail failed"
exit "$fail"
Loading