diff --git a/CMakeLists.txt b/CMakeLists.txt index 7b6821e3e..d0a52b964 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -278,6 +278,9 @@ add_test(NAME serve_port COMMAND bash ${CMAKE_SOURCE_DIR}/tests/serve_port_test. # device. Deliberately OUTSIDE if(ONEBIT_SERVE_TEST_GGUF): that row is device-gated, and the whole point is that the # classification is checked in every configure, including the ones that never load a model. add_test(NAME serve_context_limit_classify COMMAND bash ${CMAKE_SOURCE_DIR}/tests/serve_context_limit_test.sh --self-test) +# engine#1785: `serve --flash` names two settings (--ctx-size 1024, --prefix-pin on --device npu) and refuses what +# contradicts them. Every case ends in the option check, so the row needs no model, no device and no port. +add_test(NAME serve_flash_refusals COMMAND bash ${CMAKE_SOURCE_DIR}/tests/serve_flash_test.sh $) # engine#1594: every target ci.yml's `cmake --build --target …` list names must exist in the configuration ci.yml # creates. #1583 added one name for a target defined inside if(ONEBIT_NPU), which CI does not configure -- and ninja # rejects the whole command line before building anything, so `onebit` stopped being compiled for every PR. Reads the diff --git a/app/forward_serve.cpp b/app/forward_serve.cpp index effed11be..d193ee114 100644 --- a/app/forward_serve.cpp +++ b/app/forward_serve.cpp @@ -74,6 +74,16 @@ namespace onebit { namespace { +// A prompt that leaves no room for an answer inside the context (the model's, or --ctx-size). +struct ContextOverflow : std::runtime_error { + size_t prompt_tokens; + int context; + ContextOverflow(size_t n, int ctx) + : std::runtime_error("request (" + std::to_string(n) + " tokens) exceeds the available context size (" + + std::to_string(ctx) + " tokens)"), + prompt_tokens(n), context(ctx) {} +}; + using json = nlohmann::json; struct ForwardEngine { @@ -297,7 +307,7 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res // The KV cache grows with every position: the prompt and the answer together stay // inside the model's context. const int room = e.max_context - int(ids.size()); - if (room < 1) throw std::runtime_error("prompt longer than the context"); + if (room < 1) throw ContextOverflow(ids.size(), e.max_context); const int limit = std::clamp(max_tokens, 0, room); std::lock_guard lock(e.mu); @@ -729,6 +739,13 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res return true; }); } + } catch (const ContextOverflow& ex) { + // The client's request, not a server fault: 400, with the type and the wording the GGUF route's + // backend uses for the same refusal, so one client-side check covers both routes. + res.status = 400; + res.set_content(json{{"error", {{"message", ex.what()}, {"type", "exceed_context_size_error"}, + {"n_prompt_tokens", ex.prompt_tokens}, {"n_ctx", ex.context}}}}.dump(), + "application/json"); } catch (const std::exception& ex) { res.status = 500; res.set_content(json{{"error", {{"message", ex.what()}}}}.dump(), "application/json"); @@ -906,6 +923,7 @@ int run_forward_serve(int argc, char** argv) { std::string npu_target = ::npu::target::xdna::XdnaTarget::kName; npu::PrefixOptions prefix_opts; // --prefix-slots / --prefix-cache-mb / --prefix-min-gain (#650) bool prefix_pin = false; // --prefix-pin (engine#1785) + int ctx_size = 0; // --ctx-size N: prompt plus answer, at most the model's own (0 = the model's) for (int i = 0; i < argc; i++) { const std::string a = argv[i]; auto next = [&]() -> std::string { @@ -921,6 +939,7 @@ int run_forward_serve(int argc, char** argv) { else if (a == "--prefix-cache-mb") prefix_opts.cache_mb = std::stoul(next()); else if (a == "--prefix-min-gain") prefix_opts.min_gain = std::stoi(next()); else if (a == "--prefix-pin") prefix_pin = true; + else if (a == "-c" || a == "--ctx-size") ctx_size = std::stoi(next()); else if (a == "--npu-streams") streams = std::stoi(next()); else if (a == "--npu-target") npu_target = next(); else if (a == "--dump-logits") dump_logits = next(); @@ -963,6 +982,16 @@ int run_forward_serve(int argc, char** argv) { // q4nx_config_int finds the key at any depth (a text_config's too). e.max_context = q4nx_config_int(cfg.c_str(), "max_position_embeddings", 0); if (e.max_context <= 0) e.max_context = 32768; + // --ctx-size lowers the limit and never raises it: the model's positions are what they are. The same + // guard that refuses a prompt longer than the model's context then refuses one longer than this. + if (ctx_size < 0) throw std::runtime_error("--ctx-size: must be zero or more"); + if (ctx_size > 0) { + if (ctx_size > e.max_context) + std::fprintf(stderr, "1bit forward-serve: --ctx-size %d is above the model's context %d; using %d\n", ctx_size, + e.max_context, e.max_context); + e.max_context = std::min(e.max_context, ctx_size); + } + std::fprintf(stderr, "1bit forward-serve: context %d tokens (prompt plus answer)\n", e.max_context); // #963: the chat route's prompt was ChatML for every container, and a tokenizer with no // <|im_start|>/<|im_end|> added token cannot represent it -- the marker becomes ordinary pieces and diff --git a/app/serve.cpp b/app/serve.cpp index 164d6623f..69f3ec145 100644 --- a/app/serve.cpp +++ b/app/serve.cpp @@ -186,6 +186,7 @@ struct Options { int prefix_cache_mb = -1; // --prefix-cache-mb N int prefix_min_gain = -1; // --prefix-min-gain N: tokens a prompt must add to be worth keeping bool prefix_pin = false; // --prefix-pin: keep the part single-turn requests share (docs/npu.md) + bool flash = false; // --flash: the single-turn NPU profile = --ctx-size 1024 + --prefix-pin (engine#1785) std::string npu_target; // --npu-target NAME: which NPU target serves the per-op route (empty = the build's default) std::string prefill_provider; // --prefill-provider PATH (or ONEBIT_HYBRID_PREFILL): --device hybrid's prefill (docs/hybrid.md) std::string laya_model; // --laya-model DIR: route each request by the Laya scorer (docs/laya.md) @@ -3173,6 +3174,7 @@ void usage(FILE* out) { " [--prefix-slots N] [--prefix-cache-mb N] [--prefix-min-gain N] the prompt prefix cache\n" " (docs/npu.md; 0 slots turns it off). Unset keeps the forward's defaults: 8 / 4096 / 128\n" " [--prefix-pin] keep the part a request shares with the previous one (one system prompt, many questions)\n" + " [--flash] with --device npu: the single-turn profile, --ctx-size 1024 plus --prefix-pin\n" " (default 1, the device exclusive; docs/npu.md)\n" " [--load-queue-ms N] a load refused because another client holds the NPU array\n" " waits at most N ms for it to free before the driver is called dead\n" @@ -3289,6 +3291,7 @@ int run_serve(int argc, char** argv) { else if (a == "--prefix-cache-mb") o.prefix_cache_mb = std::stoi(next()); else if (a == "--prefix-min-gain") o.prefix_min_gain = std::stoi(next()); else if (a == "--prefix-pin") o.prefix_pin = true; + else if (a == "--flash") o.flash = true; else if (a == "--npu-target") o.npu_target = next(); else if (a == "--prefill-provider") o.prefill_provider = next(); else if (a == "--laya-model") o.laya_model = next(); @@ -3325,6 +3328,16 @@ int run_serve(int argc, char** argv) { else if (a == "-h" || a == "--help") { usage(stdout); return 0; } else throw std::runtime_error("unknown option " + a); } + // --flash is a name for two settings, so that the profile is one word and its parts stay visible: a 1024-token + // context and the pinned shared prefix. It refuses what contradicts it instead of quietly winning or losing. + if (o.flash) { + if (o.ctx_size != 0 && o.ctx_size != 1024) + throw std::runtime_error("--flash fixes the context at 1024 tokens; drop --ctx-size " + std::to_string(o.ctx_size)); + if (o.prefix_slots == 0) throw std::runtime_error("--flash needs the prefix cache; drop --prefix-slots 0"); + if (o.device != "npu") throw std::runtime_error("--flash is a profile of --device npu"); + o.ctx_size = 1024; + o.prefix_pin = true; + } if (!o.laya_models.empty()) { #if defined(ONEBIT_LAYA) && !defined(_WIN32) if (o.laya_model.empty() && !o.laya_auto) throw std::runtime_error("--laya-models needs --laya or --laya-model DIR"); @@ -3456,6 +3469,9 @@ int run_serve(int argc, char** argv) { if (o.prefix_cache_mb >= 0) { fargs.push_back("--prefix-cache-mb"); fargs.push_back(std::to_string(o.prefix_cache_mb)); } if (o.prefix_min_gain >= 0) { fargs.push_back("--prefix-min-gain"); fargs.push_back(std::to_string(o.prefix_min_gain)); } if (o.prefix_pin) fargs.push_back("--prefix-pin"); + // The context limit reaches this route too: it used to stop here, so `serve --ctx-size N` on a + // model-generic container was accepted and the forward kept the model's own limit. + if (o.ctx_size > 0) { fargs.push_back("--ctx-size"); fargs.push_back(std::to_string(o.ctx_size)); } std::vector fav; for (auto& a : fargs) fav.push_back(a.data()); return run_forward_serve(int(fav.size()), fav.data()); diff --git a/docs/npu.md b/docs/npu.md index 1ae6d6ae9..ef9d0ff5b 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1622,7 +1622,7 @@ The cost is one more launch on the request that makes the pin. Every decision is (`[prefix] pin slot=1 tokens=342 state_at=341 pinned`), and `usage.flm_prefix_cache.pinned` says when a hit came from the pinned slot. -Measured 2026-11-10 on `bonsai2-t2s2-perrole876` with its `kernels-m16`, `--ctx-size 1024`, one `npu-hold` +Measured 2026-11-10 on `bonsai2-t2s2-perrole876` with its `kernels-m16`, one `npu-hold` ticket, one server per arm, four requests: the same 14-rule system prompt, then three different questions and the second one again. 16 tokens generated, temperature 0. @@ -1638,6 +1638,30 @@ The default-cache arm's times are not in the table: the ticket ended during its three cache decisions (from the server log) are known. A request's time here is the whole HTTP call, prefill plus 16 generated tokens, not TTFT. +**`--flash`** (`1bit serve --device npu --flash`) is the single-turn profile as one word: `--ctx-size 1024` +plus `--prefix-pin`. It refuses what contradicts it (`--ctx-size` other than 1024, `--prefix-slots 0`, a device +other than `npu`) instead of quietly winning or losing. + +`--ctx-size N` now reaches this route. Before engine#1785's flash slice, `serve --ctx-size N` on a +model-generic container was accepted and dropped at the hand-off to `forward-serve`, which kept the model's +own `max_position_embeddings` (so the run in the table above was passed `--ctx-size 1024` and it had no effect; +its prompts were about 355 tokens, so nothing in the table depends on it). The limit is prompt plus answer, +it can lower the model's context and never raise it, and the load line says what is in force: +`1bit forward-serve: context 1024 tokens (prompt plus answer)`. + +A request that does not fit is the client's error, so it is a 400 with the type the GGUF route's backend uses +for the same refusal (it was a 500 with a bare message): + +``` +HTTP 400 {"error": {"message": "request (1597 tokens) exceeds the available context size (1024 tokens)", + "type": "exceed_context_size_error", "n_prompt_tokens": 1597, "n_ctx": 1024}} +``` + +Measured 2026-11-10 with `--flash`, same pack and kernels, one `npu-hold` ticket, `max_tokens 8`: the +1597-token request above refused in 0.02 s; then three questions under one system prompt: request 1 +`cached 0` (56.2 s), request 2 `cached 0` (48.9 s, makes the pin at 342 tokens), request 3 +`cached 341 of 355, pinned` (13.9 s). Whole HTTP calls, one run. + ### Sampling and structured output `1bit serve -m --device npu` serves through `1bit forward-serve` diff --git a/tests/serve_flash_test.sh b/tests/serve_flash_test.sh new file mode 100755 index 000000000..a3fae7b65 --- /dev/null +++ b/tests/serve_flash_test.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# `1bit serve --flash` (engine #1785) is a name for two settings: --ctx-size 1024 and --prefix-pin on --device npu. +# What contradicts the profile must be refused by name, before any model is opened or any device is touched, instead +# of one side quietly winning. +# +# No model, no device, no port: every case here ends in the option check. The model path does not exist on purpose; +# a case that got past the check would fail on it with a different message, and the pattern would not match. +# +# serve_flash_test.sh <1bit binary> +set -uo pipefail + +bin=${1:?usage: serve_flash_test.sh <1bit binary>} +[ -x "$bin" ] || { echo "serve_flash_test: $bin is not executable"; exit 1; } + +fail=0 +missing=/nonexistent/serve-flash-test-model +# refused NAME PATTERN ARGS...: the command must exit non-zero within the limit and name the reason. +refused() { + local name=$1 pattern=$2 out rc + shift 2 + out=$(timeout 20 "$bin" serve -m "$missing" "$@" 2>&1) + rc=$? + if [ "$rc" -ne 0 ] && [ "$rc" -ne 124 ] && printf '%s' "$out" | grep -qF -- "$pattern"; then + echo "ok $name" + else + echo "FAIL $name (rc=$rc): $out" + fail=$(( fail + 1 )) + fi +} +# not_refused_by_flash NAME ARGS...: the command may fail (the model is missing), but not on a --flash rule and +# not in the option parser. +not_refused_by_flash() { + local name=$1 out rc + shift + out=$(timeout 20 "$bin" serve -m "$missing" "$@" 2>&1) + rc=$? + if [ "$rc" -ne 124 ] && ! printf '%s' "$out" | grep -qF -e "--flash" -e "unknown option"; then + echo "ok $name" + else + echo "FAIL $name (rc=$rc): $out" + fail=$(( fail + 1 )) + fi +} + +refused "another context size is refused" "--flash fixes the context at 1024 tokens; drop --ctx-size 2048" \ + --device npu --flash --ctx-size 2048 +refused "and so is one given before --flash" "--flash fixes the context at 1024 tokens; drop --ctx-size 512" \ + --device npu --ctx-size 512 --flash +refused "a cache turned off is refused" "--flash needs the prefix cache; drop --prefix-slots 0" \ + --device npu --flash --prefix-slots 0 +refused "another device is refused" "--flash is a profile of --device npu" \ + --device cpu --flash +refused "no device at all is refused too" "--flash is a profile of --device npu" \ + --flash + +not_refused_by_flash "its own context size, spelled out, is accepted" --device npu --flash --ctx-size 1024 +not_refused_by_flash "--prefix-pin next to it is accepted" --device npu --flash --prefix-pin +not_refused_by_flash "without --flash none of the rules apply" --device cpu --ctx-size 2048 --prefix-slots 0 + +[ "$fail" -eq 0 ] && echo "serve_flash: ok" || echo "serve_flash: $fail failed" +exit "$fail"