diff --git a/app/serve.cpp b/app/serve.cpp index 69f3ec145..65cc89962 100644 --- a/app/serve.cpp +++ b/app/serve.cpp @@ -1151,7 +1151,16 @@ Launch launch_for(const Options& o, const std::string& device, int child_port) { // the experts stay file-backed: without these, llama.cpp copies every one into RAM // (a pinned host buffer, or a CPU repack), which is what streaming avoids // (per_layer_token_embd: Qwen3.8-Flash-Next's 26.8 GiB table, only ever row-gathered) - argv.insert(argv.end(), {"-ot", "exps=CPU,per_layer_token_embd=CPU", "--no-host", "--no-repack", "--load-mode", "mmap"}); + argv.insert(argv.end(), {"-ot", "exps=CPU,per_layer_token_embd=CPU", "--no-host", "--no-repack"}); + // engine#1668: `-lm none` is +17.5 % prefill at -1.6 % decode for ~604 MiB more GTT (docs/bench.md), + // and prefill is what a served model pays for TTFT -- but this site hard-coded `--load-mode mmap`, + // and an explicit flag beats the child's own LLAMA_ARG_LOAD_MODE, so the measured lever could not be + // selected in a server at all. Default unchanged (mmap when the operator has not asked for something + // else); setting LLAMA_ARG_LOAD_MODE=none in the environment now reaches the child, with the GTT cost + // that implies documented beside the measurement. + if (std::getenv("LLAMA_ARG_LOAD_MODE") == nullptr) { + argv.insert(argv.end(), {"--load-mode", "mmap"}); + } #endif } if (device == "hrx") { diff --git a/docs/bench.md b/docs/bench.md index 9233a739a..9dff89c8a 100644 --- a/docs/bench.md +++ b/docs/bench.md @@ -69,6 +69,13 @@ bytes rather than slack: three more ~306 MiB tensors become device-resident (3 quote a prefill delta measured across it. `-lm none` is not a lever to ship on the strength of one measurement — it costs GTT on a box where GTT is the scarce resource, and the honest statement is the mechanism above, not the percentage. +**How it is selected in a server (engine#1668).** The measurement above was taken with `llama-bench`/`llama-server` directly, and +`1bit serve` hard-coded `--load-mode mmap` for its child, so the lever could not be reached where prefill actually costs +time-to-first-token. It now passes that flag **only when the operator has not chosen a mode**: set `LLAMA_ARG_LOAD_MODE=none` +(the child's own variable, so there is one mechanism rather than two) and the flag is omitted, letting the environment value +ride. The default is unchanged, and the 604 MiB of GTT the `-lm none` row records is the price of asking — a per-model cost on +a box where the pool budget is the scarce resource (engine#1711), which is why this is an override and not a new default. + ## What is measured - **Prompt:** one long prompt (default: the first 8,000 characters of `docs/*.md`, about 1,800