server: one preemption path, swap under --preempt-ram and recompute above it

common/arg.cpp

@@ -1709,10 +1709,20 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             params.cache_ram_mib = value;
         }
     ).set_env("LLAMA_ARG_CACHE_RAM").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}));
+    add_opt(common_arg(
+        {"--preempt"},
+        {"--no-preempt"},
+        string_format("with a unified KV cache and more than one slot, park a running request and put it back later "
+            "instead of failing every request when the cache fills (default: %s)", params.preempt ? "enabled" : "disabled"),
+        [](common_params & params, bool value) {
+            params.preempt = value;
+        }
+    ).set_env("LLAMA_ARG_PREEMPT").set_examples({LLAMA_EXAMPLE_SERVER}));
     add_opt(common_arg(
         {"--preempt-ram"}, "N",
-        string_format("with a unified KV cache, park a slot in host RAM instead of failing every slot when the cache fills; "
-            "N is the maximum host RAM for parked sequences in MiB (default: %d, -1 - no limit, 0 - disable)", params.preempt_ram_mib),
+        string_format("maximum host RAM in MiB for parked (preempted) sequences; a slot that fits is parked by copying "
+            "its sequence to host RAM, one that does not is parked by dropping its cells and recomputing them later "
+            "(default: %d, -1 - no limit, 0 - never copy; use --no-preempt to turn preemption off)", params.preempt_ram_mib),
         [](common_params & params, int value) {
             params.preempt_ram_mib = value;
         }

common/common.h

@@ -614,7 +614,8 @@ struct common_params {
     int32_t n_ctx_checkpoints   = 32;    // max number of context checkpoints per slot
     int32_t checkpoint_min_step = 8192;  // minimum spacing between context checkpoints
     int32_t cache_ram_mib       = 8192;  // -1 = no limit, 0 - disable, 1 = 1 MiB, etc.
-    int32_t preempt_ram_mib     = 8192;  // host RAM for parked (preempted) sequences: -1 = no limit, 0 = disable preemption
+    int32_t preempt_ram_mib     = 8192;  // host RAM for parked (preempted) sequences: -1 = no limit, 0 = always recompute
+    bool    preempt             = true;  // with a unified KV cache, park a slot instead of failing every slot
 
     std::string hostname      = "127.0.0.1";
     std::string public_path   = "";                                                                         // NOLINT

tools/server/README.md

@@ -164,7 +164,8 @@ For the full list of features, please refer to [server's changelog](https://gith
 | `-ctxcp, --ctx-checkpoints, --swa-checkpoints N` | max number of context checkpoints to create per slot (default: 32)[(more info)](https://github.com/ggml-org/llama.cpp/pull/15293)<br/>(env: LLAMA_ARG_CTX_CHECKPOINTS) |
 | `-cms, --checkpoint-min-step N` | minimum spacing between context checkpoints in tokens (default: 8192, 0 = no minimum)<br/>(env: LLAMA_ARG_CHECKPOINT_MIN_SPACING_NT) |
 | `-cram, --cache-ram N` | set the maximum cache size in MiB (default: 8192, -1 - no limit, 0 - disable)[(more info)](https://github.com/ggml-org/llama.cpp/pull/16391)<br/>(env: LLAMA_ARG_CACHE_RAM) |
-| `--preempt-ram N` | with a unified KV cache, park a slot in host RAM instead of failing every slot when the cache fills; N is the maximum host RAM for parked sequences in MiB (default: 8192, -1 - no limit, 0 - disable)<br/>(env: LLAMA_ARG_PREEMPT_RAM) |
+| `--preempt, --no-preempt` | with a unified KV cache and more than one slot, park a running request and put it back later instead of failing every request when the cache fills (default: enabled)<br/>(env: LLAMA_ARG_PREEMPT) |
+| `--preempt-ram N` | maximum host RAM in MiB for parked (preempted) sequences; a slot that fits is parked by copying its sequence to host RAM, one that does not is parked by dropping its cells and recomputing them later (default: 8192, -1 - no limit, 0 - never copy; use `--no-preempt` to turn preemption off). Copying back is exact; recomputing costs no host RAM but can change the continuation of a paused answer. Unlike `--cache-ram`, `0` does not disable the feature<br/>(env: LLAMA_ARG_PREEMPT_RAM) |
 | `-kvu, --kv-unified, -no-kvu, --no-kv-unified` | use single unified KV buffer shared across all sequences (default: enabled if number of slots is auto)<br/>(env: LLAMA_ARG_KV_UNIFIED) |
 | `--cache-idle-slots, --no-cache-idle-slots` | save idle slots to the prompt cache on new task, and clear them when using unified KV (default: enabled, requires cache-ram)<br/>(env: LLAMA_ARG_CACHE_IDLE_SLOTS) |
 | `--context-shift, --no-context-shift` | whether to use context shift on infinite text generation (default: disabled)<br/>(env: LLAMA_ARG_CONTEXT_SHIFT) |
@@ -1139,8 +1140,11 @@ In *router mode* the query param `?model={model_id}` has to be set. This endpoin
 | `llamacpp:spec_decode_num_accepted_tokens_total` | Counter | Total draft tokens accepted by the target model (0 when spec-decode is off). |
 | `llamacpp:spec_decode_num_drafts_total` | Counter | Total speculative decoding verification steps (0 when spec-decode is off). |
 | `llamacpp:spec_decode_num_accepted_tokens_per_pos_total` | Counter | Accepted tokens per draft position (labeled `position="N"`; absent when spec-decode is off or before the first completed speculative request). |
-| `llamacpp:n_preempt_total` | Counter | Slots parked to make room in the unified KV cache (0 unless `--kv-unified` with more than one slot). |
+| `llamacpp:n_preempt_total` | Counter | Slots parked to make room in the unified KV cache (0 unless `--kv-unified` with more than one slot, and 0 with `--no-preempt`). |
+| `llamacpp:n_preempt_swap_total` | Counter | Slots parked by copying the sequence to host RAM. |
+| `llamacpp:n_preempt_recompute_total` | Counter | Slots parked by dropping the cells, to be recomputed later. |
 | `llamacpp:n_resume_total` | Counter | Parked slots put back. |
+| `llamacpp:n_recompute_tokens_total` | Counter | Tokens re-processed through the model to put parked slots back (prompt and generated). |
 | `llamacpp:requests_preempted` | Gauge | Requests currently parked, waiting for room in the unified KV cache. |
 | `llamacpp:preempt_ram_bytes` | Gauge | Host RAM held by parked sequences. |
 

tools/server/server-common.h

@@ -467,9 +467,14 @@ struct server_metrics {
     uint64_t n_decode     = 0;
     uint64_t n_busy_slots = 0;
 
-    // [TAG_PREEMPT] slots parked to make room in the unified KV pool, and put back
-    uint64_t n_preempt = 0;
-    uint64_t n_resume  = 0;
+    // [TAG_PREEMPT] slots parked to make room in the unified KV pool, split by how they were
+    // parked, put back, and the tokens re-run through the model to put a recomputed one back
+    // (n_preempt == n_preempt_swap + n_preempt_recompute)
+    uint64_t n_preempt           = 0;
+    uint64_t n_preempt_swap      = 0;
+    uint64_t n_preempt_recompute = 0;
+    uint64_t n_resume            = 0;
+    uint64_t n_recompute_tokens  = 0;
 
     uint64_t n_draft_tokens      = 0; // Total draft tokens generated
     uint64_t n_draft_accepted    = 0; // Draft tokens actually accepted