From 7efd67cc22e6d5912d41ae5b735262ee6f033069 Mon Sep 17 00:00:00 2001 From: Jay Beavers Date: Sun, 30 Aug 2026 19:41:17 -0700 Subject: [PATCH] server : keep speculative checkpoints on device qwen4exp is a recurrent hybrid, so the target context cannot partially seq_rm and the server classifies it SEQ_RM_TYPE_FULL. Every speculative round then takes a full recurrent-state checkpoint, and every rejected draft restores one. Through the host path that means serializing each GDN layer conv+state row, the 4-stream hyper-connection residual and the PLE history into a host vector with one synchronous backend read per tensor, then pushing it all back. The cost is flat in context and swamps everything else: with the MTP drafter attached, decode ran 201 ms/token against 29 ms/token on a tree that keeps the state on device, and a 120-token generation spent roughly 600 ms of each 825 ms round in checkpoint traffic. It also inflates the reported prompt eval time, which runs to first token and so absorbs the first save: 827 ms for an 11-token prompt. Request ON_DEVICE at the six speculative checkpoint sites so the state stays in device buffers. The library already implements this path; only the server never asked for it. Measured on gfx1151 Vulkan, Qwen3.8-Flash-Next UD-Q4_K_XL with a Q4_K_M MTP head: 4.77 -> 25.83 tok/s at short context, 4.33 -> 16.08 at 70k, draft acceptance unchanged at 70-80 percent. The prompt-cache checkpoints are deliberately left host-resident: they retain several historical states rather than one live round. Diagnosed by Claude Fable 5. The ON_DEVICE flag and the mechanism come from Gaetan Puleo (c8b681b6f), carried in Nathanw1014/llama.cpp as 08a3255. (cherry picked from commit 175b66c51f08583caba7eb8ef68c6b6ed83f73ce) --- tools/server/server-context.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index f5477356d61d..e7ec9aa565f2 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -2993,7 +2993,7 @@ struct server_context_impl { llama_memory_seq_pos_max(llama_get_memory(ctx_tgt), slot.id)); if (use_ckpt_dft) { - slot.spec_ckpt.update_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); + slot.spec_ckpt.update_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); } slot.spec_prompt = slot.prompt.tokens.get_text_tokens(); @@ -3032,7 +3032,7 @@ struct server_context_impl { if (ctx_dft) { if (use_ckpt_dft) { - ckpt.load_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); + ckpt.load_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); } if (!llama_memory_seq_rm(llama_get_memory(ctx_dft), slot.id, ckpt.pos_max + 1, -1)) { @@ -3051,7 +3051,7 @@ struct server_context_impl { if (use_ckpt_tgt) { //const int64_t t_start = ggml_time_us(); - ckpt.update_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); + ckpt.update_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); //const int64_t t_total = ggml_time_us() - t_start; //printf("checkpoint total: %f ms\n", t_total / 1000.0); @@ -3063,7 +3063,7 @@ struct server_context_impl { } if (use_ckpt_dft) { - ckpt.update_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); + ckpt.update_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); } } }); @@ -3915,10 +3915,10 @@ struct server_context_impl { SLT_DBG(slot, "restoring speculative checkpoint (pos_min = %d, pos_max = %d, size = %zu)\n", ckpt.pos_min, ckpt.pos_max, ckpt.size()); - ckpt.load_tgt(slot.ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); + ckpt.load_tgt(slot.ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); if (slot.ctx_dft) { - ckpt.load_dft(slot.ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); + ckpt.load_dft(slot.ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); } slot.mem.seq_rm(slot.id, ckpt.pos_max + 1, -1);