From 52dac8d026703812b5f4265602e496cee0837cc4 Mon Sep 17 00:00:00 2001 From: cnndabbler Date: Wed, 15 Jul 2026 12:37:16 -0700 Subject: [PATCH] server: do not propagate kv-mean-center bias to draft model contexts The bias GGUF is calibrated for the target model's K geometry. When a draft model is loaded (-md), copying common_params wholesale made both the VRAM-measurement probe and the real draft context inherit kv_mean_center_path, failing with 'bias tensor ... has 1024 elements, expected 512' and disabling speculative decoding. Clear the path on the draft params in both sites so bias + drafter can coexist. --- tools/server/server-context.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 45f668d7c1d9..e8e2e4fa8348 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -901,6 +901,10 @@ struct server_context_impl { params_dft.cache_type_k = params_spec.cache_type_k; params_dft.cache_type_v = params_spec.cache_type_v; params_dft.tensor_buft_overrides = params_spec.tensor_buft_overrides; + // the K-cache mean-center bias is calibrated for the target model + // (per-head/channel K layout); the draft model has a different K + // geometry, so it must not inherit the bias + params_dft.kv_mean_center_path.clear(); } else { // MTP draft context lives on the target model, only context+compute are new measure_model_bytes = false; @@ -991,6 +995,10 @@ struct server_context_impl { params_dft.n_gpu_layers = params_spec.n_gpu_layers; params_dft.cache_type_k = params_spec.cache_type_k; params_dft.cache_type_v = params_spec.cache_type_v; + // the K-cache mean-center bias is calibrated for the target model + // (per-head/channel K layout); the draft model has a different K + // geometry, so it must not inherit the bias + params_dft.kv_mean_center_path.clear(); if (params_spec.cpuparams.n_threads > 0) { params_dft.cpuparams.n_threads = params_spec.cpuparams.n_threads;