From 1fbce6287a052d575ce18199a9c77610cc9faf09 Mon Sep 17 00:00:00 2001 From: "Sabnis, Omkar" Date: Fri, 2 Oct 2026 15:18:06 -0700 Subject: [PATCH 01/20] ggml-openvino: fix SWA mask detection When the SWA cache size matches the full cache size, identify SWA layers by the number of layers using each mask instead of cache size; the larger group is SWA, and a tie leaves the layers unclassified. --- ggml/src/ggml-openvino/ggml-decoder.cpp | 26 +++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index 10e82f3e602a..f7f9d271fe14 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -803,10 +803,9 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr // from the first graph onwards, while the view grows with context depth and would invert the // comparison at shallow depth. // - // Layers whose leaf is smaller than the largest leaf are the windowed ones. When every layer - // reports the same extent there is no distinction to draw -- either the model has no windowed - // layers, or the window is at least as large as the context so the two caches coincide, in - // which case a windowed layer and a full-attention one compute the same thing. + // Layers whose leaf is smaller than the largest leaf are the windowed ones. Equal extents do not + // rule them out: the windowed cache is padded to n_swa + n_ubatch, which can equal the full + // context (gemma-4 at -c 1024 -ub 512). See the mask fallback below. // // Getting this wrong is silent and severe: with the windowed layers classified as // full-attention, permute's KV slicing uses attention_size instead of attention_size_swa. The @@ -851,6 +850,25 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr } } } + + // equal extents: tell the two masks apart by the layers that read them, the larger group is swa (gemma-4: 4 of 5 layers) + // groups of equal size can not be told apart, so leave the layers unclassified + if (model_params.swa_layers.empty()) { + std::map> mask_layers; + for (const auto & [layer, mask] : layer_mask) { + mask_layers[mask].push_back(layer); + } + if (mask_layers.size() == 2 && + mask_layers.begin()->second.size() != std::next(mask_layers.begin())->second.size()) { + auto swa = mask_layers.begin(); + auto other = std::next(swa); + if (other->second.size() > swa->second.size()) { + std::swap(swa, other); + } + model_params.swa_layers = swa->second; + model_params.swa_mask = swa->first; + } + } std::sort(model_params.swa_layers.begin(), model_params.swa_layers.end()); if (ggml_openvino_getenv_int("GGML_OPENVINO_LOG_SWA_LAYERS")) { From 671e4978d0b5512d87b11e8466ffbb1896cda14c Mon Sep 17 00:00:00 2001 From: "Sabnis, Omkar" Date: Fri, 2 Oct 2026 15:18:24 -0700 Subject: [PATCH 02/20] ggml-openvino: fix SWA KV cache write indices Give SWA KV cache index tensors unique input names (a _swa suffix, like the masks) to prevent them from being merged with the full-attention cache indices. --- ggml/src/ggml-openvino/ggml-decoder.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index f7f9d271fe14..12198982f4a3 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -229,6 +229,15 @@ static std::string get_tensor_graph_input_ov_name(const GgmlOvDecoder * decoder, return get_tensor_ov_name(cgraph, tensor) + "_swa"; } } + if (!decoder->get_model_params().swa_layers.empty() && GgmlOvDecoder::is_kv_idx(tensor, op) && + op->src[2] != nullptr) { + // same as the masks: the full and the sliding-window cache get KV write indices of the same name + const ggml_tensor * cache = op->src[2]->view_src != nullptr ? op->src[2]->view_src : op->src[2]; + auto layer = extract_layer_from_name(cache->name); + if (layer.has_value() && decoder->is_swa_layer(layer.value())) { + return get_tensor_ov_name(cgraph, tensor) + "_swa"; + } + } return get_tensor_ov_name(cgraph, tensor); } From d1018114431bd6cbbd21175e5d20043bdd952393 Mon Sep 17 00:00:00 2001 From: "Sabnis, Omkar" Date: Sun, 4 Oct 2026 16:03:33 -0700 Subject: [PATCH 03/20] ggml-openvino: classify SWA layers only when the mask shows the window Two attention masks do not imply SWA: DeepSeek sparse attention uses two full-attention masks with the same name. When cache extents match, classify the larger mask group as SWA only once its mask has fewer keys per row than the other mask. Keep the classification per KV cache buffer so new sequences reuse the same graph, and include swa_layers when checking graph reuse. --- ggml/src/ggml-openvino/ggml-decoder.cpp | 26 +++++++++++++++++++++++-- ggml/src/ggml-openvino/ggml-decoder.h | 7 ++++--- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index 12198982f4a3..fe889f501867 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -823,6 +823,7 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr { std::map layer_extent; // layer -> leaf cache_k cell count std::map layer_mask; // layer -> mask it consumes + std::map layer_buffer; // layer -> id of the buffer holding its cache_k int64_t max_extent = 0; for (int i = 0; i < cgraph->n_nodes; i++) { @@ -848,6 +849,8 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr } layer_extent[layer.value()] = leaf->ne[1]; layer_mask[layer.value()] = mask; + layer_buffer[layer.value()] = + leaf->buffer != nullptr ? ggml_backend_openvino_buffer_get_ctx_id(leaf->buffer) : 0; max_extent = std::max(max_extent, leaf->ne[1]); } @@ -862,6 +865,9 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr // equal extents: tell the two masks apart by the layers that read them, the larger group is swa (gemma-4: 4 of 5 layers) // groups of equal size can not be told apart, so leave the layers unclassified + // two masks do not imply swa (deepseek sparse attention builds two full masks), so only classify once the window + // shows: the swa mask then keeps fewer keys per row than the other. Before that the two masks are equal. + // Remember it per KV cache buffer, so that a new sequence starting below the window keeps the same graph if (model_params.swa_layers.empty()) { std::map> mask_layers; for (const auto & [layer, mask] : layer_mask) { @@ -874,8 +880,24 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr if (other->second.size() > swa->second.size()) { std::swap(swa, other); } - model_params.swa_layers = swa->second; - model_params.swa_mask = swa->first; + static std::mutex seen_mutex; + static std::map> seen_swa_layers; // cache_k buffer id -> swa layers + const size_t buffer_id = layer_buffer[swa->second.front()]; + const int swa_kept = get_swa_window_from_mask(swa->first); + const int other_kept = get_swa_window_from_mask(other->first); + bool is_swa = swa_kept > 0 && swa_kept < other_kept; + { + std::lock_guard lock(seen_mutex); + if (is_swa && buffer_id != 0) { + seen_swa_layers[buffer_id] = swa->second; + } + auto seen = seen_swa_layers.find(buffer_id); + is_swa = is_swa || (buffer_id != 0 && seen != seen_swa_layers.end() && seen->second == swa->second); + } + if (is_swa) { + model_params.swa_layers = swa->second; + model_params.swa_mask = swa->first; + } } } std::sort(model_params.swa_layers.begin(), model_params.swa_layers.end()); diff --git a/ggml/src/ggml-openvino/ggml-decoder.h b/ggml/src/ggml-openvino/ggml-decoder.h index a614d291acb5..f26a378aac2e 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.h +++ b/ggml/src/ggml-openvino/ggml-decoder.h @@ -47,14 +47,15 @@ struct ModelParams { memcmp(rope_params, other.rope_params, sizeof(int32_t) * 16) == 0; } + // swa_layers: the classification can change once the sliding window shows in the mask bool can_reuse_dynamically(const ModelParams & other) const { - return same_rope_params(other) && n_rs_slots == other.n_rs_slots && - has_rs_rollback == other.has_rs_rollback; + return same_rope_params(other) && n_rs_slots == other.n_rs_slots && has_rs_rollback == other.has_rs_rollback && + swa_layers == other.swa_layers; } bool can_reuse_statically(const ModelParams & other) const { return same_rope_params(other) && ctx == other.ctx && n_rs_slots == other.n_rs_slots && - has_rs_rollback == other.has_rs_rollback; + has_rs_rollback == other.has_rs_rollback && swa_layers == other.swa_layers; } bool kv_buffer_changed(const ModelParams & other) const { return kv_buffer_ctx_id != other.kv_buffer_ctx_id; } From 38548d91c34fb5bb3262c3938fe86d1025227521 Mon Sep 17 00:00:00 2001 From: "Sabnis, Omkar" Date: Mon, 5 Oct 2026 07:20:27 -0700 Subject: [PATCH 04/20] ggml-openvino: classify SWA layers from the first graph Waiting for the window to appear in the mask caused a mid-sequence graph recompile (~9 s for Gemma-4 at -c 1024 after position 512). SWA classification already gives each mask group its own mask, KV write indices, and attention size, which is correct whether or not the mask is a sliding window. Stateful mask rebuild recovers the window from the mask, or the causal mask for full attention. Classify the larger of two mask groups from the first graph and drop the per-KV-buffer bookkeeping. --- ggml/src/ggml-openvino/ggml-decoder.cpp | 40 +++++++------------------ ggml/src/ggml-openvino/ggml-decoder.h | 2 +- 2 files changed, 12 insertions(+), 30 deletions(-) diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index fe889f501867..1816bd06b15b 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -823,7 +823,6 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr { std::map layer_extent; // layer -> leaf cache_k cell count std::map layer_mask; // layer -> mask it consumes - std::map layer_buffer; // layer -> id of the buffer holding its cache_k int64_t max_extent = 0; for (int i = 0; i < cgraph->n_nodes; i++) { @@ -849,8 +848,6 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr } layer_extent[layer.value()] = leaf->ne[1]; layer_mask[layer.value()] = mask; - layer_buffer[layer.value()] = - leaf->buffer != nullptr ? ggml_backend_openvino_buffer_get_ctx_id(leaf->buffer) : 0; max_extent = std::max(max_extent, leaf->ne[1]); } @@ -863,11 +860,13 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr } } - // equal extents: tell the two masks apart by the layers that read them, the larger group is swa (gemma-4: 4 of 5 layers) - // groups of equal size can not be told apart, so leave the layers unclassified - // two masks do not imply swa (deepseek sparse attention builds two full masks), so only classify once the window - // shows: the swa mask then keeps fewer keys per row than the other. Before that the two masks are equal. - // Remember it per KV cache buffer, so that a new sequence starting below the window keeps the same graph + // equal extents: tell the two masks apart by the layers that read them, the larger group gets the swa role + // (gemma-4: 4 of 5 layers). Groups of equal size can not be told apart, so leave the layers unclassified. + // Two masks do not imply swa (deepseek sparse attention builds two full masks), but the role does not need + // them to: it gives each group its own mask, KV write indices and attention size, which is right either way. + // The only use of the window itself, the stateful mask rebuild, reads it back from the mask, and on a full + // mask that gives back the causal mask. Decide from the first graph, before the window shows in the mask, so + // the graph does not change, and is not recompiled, when it does. if (model_params.swa_layers.empty()) { std::map> mask_layers; for (const auto & [layer, mask] : layer_mask) { @@ -876,28 +875,11 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr if (mask_layers.size() == 2 && mask_layers.begin()->second.size() != std::next(mask_layers.begin())->second.size()) { auto swa = mask_layers.begin(); - auto other = std::next(swa); - if (other->second.size() > swa->second.size()) { - std::swap(swa, other); - } - static std::mutex seen_mutex; - static std::map> seen_swa_layers; // cache_k buffer id -> swa layers - const size_t buffer_id = layer_buffer[swa->second.front()]; - const int swa_kept = get_swa_window_from_mask(swa->first); - const int other_kept = get_swa_window_from_mask(other->first); - bool is_swa = swa_kept > 0 && swa_kept < other_kept; - { - std::lock_guard lock(seen_mutex); - if (is_swa && buffer_id != 0) { - seen_swa_layers[buffer_id] = swa->second; - } - auto seen = seen_swa_layers.find(buffer_id); - is_swa = is_swa || (buffer_id != 0 && seen != seen_swa_layers.end() && seen->second == swa->second); - } - if (is_swa) { - model_params.swa_layers = swa->second; - model_params.swa_mask = swa->first; + if (std::next(swa)->second.size() > swa->second.size()) { + swa = std::next(swa); } + model_params.swa_layers = swa->second; + model_params.swa_mask = swa->first; } } std::sort(model_params.swa_layers.begin(), model_params.swa_layers.end()); diff --git a/ggml/src/ggml-openvino/ggml-decoder.h b/ggml/src/ggml-openvino/ggml-decoder.h index f26a378aac2e..938c054ed9df 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.h +++ b/ggml/src/ggml-openvino/ggml-decoder.h @@ -47,7 +47,7 @@ struct ModelParams { memcmp(rope_params, other.rope_params, sizeof(int32_t) * 16) == 0; } - // swa_layers: the classification can change once the sliding window shows in the mask + // swa_layers: the masks and KV write indices of these layers are separate graph inputs bool can_reuse_dynamically(const ModelParams & other) const { return same_rope_params(other) && n_rs_slots == other.n_rs_slots && has_rs_rollback == other.has_rs_rollback && swa_layers == other.swa_layers; From 13d95e6d5b99d1cf7b5cbdabbc4043d4e456918c Mon Sep 17 00:00:00 2001 From: "Sabnis, Omkar" Date: Tue, 6 Oct 2026 14:48:30 -0700 Subject: [PATCH 05/20] ggml-openvino: mark SWA fallback as a model-level heuristic Add a TODO above the equal-extent fallback to document that the larger mask group is currently assumed to be the SWA group. Since stateless graphs only require the two groups to remain distinct, this heuristic may only be necessary for stateful execution and could potentially be restricted to that path. --- ggml/src/ggml-openvino/ggml-decoder.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index 1816bd06b15b..1c37e3d8d44f 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -867,6 +867,9 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr // The only use of the window itself, the stateful mask rebuild, reads it back from the mask, and on a full // mask that gives back the causal mask. Decide from the first graph, before the window shows in the mask, so // the graph does not change, and is not recompiled, when it does. + // TODO: "larger group is swa" is a model-level guess. Revisit when llama.cpp tells the backend which layers + // use a sliding window. Stateless graphs only need the two groups kept apart (separate mask, KV write index + // and attention size inputs), not the swa role itself, so the guess could be limited to stateful execution. if (model_params.swa_layers.empty()) { std::map> mask_layers; for (const auto & [layer, mask] : layer_mask) { From f3f671f5068d0c56676b3c83aa3d34d9c325138c Mon Sep 17 00:00:00 2001 From: "Sabnis, Omkar" Date: Mon, 5 Oct 2026 11:34:08 -0700 Subject: [PATCH 06/20] ggml-openvino: derive attention sizes from the KQ mask shape Dynamic CPU/GPU graphs previously passed n_seq_active and attention_size(_swa) as separate inputs. Derive these values directly from the KQ mask shape using ShapeOf + Gather inside the model. This allows the GPU plugin to infer the Q reshape and KV slice shapes from shape inference rather than runtime input values. Q's head size also becomes static, enabling the fused SDPA kernel. GGML_OPENVINO_DISABLE_SHAPE_FROM_MASK=1 restores the previous separate-input behavior. The fused SDPA path also requires two supporting changes: - Clamp the GPU mask to -30000. A key block that is entirely -inf for a row produces NaNs in the fused kernel. - Restate the head count and head size after the KV slice. The plugin loses all slice dimensions when the end index is dynamic, causing SDPA to fall back to the reference kernel. Additional graph and quantization improvements: - Skip the identity reshape of rows in SET_ROWS. - Share a single Split for one-element view slices in dynamic GPU graphs (e.g. Gemma-4 per-layer embeddings), replacing one StridedSlice per layer. - Add GGML_OPENVINO_REQUANT_EMBD=native (off by default) to preserve the file's native quantization for token_embd/output instead of requantizing to Q8_0_C. - Include both environment settings in the graph and model cache keys and document them. --- docs/backend/OPENVINO.md | 2 ++ ggml/src/ggml-openvino/README.md | 2 +- ggml/src/ggml-openvino/ggml-decoder.cpp | 26 +++++++++++++++++ ggml/src/ggml-openvino/ggml-decoder.h | 3 ++ .../src/ggml-openvino/ggml-openvino-extra.cpp | 13 +++++++++ ggml/src/ggml-openvino/ggml-openvino-extra.h | 3 ++ ggml/src/ggml-openvino/model-cache.cpp | 2 ++ ggml/src/ggml-openvino/openvino/decoder.h | 3 ++ .../openvino/op/flash_attn_ext.cpp | 9 ++++++ .../src/ggml-openvino/openvino/op/permute.cpp | 28 +++++++++++++++++-- .../ggml-openvino/openvino/op/set_rows.cpp | 17 +++++++---- .../openvino/translate_session.cpp | 16 +++++++++-- ggml/src/ggml-openvino/openvino/utils.cpp | 27 +++++++++++++++++- ggml/src/ggml-openvino/utils.cpp | 5 ++++ 14 files changed, 144 insertions(+), 12 deletions(-) diff --git a/docs/backend/OPENVINO.md b/docs/backend/OPENVINO.md index decc588a9f4c..eb629167fd62 100644 --- a/docs/backend/OPENVINO.md +++ b/docs/backend/OPENVINO.md @@ -723,12 +723,14 @@ Boolean flags follow a uniform convention: set to a **positive integer** (e.g. ` | `GGML_OPENVINO_DISABLE_CACHE` | Boolean | `0` | Disable the in-process compiled-model / decoder cache (cache is on by default). Set to `1` to disable. | | `GGML_OPENVINO_DISABLE_KV_SLICE` | Boolean | `0` | Disable the KV-cache input-tensor slicing optimization (slicing is on by default on CPU/GPU). Set to `1` to disable. | | `GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT` | Boolean | `0` | Disable the stateful KV-state sequence-axis relayout (relayout is on by default). It moves the KV state sequence axis from dim 1 to dim 2, so the GPU plugin can append new tokens in place instead of copying the whole state every token, and the reader side no longer transposes the whole accumulated state. Set to `1` to disable. | +| `GGML_OPENVINO_DISABLE_SHAPE_FROM_MASK` | Boolean | `0` | Dynamic CPU/GPU graphs read the active sequence count and the attended KV length from the shape of the KQ mask inside the model, so the GPU plugin can infer the attention shapes without waiting for input values and selects its fused SDPA kernel. Set to `1` to pass them as separate inputs instead. | | `GGML_OPENVINO_MANUAL_GQA_ATTN` | Boolean | device-based | Tri-state. When **unset**, manual GQA attention is enabled by default on `GPU` and disabled on other devices. Set to a positive integer to force-enable, or `0` to force-disable. | | `GGML_OPENVINO_MEMORY_OPTIMIZE` | Boolean | `0` | Umbrella switch for compile-time memory reductions. Enables `GGML_OPENVINO_REDUCE_COMPILE_MEM` and, on GPU, `GGML_OPENVINO_RELEASE_WEIGHTS` unless those fine-grained variables are explicitly set. | | `GGML_OPENVINO_REDUCE_COMPILE_MEM`| Boolean | inherits from `GGML_OPENVINO_MEMORY_OPTIMIZE` | Reduce compile-time host memory use by streaming weight requantization and avoiding extra weight-node materialization where possible. Set explicitly to override the umbrella switch. | | `GGML_OPENVINO_RELEASE_WEIGHTS` | Boolean | inherits from `GGML_OPENVINO_MEMORY_OPTIMIZE` on GPU | GPU-only. Release host weight buffers after the compiled model cache can reuse the device/plugin copy. Requires stable graph shapes; dynamic workloads that need recompilation should leave this disabled. | | `GGML_OPENVINO_SPILL_DIR` | String | `not set` | Directory for a disk-backed weight buffer. When set, the repacked weight buffer is mapped from an unlinked file on this path instead of anonymous memory, so its pages are reclaimable under memory pressure instead of staying pinned, cutting the load-time host memory peak. Must point at real storage; a tmpfs mount (e.g. `/tmp` on many systems) backs it with RAM and makes the peak worse. | | `GGML_OPENVINO_REQUANT_KQUANT` | String | `not set` | Requantize Q6_K/Q5_K weights (and matching MoE expert weights) to a 4-bit target instead of the default Q8_0_C, trading accuracy for less memory traffic. One of `q4_asym64` (Q6_K/Q5_K only, keeps a real zero point at group 64), `q4_asym64_all` (also requantizes Q4_K), `q4_sym128` (Q6_K/Q5_K only), `q4_sym128_all` (Q4_K too, drops its per-group zero point), or `native` (no requantization). | +| `GGML_OPENVINO_REQUANT_EMBD` | String | `not set` | `native` keeps `token_embd` and `output` in their file quantization instead of requantizing them to Q8_0_C (CPU/GPU; for tied embeddings this is also the LM head). Less memory traffic per token at a small accuracy cost. | | `GGML_OPENVINO_PROFILING` | Integer | `0` | `1` logs execution timing; `2` or higher also enables OpenVINO and OpenCL profiling. | | `GGML_OPENVINO_DEBUG_NODE` | String | `not set` | Add the named graph nodes as compiled outputs for debugging. Separate multiple names with commas. | | `GGML_OPENVINO_MOE_OP` | Boolean | `1` | On GPU, set to `0` to keep the unfused GatherMatmul path. | diff --git a/ggml/src/ggml-openvino/README.md b/ggml/src/ggml-openvino/README.md index 6a23f62bf246..a3c6af9ba8de 100644 --- a/ggml/src/ggml-openvino/README.md +++ b/ggml/src/ggml-openvino/README.md @@ -29,7 +29,7 @@ GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY=1 \ Cache-only mode allocates backend address space without filling weight pages. On Windows, this also uses system commit capacity. The model-buffer size in the loader log is this virtual size. Weight uploads only record source identity; they do not read or requantize the weights. Graph conversion and compilation are skipped. Runtime buffers are still allocated and populated normally. -Cache-only mode uses the settings provided by the current process. Keep these values exactly the same, including set versus unset: `GGML_OPENVINO_REQUANT_KQUANT`, `GGML_OPENVINO_NATIVE_SOFTPLUS`, `GGML_OPENVINO_DISABLE_KV_SLICE`, `GGML_OPENVINO_MANUAL_GQA_ATTN`, `GGML_OPENVINO_STATEFUL_EXECUTION`, `GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT`, `GGML_OPENVINO_DISABLE_REMOTE_OUTPUTS`, `GGML_OPENVINO_REDUCE_COMPILE_MEM`, `GGML_OPENVINO_MEMORY_OPTIMIZE`, `GGML_OPENVINO_PROFILING`, and `GGML_OPENVINO_DEBUG_NODE`. On GPU, also repeat `GGML_OPENVINO_MOE_OP=0` if used. `GGML_OPENVINO_SPILL_DIR` is optional on the first run and ignored in cache-only mode; host-weight release is disabled in cache-only mode. +Cache-only mode uses the settings provided by the current process. Keep these values exactly the same, including set versus unset: `GGML_OPENVINO_REQUANT_KQUANT`, `GGML_OPENVINO_REQUANT_EMBD`, `GGML_OPENVINO_NATIVE_SOFTPLUS`, `GGML_OPENVINO_DISABLE_KV_SLICE`, `GGML_OPENVINO_MANUAL_GQA_ATTN`, `GGML_OPENVINO_STATEFUL_EXECUTION`, `GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT`, `GGML_OPENVINO_DISABLE_REMOTE_OUTPUTS`, `GGML_OPENVINO_REDUCE_COMPILE_MEM`, `GGML_OPENVINO_MEMORY_OPTIMIZE`, `GGML_OPENVINO_PROFILING`, `GGML_OPENVINO_DISABLE_SHAPE_FROM_MASK`, and `GGML_OPENVINO_DEBUG_NODE`. On GPU, also repeat `GGML_OPENVINO_MOE_OP=0` if used. `GGML_OPENVINO_SPILL_DIR` is optional on the first run and ignored in cache-only mode; host-weight release is disabled in cache-only mode. A missing or incompatible graph fails with an error instead of compiling with absent weights. The fingerprint uses the dynamic graph's topology, ports, model parameters, weights, settings, and OpenVINO version; changing only dynamic token or KV sizes does not require a new entry. A different workload can still require another graph; populate it first without cache-only mode. diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index 1c37e3d8d44f..09894f0d9f69 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -974,11 +974,17 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr memcpy(&offset, cache_k_view->op_params, sizeof(size_t)); compute_params.seq_active_start = offset / seq_size; + // a mask that is itself a graph input has the shape of its OV parameter + const std::string mask_name = mask->op != GGML_OP_NONE ? std::string() : + mask == model_params.swa_mask ? get_tensor_ov_name(cgraph, mask) + "_swa" : + get_tensor_ov_name(cgraph, mask); if (layer_is_swa) { compute_params.attention_size_swa = mask->ne[0]; compute_params.swa_window = get_swa_window_from_mask(mask); + compute_params.mask_swa_name = mask_name; } else { compute_params.attention_size = mask->ne[0]; + compute_params.mask_name = mask_name; } if (is_static) { compute_params.attention_size = model_params.ctx_per_seq; @@ -992,6 +998,7 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr if (node->src[1]->op == GGML_OP_PERMUTE && node->src[1]->src[0]->op == GGML_OP_VIEW && node->src[1]->src[0]->src[0]->op == GGML_OP_ROPE) { compute_params.attention_size = node->ne[0]; + compute_params.mask_name.clear(); } } @@ -999,6 +1006,7 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr if (node->op == GGML_OP_TRANSPOSE && node->src[0]->op == GGML_OP_PERMUTE && node->src[0]->src[0]->op == GGML_OP_VIEW) { compute_params.attention_size = node->ne[0]; + compute_params.mask_name.clear(); if (is_static) { compute_params.attention_size = model_params.ctx_per_seq; } @@ -1282,6 +1290,24 @@ void GgmlOvDecoder::add_extra_inputs() { } // create_1d_input("token_len", m_compute_params.token_len_per_seq * m_compute_params.n_seq_active); + // n_seq_active and attention_size(_swa) are dimensions of the KQ masks. Read them from the mask shape inside the + // model: the GPU plugin then gets the Q Reshape and KV Slice shapes from shape inference instead of waiting for + // the input values on the device + if (!m_is_static && !m_is_stateful && ggml_openvino_shape_from_mask_enabled()) { + auto from_mask = [this](const std::string & name, const std::string & mask, int64_t axis) { + auto it = m_model_extra_inputs.find(name); + if (it != m_model_extra_inputs.end() && !mask.empty() && m_model_inputs.count(mask) != 0) { + it->second.shape_source = mask; + it->second.shape_axis = axis; + } + }; + const auto & full = m_compute_params.mask_name; + const auto & swa = m_compute_params.mask_swa_name; + from_mask("attention_size", full, 3); + from_mask("attention_size_swa", swa, 3); + from_mask("n_seq_active", full.empty() ? swa : full, 0); + } + if (m_compute_params.cache_rs_reset_idx != -1 && m_model_params.n_rs_slots != 1) { // Whether/which cache slot to reset varies per compute call (e.g. a new sequence starting // vs. continued decoding). can_reuse_statically() does not invalidate the cached static diff --git a/ggml/src/ggml-openvino/ggml-decoder.h b/ggml/src/ggml-openvino/ggml-decoder.h index 938c054ed9df..ee9788b19090 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.h +++ b/ggml/src/ggml-openvino/ggml-decoder.h @@ -67,6 +67,9 @@ struct ComputeParams { int attention_size = -1; int attention_size_swa = -1; int attention_size_static = -1; // encoder/cross-attn KV fill level (whisper) + // OV names of the KQ mask inputs that n_seq_active (mask->ne[3]) and attention_size(_swa) (mask->ne[0]) were read from + std::string mask_name; + std::string mask_swa_name; // Sliding window width, read back from the band of ggml's own SWA mask. ggml never passes // n_swa down to a backend, but fill_mask() bakes it into the mask contents, so the widest // unmasked row recovers it. Shorter than n_swa while the sequence is still short, which is diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp index 14a0b0382b0d..4c00a44ebb14 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp @@ -141,7 +141,9 @@ void ggml_openvino_device_config::init() { "GGML_OPENVINO_NATIVE_SOFTPLUS", "GGML_OPENVINO_DISABLE_REMOTE_OUTPUTS", "GGML_OPENVINO_REQUANT_KQUANT", + "GGML_OPENVINO_REQUANT_EMBD", "GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT", + "GGML_OPENVINO_DISABLE_SHAPE_FROM_MASK", // Build the precise (but O(n_nodes)) graph cache key. Needed by op tests. "GGML_OPENVINO_FULL_GRAPH_KEY", }; @@ -322,6 +324,10 @@ bool ggml_openvino_is_gpu() { return has_prefix(ggml_openvino_get_device_name(), "GPU"); } +bool ggml_openvino_shape_from_mask_enabled() { + return ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_SHAPE_FROM_MASK") == 0; +} + size_t ggml_openvino_max_alloc_size() { return ggml_openvino_get_device_config().max_alloc_size; } @@ -356,6 +362,13 @@ std::optional ggml_openvino_get_requant_type(const ggml_tensor * if (no_requant) { return std::nullopt; } + // GGML_OPENVINO_REQUANT_EMBD=native keeps token_embd/output in the file's quantization (tied models: also the lm_head) + const bool is_embd_or_output = + strncmp(tensor->name, "token_embd.weight", 17) == 0 || strncmp(tensor->name, "output.weight", 13) == 0; + const char * rq_embd = ggml_openvino_getenv_str("GGML_OPENVINO_REQUANT_EMBD"); + if (is_embd_or_output && rq_embd && strcmp(rq_embd, "native") == 0 && !ggml_openvino_is_npu()) { + return std::nullopt; + } if (strncmp(tensor->name, "token_embd.weight", 17) == 0) { return ((ggml_openvino_is_npu() && tensor->type == GGML_TYPE_Q6_K) ? ExtraQuantType::F16 : ExtraQuantType::Q8_0_C); diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.h b/ggml/src/ggml-openvino/ggml-openvino-extra.h index 3a58b2d02803..9b42ac721068 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.h +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.h @@ -120,6 +120,9 @@ bool ggml_openvino_is_npu(); // Check if running on a GPU (GPU, GPU.0, GPU.1, ...) bool ggml_openvino_is_gpu(); +// Read n_seq_active and attention_size(_swa) from the KQ mask shape (off with GGML_OPENVINO_DISABLE_SHAPE_FROM_MASK=1) +bool ggml_openvino_shape_from_mask_enabled(); + // Largest single memory object the device can allocate, SIZE_MAX when there is no known limit size_t ggml_openvino_max_alloc_size(); diff --git a/ggml/src/ggml-openvino/model-cache.cpp b/ggml/src/ggml-openvino/model-cache.cpp index c8c5fb5adb8e..91af222c6f54 100644 --- a/ggml/src/ggml-openvino/model-cache.cpp +++ b/ggml/src/ggml-openvino/model-cache.cpp @@ -166,6 +166,7 @@ bool ggml_openvino_model_cache_only() { static const char * cache_settings[] = { "GGML_OPENVINO_REQUANT_KQUANT", + "GGML_OPENVINO_REQUANT_EMBD", "GGML_OPENVINO_NATIVE_SOFTPLUS", "GGML_OPENVINO_DISABLE_KV_SLICE", "GGML_OPENVINO_MANUAL_GQA_ATTN", @@ -175,6 +176,7 @@ static const char * cache_settings[] = { "GGML_OPENVINO_REDUCE_COMPILE_MEM", "GGML_OPENVINO_MEMORY_OPTIMIZE", "GGML_OPENVINO_PROFILING", + "GGML_OPENVINO_DISABLE_SHAPE_FROM_MASK", }; void ggml_openvino_model_cache_init() { diff --git a/ggml/src/ggml-openvino/openvino/decoder.h b/ggml/src/ggml-openvino/openvino/decoder.h index ec6975282a54..1294146b60eb 100644 --- a/ggml/src/ggml-openvino/openvino/decoder.h +++ b/ggml/src/ggml-openvino/openvino/decoder.h @@ -23,6 +23,9 @@ struct ModelExtraInputInfo { Shape shape; int64_t value; bool is_parameter; + // when set, the value is read from dimension shape_axis of this model input instead of a Parameter + std::string shape_source; + int64_t shape_axis = -1; }; class GgmlDecoder : public DecoderBase { diff --git a/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp b/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp index 44c6dbc03ddb..60856d20bc72 100644 --- a/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp +++ b/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -222,6 +223,14 @@ OutputVector translate_flash_attn_ext(const NodeContext & context) { k = tile_kv(num_heads, num_heads_kv, head_size, k); v = tile_kv(num_heads, num_heads_kv, head_size, v); + // With n_seq_active read from the mask shape, the GPU plugin knows Q's head size and uses its fused SDPA kernel. + // That kernel works on blocks of keys: a block fully masked with -inf for a row gives exp(-inf - -inf) = NaN, + // which spreads to the row's result, so mask with a large finite value instead + if (has_mask && ggml_openvino_is_gpu() && !context.is_stateful() && ggml_openvino_shape_from_mask_enabled()) { + mask = std::make_shared( + mask, ov::op::v0::Constant::create(ov::element::f16, ov::Shape{}, std::vector{-30000.0f})); + } + constexpr auto causal = false; if (has_mask) { auto sdpa = std::make_shared(q, k, v, mask, scale_node, causal); diff --git a/ggml/src/ggml-openvino/openvino/op/permute.cpp b/ggml/src/ggml-openvino/openvino/op/permute.cpp index df4f038984c5..a9834d99f54a 100644 --- a/ggml/src/ggml-openvino/openvino/op/permute.cpp +++ b/ggml/src/ggml-openvino/openvino/op/permute.cpp @@ -121,7 +121,26 @@ OutputVector translate_permute(const NodeContext & context) { auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - if (op_case == 3 || op_case == 4) { + if ((op_case == 3 || op_case == 4) && n_seq == 1 && perm_values == std::vector{0, 2, 1, 3}) { + // one sequence: slice the cache [1, 1, ctx, n_heads * head_size] on its context axis, then split the + // heads. The reshape after the slice also restates the head count and head size: the GPU plugin loses + // all dimensions of a slice with a runtime end, and SDPA would then run its reference kernel + auto two = ov::op::v0::Constant::create(ov::element::i64, {1}, {2}); + auto sliced = std::make_shared(src, zero, attention_size, one, two); + if (n_heads == 1) { + res = std::make_shared( + sliced, + ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{1, 1, -1, head_size}), + false); + } else { + auto split = std::make_shared( + sliced, + ov::op::v0::Constant::create(ov::element::i64, {4}, + std::vector{1, -1, n_heads, head_size}), + false); + res = std::make_shared(split, perm); + } + } else if (op_case == 3 || op_case == 4) { auto src_reshaped = std::make_shared( src, ov::op::v0::Constant::create(ov::element::i64, {4}, {n_seq, ctx_per_seq, n_heads, head_size}), false); @@ -133,7 +152,12 @@ OutputVector translate_permute(const NodeContext & context) { std::make_shared(src_reshaped, seq_active_start, seq_active_end, one, zero); } auto slice2 = std::make_shared(after_seq_slice, zero, attention_size, one, one); - res = std::make_shared(slice2, perm); + // restate the head count and head size, see above + auto restated = std::make_shared( + slice2, + ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{0, -1, n_heads, head_size}), + true); + res = std::make_shared(restated, perm); } else { auto three = ov::op::v0::Constant::create(ov::element::i64, {1}, {3}); auto src_reshaped = std::make_shared( diff --git a/ggml/src/ggml-openvino/openvino/op/set_rows.cpp b/ggml/src/ggml-openvino/openvino/op/set_rows.cpp index 3b606c82accb..9773198f6968 100644 --- a/ggml/src/ggml-openvino/openvino/op/set_rows.cpp +++ b/ggml/src/ggml-openvino/openvino/op/set_rows.cpp @@ -68,11 +68,18 @@ OutputVector translate_set_rows(const NodeContext & context) { auto row_size = context.get_input_shape(2)[3].get_length(); auto ind_squeezed = std::make_shared( indices, ov::op::v0::Constant::create(ov::element::i64, {3}, {0, 1, 2})); - auto data_reshaped = std::make_shared( - data, - ov::op::v0::Constant::create(ov::element::i64, {4}, - {(int64_t) 1, (int64_t) 1, (int64_t) -1, (int64_t) row_size}), - false); + // skip the reshape when the rows already have that shape: it costs a GPU primitive per cache write + const auto & data_shape = data.get_partial_shape(); + const bool data_is_rows = data_shape.rank().is_static() && data_shape.rank().get_length() == 4 && + data_shape[0] == 1 && data_shape[1] == 1 && data_shape[3] == row_size; + ov::Output data_reshaped = data; + if (!data_is_rows) { + data_reshaped = std::make_shared( + data, + ov::op::v0::Constant::create(ov::element::i64, {4}, + {(int64_t) 1, (int64_t) 1, (int64_t) -1, (int64_t) row_size}), + false); + } res = std::make_shared(dst, ind_squeezed, data_reshaped, axes); } diff --git a/ggml/src/ggml-openvino/openvino/translate_session.cpp b/ggml/src/ggml-openvino/openvino/translate_session.cpp index ec3596de200d..f3183e27ee44 100644 --- a/ggml/src/ggml-openvino/openvino/translate_session.cpp +++ b/ggml/src/ggml-openvino/openvino/translate_session.cpp @@ -5,10 +5,10 @@ #include "ggml-openvino/openvino/node_context.h" #include "ggml-openvino/openvino/utils.h" #include "input_model.h" -#include "pass/fuse_argsort_topk.h" -#include "pass/fuse_moe_router.h" #include "pass/align_eltwise_ranks.h" +#include "pass/fuse_argsort_topk.h" #include "pass/fuse_moe_compressed.h" +#include "pass/fuse_moe_router.h" #include "pass/fuse_to_conv.h" #include "pass/kv_state_seq_axis.h" #include "pass/mark_decompression_convert_constant_folding.h" @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -44,6 +45,7 @@ #include #include #include +#include #include #include #include @@ -53,7 +55,6 @@ #include #include #include -#include #include namespace ov { @@ -291,6 +292,15 @@ std::shared_ptr TranslateSession::translate_graph(const frontend::InputMo } for (const auto & it : ggml_model_decoder->get_model_extra_inputs()) { + auto source = tensor_map->find(it.second.shape_source); + if (!it.second.shape_source.empty() && source != tensor_map->end()) { + auto dim = std::make_shared(std::make_shared(source->second, it.second.type), + v0::Constant::create(element::i64, {1}, {it.second.shape_axis}), + v0::Constant::create(element::i64, {}, {0})); + dim->set_friendly_name(it.first); + (*tensor_map)[it.first] = dim; + continue; + } auto input_node = create_extra_input(it.first, it.second); if (it.second.is_parameter) { params.push_back(ov::as_type_ptr(input_node)); diff --git a/ggml/src/ggml-openvino/openvino/utils.cpp b/ggml/src/ggml-openvino/openvino/utils.cpp index 40ba938a62ce..56d213f4f3b6 100644 --- a/ggml/src/ggml-openvino/openvino/utils.cpp +++ b/ggml/src/ggml-openvino/openvino/utils.cpp @@ -1,5 +1,7 @@ #include "utils.h" +#include "ggml-openvino/ggml-openvino-extra.h" + #include #include #include @@ -431,7 +433,10 @@ ov::Output process_view_input_new(const NodeContext & context, int inp // This ensures NPUW's FOLD doesn't parametrize per-layer slice indices (which // would introduce dynamic shapes). A shared Split node sits outside the repeated // subgraph boundary; each layer receives one of its output ports. - if (context.is_static() && view_input_size == 1) { + // The dynamic GPU path uses it too: a StridedSlice costs a full shape inference per run on GPU, a Split output does + // not (gemma-4 slices its per-layer embeddings this way, once per layer) + const bool dynamic_split = !context.is_static() && !context.is_stateful() && ggml_openvino_is_gpu(); + if ((context.is_static() || dynamic_split) && view_input_size == 1) { auto view_stride_v = context.get_view_input_stride(input_index, 0); auto view_src_stride_v = context.get_view_input_src_stride(input_index, 0); auto view_ggml_shape = context.get_view_input_ggml_shape(input_index, 0); @@ -471,6 +476,26 @@ ov::Output process_view_input_new(const NodeContext & context, int inp size_t relative_offset = view_offset >= view_src_offset ? view_offset - view_src_offset : 0; int64_t split_index = static_cast(relative_offset / view_src_stride_v[split_dim]); + if (split_index >= 0 && split_index < num_splits && dynamic_split) { + // the Split keeps its input alive, so only keep a weak reference to it on the input + auto src_node = input.get_node_shared_ptr(); + std::string rt_key = + "split_dim_weak_" + std::to_string(split_dim) + "_" + std::to_string(input.get_index()); + auto & rt_info = src_node->get_rt_info(); + std::shared_ptr split_node; + if (rt_info.count(rt_key) != 0) { + split_node = rt_info[rt_key].as>().lock(); + } + if (split_node == nullptr) { + auto axis_const = + ov::op::v0::Constant::create(ov::element::i64, {}, {static_cast(split_dim)}); + split_node = + std::make_shared(input, axis_const, static_cast(num_splits)); + split_node->set_friendly_name(src_node->get_friendly_name() + "_split"); + rt_info[rt_key] = std::weak_ptr(split_node); + } + return split_node->output(static_cast(split_index)); + } if (split_index >= 0 && split_index < num_splits) { auto src_node = input.get_node_shared_ptr(); std::string rt_key = "split_dim_" + std::to_string(split_dim); diff --git a/ggml/src/ggml-openvino/utils.cpp b/ggml/src/ggml-openvino/utils.cpp index f07e180a40f7..6db96abfa19e 100644 --- a/ggml/src/ggml-openvino/utils.cpp +++ b/ggml/src/ggml-openvino/utils.cpp @@ -120,6 +120,7 @@ static uint64_t ggml_openvino_model_cache_extra_cfg(bool stateful, bool recurren extra_cfg = extra_cfg * 131 + (ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_KV_SLICE") ? 1u : 0u); extra_cfg = extra_cfg * 131 + (manual_gqa_enabled ? 1u : 0u); extra_cfg = extra_cfg * 131 + (ggml_openvino_getenv_int("GGML_OPENVINO_PROFILING") >= 2 ? 1u : 0u); + extra_cfg = extra_cfg * 131 + (ggml_openvino_shape_from_mask_enabled() ? 1u : 0u); return extra_cfg; } @@ -205,6 +206,8 @@ static std::string compiled_graph_key(const ggml_cgraph * graph, if (!input.second.is_parameter) { append(input.second.value); } + append_string(input.second.shape_source); + append(input.second.shape_axis); } // Without an allocation generation, pointer reuse could select stale weights. // Such graphs still get private requests; they simply do not share compilation. @@ -256,6 +259,8 @@ static std::string dynamic_graph_signature(const ggml_cgraph * graph, if (!input.is_parameter) { append(input.value); } + append_string(input.shape_source); + append(input.shape_axis); } return key; } From 5f1326b1094c56f580ed382b2c56001b6526de09 Mon Sep 17 00:00:00 2001 From: "Sabnis, Omkar" Date: Tue, 6 Oct 2026 14:23:12 -0700 Subject: [PATCH 07/20] ggml-openvino: only reuse Split outputs for exact chunk-aligned views The dynamic GPU path reused a Split output for any single-element view by deriving the chunk index from offset / stride. This produced incorrect results for views that were not chunk-aligned or whose remaining dimensions did not preserve the source strides, as seen in test-backend-ops CONT with use_view_slice. Restrict Split reuse to cases where the view exactly matches a split chunk, meaning the offset is aligned to the split stride and the remaining non-unit dimensions preserve the original source strides. Gemma-4 per-layer embedding slices continue to use the shared Split path. --- ggml/src/ggml-openvino/openvino/utils.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-openvino/openvino/utils.cpp b/ggml/src/ggml-openvino/openvino/utils.cpp index 56d213f4f3b6..a0c0709282e0 100644 --- a/ggml/src/ggml-openvino/openvino/utils.cpp +++ b/ggml/src/ggml-openvino/openvino/utils.cpp @@ -476,7 +476,14 @@ ov::Output process_view_input_new(const NodeContext & context, int inp size_t relative_offset = view_offset >= view_src_offset ? view_offset - view_src_offset : 0; int64_t split_index = static_cast(relative_offset / view_src_stride_v[split_dim]); - if (split_index >= 0 && split_index < num_splits && dynamic_split) { + // the view must be exactly one Split output: offset on a chunk boundary, the strides of all + // other dims with more than one element equal + bool exact_chunk = relative_offset % view_src_stride_v[split_dim] == 0; + for (size_t i = 0; i < ndims && exact_chunk; ++i) { + exact_chunk = i == static_cast(split_dim) || view_ggml_shape[i] == 1 || + view_stride_v[i] == view_src_stride_v[i]; + } + if (split_index >= 0 && split_index < num_splits && dynamic_split && exact_chunk) { // the Split keeps its input alive, so only keep a weak reference to it on the input auto src_node = input.get_node_shared_ptr(); std::string rt_key = @@ -496,7 +503,7 @@ ov::Output process_view_input_new(const NodeContext & context, int inp } return split_node->output(static_cast(split_index)); } - if (split_index >= 0 && split_index < num_splits) { + if (split_index >= 0 && split_index < num_splits && !dynamic_split) { auto src_node = input.get_node_shared_ptr(); std::string rt_key = "split_dim_" + std::to_string(split_dim); auto & rt_info = src_node->get_rt_info(); From ef965f32c38ed6126168536f55e1c911b37bb15c Mon Sep 17 00:00:00 2001 From: "Sabnis, Omkar" Date: Fri, 9 Oct 2026 08:00:59 -0700 Subject: [PATCH 08/20] ggml-openvino: initialize ModelExtraInputInfo shape fields Initialize the newly added `shape_source` and `shape_axis` fields in the `ModelExtraInputInfo` brace initializer used by `add_extra_inputs()`. Without initializers for these fields, GCC emits `-Wmissing-field-initializers`, causing CI to fail with `LLAMA_FATAL_WARNINGS`. --- ggml/src/ggml-openvino/ggml-decoder.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index 09894f0d9f69..e1bdd9abe0a2 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -1266,7 +1266,7 @@ void GgmlOvDecoder::add_extra_inputs() { // 2. `n_seq_active` and `seq_active_start`, used in FLASH_ATTN_EXT to indicate the active sequences in the batch auto create_1d_input = [this](const std::string & name, int64_t value, bool force_parameter = false) { - m_model_extra_inputs[name] = {ov::element::i64, ov::Shape{1}, value, force_parameter || !m_is_static}; + m_model_extra_inputs[name] = {ov::element::i64, ov::Shape{1}, value, force_parameter || !m_is_static, {}, -1}; }; if (m_compute_params.attention_size != -1) { From 8480ea017e85d35d1ae552f83afbe2622287121e Mon Sep 17 00:00:00 2001 From: "Sabnis, Omkar" Date: Fri, 9 Oct 2026 12:24:59 -0700 Subject: [PATCH 09/20] ggml-openvino: remove GGML_OPENVINO_REQUANT_EMBD The option identifies embedding and output weights by their tensor names, which are defined by core llama.cpp. Remove it from this PR; preserving the file quantization of these weights will be addressed in a separate change that identifies them by their roles in the graph. --- docs/backend/OPENVINO.md | 1 - ggml/src/ggml-openvino/README.md | 2 +- ggml/src/ggml-openvino/ggml-openvino-extra.cpp | 8 -------- ggml/src/ggml-openvino/model-cache.cpp | 1 - 4 files changed, 1 insertion(+), 11 deletions(-) diff --git a/docs/backend/OPENVINO.md b/docs/backend/OPENVINO.md index eb629167fd62..0d5804cafc2c 100644 --- a/docs/backend/OPENVINO.md +++ b/docs/backend/OPENVINO.md @@ -730,7 +730,6 @@ Boolean flags follow a uniform convention: set to a **positive integer** (e.g. ` | `GGML_OPENVINO_RELEASE_WEIGHTS` | Boolean | inherits from `GGML_OPENVINO_MEMORY_OPTIMIZE` on GPU | GPU-only. Release host weight buffers after the compiled model cache can reuse the device/plugin copy. Requires stable graph shapes; dynamic workloads that need recompilation should leave this disabled. | | `GGML_OPENVINO_SPILL_DIR` | String | `not set` | Directory for a disk-backed weight buffer. When set, the repacked weight buffer is mapped from an unlinked file on this path instead of anonymous memory, so its pages are reclaimable under memory pressure instead of staying pinned, cutting the load-time host memory peak. Must point at real storage; a tmpfs mount (e.g. `/tmp` on many systems) backs it with RAM and makes the peak worse. | | `GGML_OPENVINO_REQUANT_KQUANT` | String | `not set` | Requantize Q6_K/Q5_K weights (and matching MoE expert weights) to a 4-bit target instead of the default Q8_0_C, trading accuracy for less memory traffic. One of `q4_asym64` (Q6_K/Q5_K only, keeps a real zero point at group 64), `q4_asym64_all` (also requantizes Q4_K), `q4_sym128` (Q6_K/Q5_K only), `q4_sym128_all` (Q4_K too, drops its per-group zero point), or `native` (no requantization). | -| `GGML_OPENVINO_REQUANT_EMBD` | String | `not set` | `native` keeps `token_embd` and `output` in their file quantization instead of requantizing them to Q8_0_C (CPU/GPU; for tied embeddings this is also the LM head). Less memory traffic per token at a small accuracy cost. | | `GGML_OPENVINO_PROFILING` | Integer | `0` | `1` logs execution timing; `2` or higher also enables OpenVINO and OpenCL profiling. | | `GGML_OPENVINO_DEBUG_NODE` | String | `not set` | Add the named graph nodes as compiled outputs for debugging. Separate multiple names with commas. | | `GGML_OPENVINO_MOE_OP` | Boolean | `1` | On GPU, set to `0` to keep the unfused GatherMatmul path. | diff --git a/ggml/src/ggml-openvino/README.md b/ggml/src/ggml-openvino/README.md index a3c6af9ba8de..a12233d2aa17 100644 --- a/ggml/src/ggml-openvino/README.md +++ b/ggml/src/ggml-openvino/README.md @@ -29,7 +29,7 @@ GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY=1 \ Cache-only mode allocates backend address space without filling weight pages. On Windows, this also uses system commit capacity. The model-buffer size in the loader log is this virtual size. Weight uploads only record source identity; they do not read or requantize the weights. Graph conversion and compilation are skipped. Runtime buffers are still allocated and populated normally. -Cache-only mode uses the settings provided by the current process. Keep these values exactly the same, including set versus unset: `GGML_OPENVINO_REQUANT_KQUANT`, `GGML_OPENVINO_REQUANT_EMBD`, `GGML_OPENVINO_NATIVE_SOFTPLUS`, `GGML_OPENVINO_DISABLE_KV_SLICE`, `GGML_OPENVINO_MANUAL_GQA_ATTN`, `GGML_OPENVINO_STATEFUL_EXECUTION`, `GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT`, `GGML_OPENVINO_DISABLE_REMOTE_OUTPUTS`, `GGML_OPENVINO_REDUCE_COMPILE_MEM`, `GGML_OPENVINO_MEMORY_OPTIMIZE`, `GGML_OPENVINO_PROFILING`, `GGML_OPENVINO_DISABLE_SHAPE_FROM_MASK`, and `GGML_OPENVINO_DEBUG_NODE`. On GPU, also repeat `GGML_OPENVINO_MOE_OP=0` if used. `GGML_OPENVINO_SPILL_DIR` is optional on the first run and ignored in cache-only mode; host-weight release is disabled in cache-only mode. +Cache-only mode uses the settings provided by the current process. Keep these values exactly the same, including set versus unset: `GGML_OPENVINO_REQUANT_KQUANT`, `GGML_OPENVINO_NATIVE_SOFTPLUS`, `GGML_OPENVINO_DISABLE_KV_SLICE`, `GGML_OPENVINO_MANUAL_GQA_ATTN`, `GGML_OPENVINO_STATEFUL_EXECUTION`, `GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT`, `GGML_OPENVINO_DISABLE_REMOTE_OUTPUTS`, `GGML_OPENVINO_REDUCE_COMPILE_MEM`, `GGML_OPENVINO_MEMORY_OPTIMIZE`, `GGML_OPENVINO_PROFILING`, `GGML_OPENVINO_DISABLE_SHAPE_FROM_MASK`, and `GGML_OPENVINO_DEBUG_NODE`. On GPU, also repeat `GGML_OPENVINO_MOE_OP=0` if used. `GGML_OPENVINO_SPILL_DIR` is optional on the first run and ignored in cache-only mode; host-weight release is disabled in cache-only mode. A missing or incompatible graph fails with an error instead of compiling with absent weights. The fingerprint uses the dynamic graph's topology, ports, model parameters, weights, settings, and OpenVINO version; changing only dynamic token or KV sizes does not require a new entry. A different workload can still require another graph; populate it first without cache-only mode. diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp index 4c00a44ebb14..785ae6f6420c 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp @@ -141,7 +141,6 @@ void ggml_openvino_device_config::init() { "GGML_OPENVINO_NATIVE_SOFTPLUS", "GGML_OPENVINO_DISABLE_REMOTE_OUTPUTS", "GGML_OPENVINO_REQUANT_KQUANT", - "GGML_OPENVINO_REQUANT_EMBD", "GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT", "GGML_OPENVINO_DISABLE_SHAPE_FROM_MASK", // Build the precise (but O(n_nodes)) graph cache key. Needed by op tests. @@ -362,13 +361,6 @@ std::optional ggml_openvino_get_requant_type(const ggml_tensor * if (no_requant) { return std::nullopt; } - // GGML_OPENVINO_REQUANT_EMBD=native keeps token_embd/output in the file's quantization (tied models: also the lm_head) - const bool is_embd_or_output = - strncmp(tensor->name, "token_embd.weight", 17) == 0 || strncmp(tensor->name, "output.weight", 13) == 0; - const char * rq_embd = ggml_openvino_getenv_str("GGML_OPENVINO_REQUANT_EMBD"); - if (is_embd_or_output && rq_embd && strcmp(rq_embd, "native") == 0 && !ggml_openvino_is_npu()) { - return std::nullopt; - } if (strncmp(tensor->name, "token_embd.weight", 17) == 0) { return ((ggml_openvino_is_npu() && tensor->type == GGML_TYPE_Q6_K) ? ExtraQuantType::F16 : ExtraQuantType::Q8_0_C); diff --git a/ggml/src/ggml-openvino/model-cache.cpp b/ggml/src/ggml-openvino/model-cache.cpp index 91af222c6f54..278f62a25a52 100644 --- a/ggml/src/ggml-openvino/model-cache.cpp +++ b/ggml/src/ggml-openvino/model-cache.cpp @@ -166,7 +166,6 @@ bool ggml_openvino_model_cache_only() { static const char * cache_settings[] = { "GGML_OPENVINO_REQUANT_KQUANT", - "GGML_OPENVINO_REQUANT_EMBD", "GGML_OPENVINO_NATIVE_SOFTPLUS", "GGML_OPENVINO_DISABLE_KV_SLICE", "GGML_OPENVINO_MANUAL_GQA_ATTN", From f083df26e7b545f8904e97ae77ce60f15419d8f0 Mon Sep 17 00:00:00 2001 From: "Sabnis, Omkar" Date: Mon, 12 Oct 2026 09:58:52 -0700 Subject: [PATCH 10/20] ggml-openvino: keep manual GQA attention 4-D The manual GQA attention path reshaped Q, K, and V to 5-D so that query heads sharing a KV head could broadcast against it. The GPU plugin cannot fuse this graph: it materializes the attention scores and copies the transposed K and V caches for every layer. Instead, stack the query heads for each KV head along the sequence axis (Q [B, H, S, D] -> [B, H_kv, factor * S, D]) and tile the mask accordingly. This keeps the graph 4-D from the MatMuls through the Softmax, allowing the GPU plugin to fuse the attention into its SDPA kernel and fold the K and V transposes into the fused operation. Clamp the mask to -30000, matching the SDPA path, because the fused kernel can return NaN when an entire block is masked with -inf. Models where the K or V head size differs from the Q head size now use the SDPA path, since the manual path assumes one head size. --- .../openvino/op/flash_attn_ext.cpp | 78 ++++++++++--------- 1 file changed, 40 insertions(+), 38 deletions(-) diff --git a/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp b/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp index 60856d20bc72..ebca86621d24 100644 --- a/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp +++ b/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -114,6 +115,7 @@ OutputVector translate_flash_attn_ext(const NodeContext & context) { // if (factor > 1 && num_heads_kv > 1) { auto q_shape = context.get_input_shape(0).to_shape(); auto k_shape = context.get_input_shape(1).to_shape(); + auto v_shape = context.get_input_shape(2).to_shape(); const int64_t num_heads = q_shape[1]; const int64_t num_heads_kv = k_shape[1]; const int64_t head_size = q_shape[3]; @@ -129,54 +131,54 @@ OutputVector translate_flash_attn_ext(const NodeContext & context) { } return ggml_openvino_is_gpu(); }(); + // The manual path reshapes Q, K, V and the output with one head size; models whose K/V head sizes differ + // from Q's (e.g. MLA) take the SDPA path. + const bool same_head_size = k_shape[3] == q_shape[3] && v_shape[3] == q_shape[3]; const bool use_manual_gqa_attention = - manual_gqa_enabled && factor > 1 && num_heads_kv > 1 && !context.is_stateful(); + manual_gqa_enabled && factor > 1 && num_heads_kv > 1 && same_head_size && !context.is_stateful(); if (use_manual_gqa_attention) { // Q, K, V arrive as [B, n_heads(_kv), S, head_size], where B is the active // batch (n_seq_active) and may be > 1 (llama-perplexity, llama-server -np > 1) - // or dynamic. Reshape to - // K_r: [B, num_heads_kv, 1, S, head_size] - // Q_r: [B, num_heads_kv, factor, S_q, head_size] - // and let MatMul broadcast across the factor dim without materialising - // an expanded K/V. The leading 0 + special_zero=true copies B at runtime, - // so this is correct for B == 1, B > 1, and dynamic B alike. Only the head - // dims and head_size are baked in as literals; the sequence dim stays -1. - auto k_5d_shape = ov::op::v0::Constant::create(ov::element::i64, {5}, - std::vector{0, num_heads_kv, 1, -1, head_size}); - auto v_5d_shape = ov::op::v0::Constant::create(ov::element::i64, {5}, - std::vector{0, num_heads_kv, 1, -1, head_size}); - auto q_5d_shape = ov::op::v0::Constant::create(ov::element::i64, {5}, - std::vector{0, num_heads_kv, factor, -1, head_size}); - - auto k_r = std::make_shared(k, k_5d_shape, true); - auto v_r = std::make_shared(v, v_5d_shape, true); - auto q_r = std::make_shared(q, q_5d_shape, true); - - // QK^T → [B, num_heads_kv, factor, S_q, S_k] - auto qk = std::make_shared(q_r, k_r, /*tA=*/false, /*tB=*/true); - auto qk_scaled = std::make_shared(qk, scale_node); - - // Mask arrives as [B, 1, S_q, S_k]. Unsqueeze a factor axis at position 2 to - // get [B, 1, 1, S_q, S_k], which NUMPY-broadcasts cleanly against the - // [B, num_heads_kv, factor, S_q, S_k] scores: B==B, then 1→num_heads_kv and - // 1→factor on the head dims. - ov::Output qk_masked; + // or dynamic. K and V are Transposes of the KV cache; they must feed the MatMuls + // directly (4-D, no Reshape in between) so that the GPU plugin folds the transpose + // into the Gemm instead of copying the whole transposed cache on every token. + // The query heads that share a KV head are stacked along the row axis: + // Q: [B, num_heads, S_q, head_size] -> [B, num_heads_kv, factor * S_q, head_size] + // (a pure reshape: the query heads of one KV head are adjacent) + // scores: [B, num_heads_kv, factor * S_q, S_k] + // The leading 0 + special_zero=true copies B at runtime, so this is correct for + // B == 1, B > 1, and dynamic B alike. + auto q_r = std::make_shared( + q, + ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{0, num_heads_kv, -1, head_size}), + true); + + // QK^T -> [B, num_heads_kv, factor * S_q, S_k] + auto qk = std::make_shared(q_r, k, /*tA=*/false, /*tB=*/true); + ov::Output probs = std::make_shared(qk, scale_node); + if (has_mask) { - auto mask_unsq1 = - std::make_shared(mask, ov::op::v0::Constant::create(ov::element::i64, {1}, {2})); - qk_masked = std::make_shared(qk_scaled, mask_unsq1); - } else { - qk_masked = qk_scaled; + // Mask arrives as [B, 1, S_q, S_k]. Row j * S_q + s of the scores belongs to query position s, so + // repeat the mask factor times along the rows, [B, 1, factor * S_q, S_k], and add it to the 4-D + // scores. Keeping the scores 4-D from the MatMul to the Softmax lets the GPU plugin fuse the scale + // and the mask into the Gemm: a reshape of the scores in between costs a full pass over them per + // layer, which is large in prefill. The tiled mask is the same in every layer. + // The GPU plugin fuses this attention into its SDPA kernel, which works on blocks of keys: a block + // fully masked with -inf for a row gives NaN (see the SDPA path below), so use a large finite value. + auto mask_finite = std::make_shared( + mask, ov::op::v0::Constant::create(ov::element::f16, ov::Shape{}, std::vector{-30000.0f})); + auto mask_tiled = std::make_shared( + mask_finite, + ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{1, 1, factor, 1})); + probs = std::make_shared(probs, mask_tiled); } + probs = std::make_shared(probs, /*axis=*/-1); - auto softmax = std::make_shared(qk_masked, /*axis=*/-1); - - // softmax @ V → [B, num_heads_kv, factor, S_q, head_size] - auto attn = std::make_shared(softmax, v_r); + // softmax @ V -> [B, num_heads_kv, factor * S_q, head_size] + auto attn = std::make_shared(probs, v); // Reshape back to [B, num_heads, S_q, head_size] (combine num_heads_kv * factor). - // Leading 0 + special_zero=true copies B at runtime. auto out_4d_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{0, num_heads, -1, head_size}); auto out_4d = std::make_shared(attn, out_4d_shape, true); From f26ac1ffa12314dd405d80b44056bfe0aa560e69 Mon Sep 17 00:00:00 2001 From: "Sabnis, Omkar" Date: Mon, 12 Oct 2026 10:00:19 -0700 Subject: [PATCH 11/20] ggml-openvino: use a 4-D stateless KV cache on GPU In stateless GPU mode, bind the entire KV cache as a `[1, ctx, n_heads_kv, head_size]` tensor, with one token per row, instead of binding the first `n_kv` rows as a `[1, 1, ctx, n_heads_kv * head_size]` tensor. Write cache updates with `ScatterUpdate` along the token axis, and read the cache through `VariadicSplit` and `Transpose`. This allows the GPU plugin's `StatelessKVFusion` to replace cache reads and writes with in-place `StatelessKV` nodes, avoiding full-cache copies on every decode step and repeated memory pool allocations as the bound cache size grows. Set `GGML_OPENVINO_DISABLE_KV_4D=1` to retain the previous layout. --- docs/backend/OPENVINO.md | 1 + ggml/src/ggml-openvino/ggml-decoder.cpp | 68 ++++++++++++++++++- ggml/src/ggml-openvino/ggml-decoder.h | 11 +++ .../src/ggml-openvino/ggml-openvino-extra.cpp | 1 + .../src/ggml-openvino/openvino/op/permute.cpp | 22 ++++++ .../ggml-openvino/openvino/op/set_rows.cpp | 16 +++++ ggml/src/ggml-openvino/openvino/utils.h | 7 ++ ggml/src/ggml-openvino/utils.cpp | 26 +++++++ 8 files changed, 150 insertions(+), 2 deletions(-) diff --git a/docs/backend/OPENVINO.md b/docs/backend/OPENVINO.md index 0d5804cafc2c..d1ec2bb74a18 100644 --- a/docs/backend/OPENVINO.md +++ b/docs/backend/OPENVINO.md @@ -722,6 +722,7 @@ Boolean flags follow a uniform convention: set to a **positive integer** (e.g. ` | `GGML_OPENVINO_STATEFUL_EXECUTION`| Boolean | `0` | Keep KV and supported recurrent caches inside the model. Single-slot CPU/GPU execution only. | | `GGML_OPENVINO_DISABLE_CACHE` | Boolean | `0` | Disable the in-process compiled-model / decoder cache (cache is on by default). Set to `1` to disable. | | `GGML_OPENVINO_DISABLE_KV_SLICE` | Boolean | `0` | Disable the KV-cache input-tensor slicing optimization (slicing is on by default on CPU/GPU). Set to `1` to disable. | +| `GGML_OPENVINO_DISABLE_KV_4D` | Boolean | `0` | GPU, stateless: keep the KV cache in the `[1, 1, ctx, n_heads_kv * head_size]` layout instead of `[1, ctx, n_heads_kv, head_size]`, which lets the GPU plugin update and read the cache in place. Set to `1` to disable. | | `GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT` | Boolean | `0` | Disable the stateful KV-state sequence-axis relayout (relayout is on by default). It moves the KV state sequence axis from dim 1 to dim 2, so the GPU plugin can append new tokens in place instead of copying the whole state every token, and the reader side no longer transposes the whole accumulated state. Set to `1` to disable. | | `GGML_OPENVINO_DISABLE_SHAPE_FROM_MASK` | Boolean | `0` | Dynamic CPU/GPU graphs read the active sequence count and the attended KV length from the shape of the KQ mask inside the model, so the GPU plugin can infer the attention shapes without waiting for input values and selects its fused SDPA kernel. Set to `1` to pass them as separate inputs instead. | | `GGML_OPENVINO_MANUAL_GQA_ATTN` | Boolean | device-based | Tri-state. When **unset**, manual GQA attention is enabled by default on `GPU` and disabled on other devices. Set to a positive integer to force-enable, or `0` to force-disable. | diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index e1bdd9abe0a2..d27f278cb39b 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -1181,9 +1181,10 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op, // do not fix ctx size to make llama-bench work across test params input_shape[2] = -1; } - if (is_stateful() && !is_flat_kv) { + if ((is_stateful() && !is_flat_kv) || is_kvcache_4d(input)) { // Convert stateless KV cache layout [1, 1, seq, n_heads_kv * head_size] - // to stateful layout [1, seq, n_heads_kv, head_size]. + // to stateful layout [1, seq, n_heads_kv, head_size]. Stateless GPU graphs use the same + // layout for the caches that qualify (see is_kvcache_4d), bound over the same memory. // NOTE: Gemma4 uses per-layer-type KV shapes, so no single scalar describes every // layer. E2B varies only the head size (sliding 256, full 512); 12B also varies the // head COUNT (sliding 8 x 256, full 1 x 512). Take the head count for this tensor's @@ -1354,6 +1355,14 @@ bool GgmlOvDecoder::node_is_used_as_src(const int node_idx) { void GgmlOvDecoder::compute_model_inputs() { m_model_inputs.clear(); m_inputs.clear(); + m_kvcache_4d.clear(); + // remember the KV caches that get_graph_input_shape gave the 4-D layout, so that they are bound with that shape + auto note_kvcache_4d = [this](const ggml_tensor * input, const ggml_tensor * op, const ov::PartialShape & shape) { + if (!m_is_stateful && is_kvcache(input, op) && shape.rank().is_static() && shape.rank().get_length() == 4 && + shape[1].is_dynamic()) { + m_kvcache_4d.insert(input); + } + }; for (int i = 0; i < m_cgraph->n_nodes; i++) { ggml_tensor * node = m_cgraph->nodes[i]; // the node op is NONE means this node maybe as input of later nodes, we should add it to model inputs for this node. @@ -1363,6 +1372,7 @@ void GgmlOvDecoder::compute_model_inputs() { m_inputs[node_name] = node; m_model_inputs[node_name] = {get_ov_type(node), get_graph_input_shape(node, nullptr, m_node_dynamic_dims[node])}; + note_kvcache_4d(node, nullptr, m_model_inputs[node_name].shape); } continue; } @@ -1416,6 +1426,7 @@ void GgmlOvDecoder::compute_model_inputs() { m_inputs[src_name] = src; m_model_inputs[src_name] = {get_ov_type(src), get_graph_input_shape(node, src, m_node_dynamic_dims[src])}; + note_kvcache_4d(src, node, m_model_inputs[src_name].shape); } } } @@ -1475,6 +1486,59 @@ const ggml_tensor * GgmlOvDecoder::get_tensor_used_op(const ggml_tensor * tensor return nullptr; } +bool GgmlOvDecoder::is_kvcache_4d(const ggml_tensor * cache) const { + static const bool disabled = ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_KV_4D") != 0; + if (disabled || m_is_static || m_is_stateful || m_model_is_splitted || !ggml_openvino_is_gpu()) { + return false; + } + if (cache->op != GGML_OP_NONE || cache->view_src != nullptr || !is_kvcache(cache, nullptr)) { + return false; + } + // one sequence, [n_heads_kv * head_size, ctx, 1, 1]; whisper's flat KV buffer has ctx == 1 + if (cache->ne[1] <= 1 || cache->ne[2] != 1 || cache->ne[3] != 1) { + return false; + } + const int n_heads_kv = get_n_heads_kv_for_tensor(cache); + if (n_heads_kv <= 0 || cache->ne[0] % n_heads_kv != 0) { + return false; + } + // check the nodes that take the cache itself as a source (views of those views, such as the PERMUTE of the read + // VIEW, are checked through them) + auto feeds_only_permutes = [this](const ggml_tensor * view) { + for (int i = 0; i < m_cgraph->n_nodes; i++) { + const auto * node = m_cgraph->nodes[i]; + for (int j = 0; j < GGML_MAX_SRC; j++) { + if (node->src[j] == view && node->op != GGML_OP_PERMUTE) { + return false; + } + } + } + return true; + }; + for (int i = 0; i < m_cgraph->n_nodes; i++) { + const auto * node = m_cgraph->nodes[i]; + bool uses_cache = false; + for (int j = 0; j < GGML_MAX_SRC; j++) { + uses_cache = uses_cache || node->src[j] == cache; + } + if (!uses_cache) { + continue; + } + // write: whole rows into the cache itself + if (node->op == GGML_OP_SET_ROWS && node->src[2] == cache && node->src[0]->ne[0] == cache->ne[0]) { + continue; + } + // read: the first rows of the cache, each row split into heads, permuted for attention + if (node->op == GGML_OP_VIEW && node->src[0] == cache && node->view_offs == 0 && + node->ne[0] * node->ne[1] == cache->ne[0] && node->nb[2] == cache->nb[1] && node->ne[3] == 1 && + feeds_only_permutes(node)) { + continue; + } + return false; + } + return true; +} + const ggml_tensor * GgmlOvDecoder::get_tensor_from_name(const std::string & name) const { for (int i = 0; i < m_cgraph->n_nodes; i++) { const auto * node = m_cgraph->nodes[i]; diff --git a/ggml/src/ggml-openvino/ggml-decoder.h b/ggml/src/ggml-openvino/ggml-decoder.h index ee9788b19090..ace5e7afb10c 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.h +++ b/ggml/src/ggml-openvino/ggml-decoder.h @@ -337,6 +337,16 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { const ggml_tensor * get_tensor_used_op(const ggml_tensor * tensor) const; + // True if the KV cache is a model input of shape [1, ctx, n_heads_kv, head_size] instead of + // [1, 1, ctx, n_heads_kv * head_size]. Used for stateless GPU graphs when the cache holds one sequence, one token + // per row, and is only written by SET_ROWS and read through views that split the rows into heads. The GPU + // plugin can then fuse the cache write and read into StatelessKV: the new rows are written in place and attention + // reads the cache without a copy. + bool is_kvcache_4d(const ggml_tensor * cache) const; + + // True if the KV cache is an input of this graph in the 4-D layout (see is_kvcache_4d) + bool is_kvcache_4d_input(const ggml_tensor * cache) const { return m_kvcache_4d.count(cache) != 0; } + const ggml_tensor * get_tensor_from_name(const std::string & name) const; void clear_model_weights() { m_model_weights.clear(); } @@ -503,6 +513,7 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { std::set m_model_output_names; std::vector m_node_info_list; std::map m_node_dynamic_dims; + std::set m_kvcache_4d; ModelParams m_model_params; ComputeParams m_compute_params; diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp index 785ae6f6420c..03701b51f328 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp @@ -129,6 +129,7 @@ void ggml_openvino_device_config::init() { "GGML_OPENVINO_ENABLE_CACHE", "GGML_OPENVINO_DISABLE_CACHE", "GGML_OPENVINO_DISABLE_KV_SLICE", + "GGML_OPENVINO_DISABLE_KV_4D", "GGML_OPENVINO_ENABLE_FALLBACK", "GGML_OPENVINO_MANUAL_GQA_ATTN", "GGML_OPENVINO_DISABLE_ELTWISE_RANK_ALIGN", diff --git a/ggml/src/ggml-openvino/openvino/op/permute.cpp b/ggml/src/ggml-openvino/openvino/op/permute.cpp index a9834d99f54a..85e635296d67 100644 --- a/ggml/src/ggml-openvino/openvino/op/permute.cpp +++ b/ggml/src/ggml-openvino/openvino/op/permute.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include namespace ov { @@ -78,6 +79,27 @@ OutputVector translate_permute(const NodeContext & context) { auto reshaped = std::make_shared(src, new_shape, true); res = std::make_shared(reshaped, perm); + } else if ((op_case == 3 || op_case == 4) && is_kvcache_4d_shape(src.get_partial_shape())) { + // cache [1, ctx, n_heads_kv, head_size] (see set_rows): take the first attention_size tokens with + // VariadicSplit, the read side of the GPU plugin's StatelessKVFusion. The fused StatelessKV then hands + // attention a view of those tokens, with no copy + FRONT_END_OP_CONVERSION_CHECK((perm_values == std::vector{0, 2, 1, 3}), + "Unsupported permutation of a 4-D KV cache"); + Output attention_size; + if (!context.has_input("attention_size")) { + attention_size = + ov::op::v0::Constant::create(ov::element::i64, {1}, {context.get_output_shape().to_shape()[2]}); + } else if (op_case == 3) { + attention_size = context.get_input("attention_size"); + } else { + attention_size = context.get_input("attention_size_swa"); + } + auto split_lengths = std::make_shared( + ov::OutputVector{attention_size, ov::op::v0::Constant::create(ov::element::i64, {1}, {-1})}, 0); + auto tokens = std::make_shared( + src, ov::op::v0::Constant::create(ov::element::i64, {}, {1}), split_lengths); + res = std::make_shared(tokens->output(0), + ov::op::v0::Constant::create(ov::element::i64, {4}, perm_values)); } else { auto perm = ov::op::v0::Constant::create(ov::element::i64, {4}, perm_values); auto cache_shape = src.get_partial_shape(); diff --git a/ggml/src/ggml-openvino/openvino/op/set_rows.cpp b/ggml/src/ggml-openvino/openvino/op/set_rows.cpp index 9773198f6968..4ead927fa4b7 100644 --- a/ggml/src/ggml-openvino/openvino/op/set_rows.cpp +++ b/ggml/src/ggml-openvino/openvino/op/set_rows.cpp @@ -53,6 +53,22 @@ OutputVector translate_set_rows(const NodeContext & context) { data = std::make_shared( data, ov::op::v0::Constant::create(ov::element::i64, {4}, {(int64_t) 1, (int64_t) -1, dim2, dim3}), false); res = std::make_shared(OutputVector{dst, data}, concat_axis); + } else if (is_kvcache_4d_shape(dst.get_partial_shape())) { + // cache [1, ctx, n_heads_kv, head_size]: write whole tokens on axis 1. This is the pattern of the GPU + // plugin's StatelessKVFusion (1-D axis, read with VariadicSplit, see permute), which writes the new rows in + // place instead of copying the whole cache first + FRONT_END_OP_CONVERSION_CHECK(!multidim_indices, "SET_ROWS into a 4-D KV cache needs one index per token"); + const int64_t n_heads_kv = dst.get_partial_shape()[2].get_length(); + const int64_t head_size = dst.get_partial_shape()[3].get_length(); + auto data_tokens = std::make_shared( + data, + ov::op::v0::Constant::create(ov::element::i64, {4}, {(int64_t) 1, (int64_t) -1, n_heads_kv, head_size}), + false); + auto ind_squeezed = std::make_shared( + indices, ov::op::v0::Constant::create(ov::element::i64, {3}, {0, 1, 2})); + res = std::make_shared(dst, ind_squeezed, data_tokens, + ov::op::v0::Constant::create(ov::element::i64, {1}, {1})); + return rename_outputs_with_suffix({res}, context.get_name()); } else if (multidim_indices) { auto updates_shape = std::make_shared(data, ov::element::i64); diff --git a/ggml/src/ggml-openvino/openvino/utils.h b/ggml/src/ggml-openvino/openvino/utils.h index effa1f843778..1b6b4ea19f20 100644 --- a/ggml/src/ggml-openvino/openvino/utils.h +++ b/ggml/src/ggml-openvino/openvino/utils.h @@ -66,6 +66,13 @@ ov::Output process_view_input(const NodeContext & context, int input_i ov::Output process_view_input_new(const NodeContext & context, int input_index); +// A stateless KV cache in the [1, ctx, n_heads_kv, head_size] layout (GgmlOvDecoder::is_kvcache_4d), as opposed to +// [1, n_seq, ctx, n_heads_kv * head_size] +inline bool is_kvcache_4d_shape(const ov::PartialShape & shape) { + return shape.rank().is_static() && shape.rank().get_length() == 4 && shape[1].is_dynamic() && + shape[2].is_static() && shape[3].is_static(); +} + namespace op { template OutputVector translate_1to1_match_2_inputs(const NodeContext & context) { num_inputs_check(context, 2, 2); diff --git a/ggml/src/ggml-openvino/utils.cpp b/ggml/src/ggml-openvino/utils.cpp index 6db96abfa19e..b6d517b149eb 100644 --- a/ggml/src/ggml-openvino/utils.cpp +++ b/ggml/src/ggml-openvino/utils.cpp @@ -108,6 +108,25 @@ std::optional try_make_kv_sliced_tensor(const std::shared_ptrdata); } +// For a KV cache in the 4-D layout (GgmlOvDecoder::is_kvcache_4d), a [1, ctx, n_heads_kv, head_size] tensor over the +// whole cache. Its shape stays the same from one inference to the next, so the GPU plugin keeps writing the cache in +// place and reading it without a copy. +std::optional try_make_kv_4d_tensor(const std::shared_ptr & ggml_decoder, + const ggml_tensor * ggml_tensor) { + if (!ggml_decoder->is_kvcache_4d_input(ggml_tensor)) { + return std::nullopt; + } + const size_t n_heads_kv = static_cast(ggml_decoder->get_n_heads_kv_for_tensor(ggml_tensor)); + const ov::Shape shape{1, static_cast(ggml_tensor->ne[1]), n_heads_kv, + static_cast(ggml_tensor->ne[0]) / n_heads_kv}; + if (ggml_openvino_buffer_is_remote(ggml_tensor)) { + auto remote_context = ggml_openvino_get_remote_context(); + auto gpu_context = remote_context->as(); + return gpu_context.create_tensor(ggml_decoder->get_ov_type(ggml_tensor), shape, ggml_tensor->data); + } + return ov::Tensor(GgmlOvDecoder::get_ov_type(ggml_tensor), shape, ggml_tensor->data); +} + static uint64_t ggml_openvino_model_cache_extra_cfg(bool stateful, bool recurrent) { const char * manual_gqa_env = ggml_openvino_getenv_str("GGML_OPENVINO_MANUAL_GQA_ATTN"); const bool manual_gqa_enabled = manual_gqa_env != nullptr ? @@ -118,6 +137,7 @@ static uint64_t ggml_openvino_model_cache_extra_cfg(bool stateful, bool recurren extra_cfg = extra_cfg * 131 + (stateful ? (recurrent ? 2u : 1u) : 0u); extra_cfg = extra_cfg * 131 + (ggml_openvino_reduce_compile_mem_enabled() ? 1u : 0u); extra_cfg = extra_cfg * 131 + (ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_KV_SLICE") ? 1u : 0u); + extra_cfg = extra_cfg * 131 + (ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_KV_4D") ? 1u : 0u); extra_cfg = extra_cfg * 131 + (manual_gqa_enabled ? 1u : 0u); extra_cfg = extra_cfg * 131 + (ggml_openvino_getenv_int("GGML_OPENVINO_PROFILING") >= 2 ? 1u : 0u); extra_cfg = extra_cfg * 131 + (ggml_openvino_shape_from_mask_enabled() ? 1u : 0u); @@ -383,6 +403,9 @@ ov::Tensor create_ov_output_tensor(const std::shared_ptr & ggml_d const std::shared_ptr & infer_request, int output_index, const ggml_tensor * ggml_tensor) { + if (auto kv_4d = try_make_kv_4d_tensor(ggml_decoder, ggml_tensor)) { + return *kv_4d; + } if (auto sliced = try_make_kv_sliced_tensor(ggml_decoder, std::string(ggml_tensor->name), ggml_tensor)) { return *sliced; } @@ -487,6 +510,9 @@ ov::Tensor make_contiguous_split_input_tensor(const struct ggml_tensor * ggml_te ov::Tensor convert_ggml_input_to_ov(const std::shared_ptr & ggml_decoder, const std::string & name) { const auto * ggml_tensor = ggml_decoder->get_input_ggml_tensor(name); + if (auto kv_4d = try_make_kv_4d_tensor(ggml_decoder, ggml_tensor)) { + return *kv_4d; + } if (auto sliced = try_make_kv_sliced_tensor(ggml_decoder, name, ggml_tensor)) { return *sliced; } From ddf23f61d0949d31ef4618ff37cedc2c1ddb1b4e Mon Sep 17 00:00:00 2001 From: Beheshti Date: Mon, 12 Oct 2026 12:17:06 -0700 Subject: [PATCH 12/20] make embedding to have dynamic shape for OV GPU --- ggml/src/ggml-openvino/ggml-decoder.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index d27f278cb39b..af6c63981c6e 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -1210,7 +1210,13 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op, // index list has a static length; on CPU/GPU it may change across compiles (defrag). input_shape = m_is_static ? ov::PartialShape{get_shape(input)} : ov::PartialShape{1, 1, 1, -1}; - } else { + } else if (input != nullptr && op != nullptr && is_inp_emb(input, op)) { + input_shape = ov::PartialShape{get_shape(input)}; + if (!m_is_static) { + input_shape[2] = -1; + } + } + else { input_shape = ov::PartialShape{get_shape(input)}; } if (dynamic_dim_index != -1 && m_model_is_splitted) { From 6b9bf1a7dc9b535763ff19e2511fd546330b6bb3 Mon Sep 17 00:00:00 2001 From: Mustafa Cavus Date: Thu, 15 Oct 2026 13:54:15 -0700 Subject: [PATCH 13/20] ggml-openvino: make the 4-D stateless KV cache opt-in --- docs/backend/OPENVINO.md | 2 +- ggml/src/ggml-openvino/ggml-decoder.cpp | 4 ++-- ggml/src/ggml-openvino/ggml-openvino-extra.cpp | 2 +- ggml/src/ggml-openvino/utils.cpp | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/backend/OPENVINO.md b/docs/backend/OPENVINO.md index d1ec2bb74a18..f720f18aa41a 100644 --- a/docs/backend/OPENVINO.md +++ b/docs/backend/OPENVINO.md @@ -722,7 +722,7 @@ Boolean flags follow a uniform convention: set to a **positive integer** (e.g. ` | `GGML_OPENVINO_STATEFUL_EXECUTION`| Boolean | `0` | Keep KV and supported recurrent caches inside the model. Single-slot CPU/GPU execution only. | | `GGML_OPENVINO_DISABLE_CACHE` | Boolean | `0` | Disable the in-process compiled-model / decoder cache (cache is on by default). Set to `1` to disable. | | `GGML_OPENVINO_DISABLE_KV_SLICE` | Boolean | `0` | Disable the KV-cache input-tensor slicing optimization (slicing is on by default on CPU/GPU). Set to `1` to disable. | -| `GGML_OPENVINO_DISABLE_KV_4D` | Boolean | `0` | GPU, stateless: keep the KV cache in the `[1, 1, ctx, n_heads_kv * head_size]` layout instead of `[1, ctx, n_heads_kv, head_size]`, which lets the GPU plugin update and read the cache in place. Set to `1` to disable. | +| `GGML_OPENVINO_ENABLE_KV_4D` | Boolean | `0` | GPU, stateless: bind the KV cache as `[1, ctx, n_heads_kv, head_size]` instead of `[1, 1, ctx, n_heads_kv * head_size]`, so the GPU plugin's `StatelessKVFusion` can update and read the cache in place. **Off by default**: it needs an OpenVINO build with openvino#37569 (2026.5 or a recent nightly) and is much slower on older releases (e.g. 2026.4.1). Set to a positive integer to enable. | | `GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT` | Boolean | `0` | Disable the stateful KV-state sequence-axis relayout (relayout is on by default). It moves the KV state sequence axis from dim 1 to dim 2, so the GPU plugin can append new tokens in place instead of copying the whole state every token, and the reader side no longer transposes the whole accumulated state. Set to `1` to disable. | | `GGML_OPENVINO_DISABLE_SHAPE_FROM_MASK` | Boolean | `0` | Dynamic CPU/GPU graphs read the active sequence count and the attended KV length from the shape of the KQ mask inside the model, so the GPU plugin can infer the attention shapes without waiting for input values and selects its fused SDPA kernel. Set to `1` to pass them as separate inputs instead. | | `GGML_OPENVINO_MANUAL_GQA_ATTN` | Boolean | device-based | Tri-state. When **unset**, manual GQA attention is enabled by default on `GPU` and disabled on other devices. Set to a positive integer to force-enable, or `0` to force-disable. | diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index af6c63981c6e..012c5a11f14a 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -1493,8 +1493,8 @@ const ggml_tensor * GgmlOvDecoder::get_tensor_used_op(const ggml_tensor * tensor } bool GgmlOvDecoder::is_kvcache_4d(const ggml_tensor * cache) const { - static const bool disabled = ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_KV_4D") != 0; - if (disabled || m_is_static || m_is_stateful || m_model_is_splitted || !ggml_openvino_is_gpu()) { + static const bool enabled = ggml_openvino_getenv_int("GGML_OPENVINO_ENABLE_KV_4D") > 0; + if (!enabled || m_is_static || m_is_stateful || m_model_is_splitted || !ggml_openvino_is_gpu()) { return false; } if (cache->op != GGML_OP_NONE || cache->view_src != nullptr || !is_kvcache(cache, nullptr)) { diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp index 03701b51f328..fc6a4c276b88 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp @@ -129,7 +129,7 @@ void ggml_openvino_device_config::init() { "GGML_OPENVINO_ENABLE_CACHE", "GGML_OPENVINO_DISABLE_CACHE", "GGML_OPENVINO_DISABLE_KV_SLICE", - "GGML_OPENVINO_DISABLE_KV_4D", + "GGML_OPENVINO_ENABLE_KV_4D", "GGML_OPENVINO_ENABLE_FALLBACK", "GGML_OPENVINO_MANUAL_GQA_ATTN", "GGML_OPENVINO_DISABLE_ELTWISE_RANK_ALIGN", diff --git a/ggml/src/ggml-openvino/utils.cpp b/ggml/src/ggml-openvino/utils.cpp index b6d517b149eb..c9866ef0eadc 100644 --- a/ggml/src/ggml-openvino/utils.cpp +++ b/ggml/src/ggml-openvino/utils.cpp @@ -137,7 +137,7 @@ static uint64_t ggml_openvino_model_cache_extra_cfg(bool stateful, bool recurren extra_cfg = extra_cfg * 131 + (stateful ? (recurrent ? 2u : 1u) : 0u); extra_cfg = extra_cfg * 131 + (ggml_openvino_reduce_compile_mem_enabled() ? 1u : 0u); extra_cfg = extra_cfg * 131 + (ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_KV_SLICE") ? 1u : 0u); - extra_cfg = extra_cfg * 131 + (ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_KV_4D") ? 1u : 0u); + extra_cfg = extra_cfg * 131 + (ggml_openvino_getenv_int("GGML_OPENVINO_ENABLE_KV_4D") > 0 ? 1u : 0u); extra_cfg = extra_cfg * 131 + (manual_gqa_enabled ? 1u : 0u); extra_cfg = extra_cfg * 131 + (ggml_openvino_getenv_int("GGML_OPENVINO_PROFILING") >= 2 ? 1u : 0u); extra_cfg = extra_cfg * 131 + (ggml_openvino_shape_from_mask_enabled() ? 1u : 0u); From c1be3f6f96fb4abcd1a2428286f383d0f6587ebc Mon Sep 17 00:00:00 2001 From: Beheshti Date: Tue, 13 Oct 2026 12:03:32 -0700 Subject: [PATCH 14/20] add support for Q5_0 data type in OV GPU backend --- ggml/src/ggml-openvino/ggml-openvino-extra.cpp | 4 ++++ ggml/src/ggml-openvino/ggml-openvino.cpp | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp index fc6a4c276b88..47a32930c529 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp @@ -405,6 +405,10 @@ std::optional ggml_openvino_get_requant_type(const ggml_tensor * return ExtraQuantType::Q4_1_64; } } + if (tensor->type == GGML_TYPE_Q5_0) { + //return ExtraQuantType::Q4_0_128; + return ExtraQuantType::Q4_0_64; + } // MoE expert weights (3D, ne[2] = n_expert) stored as Q5_1/Q8_0 are the expert-side // equivalent of Q6_K/Q5_K: kept at 8 bits by default while the rest of the model is at 4 // (gemma-4 26B-A4B keeps its down projection there). Send them to 4 bits under the same diff --git a/ggml/src/ggml-openvino/ggml-openvino.cpp b/ggml/src/ggml-openvino/ggml-openvino.cpp index 22b2f45e24a6..ff6773c864f0 100644 --- a/ggml/src/ggml-openvino/ggml-openvino.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino.cpp @@ -1782,7 +1782,7 @@ static ggml_openvino_op_support ggml_backend_openvino_device_supports_op_impl(gg static std::unordered_set supported_types{ GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_I64, GGML_TYPE_I32, GGML_TYPE_Q4_0, - GGML_TYPE_Q4_1, GGML_TYPE_Q4_K, GGML_TYPE_Q5_1, GGML_TYPE_Q5_K, GGML_TYPE_Q8_0, GGML_TYPE_Q6_K, + GGML_TYPE_Q4_1, GGML_TYPE_Q4_K, GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, GGML_TYPE_Q5_K, GGML_TYPE_Q8_0, GGML_TYPE_Q6_K, GGML_TYPE_MXFP4}; // derive supported op sets from the op_table map, keys in From 649b839b0529f28ffc291b7a4e77cf45443a4e4f Mon Sep 17 00:00:00 2001 From: Beheshti Date: Wed, 14 Oct 2026 08:40:03 -0700 Subject: [PATCH 15/20] resolve CI test error --- ggml/src/ggml-openvino/ggml-decoder.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index 012c5a11f14a..9b3c97cfbfff 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -1707,7 +1707,7 @@ std::shared_ptr GgmlOvDecoder::create_weight_node(ggml_tensor * tensor // GGML_LOG_DEBUG("%s: creating new weight node for %s\n", __func__, tensor->name); static const std::set weight_types = {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_Q8_0, - GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q5_1, GGML_TYPE_Q4_K, + GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K, GGML_TYPE_MXFP4}; if (weight_types.find(tensor->type) == weight_types.end()) { throw std::runtime_error("Unexpected weight tensor type: " + std::string(tensor->name) + " with type " + From 348e932199ac3c9447beaee5b6a904fe54df78e9 Mon Sep 17 00:00:00 2001 From: "Sabnis, Omkar" Date: Mon, 12 Oct 2026 11:25:08 -0700 Subject: [PATCH 16/20] ggml-openvino: fix -fa off and optimize grouped-query attention With -fa off, llama.cpp stores the V cache transposed, with value elements as rows and tokens along each row. The KV slicing code assumes one token per row and binds the wrong data when n_kv < kv_size. Detect transposed caches with GgmlOvDecoder::is_kvcache_token_rows() and skip slicing for them. For grouped-query attention, avoid repeating K and V for each query head. Stack the query heads for each KV head along the row dimension and let SOFT_MAX apply the scale, mask, and softmax so the GPU plugin can fuse the operation with the MatMuls. Read only the current layer's V-cache rows when layers use different cache sizes. Reject -fa off in stateful mode with a clear error and document that it requires stateless mode. --- docs/backend/OPENVINO.md | 1 + ggml/src/ggml-openvino/ggml-decoder.cpp | 15 +++++++ ggml/src/ggml-openvino/ggml-decoder.h | 4 ++ ggml/src/ggml-openvino/openvino/op/mulmat.cpp | 21 +++++++++ .../src/ggml-openvino/openvino/op/permute.cpp | 12 ++++- .../ggml-openvino/openvino/op/set_rows.cpp | 5 +++ .../src/ggml-openvino/openvino/op/softmax.cpp | 44 ++++++++++++++++++- ggml/src/ggml-openvino/openvino/utils.h | 4 ++ ggml/src/ggml-openvino/utils.cpp | 7 +-- 9 files changed, 108 insertions(+), 5 deletions(-) diff --git a/docs/backend/OPENVINO.md b/docs/backend/OPENVINO.md index f720f18aa41a..47ea5aedb70a 100644 --- a/docs/backend/OPENVINO.md +++ b/docs/backend/OPENVINO.md @@ -790,6 +790,7 @@ build\ReleaseOV\bin\llama-simple.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M. - `llama-bench`: requires `-fa 1` (flash-attention). - `llama-cli --context-shift`: stateless only (`GGML_OPENVINO_STATEFUL_EXECUTION=0`). In stateful mode the KV cache is owned by the OpenVINO model and cannot be shifted externally. +- `-fa off`: stateless only. With `-fa off` the V cache is stored transposed, which the stateful KV cache does not support. - `llama-server`: only one chat session/thread when `GGML_OPENVINO_STATEFUL_EXECUTION=1`. **GPU-specific** diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index 9b3c97cfbfff..1af9b3cd96ca 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -1545,6 +1545,21 @@ bool GgmlOvDecoder::is_kvcache_4d(const ggml_tensor * cache) const { return true; } +bool GgmlOvDecoder::is_kvcache_token_rows(const ggml_tensor * cache) const { + for (int i = 0; i < m_cgraph->n_nodes; i++) { + const auto * node = m_cgraph->nodes[i]; + if (node->op != GGML_OP_SET_ROWS) { + continue; + } + // SET_ROWS writes whole rows of src[2], which is the cache or a view of it + const auto * dst = node->src[2]; + if (dst != nullptr && (dst == cache || dst->view_src == cache) && dst->ne[0] != cache->ne[0]) { + return false; + } + } + return true; +} + const ggml_tensor * GgmlOvDecoder::get_tensor_from_name(const std::string & name) const { for (int i = 0; i < m_cgraph->n_nodes; i++) { const auto * node = m_cgraph->nodes[i]; diff --git a/ggml/src/ggml-openvino/ggml-decoder.h b/ggml/src/ggml-openvino/ggml-decoder.h index ace5e7afb10c..635a02e31219 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.h +++ b/ggml/src/ggml-openvino/ggml-decoder.h @@ -347,6 +347,10 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { // True if the KV cache is an input of this graph in the 4-D layout (see is_kvcache_4d) bool is_kvcache_4d_input(const ggml_tensor * cache) const { return m_kvcache_4d.count(cache) != 0; } + // True if each row of the KV cache holds one token. With `-fa off`, llama.cpp keeps the V cache transposed + // and writes it one element per row, so its first rows do not hold the first tokens. + bool is_kvcache_token_rows(const ggml_tensor * cache) const; + const ggml_tensor * get_tensor_from_name(const std::string & name) const; void clear_model_weights() { m_model_weights.clear(); } diff --git a/ggml/src/ggml-openvino/openvino/op/mulmat.cpp b/ggml/src/ggml-openvino/openvino/op/mulmat.cpp index 9d4315aa4abc..bb7ae8b8772f 100644 --- a/ggml/src/ggml-openvino/openvino/op/mulmat.cpp +++ b/ggml/src/ggml-openvino/openvino/op/mulmat.cpp @@ -48,6 +48,27 @@ OutputVector translate_mulmat(const NodeContext & context) { Output Z = A_batch_larger ? B : A; auto Z_shape = A_batch_larger ? B_shape : A_shape; int64_t factor = batch_large / batch_small; + if (factor > 1 && batch_small > 1 && A_batch_larger && A_shape[0] == B_shape[0]) { + // B is shared by groups of factor consecutive A matrices (grouped-query attention with `-fa off`: K and V + // have fewer heads than Q and KQ). Stack each group along the rows of A instead of repeating B, which would + // copy the whole K or V cache factor times per layer and token: + // A [n, batch_large, rows, k] -> [n, batch_small, factor * rows, k] (a pure reshape: the group is adjacent) + // A x B^T [n, batch_small, factor * rows, cols] -> [n, batch_large, rows, cols] + // The zeros copy the input dimensions (special_zero), which may be dynamic. + auto A_grouped = std::make_shared( + A, ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{0, batch_small, -1, 0}), true); + ov::Output res = std::make_shared(A_grouped, B, false, true); + res = std::make_shared( + res, ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{0, batch_large, -1, 0}), + true); + // lets SOFT_MAX work on the grouped scores (see softmax) + res.get_node()->get_rt_info()[GGML_OV_GROUPED_ROWS] = factor; + const auto output_type = context.get_output_type(); + if (res.get_element_type() != output_type) { + res = std::make_shared(res, output_type); + } + return rename_outputs_with_suffix({res}, context.get_name()); + } if (factor > 1 && batch_small > 1) { auto batch_large_node = ov::op::v0::Constant::create(ov::element::i64, {1}, std::vector{batch_large}); auto batch_small_node = ov::op::v0::Constant::create(ov::element::i64, {1}, std::vector{batch_small}); diff --git a/ggml/src/ggml-openvino/openvino/op/permute.cpp b/ggml/src/ggml-openvino/openvino/op/permute.cpp index 85e635296d67..14788bef429e 100644 --- a/ggml/src/ggml-openvino/openvino/op/permute.cpp +++ b/ggml/src/ggml-openvino/openvino/op/permute.cpp @@ -182,8 +182,18 @@ OutputVector translate_permute(const NodeContext & context) { res = std::make_shared(restated, perm); } else { auto three = ov::op::v0::Constant::create(ov::element::i64, {1}, {3}); + // the transposed V cache holds one row per value element, [n_seq, rows, ctx]. A model whose layers have + // different V sizes allocates the rows for the widest layer (gemma-4: 512, sliding-window layers use the + // first 256), so keep this layer's n_heads * head_size rows before splitting them into heads + const int64_t cache_rows = cache_shape[3].get_length(); + ov::Output v_rows = std::make_shared( + src, ov::op::v0::Constant::create(ov::element::i64, {3}, {n_seq, cache_rows, ctx_per_seq}), false); + if (cache_rows != n_heads * head_size) { + v_rows = std::make_shared( + v_rows, zero, ov::op::v0::Constant::create(ov::element::i64, {1}, {n_heads * head_size}), one, one); + } auto src_reshaped = std::make_shared( - src, ov::op::v0::Constant::create(ov::element::i64, {4}, {n_seq, n_heads, head_size, ctx_per_seq}), + v_rows, ov::op::v0::Constant::create(ov::element::i64, {4}, {n_seq, n_heads, head_size, ctx_per_seq}), false); ov::Output after_seq_slice; if (n_seq == 1) { diff --git a/ggml/src/ggml-openvino/openvino/op/set_rows.cpp b/ggml/src/ggml-openvino/openvino/op/set_rows.cpp index 4ead927fa4b7..6a69351faf60 100644 --- a/ggml/src/ggml-openvino/openvino/op/set_rows.cpp +++ b/ggml/src/ggml-openvino/openvino/op/set_rows.cpp @@ -48,6 +48,11 @@ OutputVector translate_set_rows(const NodeContext & context) { Output res; if (context.is_stateful()) { int concat_axis = 1; + // the stateful cache appends whole tokens; with `-fa off` llama.cpp writes the V cache transposed, + // one element per row + FRONT_END_OP_CONVERSION_CHECK(dst.get_partial_shape()[2].is_static() && dst.get_partial_shape()[3].is_static(), + "Stateful execution needs flash attention (-fa on): with -fa off the V cache " + "is written transposed"); int64_t dim2 = dst.get_partial_shape()[2].get_length(); int64_t dim3 = dst.get_partial_shape()[3].get_length(); data = std::make_shared( diff --git a/ggml/src/ggml-openvino/openvino/op/softmax.cpp b/ggml/src/ggml-openvino/openvino/op/softmax.cpp index b391d3f91075..f690258ec2b5 100644 --- a/ggml/src/ggml-openvino/openvino/op/softmax.cpp +++ b/ggml/src/ggml-openvino/openvino/op/softmax.cpp @@ -1,22 +1,25 @@ #include "../node_context.h" #include "../op_table.h" #include "../utils.h" +#include "ggml-openvino/ggml-openvino-extra.h" #include #include #include #include -#include #include #include +#include #include #include #include +#include #include #include #include #include #include +#include #include namespace ov { @@ -62,6 +65,45 @@ OutputVector translate_soft_max(const NodeContext & context) { const bool has_sinks = second_input_is_sinks || context.get_input_size() > 2; const size_t sinks_input_idx = second_input_is_sinks ? 1 : 2; + // Scores of a grouped-query MUL_MAT (mulmat) come as [n, n_heads_kv, factor * rows, cols], reshaped to + // [n, n_heads, rows, cols]. Apply the scale, the mask (repeated factor times along the rows) and the softmax to the + // grouped scores and reshape after the softmax: the GPU plugin then fuses them with the MatMuls, while a reshape + // in between leaves a separate pass over the scores per layer, which is large in prefill. + const auto grouped_reshape = ov::as_type_ptr(logits.get_node_shared_ptr()); + if (has_mask && !has_sinks && max_bias <= 0.0f && grouped_reshape && + grouped_reshape->get_rt_info().count(GGML_OV_GROUPED_ROWS) != 0) { + const int64_t factor = grouped_reshape->get_rt_info().at(GGML_OV_GROUPED_ROWS).as(); + ov::Output grouped = grouped_reshape->input_value(0); + if (scale != 1.0f) { + grouped = std::make_shared( + grouped, std::make_shared(grouped.get_element_type(), ov::Shape{}, + std::vector{scale})); + } + ov::Output mask = context.get_input(1); + std::string mask_name = "KQ_mask_sliced"; + if (context.get_input_names()[1].find("swa") != std::string::npos) { + mask_name = "KQ_mask_swa_sliced"; + } + if (context.has_input(mask_name)) { + mask = context.get_input(mask_name); + } + if (mask.get_element_type() != grouped.get_element_type()) { + mask = std::make_shared(mask, grouped.get_element_type()); + } + if (ggml_openvino_is_gpu()) { + // the GPU plugin can fuse this attention into its SDPA kernel, which works on blocks of keys: a block fully + // masked with -inf for a row gives NaN, so use a large finite value (as flash_attn_ext does) + mask = std::make_shared( + mask, std::make_shared(mask.get_element_type(), ov::Shape{}, + std::vector{-30000.0f})); + } + auto mask_tiled = std::make_shared( + mask, ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{1, 1, factor, 1})); + auto probs = std::make_shared(std::make_shared(grouped, mask_tiled), -1); + auto res = std::make_shared(probs, grouped_reshape->input_value(1), true); + return rename_outputs_with_suffix({res}, context.get_name()); + } + // Apply scale first: logits = src0 * scale if (scale != 1.0f) { auto scale_const = diff --git a/ggml/src/ggml-openvino/openvino/utils.h b/ggml/src/ggml-openvino/openvino/utils.h index 1b6b4ea19f20..c98f8d36d91d 100644 --- a/ggml/src/ggml-openvino/openvino/utils.h +++ b/ggml/src/ggml-openvino/openvino/utils.h @@ -73,6 +73,10 @@ inline bool is_kvcache_4d_shape(const ov::PartialShape & shape) { shape[2].is_static() && shape[3].is_static(); } +// rt_info key on the Reshape that restores the heads after a grouped-query MUL_MAT (mulmat): the number of A rows +// stacked per B matrix +constexpr const char * GGML_OV_GROUPED_ROWS = "ggml_ov_grouped_rows"; + namespace op { template OutputVector translate_1to1_match_2_inputs(const NodeContext & context) { num_inputs_check(context, 2, 2); diff --git a/ggml/src/ggml-openvino/utils.cpp b/ggml/src/ggml-openvino/utils.cpp index c9866ef0eadc..49aa5439bfdc 100644 --- a/ggml/src/ggml-openvino/utils.cpp +++ b/ggml/src/ggml-openvino/utils.cpp @@ -44,8 +44,9 @@ namespace { // for that layer) instead of the fully-allocated ctx_per_seq. Pre-conditions: // * non-static (CPU/GPU) backend, single sequence, seq_active_start == 0 // * ggml KV layout is a contiguous [1, 1, ctx_per_seq, n_heads_kv*head_size] -// so the first n_kv rows are the live prefix and shrinking the ctx axis -// gives a valid tensor over the same host storage +// with one token per row (not the transposed V cache of `-fa off`), so the +// first n_kv rows are the live prefix and shrinking the ctx axis gives a +// valid tensor over the same host storage // * not an SWA layer (ring cache): once the window has wrapped the first // n_kv rows no longer contain the live prefix // On any unmet pre-condition returns std::nullopt; the caller falls back to @@ -64,7 +65,7 @@ std::optional try_make_kv_sliced_tensor(const std::shared_ptrget_tensor_used_op(ggml_tensor); - if (!GgmlOvDecoder::is_kvcache(ggml_tensor, op)) { + if (!GgmlOvDecoder::is_kvcache(ggml_tensor, op) || !ggml_decoder->is_kvcache_token_rows(ggml_tensor)) { return std::nullopt; } From e9ab4218ab8100d6116ce1e6efd90ba7bc5acae1 Mon Sep 17 00:00:00 2001 From: "Sabnis, Omkar" Date: Mon, 12 Oct 2026 16:41:57 -0700 Subject: [PATCH 17/20] ggml-openvino: fall back to stateless execution when -fa off With `-fa off`, the graph writes the V cache in transposed form, which is not supported by the stateful KV cache. Detect this pattern in `compute_llm_params()` as a `SET_ROWS` into a KV cache through a `[1, n]` reshape. When stateful execution is enabled, log a warning and fall back to stateless execution instead of failing graph conversion. Update the `-fa off` note in `OPENVINO.md` to document this behavior. Co-authored-by: Suhas Somashekar --- docs/backend/OPENVINO.md | 2 +- ggml/src/ggml-openvino/ggml-decoder.cpp | 5 +++++ ggml/src/ggml-openvino/ggml-decoder.h | 2 ++ ggml/src/ggml-openvino/utils.cpp | 6 ++++++ 4 files changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/backend/OPENVINO.md b/docs/backend/OPENVINO.md index 47ea5aedb70a..2c0e06f3cd91 100644 --- a/docs/backend/OPENVINO.md +++ b/docs/backend/OPENVINO.md @@ -790,7 +790,7 @@ build\ReleaseOV\bin\llama-simple.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M. - `llama-bench`: requires `-fa 1` (flash-attention). - `llama-cli --context-shift`: stateless only (`GGML_OPENVINO_STATEFUL_EXECUTION=0`). In stateful mode the KV cache is owned by the OpenVINO model and cannot be shifted externally. -- `-fa off`: stateless only. With `-fa off` the V cache is stored transposed, which the stateful KV cache does not support. +- `-fa off`: stateless only. With `-fa off` the V cache is stored transposed, which the stateful KV cache does not support; with `GGML_OPENVINO_STATEFUL_EXECUTION=1` the context falls back to stateless execution and logs a warning. - `llama-server`: only one chat session/thread when `GGML_OPENVINO_STATEFUL_EXECUTION=1`. **GPU-specific** diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index 1af9b3cd96ca..a86e7f04b3fe 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -901,6 +901,11 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr bool rope_seen = false; for (int i = 0; i < cgraph->n_nodes; i++) { ggml_tensor * node = cgraph->nodes[i]; + // `-fa off` stores the V cache transposed: SET_ROWS writes it through a [1, n] reshape, one element per row + if (node->op == GGML_OP_SET_ROWS && node->src[2] != nullptr && node->src[2]->op == GGML_OP_RESHAPE && + node->src[2]->ne[0] == 1 && is_kvcache(node->src[2]->src[0], nullptr)) { + model_params.v_trans = true; + } const int attention_pattern_case = get_attention_pattern_case(node); if (attention_pattern_case != -1) { ggml_tensor * cache_k_permute = nullptr; diff --git a/ggml/src/ggml-openvino/ggml-decoder.h b/ggml/src/ggml-openvino/ggml-decoder.h index 635a02e31219..b9068d4885d6 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.h +++ b/ggml/src/ggml-openvino/ggml-decoder.h @@ -33,6 +33,8 @@ struct ModelParams { bool has_rs_rollback = false; bool mixed_rope_params = false; bool is_cacheless_attn = false; + // the graph writes a transposed V cache (`-fa off`: one value element per row, tokens along the row) + bool v_trans = false; std::vector swa_layers; // The sliding-window mask tensor, identified in compute_llm_params() by grouping attention // layers on the mask they consume. Only used to tell the two masks apart when naming OV diff --git a/ggml/src/ggml-openvino/utils.cpp b/ggml/src/ggml-openvino/utils.cpp index 49aa5439bfdc..ab06f2daaf85 100644 --- a/ggml/src/ggml-openvino/utils.cpp +++ b/ggml/src/ggml-openvino/utils.cpp @@ -890,6 +890,12 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, const std::share const int64_t graph_param_compute_time = ggml_time_us() - graph_param_compute_start_time; const bool cache_enabled = !model_is_splitted && !cache_disabled; + if (r_ctx->stateful && m_params.v_trans) { + GGML_LOG_WARN( + "ggml-openvino: disabling stateful execution for this context: the graph writes a transposed V cache " + "(flash attention is off), which the stateful KV cache does not support\n"); + r_ctx->stateful = false; + } const bool stateful_recurrent = r_ctx->stateful && m_params.state_size >= 0; const bool stateful_kv_only = r_ctx->stateful && !stateful_recurrent; if (r_ctx->stateful && From df21ad853e5ba53a18e73a39f987ba186c9f7d7b Mon Sep 17 00:00:00 2001 From: "Sabnis, Omkar" Date: Tue, 13 Oct 2026 12:06:12 -0700 Subject: [PATCH 18/20] ggml-openvino: preserve flat KV cache layout when -fa is off With -fa off, the 4-D KV cache layout introduced in #344 breaks MUL_MAT attention for Phi-3.5 mini and Phi-4 mini, causing oneDNN to fail with "could not construct a memory descriptor using strides". Preserve the [1, 1, ctx, n_heads_kv * head_size] layout when the graph writes a transposed V cache. --- ggml/src/ggml-openvino/ggml-decoder.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index a86e7f04b3fe..cc42b3bb0562 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -1502,6 +1502,11 @@ bool GgmlOvDecoder::is_kvcache_4d(const ggml_tensor * cache) const { if (!enabled || m_is_static || m_is_stateful || m_model_is_splitted || !ggml_openvino_is_gpu()) { return false; } + // `-fa off` (transposed V cache): keep the [1, 1, ctx, n_heads_kv * head_size] layout. The 4-D K cache breaks the + // MUL_MAT attention of the Phi-3 models (Phi-3.5 mini, Phi-4 mini). + if (m_model_params.v_trans) { + return false; + } if (cache->op != GGML_OP_NONE || cache->view_src != nullptr || !is_kvcache(cache, nullptr)) { return false; } From 94ea766460285cda0f9569226e63663b28f613f1 Mon Sep 17 00:00:00 2001 From: ravi9 Date: Tue, 20 Oct 2026 23:29:33 +0530 Subject: [PATCH 19/20] ggml-openvino: extract Q5_0 weights when requantization is off Q5_0 weights are requantized to Q4_0, but test-backend-ops builds naive graphs, where requantization is skipped. Q5_0 then reached the normal weight extraction, which had no Q5_0 case, so every Q5_0 MUL_MAT and GET_ROWS op test failed with "Unsupported quantized type: q5_0". Extract Q5_0 as symmetric i8 weights (q - 16) with one f16 scale per 32 weights, the same layout as Q8_0. Models are unchanged: they still requantize Q5_0 to Q4_0. --- .../src/ggml-openvino/ggml-openvino-extra.cpp | 5 +++ ggml/src/ggml-openvino/ggml-quants.cpp | 36 +++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp index 47a32930c529..1c631ae5a5d7 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp @@ -589,6 +589,11 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten layout.is_symmetric = true; break; + case GGML_TYPE_Q5_0: + // i8 weights (5-bit values minus 16), symmetric (scale only) + layout.is_symmetric = true; + break; + case GGML_TYPE_Q5_1: // u8 weights (5-bit values), asymmetric (scale + zero point) break; diff --git a/ggml/src/ggml-openvino/ggml-quants.cpp b/ggml/src/ggml-openvino/ggml-quants.cpp index 824d24478290..f4e21eac5595 100644 --- a/ggml/src/ggml-openvino/ggml-quants.cpp +++ b/ggml/src/ggml-openvino/ggml-quants.cpp @@ -171,6 +171,38 @@ void extract_q4_1_data(const ggml_tensor * tensor, } } +// Extracts (weight, scales) from Q5_0 tensors. +// Data layout is: |16 bit scale|32 bit qh (5th bits)|32 x 4bit low nibbles|. +// Reconstructed quant q in [0,31]: q = (low nibble) | (qh_bit << 4). Dequant: (q - 16) * d. +// Weights are stored as signed i8 (q - 16), symmetric like Q8_0, so there is no zero point. +void extract_q5_0_data(const ggml_tensor * tensor, ov::Tensor & weights_arr, ov::Tensor & scales_arr) { + const uint64_t bytes_per_block = 22; // 2 scale + 4 qh + 16 (32x0.5) weights + const int qk = 32; + + OPENVINO_ASSERT(weights_arr.get_element_type() == ov::element::i8, "Q5_0 weights are extracted as i8"); + + auto * data = static_cast(tensor->data); + auto * weights = static_cast(weights_arr.data()); + auto * scales = scales_arr.data::value_type>(); + + ov::parallel_for(scales_arr.get_size(), [&](size_t i) { + const uint8_t * block = data + i * bytes_per_block; + uint16_t d; + memcpy(&d, block, sizeof(d)); + uint32_t qh; + memcpy(&qh, block + 2, sizeof(qh)); + const uint8_t * qs = block + 6; + int8_t * dst = weights + i * qk; + scales[i] = ov::float16::from_bits(d); + for (int j = 0; j < qk / 2; ++j) { + const uint8_t bit_lo = (qh >> j) & 1; + const uint8_t bit_hi = (qh >> (j + qk / 2)) & 1; + dst[j] = static_cast(((qs[j] & 0x0F) | (bit_lo << 4)) - 16); // first 16 weights + dst[j + qk / 2] = static_cast(((qs[j] >> 4) | (bit_hi << 4)) - 16); // last 16 weights + } + }); +} + // Extracts (weight, scales, zp) from Q5_1 tensors. // Data layout is: |16 bit scale|16 bit min|32 bit qh (5th bits)|32 x 4bit low nibbles|. // Reconstructed quant q in [0,31]: q = (low nibble) | (qh_bit << 4). Dequant: w*d + m. @@ -1044,6 +1076,7 @@ std::shared_ptr extract_quantized_weights(const ggml_tensor * tensor, weights_per_block = 32; break; case GGML_TYPE_Q8_0: + case GGML_TYPE_Q5_0: case GGML_TYPE_Q5_1: case GGML_TYPE_Q5_K: is_u4 = false; @@ -1076,6 +1109,9 @@ std::shared_ptr extract_quantized_weights(const ggml_tensor * tensor, case GGML_TYPE_Q4_K: extract_q4_k_data(&temp_tensor, weights, scales, zp, use_bias); break; + case GGML_TYPE_Q5_0: + extract_q5_0_data(&temp_tensor, weights, scales); + break; case GGML_TYPE_Q5_1: extract_q5_1_data(&temp_tensor, weights, scales, zp, use_bias); break; From 300a95325527c3c4881f83cd91dc2dde5efba692 Mon Sep 17 00:00:00 2001 From: "Sabnis, Omkar" Date: Thu, 8 Oct 2026 14:05:51 -0700 Subject: [PATCH 20/20] ggml-openvino: add opt-in exact zero points for asymmetric weights Asymmetric quantized weights (Q4_K, Q4_1, Q5_1, and non-requantized Q5_K) represent each group as `w = d * q - m`. The backend currently converts these to u4/u8 weights with an f16 scale and an integer zero point `round(m / d)`, which can shift all weights in a group by up to half a quantization step. Add `GGML_OPENVINO_EXACT_ZERO_POINT=1` to preserve the zero point in f16 instead, using the same path already used for MoE expert weights. This is disabled by default because, with current OpenVINO releases, an f16 zero point moves many GPU fully-connected layers off oneDNN, resulting in slower prompt processing. --- docs/backend/OPENVINO.md | 1 + ggml/src/ggml-openvino/README.md | 2 +- .../src/ggml-openvino/ggml-openvino-extra.cpp | 21 ++++++++++--------- ggml/src/ggml-openvino/ggml-openvino-extra.h | 4 ++++ ggml/src/ggml-openvino/ggml-quants.cpp | 19 +++++++---------- ggml/src/ggml-openvino/model-cache.cpp | 1 + 6 files changed, 25 insertions(+), 23 deletions(-) diff --git a/docs/backend/OPENVINO.md b/docs/backend/OPENVINO.md index 2c0e06f3cd91..d41cec4c7c7a 100644 --- a/docs/backend/OPENVINO.md +++ b/docs/backend/OPENVINO.md @@ -734,6 +734,7 @@ Boolean flags follow a uniform convention: set to a **positive integer** (e.g. ` | `GGML_OPENVINO_PROFILING` | Integer | `0` | `1` logs execution timing; `2` or higher also enables OpenVINO and OpenCL profiling. | | `GGML_OPENVINO_DEBUG_NODE` | String | `not set` | Add the named graph nodes as compiled outputs for debugging. Separate multiple names with commas. | | `GGML_OPENVINO_MOE_OP` | Boolean | `1` | On GPU, set to `0` to keep the unfused GatherMatmul path. | +| `GGML_OPENVINO_EXACT_ZERO_POINT` | Boolean | `0` | Keep the exact zero point (`min / scale`, f16) of asymmetric weights that are not requantized (Q4_1, Q4_K, Q5_1, Q5_K) instead of rounding it to an integer. Improves accuracy. On GPU, use it only with an OpenVINO version whose oneDNN fully-connected path accepts f16 zero points; older versions run these weights on a slower kernel. MoE expert weights always use the exact zero point. | | `GGML_OPENVINO_DUMP_CGRAPH` | Boolean | `0` | Dump the GGML compute graph to `cgraph_ov.txt`. | | `GGML_OPENVINO_DUMP_IR` | Boolean | `0` | Serialize OpenVINO IR files with timestamps. | | `GGML_OPENVINO_DEBUG_INPUT` | Boolean | `0` | Enable input debugging and print input tensor info. | diff --git a/ggml/src/ggml-openvino/README.md b/ggml/src/ggml-openvino/README.md index a12233d2aa17..b9067e76c0ff 100644 --- a/ggml/src/ggml-openvino/README.md +++ b/ggml/src/ggml-openvino/README.md @@ -29,7 +29,7 @@ GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY=1 \ Cache-only mode allocates backend address space without filling weight pages. On Windows, this also uses system commit capacity. The model-buffer size in the loader log is this virtual size. Weight uploads only record source identity; they do not read or requantize the weights. Graph conversion and compilation are skipped. Runtime buffers are still allocated and populated normally. -Cache-only mode uses the settings provided by the current process. Keep these values exactly the same, including set versus unset: `GGML_OPENVINO_REQUANT_KQUANT`, `GGML_OPENVINO_NATIVE_SOFTPLUS`, `GGML_OPENVINO_DISABLE_KV_SLICE`, `GGML_OPENVINO_MANUAL_GQA_ATTN`, `GGML_OPENVINO_STATEFUL_EXECUTION`, `GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT`, `GGML_OPENVINO_DISABLE_REMOTE_OUTPUTS`, `GGML_OPENVINO_REDUCE_COMPILE_MEM`, `GGML_OPENVINO_MEMORY_OPTIMIZE`, `GGML_OPENVINO_PROFILING`, `GGML_OPENVINO_DISABLE_SHAPE_FROM_MASK`, and `GGML_OPENVINO_DEBUG_NODE`. On GPU, also repeat `GGML_OPENVINO_MOE_OP=0` if used. `GGML_OPENVINO_SPILL_DIR` is optional on the first run and ignored in cache-only mode; host-weight release is disabled in cache-only mode. +Cache-only mode uses the settings provided by the current process. Keep these values exactly the same, including set versus unset: `GGML_OPENVINO_REQUANT_KQUANT`, `GGML_OPENVINO_EXACT_ZERO_POINT`, `GGML_OPENVINO_NATIVE_SOFTPLUS`, `GGML_OPENVINO_DISABLE_KV_SLICE`, `GGML_OPENVINO_MANUAL_GQA_ATTN`, `GGML_OPENVINO_STATEFUL_EXECUTION`, `GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT`, `GGML_OPENVINO_DISABLE_REMOTE_OUTPUTS`, `GGML_OPENVINO_REDUCE_COMPILE_MEM`, `GGML_OPENVINO_MEMORY_OPTIMIZE`, `GGML_OPENVINO_PROFILING`, `GGML_OPENVINO_DISABLE_SHAPE_FROM_MASK`, and `GGML_OPENVINO_DEBUG_NODE`. On GPU, also repeat `GGML_OPENVINO_MOE_OP=0` if used. `GGML_OPENVINO_SPILL_DIR` is optional on the first run and ignored in cache-only mode; host-weight release is disabled in cache-only mode. A missing or incompatible graph fails with an error instead of compiling with absent weights. The fingerprint uses the dynamic graph's topology, ports, model parameters, weights, settings, and OpenVINO version; changing only dynamic token or KV sizes does not require a new entry. A different workload can still require another graph; populate it first without cache-only mode. diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp index 1c631ae5a5d7..3a30a546f5f0 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp @@ -134,6 +134,7 @@ void ggml_openvino_device_config::init() { "GGML_OPENVINO_MANUAL_GQA_ATTN", "GGML_OPENVINO_DISABLE_ELTWISE_RANK_ALIGN", "GGML_OPENVINO_MOE_OP", + "GGML_OPENVINO_EXACT_ZERO_POINT", "GGML_OPENVINO_MEMORY_OPTIMIZE", "GGML_OPENVINO_RELEASE_WEIGHTS", "GGML_OPENVINO_REDUCE_COMPILE_MEM", @@ -455,6 +456,10 @@ std::optional ggml_openvino_get_requant_type(const ggml_tensor * } } +bool ggml_openvino_use_exact_zero_point(const ggml_tensor * tensor) { + return tensor->ne[2] > 1 || ggml_openvino_getenv_int("GGML_OPENVINO_EXACT_ZERO_POINT") > 0; +} + // ===================================================== // Extracted Layout Calculation // ===================================================== @@ -474,11 +479,9 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten return layout; } - // 3D MoE expert weights that are not requantized (see below) always use the exact f16 - // zero-point extraction (see extract_quantized_weights), which needs a wider zp slot than - // the packed integer zero point -- must be kept in sync with that function so the buffer - // sizing here matches what process_weight_tensor actually writes. - const bool for_gather_matmul = tensor->ne[2] > 1; + // An exact zero point (see extract_quantized_weights) needs an f16 zp slot instead of the + // packed integer one. Keep in sync with process_weight_tensor, which fills the slot. + const bool exact_zp = ggml_openvino_use_exact_zero_point(tensor); int64_t n_elements = ggml_nelements(tensor); const size_t alignment = 64; // Good for SIMD @@ -618,13 +621,11 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten // Scales: F16 per block, except MXFP4 which stores one E8M0 byte per block. int64_t n_blocks = n_elements / layout.weights_per_block; layout.scales_size = n_blocks * (tensor->type == GGML_TYPE_MXFP4 ? sizeof(uint8_t) : sizeof(uint16_t)); - // For symmetric quantization, no zp needed (weights stored as signed). Asymmetric - // for_gather_matmul (3D MoE expert) weights use an exact f16 zero point (see - // extract_quantized_weights/make_int8_weights/make_int4_weights), which needs one f16 per - // block instead of a packed u4/u8 integer zero point. + // For symmetric quantization, no zp needed (weights stored as signed). An exact zero point + // needs one f16 per block instead of a packed u4/u8 integer zero point. if (layout.is_symmetric) { layout.zp_size = 0; - } else if (use_bias || for_gather_matmul) { + } else if (use_bias || exact_zp) { layout.zp_size = n_blocks * sizeof(uint16_t); } else { layout.zp_size = layout.is_u4 ? ((n_blocks + 1) / 2) : n_blocks; diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.h b/ggml/src/ggml-openvino/ggml-openvino-extra.h index 9b42ac721068..cb0766eb4b21 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.h +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.h @@ -137,6 +137,10 @@ bool ggml_openvino_weight_buffers_released(); // Get requantization type for a tensor type (returns nullopt if no requant needed) std::optional ggml_openvino_get_requant_type(const ggml_tensor * tensor, bool no_requant = false); +// True if an asymmetric weight keeps an exact f16 zero point instead of a rounded integer one. +// Always true for 3D MoE expert weights. Other weights need GGML_OPENVINO_EXACT_ZERO_POINT. +bool ggml_openvino_use_exact_zero_point(const ggml_tensor * tensor); + // ===================================================== // OpenVINO Tensor Extra Types // ===================================================== diff --git a/ggml/src/ggml-openvino/ggml-quants.cpp b/ggml/src/ggml-openvino/ggml-quants.cpp index f4e21eac5595..bb84b3a9e880 100644 --- a/ggml/src/ggml-openvino/ggml-quants.cpp +++ b/ggml/src/ggml-openvino/ggml-quants.cpp @@ -1091,12 +1091,11 @@ std::shared_ptr extract_quantized_weights(const ggml_tensor * tensor, std::string(ggml_type_name(tensor->type))); } - // 3D MoE expert weights (for_gather_matmul) always use the exact f16 zero-point extraction - // (see make_int8_weights/make_int4_weights) rather than the rounded integer zero point -- - // round(min/scale) error is what corrupts Q4_K/Q5_1 experts, and the f16-zp form still fuses - // into GatherMatmulCompressed since it stays a Subtract, not an Add. + // An exact f16 zero point avoids the round(min/scale) error of the integer one, which corrupts + // Q4_K/Q5_1 MoE experts and costs accuracy on dense weights. It stays a Subtract, so the + // weights still fuse into GatherMatmulCompressed / compressed FullyConnected. const bool for_gather_matmul = tensor->ne[2] > 1; - use_bias = use_bias || for_gather_matmul; + use_bias = use_bias || ggml_openvino_use_exact_zero_point(tensor); // Extract quantized data switch (tensor->type) { @@ -1294,13 +1293,9 @@ OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, vo OPENVINO_THROW("Unsupported quantized type: ", ggml_type_name(tensor->type)); } - // 3D MoE expert weights (for_gather_matmul) always use the exact f16 zero-point path (see - // extract_quantized_weights) -- must be kept in sync with the "use_bias || for_gather_matmul" - // check in ggml_openvino_get_extracted_layout, which sizes/offsets the zp slot accordingly. - // Requantized tensors (layout.is_requant) are handled by requantize_to_buffers instead, whose - // zp sizing/type is unaffected by for_gather_matmul, so they are excluded here. - const bool for_gather_matmul = tensor->ne[2] > 1; - const bool zp_is_f16 = !layout.is_requant && (use_bias || for_gather_matmul); + // Keep in sync with ggml_openvino_get_extracted_layout, which sizes the zp slot. Requantized + // tensors are handled by requantize_to_buffers and keep an integer zero point. + const bool zp_is_f16 = !layout.is_requant && (use_bias || ggml_openvino_use_exact_zero_point(tensor)); const bool is_3d_mxfp4_moe = tensor->type == GGML_TYPE_MXFP4 && (tensor->ne[2] > 1 || tensor->ne[3] > 1); if (is_3d_mxfp4_moe) { diff --git a/ggml/src/ggml-openvino/model-cache.cpp b/ggml/src/ggml-openvino/model-cache.cpp index 278f62a25a52..4ddd893eadb0 100644 --- a/ggml/src/ggml-openvino/model-cache.cpp +++ b/ggml/src/ggml-openvino/model-cache.cpp @@ -169,6 +169,7 @@ static const char * cache_settings[] = { "GGML_OPENVINO_NATIVE_SOFTPLUS", "GGML_OPENVINO_DISABLE_KV_SLICE", "GGML_OPENVINO_MANUAL_GQA_ATTN", + "GGML_OPENVINO_EXACT_ZERO_POINT", "GGML_OPENVINO_STATEFUL_EXECUTION", "GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT", "GGML_OPENVINO_DISABLE_REMOTE_OUTPUTS",