From 1b938d012f96cd701fc6cee8598627439fa74686 Mon Sep 17 00:00:00 2001 From: Beheshti Date: Mon, 12 Oct 2026 12:17:06 -0700 Subject: [PATCH 1/4] make embedding to have dynamic shape for OV GPU --- ggml/src/ggml-openvino/ggml-decoder.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index e1bdd9abe0a2..0dc17bfda3b7 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -1209,7 +1209,13 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op, // index list has a static length; on CPU/GPU it may change across compiles (defrag). input_shape = m_is_static ? ov::PartialShape{get_shape(input)} : ov::PartialShape{1, 1, 1, -1}; - } else { + } else if (input != nullptr && op != nullptr && is_inp_emb(input, op)) { + input_shape = ov::PartialShape{get_shape(input)}; + if (!m_is_static) { + input_shape[2] = -1; + } + } + else { input_shape = ov::PartialShape{get_shape(input)}; } if (dynamic_dim_index != -1 && m_model_is_splitted) { From 375f0b76576d1a83238877b2668ede45a14bfaf2 Mon Sep 17 00:00:00 2001 From: Mustafa Cavus Date: Thu, 15 Oct 2026 13:54:15 -0700 Subject: [PATCH 2/4] ggml-openvino: make the 4-D stateless KV cache opt-in --- docs/backend/OPENVINO.md | 2 +- ggml/src/ggml-openvino/ggml-decoder.cpp | 4 ++-- ggml/src/ggml-openvino/ggml-openvino-extra.cpp | 2 +- ggml/src/ggml-openvino/utils.cpp | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/backend/OPENVINO.md b/docs/backend/OPENVINO.md index d1ec2bb74a18..f720f18aa41a 100644 --- a/docs/backend/OPENVINO.md +++ b/docs/backend/OPENVINO.md @@ -722,7 +722,7 @@ Boolean flags follow a uniform convention: set to a **positive integer** (e.g. ` | `GGML_OPENVINO_STATEFUL_EXECUTION`| Boolean | `0` | Keep KV and supported recurrent caches inside the model. Single-slot CPU/GPU execution only. | | `GGML_OPENVINO_DISABLE_CACHE` | Boolean | `0` | Disable the in-process compiled-model / decoder cache (cache is on by default). Set to `1` to disable. | | `GGML_OPENVINO_DISABLE_KV_SLICE` | Boolean | `0` | Disable the KV-cache input-tensor slicing optimization (slicing is on by default on CPU/GPU). Set to `1` to disable. | -| `GGML_OPENVINO_DISABLE_KV_4D` | Boolean | `0` | GPU, stateless: keep the KV cache in the `[1, 1, ctx, n_heads_kv * head_size]` layout instead of `[1, ctx, n_heads_kv, head_size]`, which lets the GPU plugin update and read the cache in place. Set to `1` to disable. | +| `GGML_OPENVINO_ENABLE_KV_4D` | Boolean | `0` | GPU, stateless: bind the KV cache as `[1, ctx, n_heads_kv, head_size]` instead of `[1, 1, ctx, n_heads_kv * head_size]`, so the GPU plugin's `StatelessKVFusion` can update and read the cache in place. **Off by default**: it needs an OpenVINO build with openvino#37569 (2026.5 or a recent nightly) and is much slower on older releases (e.g. 2026.4.1). Set to a positive integer to enable. | | `GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT` | Boolean | `0` | Disable the stateful KV-state sequence-axis relayout (relayout is on by default). It moves the KV state sequence axis from dim 1 to dim 2, so the GPU plugin can append new tokens in place instead of copying the whole state every token, and the reader side no longer transposes the whole accumulated state. Set to `1` to disable. | | `GGML_OPENVINO_DISABLE_SHAPE_FROM_MASK` | Boolean | `0` | Dynamic CPU/GPU graphs read the active sequence count and the attended KV length from the shape of the KQ mask inside the model, so the GPU plugin can infer the attention shapes without waiting for input values and selects its fused SDPA kernel. Set to `1` to pass them as separate inputs instead. | | `GGML_OPENVINO_MANUAL_GQA_ATTN` | Boolean | device-based | Tri-state. When **unset**, manual GQA attention is enabled by default on `GPU` and disabled on other devices. Set to a positive integer to force-enable, or `0` to force-disable. | diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index d27f278cb39b..f28bd8d539f8 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -1487,8 +1487,8 @@ const ggml_tensor * GgmlOvDecoder::get_tensor_used_op(const ggml_tensor * tensor } bool GgmlOvDecoder::is_kvcache_4d(const ggml_tensor * cache) const { - static const bool disabled = ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_KV_4D") != 0; - if (disabled || m_is_static || m_is_stateful || m_model_is_splitted || !ggml_openvino_is_gpu()) { + static const bool enabled = ggml_openvino_getenv_int("GGML_OPENVINO_ENABLE_KV_4D") > 0; + if (!enabled || m_is_static || m_is_stateful || m_model_is_splitted || !ggml_openvino_is_gpu()) { return false; } if (cache->op != GGML_OP_NONE || cache->view_src != nullptr || !is_kvcache(cache, nullptr)) { diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp index 03701b51f328..fc6a4c276b88 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp @@ -129,7 +129,7 @@ void ggml_openvino_device_config::init() { "GGML_OPENVINO_ENABLE_CACHE", "GGML_OPENVINO_DISABLE_CACHE", "GGML_OPENVINO_DISABLE_KV_SLICE", - "GGML_OPENVINO_DISABLE_KV_4D", + "GGML_OPENVINO_ENABLE_KV_4D", "GGML_OPENVINO_ENABLE_FALLBACK", "GGML_OPENVINO_MANUAL_GQA_ATTN", "GGML_OPENVINO_DISABLE_ELTWISE_RANK_ALIGN", diff --git a/ggml/src/ggml-openvino/utils.cpp b/ggml/src/ggml-openvino/utils.cpp index b6d517b149eb..c9866ef0eadc 100644 --- a/ggml/src/ggml-openvino/utils.cpp +++ b/ggml/src/ggml-openvino/utils.cpp @@ -137,7 +137,7 @@ static uint64_t ggml_openvino_model_cache_extra_cfg(bool stateful, bool recurren extra_cfg = extra_cfg * 131 + (stateful ? (recurrent ? 2u : 1u) : 0u); extra_cfg = extra_cfg * 131 + (ggml_openvino_reduce_compile_mem_enabled() ? 1u : 0u); extra_cfg = extra_cfg * 131 + (ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_KV_SLICE") ? 1u : 0u); - extra_cfg = extra_cfg * 131 + (ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_KV_4D") ? 1u : 0u); + extra_cfg = extra_cfg * 131 + (ggml_openvino_getenv_int("GGML_OPENVINO_ENABLE_KV_4D") > 0 ? 1u : 0u); extra_cfg = extra_cfg * 131 + (manual_gqa_enabled ? 1u : 0u); extra_cfg = extra_cfg * 131 + (ggml_openvino_getenv_int("GGML_OPENVINO_PROFILING") >= 2 ? 1u : 0u); extra_cfg = extra_cfg * 131 + (ggml_openvino_shape_from_mask_enabled() ? 1u : 0u); From 592d83696fbe359120e230238c8679c50babf20e Mon Sep 17 00:00:00 2001 From: yzhou51 Date: Fri, 16 Oct 2026 09:50:43 +0800 Subject: [PATCH 3/4] ggml-openvino: Add npuw dump envs for profiling --- .../src/ggml-openvino/ggml-openvino-extra.cpp | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp index fc6a4c276b88..34349a09fac9 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp @@ -122,6 +122,12 @@ void ggml_openvino_device_config::init() { "GGML_OPENVINO_DUMP_IR", "GGML_OPENVINO_DEBUG_INPUT", "GGML_OPENVINO_DEBUG_OUTPUT", + // NPUW dump options require an OpenVINO developer build (NPU_PLUGIN_DEVELOPER_BUILD). + // DUMP_SUBS accepts YES, MIN (unique function bodies), last, or indices such as 0,2. + "GGML_OPENVINO_NPUW_DUMP_SUBS", + "GGML_OPENVINO_NPUW_DUMP_SUBS_DIR", + "GGML_OPENVINO_NPUW_DUMP_SUBS_ON_FAIL", + "GGML_OPENVINO_NPUW_DUMP_IO", // Force the static (NPU-shape) compute path on any device, e.g. GGML_OPENVINO_DEVICE=CPU, // to test the static-shape translation without NPUW/real NPU hardware in the loop. "GGML_OPENVINO_FORCE_STATIC", @@ -183,6 +189,22 @@ void ggml_openvino_device_config::init() { if (compilation_mode_params && strlen(compilation_mode_params) > 0) { compile_config["NPU_COMPILATION_MODE_PARAMS"] = compilation_mode_params; } + const char * ggml_npuw_dump_subs = ggml_openvino_getenv_str("GGML_OPENVINO_NPUW_DUMP_SUBS"); + if (ggml_npuw_dump_subs && strlen(ggml_npuw_dump_subs) > 0) { + compile_config["NPUW_DUMP_SUBS"] = ggml_npuw_dump_subs; + } + const char * ggml_npuw_dump_subs_dir = ggml_openvino_getenv_str("GGML_OPENVINO_NPUW_DUMP_SUBS_DIR"); + if (ggml_npuw_dump_subs_dir && strlen(ggml_npuw_dump_subs_dir) > 0) { + compile_config["NPUW_DUMP_SUBS_DIR"] = ggml_npuw_dump_subs_dir; + } + const char * ggml_npuw_dump_subs_on_fail = ggml_openvino_getenv_str("GGML_OPENVINO_NPUW_DUMP_SUBS_ON_FAIL"); + if (ggml_npuw_dump_subs_on_fail && strlen(ggml_npuw_dump_subs_on_fail) > 0) { + compile_config["NPUW_DUMP_SUBS_ON_FAIL"] = ggml_npuw_dump_subs_on_fail; + } + const char * ggml_npuw_dump_io = ggml_openvino_getenv_str("GGML_OPENVINO_NPUW_DUMP_IO"); + if (ggml_npuw_dump_io && strlen(ggml_npuw_dump_io) > 0) { + compile_config["NPUW_DUMP_IO"] = ggml_npuw_dump_io; + } } else if (cache_dir && strlen(cache_dir) > 0) { compile_config.insert(ov::cache_dir(cache_dir)); compile_config.insert(ov::cache_mode(ov::CacheMode::OPTIMIZE_SIZE)); From f31343d441a491682317cd13e0bfceda9836ffe8 Mon Sep 17 00:00:00 2001 From: yzhou51 Date: Wed, 21 Oct 2026 09:10:51 +0800 Subject: [PATCH 4/4] Add the comments of new npu dump envs into doc --- docs/backend/OPENVINO.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/backend/OPENVINO.md b/docs/backend/OPENVINO.md index 2c0e06f3cd91..87d5182bbc63 100644 --- a/docs/backend/OPENVINO.md +++ b/docs/backend/OPENVINO.md @@ -719,6 +719,10 @@ Boolean flags follow a uniform convention: set to a **positive integer** (e.g. ` | `GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY` | Boolean | `0` | Require an existing compiled blob and skip weight uploads and compilation. Requires Linux or Windows mmap loading and a full dynamic CPU/GPU graph on OpenVINO. | | `GGML_OPENVINO_PREFILL_CHUNK_SIZE`| Integer | `256` | Token chunk size for **NPU** prefill (NPU-only; ignored on CPU/GPU). Must be a positive integer; otherwise the default is used. | | `GGML_OPENVINO_NPU_COMPILE_CONFIG` | String | `not set` | NPU-only compiler mode parameters forwarded to OpenVINO as `NPU_COMPILATION_MODE_PARAMS`, for example `optimization-level=3`. | +| `GGML_OPENVINO_NPUW_DUMP_SUBS` | String | `not set` | NPU-only. Dump NPUW subgraphs for profiling. Requires an OpenVINO developer build with `NPU_PLUGIN_DEVELOPER_BUILD`. Accepts `YES` (all), `MIN` (unique function bodies), `last`, or comma-separated indices such as `0,2`. | +| `GGML_OPENVINO_NPUW_DUMP_SUBS_DIR` | String | `not set` | NPU-only. Directory for NPUW subgraph dumps. Requires an OpenVINO developer build with `NPU_PLUGIN_DEVELOPER_BUILD`. | +| `GGML_OPENVINO_NPUW_DUMP_SUBS_ON_FAIL` | String | `not set` | NPU-only. Forwarded unchanged to OpenVINO as `NPUW_DUMP_SUBS_ON_FAIL` to control subgraph dumps on failure. Requires an OpenVINO developer build with `NPU_PLUGIN_DEVELOPER_BUILD`. | +| `GGML_OPENVINO_NPUW_DUMP_IO` | String | `not set` | NPU-only. Forwarded unchanged to OpenVINO as `NPUW_DUMP_IO` to configure input/output dumps for profiling. Requires an OpenVINO developer build with `NPU_PLUGIN_DEVELOPER_BUILD`. | | `GGML_OPENVINO_STATEFUL_EXECUTION`| Boolean | `0` | Keep KV and supported recurrent caches inside the model. Single-slot CPU/GPU execution only. | | `GGML_OPENVINO_DISABLE_CACHE` | Boolean | `0` | Disable the in-process compiled-model / decoder cache (cache is on by default). Set to `1` to disable. | | `GGML_OPENVINO_DISABLE_KV_SLICE` | Boolean | `0` | Disable the KV-cache input-tensor slicing optimization (slicing is on by default on CPU/GPU). Set to `1` to disable. | @@ -745,6 +749,7 @@ Boolean flags follow a uniform convention: set to a **positive integer** (e.g. ` > - `GGML_OPENVINO_STATEFUL_EXECUTION` is an **Experimental** feature for managing caches internally inside the OpenVINO model on CPUs and GPUs. Use a single slot (`-np 1`). KV caches retain the append-based state layout and sequence-axis optimization. Qwen3.5 adds recurrent cache states in their GGML layouts. Qwen3.5 requires an unsplit graph with model caching enabled and no recurrent rollback. A prompt starting at position 0 resets all states. State save/restore, sequence rewind, context shift, and mid-sequence graph replacement are unsupported. Stateful execution is not effective on NPUs. > - `GGML_OPENVINO_LOG_UNSUPPORTED_OPS` emits logs at `WARN` level (`GGML_LOG_WARN`), which requires application log verbosity `--log-verbosity >= 2` (or `-lv 2`). > - With `GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY=1`, use the same compilation settings as the export run. One directory can hold blobs for different models and settings; `GGML_OPENVINO_SPILL_DIR` does not affect the cache key and is ignored in cache-only mode. See [Compiled model cache](../../ggml/src/ggml-openvino/README.md) for the workflow and restrictions. +> - The `GGML_OPENVINO_NPUW_DUMP_*` variables are for NPU profiling and require an OpenVINO developer build with `NPU_PLUGIN_DEVELOPER_BUILD`. Set `GGML_OPENVINO_DEVICE=NPU`, use `GGML_OPENVINO_NPUW_DUMP_SUBS` to select subgraphs, and optionally set `GGML_OPENVINO_NPUW_DUMP_SUBS_DIR` to choose the output directory. The `DUMP_SUBS_ON_FAIL` and `DUMP_IO` values are passed to OpenVINO unchanged. ### Example Usage