diff --git a/docs/backend/OPENVINO.md b/docs/backend/OPENVINO.md index e77a324a1d35..99a33eaeaae3 100644 --- a/docs/backend/OPENVINO.md +++ b/docs/backend/OPENVINO.md @@ -719,6 +719,10 @@ Boolean flags follow a uniform convention: set to a **positive integer** (e.g. ` | `GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY` | Boolean | `0` | Require an existing compiled blob and skip weight uploads and compilation. Requires Linux or Windows mmap loading and a full dynamic CPU/GPU graph on OpenVINO. | | `GGML_OPENVINO_PREFILL_CHUNK_SIZE`| Integer | `256` | Token chunk size for **NPU** prefill (NPU-only; ignored on CPU/GPU). Must be a positive integer; otherwise the default is used. | | `GGML_OPENVINO_NPU_COMPILE_CONFIG` | String | `not set` | NPU-only compiler mode parameters forwarded to OpenVINO as `NPU_COMPILATION_MODE_PARAMS`, for example `optimization-level=3`. | +| `GGML_OPENVINO_NPUW_DUMP_SUBS` | String | `not set` | NPU-only. Dump NPUW subgraphs for profiling. Requires an OpenVINO developer build with `NPU_PLUGIN_DEVELOPER_BUILD`. Accepts `YES` (all), `MIN` (unique function bodies), `last`, or comma-separated indices such as `0,2`. | +| `GGML_OPENVINO_NPUW_DUMP_SUBS_DIR` | String | `not set` | NPU-only. Directory for NPUW subgraph dumps. Requires an OpenVINO developer build with `NPU_PLUGIN_DEVELOPER_BUILD`. | +| `GGML_OPENVINO_NPUW_DUMP_SUBS_ON_FAIL` | String | `not set` | NPU-only. Forwarded unchanged to OpenVINO as `NPUW_DUMP_SUBS_ON_FAIL` to control subgraph dumps on failure. Requires an OpenVINO developer build with `NPU_PLUGIN_DEVELOPER_BUILD`. | +| `GGML_OPENVINO_NPUW_DUMP_IO` | String | `not set` | NPU-only. Forwarded unchanged to OpenVINO as `NPUW_DUMP_IO` to configure input/output dumps for profiling. Requires an OpenVINO developer build with `NPU_PLUGIN_DEVELOPER_BUILD`. | | `GGML_OPENVINO_GDN_Q_SCALE` | Float | unset (disabled) | NPU static path: enable GDN q pre-scaling and set its scale. A positive integer value enables the optimization; unset, zero, negative, or non-numeric values disable it. Values below `64` are clamped to `64`. | | `GGML_OPENVINO_STATEFUL_EXECUTION`| Boolean | `0` | Keep KV and supported recurrent caches inside the model. Single-slot CPU/GPU execution only. | | `GGML_OPENVINO_DISABLE_CACHE` | Boolean | `0` | Disable the in-process compiled-model / decoder cache (cache is on by default). Set to `1` to disable. | @@ -747,6 +751,7 @@ Boolean flags follow a uniform convention: set to a **positive integer** (e.g. ` > - `GGML_OPENVINO_GDN_Q_SCALE` is for eligible GDN graphs on the NPU static path. Set it to a positive integer such as `64` to enable q pre-scaling; the factor is clamped to at least `64`. Leave it unset or set it to `0` to disable the optimization. > - `GGML_OPENVINO_LOG_UNSUPPORTED_OPS` emits logs at `WARN` level (`GGML_LOG_WARN`), which requires application log verbosity `--log-verbosity >= 2` (or `-lv 2`). > - With `GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY=1`, use the same compilation settings as the export run. One directory can hold blobs for different models and settings; `GGML_OPENVINO_SPILL_DIR` does not affect the cache key and is ignored in cache-only mode. See [Compiled model cache](../../ggml/src/ggml-openvino/README.md) for the workflow and restrictions. +> - The `GGML_OPENVINO_NPUW_DUMP_*` variables are for NPU profiling and require an OpenVINO developer build with `NPU_PLUGIN_DEVELOPER_BUILD`. Set `GGML_OPENVINO_DEVICE=NPU`, use `GGML_OPENVINO_NPUW_DUMP_SUBS` to select subgraphs, and optionally set `GGML_OPENVINO_NPUW_DUMP_SUBS_DIR` to choose the output directory. The `DUMP_SUBS_ON_FAIL` and `DUMP_IO` values are passed to OpenVINO unchanged. ### Example Usage diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp index 42f1806d9f54..763334ad2429 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp @@ -122,6 +122,12 @@ void ggml_openvino_device_config::init() { "GGML_OPENVINO_DUMP_IR", "GGML_OPENVINO_DEBUG_INPUT", "GGML_OPENVINO_DEBUG_OUTPUT", + // NPUW dump options require an OpenVINO developer build (NPU_PLUGIN_DEVELOPER_BUILD). + // DUMP_SUBS accepts YES, MIN (unique function bodies), last, or indices such as 0,2. + "GGML_OPENVINO_NPUW_DUMP_SUBS", + "GGML_OPENVINO_NPUW_DUMP_SUBS_DIR", + "GGML_OPENVINO_NPUW_DUMP_SUBS_ON_FAIL", + "GGML_OPENVINO_NPUW_DUMP_IO", // Force the static (NPU-shape) compute path on any device, e.g. GGML_OPENVINO_DEVICE=CPU, // to test the static-shape translation without NPUW/real NPU hardware in the loop. "GGML_OPENVINO_FORCE_STATIC", @@ -184,6 +190,22 @@ void ggml_openvino_device_config::init() { if (compilation_mode_params && strlen(compilation_mode_params) > 0) { compile_config["NPU_COMPILATION_MODE_PARAMS"] = compilation_mode_params; } + const char * ggml_npuw_dump_subs = ggml_openvino_getenv_str("GGML_OPENVINO_NPUW_DUMP_SUBS"); + if (ggml_npuw_dump_subs && strlen(ggml_npuw_dump_subs) > 0) { + compile_config["NPUW_DUMP_SUBS"] = ggml_npuw_dump_subs; + } + const char * ggml_npuw_dump_subs_dir = ggml_openvino_getenv_str("GGML_OPENVINO_NPUW_DUMP_SUBS_DIR"); + if (ggml_npuw_dump_subs_dir && strlen(ggml_npuw_dump_subs_dir) > 0) { + compile_config["NPUW_DUMP_SUBS_DIR"] = ggml_npuw_dump_subs_dir; + } + const char * ggml_npuw_dump_subs_on_fail = ggml_openvino_getenv_str("GGML_OPENVINO_NPUW_DUMP_SUBS_ON_FAIL"); + if (ggml_npuw_dump_subs_on_fail && strlen(ggml_npuw_dump_subs_on_fail) > 0) { + compile_config["NPUW_DUMP_SUBS_ON_FAIL"] = ggml_npuw_dump_subs_on_fail; + } + const char * ggml_npuw_dump_io = ggml_openvino_getenv_str("GGML_OPENVINO_NPUW_DUMP_IO"); + if (ggml_npuw_dump_io && strlen(ggml_npuw_dump_io) > 0) { + compile_config["NPUW_DUMP_IO"] = ggml_npuw_dump_io; + } } else if (cache_dir && strlen(cache_dir) > 0) { compile_config.insert(ov::cache_dir(cache_dir)); compile_config.insert(ov::cache_mode(ov::CacheMode::OPTIMIZE_SIZE));