diff --git a/.devops/openvino.Dockerfile b/.devops/openvino.Dockerfile
index a43e5c4993f8..13301ba287dd 100644
--- a/.devops/openvino.Dockerfile
+++ b/.devops/openvino.Dockerfile
@@ -1,12 +1,12 @@
-ARG OPENVINO_VERSION_MAJOR=2026.3
-ARG OPENVINO_VERSION_FULL=2026.3.0.22451.bd8d6542e3c
+ARG OPENVINO_VERSION_MAJOR=2026.3.1
+ARG OPENVINO_VERSION_FULL=2026.3.1.22476.56d9685302d
ARG UBUNTU_VERSION=24.04
# Intel GPU driver versions. https://github.com/intel/compute-runtime/releases
-ARG IGC_VERSION=v2.38.2
-ARG IGC_VERSION_FULL=2_2.38.2+22051
-ARG COMPUTE_RUNTIME_VERSION=26.27.39122.11
-ARG COMPUTE_RUNTIME_VERSION_FULL=26.27.39122.11-0
+ARG IGC_VERSION=v2.40.13
+ARG IGC_VERSION_FULL=2_2.40.13+22418
+ARG COMPUTE_RUNTIME_VERSION=26.31.39395.13
+ARG COMPUTE_RUNTIME_VERSION_FULL=26.31.39395.13-0
ARG IGDGMM_VERSION=22.10.0
# Intel NPU driver versions. https://github.com/intel/linux-npu-driver/releases
diff --git a/.github/workflows/build-cache.yml b/.github/workflows/build-cache.yml
index 187427a8d4b0..4a23ec2d4d36 100644
--- a/.github/workflows/build-cache.yml
+++ b/.github/workflows/build-cache.yml
@@ -41,8 +41,8 @@ jobs:
env:
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
- OPENVINO_VERSION_MAJOR: "2026.3"
- OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
+ OPENVINO_VERSION_MAJOR: "2026.3.1"
+ OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
steps:
- name: Clone
@@ -69,8 +69,8 @@ jobs:
env:
# Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
- OPENVINO_VERSION_MAJOR: "2026.3"
- OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
+ OPENVINO_VERSION_MAJOR: "2026.3.1"
+ OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
steps:
- name: Clone
diff --git a/.github/workflows/build-openvino.yml b/.github/workflows/build-openvino.yml
index 0316e7ad97e3..8e0326f4a44f 100644
--- a/.github/workflows/build-openvino.yml
+++ b/.github/workflows/build-openvino.yml
@@ -32,6 +32,8 @@ env:
LLAMA_ARG_LOG_COLORS: 1
LLAMA_ARG_LOG_PREFIX: 1
LLAMA_ARG_LOG_TIMESTAMPS: 1
+ # TODO: fix and re-enable the `test-llama-archs` and `test-recurrent-state-rollback`
+ CTEST_EXCLUDE: "test-llama-archs|^test-recurrent-state-rollback"
jobs:
ubuntu-24-openvino:
@@ -39,8 +41,8 @@ jobs:
env:
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
- OPENVINO_VERSION_MAJOR: "2026.3"
- OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
+ OPENVINO_VERSION_MAJOR: "2026.3.1"
+ OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
steps:
- name: Clone
@@ -78,26 +80,24 @@ jobs:
- name: Test (CPU)
id: cmake_test_cpu
- # TODO: fix and re-enable the `test-llama-archs` test below
run: |
cd ${{ github.workspace }}
- ctest --test-dir build/ReleaseOV -L main -E "test-llama-archs|test-recurrent-state-rollback-nemotron-h" --verbose --timeout 2000
+ ctest --test-dir build/ReleaseOV -L main -E "${{ env.CTEST_EXCLUDE }}" --verbose --timeout 3000
- name: Test (GPU)
id: cmake_test_gpu
- # TODO: fix and re-enable the `test-llama-archs` test below
run: |
cd ${{ github.workspace }}
export GGML_OPENVINO_DEVICE=GPU
- ctest --test-dir build/ReleaseOV -L main -E "test-llama-archs|test-recurrent-state-rollback-nemotron-h" --verbose --timeout 3000
+ ctest --test-dir build/ReleaseOV -L main -E "${{ env.CTEST_EXCLUDE }}" --verbose --timeout 3000
openvino-windows-2022:
runs-on: windows-2022
env:
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
- OPENVINO_VERSION_MAJOR: "2026.3"
- OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
+ OPENVINO_VERSION_MAJOR: "2026.3.1"
+ OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
steps:
- name: Clone
@@ -159,14 +159,13 @@ jobs:
- name: Test (CPU)
id: cmake_test_cpu
shell: cmd
- # TODO: fix and re-enable the `test-llama-archs` test below
run: |
REM Find extracted OpenVINO folder dynamically
for /d %%i in (openvino_toolkit\*) do set OPENVINO_ROOT=%%i
call "%OPENVINO_ROOT%\setupvars.bat"
cd build
- ctest --test-dir ReleaseOV -L main -E "test-llama-archs|test-recurrent-state-rollback-nemotron-h" -C Release --verbose --timeout 3000
+ ctest --test-dir ReleaseOV -L main -E "${{ env.CTEST_EXCLUDE }}" -C Release --verbose --timeout 3000
- name: ccache-clear
uses: ./.github/actions/ccache-clear
diff --git a/.github/workflows/build-self-hosted.yml b/.github/workflows/build-self-hosted.yml
index fe2ab815473c..ccfe2a604645 100644
--- a/.github/workflows/build-self-hosted.yml
+++ b/.github/workflows/build-self-hosted.yml
@@ -288,8 +288,8 @@ jobs:
env:
# Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
- OPENVINO_VERSION_MAJOR: "2026.3"
- OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
+ OPENVINO_VERSION_MAJOR: "2026.3.1"
+ OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
steps:
- name: Clone
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index ab5d3389bd58..76717d064bd0 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -415,8 +415,8 @@ jobs:
env:
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
- OPENVINO_VERSION_MAJOR: "2026.3"
- OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
+ OPENVINO_VERSION_MAJOR: "2026.3.1"
+ OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
steps:
- name: Set OpenVINO version output
@@ -529,8 +529,8 @@ jobs:
env:
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
- OPENVINO_VERSION_MAJOR: "2026.3"
- OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
+ OPENVINO_VERSION_MAJOR: "2026.3.1"
+ OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
steps:
- name: Set OpenVINO version output
@@ -714,10 +714,10 @@ jobs:
with:
key: release-windows-2025-vs2026-${{ matrix.arch }}-cpu
- # TODO: build only the ggml-hip backend like the other windows backend jobs
- # (windows-cuda, windows-sycl), then drop the ui-build dependency
+ # note: builds only the ggml-hip backend - llama-server is injected from the
+ # windows-cpu zip during the release "Merge artifacts" step
windows-rocm:
- needs: [check-release, ui-build]
+ needs: [check-release]
if: ${{ needs.check-release.outputs.should_release == 'true' }}
runs-on: windows-2022
@@ -736,12 +736,6 @@ jobs:
with:
fetch-depth: 0
- - name: Download UI build
- uses: actions/download-artifact@v7
- with:
- name: llama-ui.zip
- path: tools/ui/dist
-
- name: Install Ninja
run: |
choco install ninja
@@ -804,17 +798,15 @@ jobs:
-DCMAKE_PREFIX_PATH="${env:HIP_PATH}" `
-DGGML_BACKEND_DL=ON `
-DGGML_NATIVE=OFF `
- -DGGML_CPU=ON `
- -DGGML_CPU_ALL_VARIANTS=ON `
+ -DGGML_CPU=OFF `
-DGGML_HIP=ON `
-DCMAKE_C_COMPILER="${env:HIP_PATH}\lib\llvm\bin\clang.exe" `
-DCMAKE_CXX_COMPILER="${env:HIP_PATH}\lib\llvm\bin\clang++.exe" `
-DCMAKE_C_FLAGS="-Wno-error=incompatible-pointer-types" `
-DCMAKE_HIP_COMPILER="${env:HIP_PATH}\lib\llvm\bin\clang.exe" `
-DHIP_PATH="${env:HIP_PATH}" `
- -DGGML_HIP_ROCWMMA_FATTN=ON `
-DAMDGPU_TARGETS="${{ matrix.gpu_targets }}"
- cmake --build build --config Release --parallel ${env:NUMBER_OF_PROCESSORS}
+ cmake --build build --config Release --parallel ${env:NUMBER_OF_PROCESSORS} --target ggml-hip
- name: Verify HIP backend was built
run: |
@@ -866,8 +858,11 @@ jobs:
- name: Pack artifacts
run: |
- cp "LICENSE" "build\bin\Release\"
- 7z a -snl llama-bin-win-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.zip .\build\bin\Release\*
+ 7z a -snl llama-bin-win-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.zip `
+ .\build\bin\Release\ggml-hip.dll `
+ .\build\bin\Release\amdhip64_7.dll `
+ .\build\bin\Release\rocm_kpack.dll `
+ .\build\bin\Release\amd_comgr.dll
- name: Upload artifacts
uses: actions/upload-artifact@v6
diff --git a/ci/run.sh b/ci/run.sh
index 1f1e4bc033c9..1701bc7ed058 100755
--- a/ci/run.sh
+++ b/ci/run.sh
@@ -189,8 +189,8 @@ if [ ! -z ${GG_BUILD_OPENVINO} ]; then
fi
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_OPENVINO=ON"
- # TODO: fix and re-enable the `test-llama-archs` test below
- CTEST_EXTRA="-E test-llama-archs|test-recurrent-state-rollback-nemotron-h"
+ # TODO: fix and re-enable the `test-llama-archs` and `test-recurrent-state-rollback*`
+ CTEST_EXTRA="-E test-llama-archs|^test-recurrent-state-rollback"
fi
## helpers
diff --git a/common/arg.cpp b/common/arg.cpp
index e4f694b84b3e..f559837f1b9b 100644
--- a/common/arg.cpp
+++ b/common/arg.cpp
@@ -1643,6 +1643,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
}
}
).set_env("LLAMA_ARG_CTX_SIZE"));
+ add_opt(common_arg(
+ { "--kv-unified-per-slot" }, "N",
+ "context limit per parallel slot (default: unset, behavior unchanged).\n"
+ "when set without -c/--ctx-size, the shared KV pool is sized to n_parallel*N",
+ [](common_params & params, int value) {
+ params.kv_unified_per_slot = value;
+ }
+ ).set_env("LLAMA_ARG_KV_UNIFIED_PER_SLOT").set_examples({ LLAMA_EXAMPLE_SERVER }));
add_opt(common_arg(
{"-n", "--predict", "--n-predict"}, "N",
string_format(
@@ -2721,18 +2729,18 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
}
).set_env("LLAMA_ARG_LOAD_MODE"));
add_opt(common_arg(
- {"--tensor-read-lazy"}, "MODE",
+ {"-lzm", "--lazy-mode"}, "MODE",
"on-demand reading of certain tensors, for example per-layer embeddings (default: auto)\n"
"- on: read the rows of such tensors from disk on demand instead of keeping them resident (requires mmap)\n"
"- auto: on, but only for tensors larger than 4 GiB\n"
"- off: always keep them resident",
[](common_params & params, const std::string & value) {
- /**/ if (value == "on") { params.tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_ON; }
- else if (value == "auto") { params.tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_AUTO; }
- else if (value == "off") { params.tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_OFF; }
+ /**/ if (value == "on") { params.lazy_mode = LLAMA_LAZY_MODE_ON; }
+ else if (value == "auto") { params.lazy_mode = LLAMA_LAZY_MODE_AUTO; }
+ else if (value == "off") { params.lazy_mode = LLAMA_LAZY_MODE_OFF; }
else { throw std::invalid_argument("invalid value"); }
}
- ).set_env("LLAMA_ARG_TENSOR_READ_LAZY"));
+ ).set_env("LLAMA_ARG_LAZY_MODE"));
add_opt(common_arg(
{"--numa"}, "TYPE",
"attempt optimizations that help on some NUMA systems\n"
diff --git a/common/common.cpp b/common/common.cpp
index 347e8e9fc416..d162a38800e0 100644
--- a/common/common.cpp
+++ b/common/common.cpp
@@ -1688,7 +1688,7 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {
mparams.main_gpu = params.main_gpu;
mparams.split_mode = params.split_mode;
mparams.load_mode = params.load_mode;
- mparams.tensor_read_lazy = params.tensor_read_lazy;
+ mparams.lazy_mode = params.lazy_mode;
mparams.tensor_split = params.tensor_split;
mparams.check_tensors = params.check_tensors;
mparams.use_extra_bufts = !params.no_extra_bufts;
diff --git a/common/common.h b/common/common.h
index 478825b6b3ff..b865b6df324b 100644
--- a/common/common.h
+++ b/common/common.h
@@ -505,7 +505,7 @@ struct common_params {
enum llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER; // how to split the model across GPUs
enum llama_load_mode load_mode = LLAMA_LOAD_MODE_AUTO; // how to load the model
- enum llama_tensor_read_lazy tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_AUTO; // on-demand reading of tensors marked by the arch
+ enum llama_lazy_mode lazy_mode = LLAMA_LAZY_MODE_AUTO; // on-demand reading of tensors marked by the arch
common_cpu_params cpuparams;
common_cpu_params cpuparams_batch;
@@ -649,6 +649,7 @@ struct common_params {
bool cache_prompt = true; // whether to enable prompt caching
bool cache_idle_slots = true; // save and clear idle slots upon starting a new task
int32_t n_ctx_checkpoints = 32; // max number of context checkpoints per slot
+ int32_t kv_unified_per_slot = 0; // max context per parallel slot; 0 = unset
int32_t checkpoint_min_step = 8192; // minimum spacing between context checkpoints
int32_t cache_ram_mib = 8192; // -1 = no limit, 0 - disable, 1 = 1 MiB, etc.
diff --git a/common/speculative.cpp b/common/speculative.cpp
index 2e90d8e59a43..f4968994a83e 100644
--- a/common/speculative.cpp
+++ b/common/speculative.cpp
@@ -1016,6 +1016,9 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
// dspark speculators
bool sample_from_anchor = true;
+ // block-internal attention
+ bool causal_attn = false;
+
const int32_t * target_layer_ids = nullptr; // model_dft's extract layer indices
uint32_t target_layer_ids_n = 0;
@@ -1053,12 +1056,25 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
if (llama_model_meta_val_str(model_dft, "dflash.sample_from_anchor", buf, sizeof(buf)) >= 0) {
sample_from_anchor = std::strcmp(buf, "true") == 0;
}
+ if (llama_model_meta_val_str(model_dft, "dflash.attention.causal", buf, sizeof(buf)) >= 0) {
+ causal_attn = std::strcmp(buf, "true") == 0;
+ }
}
selector_top_k = llama_model_dflash_selector_top_k(model_dft);
is_dflash2 = selector_top_k > 0;
mask_token_id = llama_vocab_mask(llama_model_get_vocab(model_dft));
+ if (is_dspark && this->params.p_min > 0.0f) {
+ char buf[16] = {};
+ const bool has_conf =
+ llama_model_meta_val_str(model_dft, "dflash.has_confidence_head", buf, sizeof(buf)) < 0 ||
+ std::strcmp(buf, "true") == 0;
+ if (!has_conf) {
+ throw std::runtime_error("DSpark draft has no confidence head: please set --spec-draft-p-min 0");
+ }
+ }
+
LOG_INF("%s: adding speculative implementation '%s'\n", __func__, common_speculative_type_to_str(type).c_str());
adaptive_n = this->params.adaptive;
LOG_INF("%s: - n_max=%d, n_min=%d, p_min=%.2f\n", __func__, this->params.n_max, this->params.n_min, this->params.p_min);
@@ -1118,7 +1134,7 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
// DFlash2 reads its selector lattice from h_nextn and never consumes raw logits.
llama_set_embeddings_nextn(ctx_dft, true, /*masked*/ !is_dflash2);
- llama_set_causal_attn(ctx_dft, false); // DFlash needs non-causal attention
+ llama_set_causal_attn(ctx_dft, causal_attn); // DFlash needs non-causal attention unless the model says otherwise
}
~common_speculative_impl_draft_dflash() override {
diff --git a/conversion/qwen.py b/conversion/qwen.py
index c5297418c99b..419611896fc3 100644
--- a/conversion/qwen.py
+++ b/conversion/qwen.py
@@ -709,14 +709,20 @@ def set_gguf_parameters(self):
extract_layer_ids = [i + 1 for i in target_layer_ids]
self.gguf_writer.add_target_layers(extract_layer_ids)
- use_sliding_window = self.hparams.get("use_sliding_window", False)
- sliding_window = self.hparams.get("sliding_window")
+ use_sliding_window = self.hparams.get("use_sliding_window", False) or dflash_config.get("use_swa", False)
+ sliding_window = dflash_config.get("swa_window_size") or self.hparams.get("sliding_window")
layer_types = self.hparams.get("layer_types")
if use_sliding_window and sliding_window and layer_types:
is_swa = [lt == "sliding_attention" for lt in layer_types]
self.gguf_writer.add_sliding_window(sliding_window)
self.gguf_writer.add_sliding_window_pattern(is_swa)
+ causal = self.hparams.get("is_causal")
+ if causal is None:
+ causal = dflash_config.get("causal")
+ if causal is not None:
+ self.gguf_writer.add_causal_attention(bool(causal))
+
# M-RoPE target: the draft ropes on the temporal dim only, so write
# degenerate sections [n_rot/2, 0, 0, 0]
if self._target_uses_mrope():
@@ -737,6 +743,8 @@ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Ca
name, gen = item
if not name.startswith("model."):
name = "model." + name
+ if "sink" in name and not name.endswith(".weight"):
+ name += ".weight"
return super().filter_tensors((name, gen))
_ROPE_PERMUTE_SUFFIXES = (
@@ -815,6 +823,10 @@ def set_gguf_parameters(self):
super().set_gguf_parameters()
self.gguf_writer.add_sample_from_anchor(self._sample_from_anchor)
+ # confidence head is optional: vanilla-markov exports ship without it
+ has_conf = any("confidence_head.proj" in name for name in self.model_tensors)
+ self.gguf_writer.add_has_confidence_head(has_conf)
+
@classmethod
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
if item[0] == "t2d": # not used at runtime
@@ -833,7 +845,7 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter
self._d2t = data_torch
return
- if self._n_vocab_draft == self.hparams["vocab_size"] and name.endswith(("embed_tokens.weight", "lm_head.weight")):
+ if self._n_vocab_draft == self.hparams["vocab_size"] and name.endswith("lm_head.weight"):
return
# interleaved-rope checkpoints (rope_is_neox_style = false) -> NeoX layout: per head, even dims first then odd
diff --git a/docs/backend/OPENVINO.md b/docs/backend/OPENVINO.md
index 3cdf631cebc2..9b43807d36b3 100644
--- a/docs/backend/OPENVINO.md
+++ b/docs/backend/OPENVINO.md
@@ -22,8 +22,8 @@ The OpenVINO backend is implemented in `ggml/src/ggml-openvino` and provides a t
- [0. Prerequisites](#0-prerequisites)
- [1. Install OpenVINO Runtime](#1-install-openvino-runtime)
- [2. Build llama.cpp with OpenVINO Backend](#2-build-llamacpp-with-openvino-backend)
- - [Automated Ubuntu Build Script](#automated-ubuntu-build-script)
- - [Automated Windows Build Script](#automated-windows-build-script)
+ - [Ubuntu Build Script](#ubuntu-build-script)
+ - [Windows Build Script](#windows-build-script)
- [3. Download Sample Model](#3-download-sample-model)
- [4. Run Inference with OpenVINO Backend](#4-run-inference-with-openvino-backend)
- [5. Docker Build](#5-docker-build)
@@ -96,7 +96,7 @@ Although, the validated models below were tested with `llama-cli` using the `Q4_
- **SL** = Stateless (`GGML_OPENVINO_STATEFUL_EXECUTION=0`)
- **SF** = Stateful (`GGML_OPENVINO_STATEFUL_EXECUTION=1`)
- Note: The NPU operates in stateless mode only.
-- **Validation system:** Intel® Core™ Ultra 5 238V (Lunar Lake) | 32 GB RAM | Ubuntu 24.04 | Intel OpenCL GPU Driver 26.18.38308.1 | Intel NPU Driver 1.33.0.
+- **Validation system:** Intel® Core™ Ultra 5 238V (Lunar Lake) | 32 GB RAM | Ubuntu 24.04 | Intel OpenCL GPU Driver 26.31.39395.13-0 | Intel NPU Driver 1.35.0.
- See [Known Limitations](#known-limitations) for context on observed failures.
| Model | CPU (SL / SF) | GPU (SL / SF) | NPU (SL) |
@@ -105,27 +105,32 @@ Although, the validated models below were tested with `llama-cli` using the `Q4_
| [bartowski/Llama-3.2-3B-Instruct-Q4_K_M](https://huggingface.co/bartowski/Llama-3.2-3B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| [bartowski/Meta-Llama-3.1-8B-Instruct-Q4_K_M](https://huggingface.co/bartowski/Meta-Llama-3.1-8B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| | | | |
-| [Qwen/qwen2.5-1.5b-instruct-q4_k_m](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
-| [Qwen/qwen2.5-coder-7b-instruct-q4_k_m](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
-| [bartowski/Qwen_Qwen3-0.6B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3-0.6B-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
-| [bartowski/Qwen_Qwen3-1.7B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3-1.7B-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
-| [Qwen/Qwen3-4B-Q4_K_M](https://huggingface.co/Qwen/Qwen3-4B-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
-| [lm-kit/Qwen3-8B-Q4_K_M](https://huggingface.co/lm-kit/qwen-3-8b-instruct-gguf) | ✓ / ✓ | ✓ / ✗ | ✓ |
+| [Qwen/qwen2.5-1.5b-instruct-q4_k_m](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
+| [Qwen/qwen2.5-coder-7b-instruct-q4_k_m](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
+| [bartowski/Qwen_Qwen3-0.6B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3-0.6B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
+| [bartowski/Qwen_Qwen3-1.7B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3-1.7B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
+| [Qwen/Qwen3-4B-Q4_K_M](https://huggingface.co/Qwen/Qwen3-4B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
+| [lm-kit/Qwen3-8B-Q4_K_M](https://huggingface.co/lm-kit/qwen-3-8b-instruct-gguf) | ✓ / ✓ | ✓ / ✓ | ✓ |
+| [bartowski/Qwen_Qwen3.5-0.8B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3.5-0.8B-GGUF) | ✓ / ✗ | ✓ / ✗ | ✗ |
+| [bartowski/Qwen_Qwen3.5-2B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3.5-2B-GGUF) | ✓ / ✗ | ✓ / ✗ | ✗ |
+| [bartowski/Qwen_Qwen3.5-4B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3.5-4B-GGUF) | ✓ / ✗ | ✓ / ✗ | ✗ |
+| [lmstudio-community/Qwen3.5-9B-Q4_K_M](https://huggingface.co/lmstudio-community/Qwen3.5-9B-GGUF) | ✓ / ✗ | ✓ / ✗ | ✗ |
| | | | |
-| [unsloth/gemma-3-4b-it-Q4_K_M](https://huggingface.co/unsloth/gemma-3-4b-it-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
-| [bartowski/google_gemma-4-E2B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E2B-it-GGUF) | ✓ / ✗ | ✓ / ✗ | ✓ |
+| [unsloth/gemma-3-4b-it-Q4_K_M](https://huggingface.co/unsloth/gemma-3-4b-it-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
+| [bartowski/google_gemma-4-E2B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E2B-it-GGUF) | ✓ / ✗ | ✓ / ✗ | ✗ |
| [bartowski/google_gemma-4-E4B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E4B-it-GGUF) | ✓ / ✗ | ✓ / ✗ | ✓ |
-| [bartowski/gemma-4-12B-it-Q4_K_M](https://huggingface.co/bartowski/gemma-4-12B-it-GGUF) | ✓ / ✗ | ✓ / ✗ | ✗ |
+| [bartowski/gemma-4-12B-it-Q4_K_M](https://huggingface.co/bartowski/gemma-4-12B-it-GGUF) | ✓ / ✗ | ✓ / ✗ | ✓ |
| | | | |
-| [bartowski/Phi-3-mini-4k-instruct-Q4_K_M](https://huggingface.co/bartowski/Phi-3-mini-4k-instruct-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
-| [bartowski/Phi-3.5-mini-instruct-Q4_K_M](https://huggingface.co/bartowski/Phi-3.5-mini-instruct-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
+| [bartowski/Phi-3-mini-4k-instruct-Q4_K_M](https://huggingface.co/bartowski/Phi-3-mini-4k-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
+| [bartowski/Phi-3.5-mini-instruct-Q4_K_M](https://huggingface.co/bartowski/Phi-3.5-mini-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
+| [bartowski/microsoft_Phi-4-mini-instruct-Q4_K_M](https://huggingface.co/bartowski/microsoft_Phi-4-mini-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| | | | |
| [bartowski/Mistral-7B-Instruct-v0.3-Q4_K_M](https://huggingface.co/bartowski/Mistral-7B-Instruct-v0.3-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| [QuantFactory/Ministral-3b-instruct.Q4_K_M](https://huggingface.co/QuantFactory/Ministral-3b-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| [bartowski/Ministral-8B-Instruct-2410-Q4_K_M](https://huggingface.co/bartowski/Ministral-8B-Instruct-2410-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| | | | |
| [bartowski/DeepSeek-R1-Distill-Llama-8B-Q4_K_M](https://huggingface.co/bartowski/DeepSeek-R1-Distill-Llama-8B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
-| [bartowski/DeepSeek-R1-Distill-Qwen-7B-Q4_K_M](https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
+| [bartowski/DeepSeek-R1-Distill-Qwen-7B-Q4_K_M](https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| | | | |
| [ibm-granite/granite-4.0-350m-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-350m-GGUF) | ✓ / ✓ | ✗ / ✗ | ✓ |
| [ibm-granite/granite-4.0-micro-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-micro-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
@@ -133,10 +138,10 @@ Although, the validated models below were tested with `llama-cli` using the `Q4_
| [ibm-research/granite-3.2-8b-instruct-Q4_K_M](https://huggingface.co/ibm-research/granite-3.2-8b-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| | | | |
| [HuggingFaceTB/smollm2-1.7b-instruct-q4_k_m](https://huggingface.co/HuggingFaceTB/SmolLM2-1.7B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
-| [openbmb/MiniCPM-V-2_6-Q4_K_M](https://huggingface.co/openbmb/MiniCPM-V-2_6-gguf) | ✓ / ✓ | ✓ / ✗ | ✓ |
-| [bartowski/tencent_Hunyuan-7B-Instruct-Q4_K_M](https://huggingface.co/bartowski/tencent_Hunyuan-7B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
-| [LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct-Q4_K_M](https://huggingface.co/LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
-| [bartowski/prism-ml_Bonsai-8B-unpacked-Q4_K_M](https://huggingface.co/bartowski/prism-ml_Bonsai-8B-unpacked-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
+| [openbmb/MiniCPM-V-2_6-Q4_K_M](https://huggingface.co/openbmb/MiniCPM-V-2_6-gguf) | ✓ / ✓ | ✓ / ✓ | ✓ |
+| [bartowski/tencent_Hunyuan-7B-Instruct-Q4_K_M](https://huggingface.co/bartowski/tencent_Hunyuan-7B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
+| [LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct-Q4_K_M](https://huggingface.co/LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
+| [bartowski/prism-ml_Bonsai-8B-unpacked-Q4_K_M](https://huggingface.co/bartowski/prism-ml_Bonsai-8B-unpacked-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| | | | |
| [gpustack/bge-m3-Q4_K_M.gguf](https://huggingface.co/gpustack/bge-m3-GGUF) | ✓ | ✗ | ✗ |
@@ -217,18 +222,18 @@ cmake --build build\ReleaseOV --parallel
> [!NOTE]
> The Windows install path is `C:\Intel\openvino` (no spaces) to avoid quoting problems some CMake/Ninja toolchains have with `C:\Program Files (x86)\...`. Adjust to wherever you installed OpenVINO Runtime. From `cmd`, run `C:\Intel\openvino\setupvars.bat`; from PowerShell, run `& "C:\Intel\openvino\setupvars.ps1"` instead. Once the build is finished you can launch the binaries from any `cmd` or `PowerShell` window after sourcing the matching `setupvars` script for that shell.
-#### Automated Ubuntu Build Script
+#### Ubuntu Build Script
For Ubuntu24 users, the following shell script automates the prerequisite installs (build tools, OpenCL ICD), the OpenVINO Runtime download/extract/setup, and the Ninja-based llama.cpp build.
-Save the following as `ubuntu-llamacpp-ov-install.sh` next to where you want the `llama.cpp` folder to land, then run it:
+Save the following as `build-llamacpp-ov.sh` next to where you want the `llama.cpp` folder to land, then run it:
```bash
-chmod +x ubuntu-llamacpp-ov-install.sh
-./ubuntu-llamacpp-ov-install.sh
+chmod +x build-llamacpp-ov.sh
+./build-llamacpp-ov.sh
```
-Click to expand ubuntu-llamacpp-ov-install.sh
+Click to expand build-llamacpp-ov.sh
```bash
#!/usr/bin/env bash
@@ -237,8 +242,8 @@ chmod +x ubuntu-llamacpp-ov-install.sh
# ============================================
set -euo pipefail
-OPENVINO_VERSION_MAJOR="2026.3"
-OPENVINO_VERSION_FULL="2026.3.0.22451.bd8d6542e3c"
+OPENVINO_VERSION_MAJOR="2026.3.1"
+OPENVINO_VERSION_FULL="2026.3.1.22476.56d9685302d"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
OPENVINO_INSTALL_DIR="/opt/intel/openvino_${OPENVINO_VERSION_MAJOR}"
@@ -313,8 +318,9 @@ fi
echo "============================================"
echo "Configuring with CMake..."
echo "============================================"
-# shellcheck disable=SC1091
+set +u
source "${OPENVINO_ROOT}/setupvars.sh"
+set -u
cmake -B build/ReleaseOV -G Ninja \
-DCMAKE_BUILD_TYPE=Release \
@@ -334,27 +340,27 @@ echo " ./build/ReleaseOV/bin/llama-cli -m model.gguf"
```
> [!NOTE]
-> The script pins OpenVINO `2026.3` via the `OPENVINO_VERSION_MAJOR` / `OPENVINO_VERSION_FULL` variables at the top — edit them to track a different release.
+> The script pins OpenVINO `2026.3.1` via the `OPENVINO_VERSION_MAJOR` / `OPENVINO_VERSION_FULL` variables at the top — edit them to track a different release.
-#### Automated Windows Build Script
+#### Windows Build Script
For Windows users, the following `.bat` script automates the prerequisite installs (Git, Ninja, CMake, Visual Studio 2022 Build Tools, vcpkg + OpenCL), the OpenVINO Runtime download/extract, and the Ninja-based llama.cpp build.
-Save the following as `windows-llamacpp-ov-install.bat` next to where you want the `llama.cpp` to land, then run it from either **Command Prompt** or **PowerShell**:
+Save the following as `build-llamacpp-ov.bat` next to where you want the `llama.cpp` to land, then run it from either **Command Prompt** or **PowerShell**:
```cmd
:: Command Prompt
-windows-llamacpp-ov-install.bat
+build-llamacpp-ov.bat
```
```powershell
# PowerShell
-.\windows-llamacpp-ov-install.bat
+.\build-llamacpp-ov.bat
```
-Click to expand windows-llamacpp-ov-install.bat
+Click to expand build-llamacpp-ov.bat
```bat
@echo off
@@ -364,8 +370,8 @@ REM ============================================
REM llama.cpp OpenVINO Build Script (Ninja)
REM ============================================
-set "OPENVINO_VERSION_MAJOR=2026.3"
-set "OPENVINO_VERSION_FULL=2026.3.0.22451.bd8d6542e3c"
+set "OPENVINO_VERSION_MAJOR=2026.3.1"
+set "OPENVINO_VERSION_FULL=2026.3.1.22476.56d9685302d"
set "SCRIPT_DIR=%~dp0"
set "VCPKG_DIR=C:\vcpkg"
@@ -453,9 +459,6 @@ if exist "%OPENVINO_INSTALL_DIR%\setupvars.bat" (
)
REM Move the single top-level folder contents into the versioned install dir.
- REM NOTE: delayed expansion (!VAR!) is required because the surrounding else( ... )
- REM block is parsed once up-front, so %OPENVINO_EXTRACTED% would expand to "" here
- REM and xcopy would then treat "\*" as C:\* and fail with "Cannot perform a cyclic copy".
set "OPENVINO_EXTRACTED="
for /d %%i in ("%OPENVINO_EXTRACT_TMP%\*") do set "OPENVINO_EXTRACTED=%%i"
if not defined OPENVINO_EXTRACTED (
@@ -547,7 +550,7 @@ endlocal
```
> [!NOTE]
-> The script pins OpenVINO `2026.3` via the `OPENVINO_VERSION_MAJOR` / `OPENVINO_VERSION_FULL` variables at the top — edit them to track a different release. From any new shell, source the matching `setupvars` script via the junction — `call "C:\Intel\openvino\setupvars.bat"` from `cmd`, or `& "C:\Intel\openvino\setupvars.ps1"` from PowerShell. If `winget` cannot register Visual Studio Build Tools on first run, install them once manually and re-run the script from an elevated **Developer Command Prompt for VS 2022**.
+> The script pins OpenVINO `2026.3.1` via the `OPENVINO_VERSION_MAJOR` / `OPENVINO_VERSION_FULL` variables at the top — edit them to track a different release. From any new shell, source the matching `setupvars` script via the junction — `call "C:\Intel\openvino\setupvars.bat"` from `cmd`, or `& "C:\Intel\openvino\setupvars.ps1"` from PowerShell. If `winget` cannot register Visual Studio Build Tools on first run, install them once manually and re-run the script from an elevated **Developer Command Prompt for VS 2022**.
@@ -712,6 +715,7 @@ Boolean flags follow a uniform convention: set to a **positive integer** (e.g. `
| `GGML_OPENVINO_CACHE_DIR` | String | `not set` | Directory for OpenVINO model caching (recommended: `/tmp/ov_cache`). Enables model caching when set. **Not supported on NPU devices.** |
| `GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR` | String | `not set` | Directory for the frontend compiled-model cache. When set, OpenVINO compiled models are exported as blobs and imported on later runs to skip weight requantization, graph conversion, and compilation for matching single-graph models. |
| `GGML_OPENVINO_PREFILL_CHUNK_SIZE`| Integer | `256` | Token chunk size for **NPU** prefill (NPU-only; ignored on CPU/GPU). Must be a positive integer; otherwise the default is used. |
+| `GGML_OPENVINO_NPU_COMPILE_CONFIG` | String | `not set` | NPU-only compiler mode parameters forwarded to OpenVINO as `NPU_COMPILATION_MODE_PARAMS`, for example `optimization-level=3`. |
| `GGML_OPENVINO_STATEFUL_EXECUTION`| Boolean | `0` | Enable stateful KV cache for better performance. Recommended on CPU, GPU. |
| `GGML_OPENVINO_DISABLE_CACHE` | Boolean | `0` | Disable the in-process compiled-model / decoder cache (cache is on by default). Set to `1` to disable. |
| `GGML_OPENVINO_DISABLE_KV_SLICE` | Boolean | `0` | Disable the KV-cache input-tensor slicing optimization (slicing is on by default on CPU/GPU). Set to `1` to disable. |
@@ -725,9 +729,11 @@ Boolean flags follow a uniform convention: set to a **positive integer** (e.g. `
| `GGML_OPENVINO_DEBUG_INPUT` | Boolean | `0` | Enable input debugging and print input tensor info. |
| `GGML_OPENVINO_DEBUG_OUTPUT` | Boolean | `0` | Enable output debugging and print output tensor info. |
| `GGML_OPENVINO_PRINT_CGRAPH_TENSOR_ADDRESS` | Boolean | `0` | Print tensor address map once. |
+| `GGML_OPENVINO_LOG_UNSUPPORTED_OPS`| Boolean | `0` | Log warning messages with tensor details and rejection reasons for any ops not supported by the OpenVINO backend. Emits at `WARN` level (requires `--log-verbosity >= 2`, enabled by default). |
> [!NOTE]
->`GGML_OPENVINO_STATEFUL_EXECUTION` is an **Experimental** feature to allow stateful execution for managing the KV cache internally inside the OpenVINO model, improving performance on CPUs and GPUs. Stateful execution is not effective on NPUs, and not all models currently support this feature. This feature is experimental and has been validated only with the llama-simple, llama-cli, llama-bench, and llama-run applications and is recommended to enable for the best performance. Other applications, such as llama-server and llama-perplexity, are not yet supported.
+> - `GGML_OPENVINO_STATEFUL_EXECUTION` is an **Experimental** feature to allow stateful execution for managing the KV cache internally inside the OpenVINO model, improving performance on CPUs and GPUs. Stateful execution is not effective on NPUs, and not all models currently support this feature. This feature is experimental and has been validated only with the llama-simple, llama-cli, llama-bench, and llama-run applications and is recommended to enable for the best performance. Other applications, such as llama-server and llama-perplexity, are not yet supported.
+> - `GGML_OPENVINO_LOG_UNSUPPORTED_OPS` emits logs at `WARN` level (`GGML_LOG_WARN`), which requires application log verbosity `--log-verbosity >= 2` (or `-lv 2`).
### Example Usage
diff --git a/docs/backend/snapdragon/windows.md b/docs/backend/snapdragon/windows.md
index 3f7d60dd95f8..886cfda3f653 100644
--- a/docs/backend/snapdragon/windows.md
+++ b/docs/backend/snapdragon/windows.md
@@ -24,7 +24,18 @@ must be included in the .cat file digitally signed with a trusted certificate.
This document covers details on how to generate personal certificate files (.pfx) and how to configure the system
to allow for test signatures (aka test-signing).
-## Install the latest Adreno OpenCL SDK
+## Install Windows SDKs
+
+The recommended method is `setup-sdk.py`:
+
+```
+> python scripts\snapdragon\setup-sdk.py --list-sdk-releases
+> python scripts\snapdragon\setup-sdk.py --hexagon --opencl
+```
+
+It installs the selected SDKs under `C:\Qualcomm` and sets their corresponding environment variables for the current user. Start a new terminal after it completes; native Windows builds check all SDK paths before CMake runs.
+
+Select the SDKs to install with `--hexagon` and `--opencl`; use both to prepare a dual-backend build. To select a different available version, pass it to the SDK option, for example `--hexagon 6.4.0.2`. SDK versions install side by side, so you can switch versions without deleting an existing installation. Use `--force` to reinstall the selected SDKs. Use a new CMake build directory after each switch because CMake caches the SDK paths.
Either use the trimmed down version (optimized for CI) from
diff --git a/docs/development/DSV4-vulkan-lightning-indexer-progress.md b/docs/development/DSV4-vulkan-lightning-indexer-progress.md
new file mode 100644
index 000000000000..d19fbd6903aa
--- /dev/null
+++ b/docs/development/DSV4-vulkan-lightning-indexer-progress.md
@@ -0,0 +1,158 @@
+# DeepSeek V4 Vulkan Lightning Indexer progress
+
+This is a restart note for the Strix Halo Lightning Indexer optimization. It is a development scratch pad and can be removed before the final PR.
+
+## Repository state
+
+- Main repository: `/home/jaap/Projects/git/llama.cpp`
+- Optimization worktree: `/tmp/llama-strix-beta-bench`
+- Branch: `strix-halo-vulkan-lightning-indexer`
+- Base commit: `316c72ee9eab590f5891089d3b6bfc0d01d00d19`
+- Base branch: Nathan's `strix-halo-vulkan-beta`
+- Decode microbench work is stored in the main worktree as `stash@{0}: On strix-halo-vulkan: wip: DSV4 decode microbench depth matrix`.
+- The Indexer changes are uncommitted. Do not commit without explicit user approval. An assisted commit needs an `Assisted-by:` trailer.
+- Do not run builds and GPU benchmarks together. The APU shares its power and memory-bandwidth budget.
+- GPU commands need sandbox escalation.
+
+## Objective and result
+
+After sparse prefill attention was flattened, the context-dependent Lightning Indexer became the next prefill bottleneck. The old cooperative-matrix shader used one wave64 subgroup per workgroup, processed one 16-key tile, and loaded one query head at a time.
+
+The new wide pipeline uses eight wave64 subgroups per workgroup. Each subgroup processes a separate 16-key tile, so one workgroup covers 128 keys. It stages four query heads and their weights together, reuses them across all eight subgroups, and uses subgroup-scoped synchronization between cooperative-matrix result stores. A workgroup barrier remains between four-head groups because all subgroups reuse the shared query storage.
+
+The optimized shader requires 512 workgroup invocations and 64 KiB shared memory. Pipeline creation is capability-based. Devices without those limits use a one-wave, one-head cooperative-matrix specialization. The scalar implementation remains the fallback when cooperative matrices are unavailable. The decode-specific cooperative-matrix pipeline is unchanged.
+
+At the 32k-equivalent prefill microbench shape:
+
+| Version | Time per layer | Throughput |
+| --- | ---: | ---: |
+| Baseline | 50.51 ms | 5.83 TFLOPS |
+| Optimized | 31.81 ms | 9.25 TFLOPS |
+
+This is a 37.0% reduction in Lightning Indexer kernel time.
+
+The canonical 32k llama-bench improved from 209.45 to 216.32 tokens/s. Total Vulkan time fell from 9.73729 to 9.42448 seconds. Total Lightning Indexer time fell from 1.11860 to 0.697276 seconds. Sparse attention and top-K were effectively unchanged.
+
+## Changed files
+
+- `ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer_cm.comp`: parameterizes the shader, adds the eight-wave four-head implementation, and remains usable for the small fallback.
+- `ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp`: generates wide `N_WAVES=8`, `HEADS_PER_TILE=4` and small `N_WAVES=1`, `HEADS_PER_TILE=1` variants.
+- `ggml/src/ggml-vulkan/ggml-vulkan.cpp`: creates and selects the capability-gated wide pipeline and the small cooperative-matrix fallback.
+- `tests/test-backend-ops.cpp`: adds 127, 128, and 129-key correctness boundaries and PP2048 performance shapes through 512k simulated source context.
+
+## Performance data
+
+The performance rows model the actual PP2048 Indexer shapes after the source context is filled in 2048-token batches. `kv=8704` is the measured shape near 32k source context. It differs from 32768 because the Indexer compresses source tokens into rows.
+
+| Source depth | `kv` | Baseline | Optimized | Reduction |
+| ---: | ---: | ---: | ---: | ---: |
+| 0 | 512 | 4.23 ms | 2.14 ms | 49.5% |
+| 8k | 2560 | 17.10 ms | 10.09 ms | 41.0% |
+| 16k | 4608 | 29.97 ms | 17.62 ms | 41.2% |
+| 32k | 8704 | 50.51 ms | 32.94 ms | 34.8% |
+| 64k | 16896 | 94.36 ms | 64.87 ms | 31.3% |
+| 128k | 33280 | 174.96 ms | 128.76 ms | 26.4% |
+| 256k | 66048 | 352.73 ms | 250.88 ms | 28.9% |
+| 512k | 131584 | 696.79 ms | 495.90 ms | 28.8% |
+
+The final isolated 32k run after cleanup measured 31.81159 ms. Small matrix differences are normal laptop GPU clock variation.
+
+| Canonical 32k metric | Baseline | Optimized |
+| --- | ---: | ---: |
+| PP2048 | 209.45 tokens/s | 216.32 tokens/s |
+| Total Vulkan | 9.73729 s | 9.42448 s |
+| Lightning Indexer | 1.11860 s | 0.697276 s |
+| Sparse FA raw | 0.198438 s | 0.195367 s |
+| Sparse FA selected | 0.835110 s | 0.843385 s |
+| Sparse FA reduce | 0.088463 s | 0.086678 s |
+| TOP_K | 0.075473 s | 0.075350 s |
+
+Logs:
+
+- Baseline matrix: `/tmp/dsv4-lightning-prefill-baseline.log`
+- Optimized matrix: `/tmp/dsv4-lightning-four-head-matrix.log`
+- Final selected-pipeline 32k microbench: `/tmp/dsv4-lightning-final-selected-32k.log`
+- Baseline canonical 32k llama-bench: `/tmp/dsv4-nathan-beta-32k-rerun-new-first.log`
+- Optimized canonical 32k llama-bench: `/tmp/dsv4-lightning-final-32k-llama-bench.log`
+
+## Correctness and resources
+
+- Wide pipeline: all 20 focused F16 cases passed, including 127, 128, and 129-key boundaries.
+- Small cooperative-matrix fallback: temporarily forced and all the same 20 cases passed.
+- Wide shader on gfx1151: 168 VGPRs, 63,488 bytes LDS, no spills, eight subgroups per SIMD.
+- The final pipeline-statistics run confirmed `lightning_indexer_cm_f16` was selected.
+- No NaN, Inf, or comparison failures were reported.
+
+Correctness logs:
+
+- Wide: `/tmp/dsv4-lightning-consolidated-wide-correctness.log`
+- Small fallback: `/tmp/dsv4-lightning-consolidated-small-correctness.log`
+
+## Experiments and decisions
+
+- Four waves improved the 32k shape about 3% and became worse at deep simulated contexts.
+- Eight waves improved it about 8% before the other changes.
+- Subgroup-scoped synchronization after cooperative-matrix stores improved the eight-wave version.
+- Staging four query heads and weights produced the large gain by reducing redundant loads and barriers.
+- Using only a subgroup barrier between head groups failed four boundary tests. One subgroup could overwrite shared query data while another still read it. A workgroup barrier is required there.
+- A separate fallback shader source was avoided. Generator definitions create both variants from one file.
+
+## Commands
+
+Build only, with no GPU benchmark running:
+
+```sh
+cd /tmp/llama-strix-beta-bench
+git diff --check
+cmake --build build --config Release --target test-backend-ops llama-bench -j "$(nproc)"
+```
+
+Focused correctness:
+
+```sh
+cd /tmp/llama-strix-beta-bench
+./build/bin/test-backend-ops test -b Vulkan0 -o LIGHTNING_INDEXER -p 'type_K=f16' > /tmp/dsv4-lightning-correctness.log 2>&1
+tail -n 30 /tmp/dsv4-lightning-correctness.log
+```
+
+Final 32k-equivalent microbench and pipeline selection:
+
+```sh
+cd /tmp/llama-strix-beta-bench
+GGML_VK_PIPELINE_STATS=lightning_indexer_cm_f16 ./build/bin/test-backend-ops perf -b Vulkan0 -o LIGHTNING_INDEXER -p 'kv=8704' > /tmp/dsv4-lightning-final-selected-32k.log 2>&1
+tail -n 16 /tmp/dsv4-lightning-final-selected-32k.log
+```
+
+Full Indexer depth matrix:
+
+```sh
+cd /tmp/llama-strix-beta-bench
+./build/bin/test-backend-ops perf -b Vulkan0 -o LIGHTNING_INDEXER -p 'nb=2048,nh=64,ns=1,nm=1,type_K=f16' > /tmp/dsv4-lightning-matrix.log 2>&1
+rg 'kv=(512|2560|4608|8704|16896|33280|66048|131584),nb=2048' /tmp/dsv4-lightning-matrix.log
+```
+
+Canonical 32k llama-bench. Run it only when needed, never while compiling, and inspect only the final block:
+
+```sh
+cd /tmp/llama-strix-beta-bench
+GGML_VK_PERF_LOGGER=1 ./build/bin/llama-bench -m /home/jaap/Projects/docker/localLLaMA/models/models--unsloth--DeepSeek-V4-Flash-0731-GGUF/snapshots/109848da2469efe1f1aab9e11acea08a065ccd4f/UD-IQ3_XXS/DeepSeek-V4-Flash-0731-UD-IQ3_XXS-00001-of-00004.gguf -r 1 -d 32768 -p 2048 -ub 2048 -fa 1 -n 0 > /tmp/dsv4-lightning-32k-llama-bench.log 2>&1
+last=$(grep -n 'Vulkan Timings:' /tmp/dsv4-lightning-32k-llama-bench.log | tail -n 1 | cut -d: -f1)
+sed -n "${last},\$p" /tmp/dsv4-lightning-32k-llama-bench.log | tail -n 180
+```
+
+Patch inspection:
+
+```sh
+cd /tmp/llama-strix-beta-bench
+git diff --check
+git diff --stat
+git diff
+git status --short
+```
+
+## Next actions
+
+1. The user reviews and understands the four-file implementation and result summary.
+2. Commit only after explicit user approval for that commit action.
+3. Remove this scratch pad before a PR if it is not useful as permanent documentation.
+4. Restore the separate decode microbench stash from the main worktree only if that work resumes.
diff --git a/docs/development/DSV4-vulkan-sparse-prefill-progress.md b/docs/development/DSV4-vulkan-sparse-prefill-progress.md
new file mode 100644
index 000000000000..e880ae63e99e
--- /dev/null
+++ b/docs/development/DSV4-vulkan-sparse-prefill-progress.md
@@ -0,0 +1,587 @@
+# DeepSeek V4 Vulkan sparse prefill progress
+
+This file is a self-contained handoff for the DeepSeek V4 sparse-attention prompt-processing optimization on AMD Strix Halo. Read the repository `AGENTS.md` and `CONTRIBUTING.md` before continuing.
+
+## Repository state
+
+- Repository: `https://github.com/Nathanw1014/llama.cpp`
+- Branch: `strix-halo-vulkan`
+- Starting commit: `baf0025de861c6f6ea3720fa81c52ae1b2e6c078`
+- Target GPU: AMD Radeon 8060S / gfx1151, RADV, Vulkan, wave64
+- The device reports `GL_KHR_cooperative_matrix`, f16 inputs with f32 accumulation, 64 KiB shared memory, and a maximum 512-thread workgroup used by this path.
+
+The implementation is not upstream `ggml-org/llama.cpp`. It builds on this branch's DeepSeek V4 graph, Lightning Indexer, sparse top-K hint, decode gather path, fused HC kernels, and Vulkan profiler changes.
+
+## Important execution constraints
+
+The system is an APU. CPU compilation and GPU benchmarking share power and memory bandwidth. Never build and benchmark at the same time. Serialize all builds, correctness tests, and performance tests.
+
+GPU commands must run with host GPU access. In an agent sandbox, request elevated/out-of-sandbox execution. A sandboxed benchmark showed only CPU activity and is invalid.
+
+Redirect the full model benchmark to a log. Inspect only the last profiler block with `tail`; do not load the full log into agent context.
+
+## Build and ccache
+
+The build directory is `build`, configured as Release with Vulkan enabled. The default ccache directory was read-only in the agent environment, so use a writable directory:
+
+```bash
+cmake -S . -B build \
+ -DGGML_VULKAN=ON \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+
+CCACHE_DIR=/tmp/llama-cpp-ccache cmake --build build --config Release \
+ --target llama-bench test-backend-ops -j "$(nproc)"
+
+CCACHE_DIR=/tmp/llama-cpp-ccache ccache -s
+```
+
+ccache was verified active. The final build reported direct hits. Note that an incremental change to `ggml-vulkan.cpp` is one large C++ translation unit and therefore uses one compiler core even with `-j`. Shader object regeneration can run in parallel.
+
+## Canonical benchmark command
+
+Do not change or omit switches for the 32k acceptance run:
+
+```bash
+GGML_VK_PERF_LOGGER=1 ./build/bin/llama-bench \
+ -m ~/Projects/docker/localLLaMA/models/models--unsloth--DeepSeek-V4-Flash-0731-GGUF/snapshots/109848da2469efe1f1aab9e11acea08a065ccd4f/UD-IQ3_XXS/DeepSeek-V4-Flash-0731-UD-IQ3_XXS-00001-of-00004.gguf \
+ -r 1 -d 32768 -p 2048 -ub 2048 -fa 1 -n 0 \
+ > /tmp/dsv4-vulkan-32k.log 2>&1
+
+tail -n 180 /tmp/dsv4-vulkan-32k.log
+```
+
+The known model path exists on the target system.
+
+## Graph and dispatch findings
+
+DeepSeek V4 builds the Lightning Indexer and sparse attention in `src/models/deepseek4.cpp`:
+
+1. `build_lid_top_k()` creates indexer Q/K/weights and calls `ggml_lightning_indexer()`.
+2. `ggml_top_k()` selects up to `hparams.indexer_top_k` compressed-cache indices for every query token.
+3. `build_csa_lid_attention()` concatenates the raw SWA K prefix with compressed CSA K, builds a dense mask carrying the same sparse selection, and calls `build_attn_mha(..., top_k, raw_k->ne[2])`.
+4. `build_attn_mha()` attaches `top_k` and `n_kv_raw` to `GGML_OP_FLASH_ATTN_EXT` through `ggml_flash_attn_ext_add_top_k()`.
+
+At the final 32k PP2048 batch:
+
+- total K rows: 11,008
+- `n_kv_raw`: 2,304 raw SWA rows, always attended subject to the mask
+- `n_top_k`: 512 selected compressed rows per query token
+- active rows: 2,816
+- selectable compressed region: 8,704
+
+The custom Vulkan path is selected by `ggml_vk_flash_attn_top_k()` before ordinary FA. Its gate requires the DeepSeek V4 shape and `total_k >= 3 * (n_kv_raw + n_top_k)`. The final shape satisfies `11008 >= 3 * 2816`.
+
+The old `flash_attn_top_k.comp` shader is scalar/subgroup code. One 512-thread workgroup covers eight heads for one query token. It stages 16 selected 512-wide K/V rows, computes QK with scalar FMAs and `subgroupAdd`, updates online softmax one key at a time, and accumulates PV manually. It does not use cooperative matrices.
+
+The top-K set differs by query token but is shared by all 64 query heads for that token. This makes the attention for one token a regular matrix problem across heads and selected keys despite sparse per-token indexing.
+
+## Root cause evidence
+
+The existing Vulkan timestamp infrastructure was extended with `ggml_vk_perf_mark_subop()` after the sparse dispatch. This reports the sparse kernel separately as `FA_TOP_K_SPARSE (sub-op)` or `FA_TOP_K_CM (sub-op)`.
+
+Focused test shape:
+
+```bash
+GGML_VK_PERF_LOGGER=1 ./build/bin/test-backend-ops perf \
+ -b Vulkan0 -o FLASH_ATTN_EXT \
+ -p 'kv=32768,nb=512,n_kv_raw=1024,n_top_k=512,sinks=0'
+```
+
+Results:
+
+- old scalar sparse kernel: 61.42 ms, 1.68 TFLOPS of useful active-set work
+- ordinary dense FA diagnostic (`GGML_VK_FA_TOPK=0`): 255.97 ms, about 8.7 TFLOPS over the full dense work
+- final cooperative sparse kernel: 32.55 ms, 3.17 TFLOPS of useful active-set work
+
+The residual `FLASH_ATTN_EXT` interval after the sparse timestamp is only about 4-7 us. The cost is inside the shader, not dispatch or surrounding synchronization.
+
+A temporary uniform stage-profiling mode was used and removed. For the final 32-head tile:
+
+- selected K gather + cooperative QK: 14.30 ms
+- gather + QK + serial softmax: 26.34 ms
+- gather + QK + parallel softmax: 15.53 ms
+- full kernel: 32.55 ms
+- the remaining cooperative PV/output portion is about 17.0 ms
+
+The old scalar shader was compute/issue inefficient. Dense FA proved matrix hardware is much faster but was still too expensive because it processes all K rows. The final implementation preserves sparsity and uses the matrix hardware for both QK and PV.
+
+## Implementation
+
+Files changed:
+
+- `ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_top_k_cm.comp`
+ - New cooperative-matrix sparse prefill shader.
+ - One 512-thread workgroup covers 32 query heads for one token.
+ - Eight wave64 subgroups cover two 16-head tiles by four 16-key or 16-output-dimension tiles.
+ - Processes 64 selected keys per online-softmax block.
+ - Stages only indexed selected K/V tiles, never the full K range.
+ - Uses f16 cooperative-matrix inputs and f32 accumulation for QK and PV.
+ - Uses a 16-lane segmented softmax per head. XOR subgroup shuffles reduce max and sum for four independent heads per wave without workgroup barriers.
+ - Keeps f32 output accumulators and normalizes after all active blocks.
+ - Preserves the raw prefix, top-K index validation, mask, sinks, stream strides, and K == V latent behavior.
+- `ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp`
+ - Embeds the new shader when cooperative-matrix shader support is available.
+- `ggml/src/ggml-vulkan/ggml-vulkan.cpp`
+ - Adds the cooperative sparse pipeline when the device supports the required 16x16x16 f16/f32 cooperative matrix shape.
+ - Selects it by capability and keeps the scalar shader as fallback.
+ - Adds `GGML_VK_FA_TOPK=0` to force ordinary dense FA for diagnostics.
+ - Adds `GGML_VK_FA_TOPK_CM=0` to force the old scalar sparse shader for A/B tests.
+ - Adds sparse sub-operation timestamps through the existing profiler.
+
+The path is capability-based, not hardcoded to Strix Halo. The current sparse shape gate remains DeepSeek V4-specific. Devices without the required cooperative matrix support keep the correct scalar sparse or dense fallback.
+
+Two discarded prototypes are useful context:
+
+- A 16-head, 64-key streaming cooperative tile was correct and reduced the focused test from 61.4 to 44.6 ms.
+- Keeping 32 complete 512-wide K/V rows in LDS grew shared memory to about 41 KiB, reduced residency, doubled block/barrier count, and regressed to 73.8 ms. Do not retry full-row LDS staging without solving occupancy.
+- A 32-head, 64-key streaming tile halved irregular row loads but initially stayed near 44.7 ms because serial softmax cost about 11.5 ms. Parallel segmented softmax produced the final 32.55 ms result.
+
+## Correctness validation
+
+Run:
+
+```bash
+./build/bin/test-backend-ops test -b Vulkan0 -o FLASH_ATTN_EXT -p 'n_top_k='
+```
+
+Final result: 8/8 sparse top-K FA cases passed against the CPU reference. Cases include:
+
+- decode and short batches that use dense/gather fallback
+- sparse prefill batch sizes 64 and 128
+- `n_kv_raw` plus top-K selection
+- invalid top-K index handling from the test fixture
+- sinks enabled and disabled
+- sparse threshold transitions
+- an active-key count of 193, which exercises a partial final 64-key block
+
+The test uses the existing FA tolerance of NMSE <= `5e-4`. No NaN or Inf failure occurred. The implementation changes Q and probability inputs to f16 cooperative-matrix operands with f32 accumulation, matching the precision strategy of ordinary Vulkan cooperative FA.
+
+Still desirable before broader submission:
+
+- compare model logits on controlled prompts between `GGML_VK_FA_TOPK_CM=0` and the default cooperative path
+
+## Canonical 32k results
+
+Exact clean runs, same command and machine, no concurrent build:
+
+```text
+32k context, PP 2048, ub 2048
+
+Before (commit baf0025de):
+112.29 tok/s
+Total Vulkan: 18.1957 s
+Sparse FA: 8.84496 s, 421.189 ms/layer
+Lightning Indexer: 1.19438 s
+TOP_K: 0.076948 s
+
+After:
+152.32 tok/s
+Total Vulkan: 13.4032 s
+Sparse FA: 4.44701 s, 211.762 ms/layer
+Lightning Indexer: 1.13156 s
+TOP_K: 0.073945 s
+
+Change:
+Throughput: +35.65%
+Total Vulkan time: -26.34%
+Sparse FA time: -49.72%
+Sparse FA saved: 4.398 s
+Total GPU time saved: 4.793 s
+```
+
+The profiler now lists the optimized dispatch as `FA_TOP_K_CM (sub-op)`. The following residual `FLASH_ATTN_EXT` line is only the post-mark interval and must not be interpreted as the kernel time.
+
+## Context-depth measurements
+
+All points use PP2048, ub2048, FA enabled, one repetition, and no token generation. They were run sequentially with no compiler active:
+
+```text
+Existing depth tok/s Total Vulkan Final large FA Lightning Indexer TOP_K
+0 253.44 8.041 s 0.294 s 0.095 s 0.001 s
+8192 211.01 9.666 s 1.625 s 0.379 s 0.022 s
+16384 177.19 11.518 s 3.031 s 0.678 s 0.044 s
+32768 152.32 13.403 s 4.447 s 1.132 s 0.074 s
+```
+
+At 0, 8k, and 16k, total K is below the existing sparse-path gate `total_k >= 3 * (n_kv_raw + n_top_k)`. These points use the unchanged ordinary dense FA implementation, so the cooperative sparse change does not affect or regress them. At 32k, total K is 11,008 and the cooperative sparse path engages. The 32k `Final large FA` value is the `FA_TOP_K_CM (sub-op)` total; the lower-depth values are the large ordinary `FLASH_ATTN_EXT` totals.
+
+Logs:
+
+- `/tmp/dsv4-vulkan-cm-0k.log`
+- `/tmp/dsv4-vulkan-cm-8k.log`
+- `/tmp/dsv4-vulkan-cm-16k.log`
+- `/tmp/dsv4-vulkan-cm-32k.log`
+- `/tmp/dsv4-vulkan-baseline-clean.log`
+- `/tmp/dsv4-fa-cm-correctness-final.log`
+
+## Next optimization target
+
+The cooperative sparse FA remains the largest context-dependent cost at about 4.45 s total. Stage profiling indicates approximately 14.3 ms of focused-test time in gather/QK, about 1.2 ms in parallel softmax, and about 17 ms in PV/output.
+
+The next useful work is PV and output accumulation, not TOP_K. Investigate:
+
+- reducing repeated selected V staging across the two 32-head workgroups per token without increasing LDS enough to lose occupancy
+- reducing the eight output-dimension passes or retaining more PV state in cooperative fragments/registers
+- checking register count and spills for the 32 f32 output accumulators per invocation using RADV shader statistics
+- alternate 32-head layouts that keep the same eight-wave occupancy but improve PV scheduling
+- query-tile overlap/union gathering only if measured top-K overlap is high enough; a whole-2048-query union is unlikely to help
+
+Do not optimize TOP_K first. At 32k it is only about 74 ms total. Lightning Indexer is about 1.13 s and is the next context-dependent target only after sparse FA improves further.
+
+## Useful diagnostics
+
+Force old scalar sparse path:
+
+```bash
+GGML_VK_FA_TOPK_CM=0 GGML_VK_PERF_LOGGER=1 ./build/bin/test-backend-ops perf \
+ -b Vulkan0 -o FLASH_ATTN_EXT \
+ -p 'kv=32768,nb=512,n_kv_raw=1024,n_top_k=512,sinks=0'
+```
+
+Force ordinary dense FA:
+
+```bash
+GGML_VK_FA_TOPK=0 GGML_VK_PERF_LOGGER=1 ./build/bin/test-backend-ops perf \
+ -b Vulkan0 -o FLASH_ATTN_EXT \
+ -p 'kv=32768,nb=512,n_kv_raw=1024,n_top_k=512,sinks=0'
+```
+
+Default cooperative sparse path:
+
+```bash
+GGML_VK_PERF_LOGGER=1 ./build/bin/test-backend-ops perf \
+ -b Vulkan0 -o FLASH_ATTN_EXT \
+ -p 'kv=32768,nb=512,n_kv_raw=1024,n_top_k=512,sinks=0'
+```
+
+Always run these sequentially. Do not run a compiler concurrently on this APU.
+
+## Experimental raw-prefix split prototype
+
+An uncommitted follow-up prototype was tested after commit `24c7ead76cc1a9631fa1b42b0bfa53a15169e1ba`. Check `git status` before continuing. It was initially tested with `GGML_VK_FA_TOPK_SPLIT=1`. After the successful 32k run, the split path was changed to default-on for a llama-server coherence test. Set `GGML_VK_FA_TOPK_SPLIT=0` to restore the single cooperative sparse kernel.
+
+Stage profiling on the previously used 19,200-row sparse shape (`kv=19200,nb=2048,n_kv_raw=2304,n_top_k=512,sinks=0`) showed:
+
+```text
+cooperative QK and selected-K gather: 88.885 ms
+softmax increment: 4.909 ms
+cooperative PV and output increment: 128.399 ms
+full cooperative sparse kernel: 222.193 ms
+```
+
+PV and output were 57.8% of the kernel. More importantly, most active keys are not sparse: all 2,304 raw-prefix rows are contiguous, while only 512 compressed rows use top-K indices. The prototype therefore makes two attention partitions:
+
+1. Ordinary optimized Vulkan cooperative FA processes the contiguous raw prefix.
+2. The cooperative top-K shader processes only the 512 selected compressed rows.
+3. The existing split-K reduction combines both online-softmax partitions and applies sinks.
+
+This preserves the exact raw-prefix, sparse top-K, causal mask, and softmax semantics. It reuses the existing ordinary FA and split-K reduction rather than adding a new subsystem.
+
+The exact-shape microbenchmark improved from 222.19 ms to 111.33 ms. Its steady split stages were about 62-64 ms raw-prefix FA, 43-46 ms selected sparse FA, and 3.6-3.9 ms reduction.
+
+Correctness validation:
+
+```bash
+GGML_VK_FA_TOPK_SPLIT=1 ./build/bin/test-backend-ops test \
+ -b Vulkan0 -o FLASH_ATTN_EXT -p 'n_top_k='
+
+./build/bin/test-backend-ops test -b Vulkan0 -o FLASH_ATTN_EXT
+```
+
+Results were 8/8 sparse top-K cases and 13,296/13,296 complete Vulkan FA cases against the CPU reference. No NaN or Inf failure occurred.
+
+Canonical 32k prototype command:
+
+```bash
+GGML_VK_FA_TOPK_SPLIT=1 GGML_VK_PERF_LOGGER=1 ./build/bin/llama-bench \
+ -m ~/Projects/docker/localLLaMA/models/models--unsloth--DeepSeek-V4-Flash-0731-GGUF/snapshots/109848da2469efe1f1aab9e11acea08a065ccd4f/UD-IQ3_XXS/DeepSeek-V4-Flash-0731-UD-IQ3_XXS-00001-of-00004.gguf \
+ -r 1 -d 32768 -p 2048 -ub 2048 -fa 1 -n 0 \
+ > /tmp/dsv4-vulkan-split-32k.log 2>&1
+```
+
+Final 32k result:
+
+```text
+32k context, PP 2048, ub 2048
+
+Old scalar: 112.29 tok/s, 18.1957 s total, 8.84496 s sparse FA
+Committed coopmat: 152.32 tok/s, 13.4032 s total, 4.44701 s sparse FA
+Experimental split: 208.70 tok/s, 9.7704 s total, 1.20170 s split sparse FA
+
+Experimental split stages:
+raw-prefix FA: 0.210775 s total, 10.037 ms/layer
+selected sparse FA: 0.908190 s total, 43.247 ms/layer
+split reduction: 0.082732 s total, 3.940 ms/layer
+Lightning Indexer: 1.110410 s total, 52.877 ms/layer
+TOP_K: 0.077394 s total, 3.685 ms/layer
+```
+
+Relative to the committed cooperative path, throughput improved 37.0%, total Vulkan time fell 27.1%, and sparse FA time fell 73.0%. Relative to the old scalar path, throughput improved 85.9%, total Vulkan time fell 46.3%, and sparse FA time fell 86.4%.
+
+The initial split implementation used 538,968,064 bytes (514 MiB) of scratch at PP2048 because each of two partitions stored an f32 partial output for 512 dimensions x 64 heads x 2048 queries, plus L/M data. This was subsequently reduced by query tiling as described below.
+
+The old sparse selection heuristic required `total_k >= 3 * active_k`. For the coherence test it now selects sparse attention whenever `total_k > active_k`; equality still uses dense FA because no keys are pruned. The shape, capability, and allocation gates remain unchanged.
+
+Prototype logs:
+
+- `/tmp/dsv4-fa-exact-qk.log`
+- `/tmp/dsv4-fa-exact-softmax.log`
+- `/tmp/dsv4-fa-exact-full.log`
+- `/tmp/dsv4-fa-exact-split.log`
+- `/tmp/dsv4-fa-split-correctness.log`
+- `/tmp/dsv4-fa-all-correctness.log`
+- `/tmp/dsv4-vulkan-split-32k.log`
+
+## Llama-server coherence check
+
+After making the split path default-on and changing the sparse crossover to `total_k > active_k`, `llama-server` produced a coherent response from a 2,044-token prompt at significant context depth. The final profiler block confirmed that all 21 sparse-attention layers used the split path:
+
+```text
+FA_TOP_K_RAW: 0.179223 s total, 8.534 ms/layer
+FA_TOP_K_SELECTED: 0.886629 s total, 42.220 ms/layer
+FA_TOP_K_REDUCE: 0.083240 s total, 3.964 ms/layer
+Split sparse FA: 1.149092 s total, 54.719 ms/layer
+Lightning Indexer: 1.054610 s total, 50.220 ms/layer
+TOP_K: 0.044540 s total, 2.121 ms/layer
+Total Vulkan: 9.797560 s
+```
+
+This closely matches the canonical PP2048 llama-bench result of 9.770 s total and 1.202 s split sparse FA. The focused sparse CPU-reference test was rerun after the crossover change and passed 8/8 cases.
+
+## Tiled split scratch optimization
+
+Commit `4bbe53e4775f0707de8158e4977a16ab770829da` used two full PP2048 output partitions. The same two-partition algorithm now processes at most 256 query tokens per tile and reuses the split scratch between tiles. Q, mask, top-K, and destination descriptors are offset to the tile while K/V remain shared. A Vulkan pipeline barrier separates reuse of each scratch tile.
+
+Scratch at PP2048 changed from:
+
+```text
+Before: 538,968,064 bytes (514 MiB)
+After: 67,371,008 bytes (64.25 MiB)
+Change: 8x reduction
+```
+
+The previously used 19,200-row microbenchmark (`kv=19200,nb=2048,n_kv_raw=2304,n_top_k=512,sinks=0`) measured:
+
+```text
+Full-batch two partitions: 114.65 ms
+256-query tiled path: 113.70 ms
+```
+
+Two one-partition alternatives were tested and discarded. Merging the raw partial directly inside the cooperative selected shader measured 119.17 ms. Writing selected output separately and using a lightweight merge kernel measured 120.45 ms. Both cut scratch in half but regressed because writing selected output outside the contiguous split layout increased the selected stage from about 44 ms to about 51 ms. Query tiling preserves the faster memory layout.
+
+Canonical 32k result after tiling:
+
+```text
+32k context, PP 2048, ub 2048
+
+Full-batch split: 208.70 tok/s, 9.77039 s total, 1.20170 s sparse FA
+Tiled split: 209.62 tok/s, 9.72897 s total, 1.18721 s sparse FA
+
+Tiled split stages:
+raw-prefix FA: 0.192879 s total, 168 tile dispatches
+selected sparse FA: 0.910153 s total, 168 tile dispatches
+split reduction: 0.084179 s total, 168 tile dispatches
+Lightning Indexer: 1.100230 s total
+TOP_K: 0.075158 s total
+```
+
+The 168 dispatch count is eight tiles x 21 sparse-attention layers. Relative to the full-batch split path, throughput improved 0.44%, total Vulkan time fell 0.42%, and sparse FA time fell 1.21%. The main result is the 8x scratch reduction without a performance regression.
+
+Final correctness after removing the discarded merge prototypes:
+
+- focused sparse top-K suite: 9/9 passed against the CPU reference, including a 257-query tile-boundary case
+- complete Vulkan Flash Attention suite: 13,296/13,296 passed
+- no NaN or Inf failure
+
+Logs:
+
+- `/tmp/dsv4-fa-exact-fused.log`
+- `/tmp/dsv4-fa-exact-merge.log`
+- `/tmp/dsv4-fa-exact-two-part-current.log`
+- `/tmp/dsv4-fa-exact-tiled.log`
+- `/tmp/dsv4-fa-tiled-final-correctness.log`
+- `/tmp/dsv4-fa-tiled-boundary-correctness.log`
+- `/tmp/dsv4-fa-tiled-all-correctness.log`
+- `/tmp/dsv4-vulkan-tiled-32k.log`
+
+## Selected PV probability reuse
+
+The selected cooperative-matrix stage remained the largest split sparse-FA component. Profiling the tiled 19,200-row shape showed:
+
+```text
+selected K gather and cooperative QK: about 20.0 ms
+QK plus softmax: about 22.3 ms
+full selected stage: about 45.5 ms
+cooperative PV and output increment: about 23.2 ms
+```
+
+PV and output were about 51% of the selected stage. The shader previously loaded each of four 16x16 probability cooperative-matrix fragments again for every one of the eight 64-dimension PV passes. The optimized shader loads these four fragments once per 64-key block and retains them across all PV dimension passes. This follows the probability-fragment lifetime used by ordinary cooperative Vulkan FA.
+
+The shader also aliases the shared score and PV-output matrices because their lifetimes do not overlap. Shader resource statistics changed as follows:
+
+```text
+ Before After
+VGPRs 192 192
+VGPR spills 0 0
+LDS 36,864 28,672 bytes
+static instructions 11,524 11,358
+```
+
+The 19,200-row microbenchmark changed from 113.70 ms for the committed tiled path to 110.11 ms. The selected stage fell from about 45.5 ms to about 43.7 ms. The raw-prefix and reduction implementations are unchanged.
+
+Two canonical 32k runs after this change measured:
+
+```text
+32k context, PP 2048, ub 2048
+
+Committed tiled reference:
+209.62 tok/s
+Total Vulkan: 9.72897 s
+Split sparse FA: 1.18721 s
+ raw-prefix FA: 0.192879 s
+ selected sparse FA: 0.910153 s
+ split reduction: 0.084179 s
+Lightning Indexer: 1.100230 s
+TOP_K: 0.075158 s
+
+Probability reuse, run 1:
+211.03 tok/s
+Total Vulkan: 9.66363 s
+Split sparse FA: 1.12776 s
+ raw-prefix FA: 0.192078 s
+ selected sparse FA: 0.849862 s
+ split reduction: 0.085821 s
+Lightning Indexer: 1.098980 s
+TOP_K: 0.071036 s
+
+Probability reuse, run 2:
+209.57 tok/s
+Total Vulkan: 9.72968 s
+Split sparse FA: 1.13553 s
+ raw-prefix FA: 0.192987 s
+ selected sparse FA: 0.859890 s
+ split reduction: 0.082653 s
+Lightning Indexer: 1.102550 s
+TOP_K: 0.071474 s
+```
+
+The two-run selected-stage improvement is 5.5-6.6%, and the complete split sparse-FA improvement is 4.4-5.0%. The two-run throughput mean is 210.30 tok/s, 0.32% above the 209.62 tok/s reference. End-to-end noise in unrelated model kernels is larger than this small total-throughput change, but both profiler runs isolate a consistent gain in the modified selected stage.
+
+Correctness after the change:
+
+- focused sparse top-K suite: 9/9 passed against the CPU reference
+- complete Vulkan Flash Attention suite: 13,297/13,297 passed
+- pipeline statistics probe: 1/1 passed, with no SGPR or VGPR spills
+- no NaN or Inf failure
+
+Logs:
+
+- `/tmp/dsv4-fa-selected-qk-tiled.log`
+- `/tmp/dsv4-fa-selected-softmax-tiled.log`
+- `/tmp/dsv4-fa-lds-alias-stats.log`
+- `/tmp/dsv4-fa-pmat-stats.log`
+- `/tmp/dsv4-fa-exact-lds-alias.log`
+- `/tmp/dsv4-fa-exact-pmat.log`
+- `/tmp/dsv4-fa-pmat-correctness.log`
+- `/tmp/dsv4-fa-pmat-all-correctness.log`
+- `/tmp/dsv4-vulkan-32k-pmat.log`
+- `/tmp/dsv4-vulkan-32k-pmat-repeat.log`
+
+The next sparse-FA optimization should continue to target the selected PV/output half. The retained probability fragments remove redundant cooperative loads without increasing reported VGPR allocation. More invasive changes such as doubling the PV dimension tile can reduce barriers but must be designed around the eight available wave64 subgroups, 64 KiB LDS limit, and already high 192-VGPR allocation. Do not build a crossover matrix until the next kernel layout is settled because an optimization can shift the crossover.
+
+## Selected mask caching and deep-context micro matrix
+
+The sparse FA `kv` dimension is compressed K/V rows, not source-token context depth. For the tested DeepSeek V4 graph, a PP2048 batch uses 2,304 raw rows and approximately one compressed row per four source tokens. The useful synthetic mapping is:
+
+```text
+Source context Sparse FA kv rows
+32k 11,008
+64k 19,200
+128k 35,584
+256k 68,352
+512k 133,888
+```
+
+The performance test registry now contains PP2048 cases at all five K extents with `n_kv_raw=2304` and `n_top_k=512`. Use this command template and replace `KV` with a value from the table:
+
+```bash
+GGML_VK_PERF_LOGGER=1 ./build/bin/test-backend-ops perf \
+ -b Vulkan0 -o FLASH_ATTN_EXT \
+ -p 'kv=KV,nb=2048,n_kv_raw=2304,n_top_k=512,sinks=0'
+```
+
+These are synthetic sparse-FA depth tests. They do not include the Lightning Indexer or the rest of the model graph and do not replace the canonical 32k llama-bench.
+
+The selected shader previously loaded the same selected-key mask value independently for all 32 heads during both softmax passes. The shader now loads each mask value once per 64-key block and reuses it from LDS. Q and probability storage also share one LDS allocation, while K and V staging share another because both pairs have disjoint lifetimes.
+
+Shader resources changed from the probability-reuse commit:
+
+```text
+ Before After
+VGPRs 192 192
+VGPR spills 0 0
+LDS 28,672 24,576 bytes
+static instructions 11,358 11,233
+```
+
+A 128-dimension V staging experiment was correct but rejected. With operand aliasing it used exactly 32 KiB LDS. It was 2.3% slower at simulated 32k, approximately equal at 64k, and 1.6% faster at 128k. The retained 64-dimension layout is better for the canonical 32k target and is simpler.
+
+Selected-mask caching improved every synthetic depth relative to the same 64-dimension operand-alias layout:
+
+```text
+Depth Selected before Selected after Split total before Split total after
+32k 43.16 ms 41.23 ms 109.45 ms 108.07 ms
+64k 43.22 ms 41.42 ms 110.45 ms 109.11 ms
+128k 52.26 ms 48.74 ms 119.15 ms 116.92 ms
+256k 51.64 ms 49.02 ms 118.06 ms 116.93 ms
+512k 51.33 ms 49.20 ms 117.87 ms 116.27 ms
+```
+
+The 256k case exposed a separate path-selection cutoff. The raw-prefix ordinary FA dispatch encoded its mask row stride in 16 bits and disabled split sparse FA above 65,535 K rows. It now uses a flagged convention that carries the full 32-bit mask stride in the existing `split_kv` push constant for this one-partition partial-output dispatch. No push-constant structure was enlarged. At simulated 256k, this changes the selected path from unsplit cooperative sparse FA at 206.22 ms to split sparse FA at 118.06 ms before mask caching, a 42.8% reduction. The 512k case also uses the split path successfully.
+
+Canonical 32k PP2048 after operand aliasing and selected-mask caching:
+
+```text
+Previous commit, two runs:
+209.57-211.03 tok/s
+Total Vulkan: 9.664-9.730 s
+Split sparse FA: 1.128-1.136 s
+ raw-prefix FA: 0.192-0.193 s
+ selected sparse FA: 0.850-0.860 s
+ split reduction: 0.083-0.086 s
+
+Current result:
+210.47 tok/s
+Total Vulkan: 9.68921 s
+Split sparse FA: 1.10160 s
+ raw-prefix FA: 0.192860 s
+ selected sparse FA: 0.821381 s
+ split reduction: 0.087354 s
+Lightning Indexer: 1.102230 s
+TOP_K: 0.072488 s
+```
+
+The selected stage improves 3.4-4.5% and complete split sparse FA improves 2.3-3.0% relative to the previous commit's two canonical runs. End-to-end throughput remains within model-wide benchmark noise.
+
+Correctness:
+
+- focused sparse top-K suite: 9/9 passed
+- complete Vulkan Flash Attention suite: 13,297/13,297 passed
+- no NaN or Inf failure
+- simulated 256k and 512k performance cases both selected the split path and completed successfully
+
+Benchmark policy for this laptop APU: do not repeat llama-bench when the exact-shape microbenchmarks and the first canonical profiler block agree. Sustained load can lower GPU clocks and bias a repeat. Repeat only when the first result is anomalous or contradicts the microbenchmarks. Never build and benchmark concurrently.
+
+Logs:
+
+- `/tmp/dsv4-fa-mask-cache-32k.log`
+- `/tmp/dsv4-fa-mask-cache-64k.log`
+- `/tmp/dsv4-fa-mask-cache-128k.log`
+- `/tmp/dsv4-fa-mask-cache-256k.log`
+- `/tmp/dsv4-fa-mask-cache-512k.log`
+- `/tmp/dsv4-fa-final-micro-32k.log`
+- `/tmp/dsv4-fa-final-focused-correctness.log`
+- `/tmp/dsv4-fa-mask-cache-all-correctness.log`
+- `/tmp/dsv4-vulkan-32k-mask-cache.log`
diff --git a/ggml/include/ggml-backend.h b/ggml/include/ggml-backend.h
index cc3f8cd36e35..27375bd0a51e 100644
--- a/ggml/include/ggml-backend.h
+++ b/ggml/include/ggml-backend.h
@@ -424,6 +424,10 @@ extern "C" {
// Compare the output of two backends
GGML_API bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t backend2, struct ggml_cgraph * graph, ggml_backend_eval_callback callback, void * user_data, struct ggml_tensor const * const * test_nodes, size_t num_test_nodes);
+ // returns true for ops that may require additional memory for fleeting data on some backends,
+ // i.e. the backend's get_alloc_size may return more than ggml_nbytes for the output tensor
+ GGML_API bool ggml_backend_op_alloc_size_may_expand(enum ggml_op op);
+
// Tensor initialization
GGML_API enum ggml_status ggml_backend_tensor_alloc(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, void * addr);
GGML_API enum ggml_status ggml_backend_view_init(struct ggml_tensor * tensor);
diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h
index 5f6774a630c0..9071e3c7056a 100644
--- a/ggml/include/ggml.h
+++ b/ggml/include/ggml.h
@@ -2450,6 +2450,16 @@ extern "C" {
struct ggml_tensor * a,
struct ggml_tensor * sinks);
+ // sparse attention hint: attend only to the first n_kv_raw keys (dense prefix) plus the
+ // keys selected by top_k. top_k is I32 [n_top_k, n_tokens, 1, n_streams]; each index i
+ // selects absolute key n_kv_raw + i. Negative or out-of-range indices are ignored.
+ // Backends may ignore the hint: the kq_mask must still encode the same selection, so a
+ // dense fallback computes the identical result.
+ GGML_API void ggml_flash_attn_ext_add_top_k(
+ struct ggml_tensor * a,
+ struct ggml_tensor * top_k,
+ int64_t n_kv_raw);
+
// TODO: needs to be adapted to ggml_flash_attn_ext
GGML_API struct ggml_tensor * ggml_flash_attn_back(
struct ggml_context * ctx,
diff --git a/ggml/src/ggml-backend-impl.h b/ggml/src/ggml-backend-impl.h
index 40cea024c3d2..56f0090cce66 100644
--- a/ggml/src/ggml-backend-impl.h
+++ b/ggml/src/ggml-backend-impl.h
@@ -103,6 +103,16 @@ extern "C" {
// Backend (stream)
//
+ // passed to graph_optimize so the backend can add allocation dependencies:
+ // if the backend executes parts of the graph out of order (e.g. on concurrent streams),
+ // it must keep the affected tensors allocated until a node where execution is known to have joined
+ struct ggml_backend_graph_optimize_params {
+ // keep `tensor` allocated at least until `until` (a node of the same graph) has been computed
+ // can be called multiple times for the same tensor: the longest lifetime applies
+ void (*add_alloc_dep)(void * user_data, struct ggml_tensor * tensor, struct ggml_tensor * until);
+ void * user_data;
+ };
+
struct ggml_backend_i {
const char * (*get_name)(ggml_backend_t backend);
@@ -137,7 +147,7 @@ extern "C" {
void (*event_wait) (ggml_backend_t backend, ggml_backend_event_t event);
// (optional) sort/optimize the nodes in the graph
- void (*graph_optimize) (ggml_backend_t backend, struct ggml_cgraph * cgraph);
+ void (*graph_optimize) (ggml_backend_t backend, struct ggml_cgraph * cgraph, struct ggml_backend_graph_optimize_params * params);
};
struct ggml_backend {
diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp
index e519bdf50a1b..c3a0a12889b5 100644
--- a/ggml/src/ggml-backend.cpp
+++ b/ggml/src/ggml-backend.cpp
@@ -20,6 +20,7 @@
#include
#include
#include
+#include
#include
#ifdef __APPLE__
@@ -64,6 +65,13 @@ size_t ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type_t buft, const s
if (buft->iface.get_alloc_size) {
size_t size = buft->iface.get_alloc_size(buft, tensor);
assert(size >= ggml_nbytes(tensor));
+
+ // [TAG_ALLOC_SIZE_EXPAND]
+ // if you hit this assert, update ggml_backend_op_alloc_size_may_expand() accordingly
+ GGML_ASSERT(size <= ggml_nbytes(tensor) ||
+ ggml_op_is_empty(tensor->op) ||
+ ggml_backend_op_alloc_size_may_expand(tensor->op));
+
return size;
}
return ggml_nbytes(tensor);
@@ -558,10 +566,10 @@ void ggml_backend_event_wait(ggml_backend_t backend, ggml_backend_event_t event)
backend->iface.event_wait(backend, event);
}
-static void ggml_backend_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * cgraph) {
+static void ggml_backend_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * cgraph, struct ggml_backend_graph_optimize_params * params) {
GGML_ASSERT(backend);
if (backend->iface.graph_optimize != NULL) {
- backend->iface.graph_optimize(backend, cgraph);
+ backend->iface.graph_optimize(backend, cgraph, params);
}
}
@@ -1331,7 +1339,10 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra
}
// check if the split has too many inputs
// FIXME: count the number of inputs instead of only checking when full
- if (split->n_inputs >= split->inputs_capacity) {
+ // cut on the constant, not on inputs_capacity: capacity doubles on demand and
+ // is never reset, so using it lets the cut point drift up and keeps every input
+ // copy of an ever-longer split live at once
+ if (split->n_inputs >= GGML_SCHED_MAX_SPLIT_INPUTS) {
const size_t id = hash_id(src);
int src_backend_id = sched->hv_tensor_backend_ids[id];
bool supported = ggml_backend_sched_buffer_supported(sched, src, cur_backend_id);
@@ -1441,11 +1452,40 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra
sched->prev_leaf_backend_ids = tmp;
}
+ // optimize the split graphs and collect the allocation dependencies added by the backends
+ // this needs to happen before we make graph_copy, so they are in sync
+ // TODO: this may create many small allocations in the scheduler, restructure to use a flat array
+ std::unordered_map> alloc_deps;
+
+ struct ggml_backend_graph_optimize_params opt_params = {
+ /* .add_alloc_dep = */ [](void * user_data, ggml_tensor * tensor, ggml_tensor * until) {
+ auto & deps = *(std::unordered_map> *) user_data;
+ std::vector & keep = deps[until];
+ if (std::find(keep.begin(), keep.end(), tensor) == keep.end()) {
+ keep.push_back(tensor);
+ }
+ },
+ /* .user_data = */ &alloc_deps,
+ };
+
+ for (int i = 0; i < sched->n_splits; i++) {
+ struct ggml_backend_sched_split * split = &sched->splits[i];
+ split->graph = ggml_graph_view(graph, split->i_start, split->i_end);
+
+ ggml_backend_graph_optimize(sched->backends[split->backend_id], &split->graph, &opt_params);
+ }
+
+ // each dep is added to graph_copy as a GGML_OP_NONE node with the kept tensors as srcs
+ int n_dep_nodes = 0;
+ for (const auto & it : alloc_deps) {
+ n_dep_nodes += (it.second.size() + GGML_MAX_SRC - 1) / GGML_MAX_SRC;
+ }
+
int total_inputs = sched->n_graph_inputs;
for (int i = 0; i < sched->n_splits; i++) {
total_inputs += sched->splits[i].n_inputs;
}
- int graph_size = std::max(graph->n_nodes, graph->n_leafs) + total_inputs * 2 * sched->n_copies;
+ int graph_size = std::max(graph->n_nodes, graph->n_leafs) + total_inputs * 2 * sched->n_copies + n_dep_nodes;
// remember the actual graph_size for performing reallocation checks later [GGML_SCHED_DEBUG_REALLOC]
sched->debug_prev_graph_size = sched->debug_graph_size;
@@ -1463,13 +1503,10 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra
struct ggml_cgraph * graph_copy = &sched->graph;
+ int n_dep_nodes_added = 0;
+
for (int i = 0; i < sched->n_splits; i++) {
struct ggml_backend_sched_split * split = &sched->splits[i];
- split->graph = ggml_graph_view(graph, split->i_start, split->i_end);
-
- // Optimize this split of the graph. This needs to happen before we make graph_copy,
- // so they are in sync.
- ggml_backend_graph_optimize(sched->backends[split->backend_id], &split->graph);
// add inputs to the graph copy so that they are allocated by ggml-alloc at the start of the split
for (int j = 0; j < split->n_inputs; j++) {
@@ -1494,9 +1531,32 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra
assert(graph_copy->size > graph_copy->n_nodes);
sched->node_backend_ids[graph_copy->n_nodes] = tensor_backend_id(graph->nodes[j]);
graph_copy->nodes[graph_copy->n_nodes++] = graph->nodes[j];
+
+ if (alloc_deps.empty()) {
+ continue;
+ }
+
+ // add a dependency node so that the kept tensors are not freed before this node is computed
+ auto it = alloc_deps.find(graph->nodes[j]);
+ if (it != alloc_deps.end()) {
+ const std::vector & keep = it->second;
+ for (size_t k = 0; k < keep.size(); k += GGML_MAX_SRC) {
+ struct ggml_tensor * dep = ggml_view_tensor(sched->ctx, keep[k]);
+ for (size_t s = 0; s < GGML_MAX_SRC && k + s < keep.size(); s++) {
+ dep->src[s] = keep[k + s];
+ }
+ assert(graph_copy->size > graph_copy->n_nodes);
+ sched->node_backend_ids[graph_copy->n_nodes] = split->backend_id;
+ graph_copy->nodes[graph_copy->n_nodes++] = dep;
+ n_dep_nodes_added++;
+ }
+ }
}
}
+ // a mismatch means a backend added a dep with an `until` tensor that is not a node of the optimized graph
+ GGML_ASSERT(n_dep_nodes_added == n_dep_nodes);
+
if (sched->n_copies > 1) {
// add input copies as leafs so that they are allocated first
for (int i = 0; i < sched->n_graph_inputs; i++) {
@@ -2051,6 +2111,22 @@ ggml_backend_t ggml_backend_sched_get_tensor_backend(ggml_backend_sched_t sched,
// utils
+// [TAG_ALLOC_SIZE_EXPAND]
+// returns true for ops that may require additional memory for fleeting data on some backends,
+// i.e. the backend's get_alloc_size may return more than ggml_nbytes for the output tensor
+bool ggml_backend_op_alloc_size_may_expand(enum ggml_op op) {
+ switch (op) {
+ case GGML_OP_FLASH_ATTN_EXT:
+ case GGML_OP_MUL_MAT_ID:
+ case GGML_OP_CUMSUM:
+ case GGML_OP_ARGSORT:
+ case GGML_OP_TOP_K:
+ return true;
+ default:
+ return false;
+ }
+}
+
enum ggml_status ggml_backend_view_init(struct ggml_tensor * tensor) {
GGML_ASSERT(tensor);
GGML_ASSERT(tensor->buffer == NULL);
diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c
index 87ac0a702efc..b9c0fa3ddc01 100644
--- a/ggml/src/ggml-cpu/ggml-cpu.c
+++ b/ggml/src/ggml-cpu/ggml-cpu.c
@@ -2936,12 +2936,13 @@ struct ggml_cplan ggml_graph_plan(
const int64_t ne10 = node->src[1]->ne[0]; // W
const int64_t ne11 = node->src[1]->ne[1]; // H
const int64_t ne12 = node->src[1]->ne[2]; // Channels In
+ const int64_t ne13 = node->src[1]->ne[3]; // Batch
GGML_ASSERT(node->src[0]->type == GGML_TYPE_F16 || node->src[0]->type == GGML_TYPE_F32);
GGML_ASSERT(node->src[1]->type == GGML_TYPE_F32);
cur += ggml_type_size(node->src[0]->type) * ne00 * ne01 * ne02 * ne03;
- cur += ggml_type_size(node->src[0]->type) * ne10 * ne11 * ne12;
+ cur += ggml_type_size(node->src[0]->type) * ne10 * ne11 * ne12 * ne13;
} break;
case GGML_OP_TOP_K:
diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
index b869f4bddde0..b47ce5463c61 100644
--- a/ggml/src/ggml-cpu/ops.cpp
+++ b/ggml/src/ggml-cpu/ops.cpp
@@ -7267,18 +7267,21 @@ static void ggml_compute_forward_conv_transpose_2d_impl(
}
}
- // permute source data (src1) from (Sw x Sh x Cin) to (Cin x Sw x Sh)
+ // permute source data (src1) from (Sw x Sh x Cin) to (Cin x Sw x Sh), for all batches
{
kernel_t * const wdata = (kernel_t *) params->wdata + nk;
- for (int i12 = 0; i12 < ne12; i12++) {
- for (int i11 = 0; i11 < ne11; i11++) {
- const float * const src = (float *)((char *) src1->data + i12*nb12 + i11*nb11);
- kernel_t * dst_data = wdata + i11*ne10*ne12;
- for (int i10 = 0; i10 < ne10; i10++) {
- if constexpr (std::is_same_v) {
- dst_data[i10*ne12 + i12] = GGML_CPU_FP32_TO_FP16(src[i10]);
- } else {
- dst_data[i10*ne12 + i12] = src[i10];
+ for (int i13 = 0; i13 < ne13; i13++) {
+ kernel_t * const wdata_b = wdata + i13*ne10*ne11*ne12;
+ for (int i12 = 0; i12 < ne12; i12++) {
+ for (int i11 = 0; i11 < ne11; i11++) {
+ const float * const src = (float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11);
+ kernel_t * dst_data = wdata_b + i11*ne10*ne12;
+ for (int i10 = 0; i10 < ne10; i10++) {
+ if constexpr (std::is_same_v) {
+ dst_data[i10*ne12 + i12] = GGML_CPU_FP32_TO_FP16(src[i10]);
+ } else {
+ dst_data[i10*ne12 + i12] = src[i10];
+ }
}
}
}
@@ -7305,24 +7308,27 @@ static void ggml_compute_forward_conv_transpose_2d_impl(
kernel_t * const wdata_src = wdata + nk;
for (int i2 = ip0; i2 < ip1; i2++) { // Cout
- float * dst_data = (float *)((char *) dst->data + i2*nb2);
kernel_t * wdata_kernel = wdata + i2*ne01*ne00*ne03;
- for (int i11 = 0; i11 < ne11; i11++) {
- for (int i10 = 0; i10 < ne10; i10++) {
- const int i1n = i11*ne10*ne12 + i10*ne12;
- for (int i01 = 0; i01 < ne01; i01++) {
- for (int i00 = 0; i00 < ne00; i00++) {
- float v = 0;
- if constexpr (std::is_same_v) {
- ggml_vec_dot_f16(ne03, &v, 0,
- wdata_src + i1n, 0,
- wdata_kernel + i01*ne00*ne03 + i00*ne03, 0, 1);
- } else {
- ggml_vec_dot_f32(ne03, &v, 0,
- wdata_src + i1n, 0,
- wdata_kernel + i01*ne00*ne03 + i00*ne03, 0, 1);
+ for (int i3 = 0; i3 < ne3; i3++) { // batch
+ float * dst_data = (float *)((char *) dst->data + i3*nb3 + i2*nb2);
+ kernel_t * wdata_src_b = wdata_src + i3*ne10*ne11*ne12;
+ for (int i11 = 0; i11 < ne11; i11++) {
+ for (int i10 = 0; i10 < ne10; i10++) {
+ const int i1n = i11*ne10*ne12 + i10*ne12;
+ for (int i01 = 0; i01 < ne01; i01++) {
+ for (int i00 = 0; i00 < ne00; i00++) {
+ float v = 0;
+ if constexpr (std::is_same_v) {
+ ggml_vec_dot_f16(ne03, &v, 0,
+ wdata_src_b + i1n, 0,
+ wdata_kernel + i01*ne00*ne03 + i00*ne03, 0, 1);
+ } else {
+ ggml_vec_dot_f32(ne03, &v, 0,
+ wdata_src_b + i1n, 0,
+ wdata_kernel + i01*ne00*ne03 + i00*ne03, 0, 1);
+ }
+ dst_data[(i11*stride + i01)*ne0 + i10*stride + i00] += v;
}
- dst_data[(i11*stride + i01)*ne0 + i10*stride + i00] += v;
}
}
}
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
index 2456f7dcc621..bd9754c2ffdf 100644
--- a/ggml/src/ggml-cuda/ggml-cuda.cu
+++ b/ggml/src/ggml-cuda/ggml-cuda.cu
@@ -4328,7 +4328,9 @@ static void ggml_backend_cuda_event_wait(ggml_backend_t backend, ggml_backend_ev
}
}
-static void ggml_backend_cuda_graph_optimize(ggml_backend_t backend, ggml_cgraph * cgraph) {
+static void ggml_backend_cuda_graph_optimize(ggml_backend_t backend, ggml_cgraph * cgraph, ggml_backend_graph_optimize_params * params) {
+ GGML_UNUSED(params);
+
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;
#ifdef USE_CUDA_GRAPH
diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp
index c1e9f919d92a..87e6989bd2f8 100644
--- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp
+++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp
@@ -69,30 +69,15 @@ using u32vec = std::vector;
#define GGML_HEXAGON_FENCE_SLOT_SIZE 128
struct ggml_hexagon_device_config {
- int physical_idx = 0;
- int virtual_idx = 0;
+ int physical_idx = 0;
+ int virtual_idx = 0;
+ int domain_id = 0;
+ std::string domain_name;
std::string name;
};
static ggml_hexagon_device_config opt_device_configs[GGML_HEXAGON_MAX_SESSIONS];
-static int get_domain_id(int physical_idx) {
- switch (physical_idx) {
- case 0: return 3; // CDSP0 (all devices)
- case 1: return 4; // CDSP1 (IQ9, IQ10)
- case 2: return 18; // CDSP2 (IQ10)
- case 3: return 19; // CDSP3 (IQ10)
- default: return CDSP_DOMAIN_ID + physical_idx;
- }
-}
-
-static std::string get_domain_name(int physical_idx) {
- if (physical_idx == 0) {
- return CDSP_DOMAIN_NAME;
- }
- return std::string("cdsp") + std::to_string(physical_idx);
-}
-
static int opt_arch = 0; // autodetect
static size_t opt_ndev = 1;
static size_t opt_nhvx = 0; // use all
@@ -361,7 +346,6 @@ struct ggml_hexagon_session {
uint32_t session_id;
uint32_t domain_id;
uint64_t queue_id;
- int dev_id;
int phys_idx;
int virt_idx;
bool valid_session;
@@ -376,9 +360,6 @@ struct ggml_hexagon_session {
std::unordered_map> cloned_buffers;
std::unordered_set sync_peers;
- ggml_backend_buffer_type buffer_type = {};
- ggml_backend_buffer_type host_buffer_type = {};
-
uint32_t n_threads = 0;
uint32_t n_hvx = 0;
uint32_t n_hmx = 0;
@@ -392,12 +373,12 @@ struct ggml_hexagon_session {
mutable std::unordered_set needs_repack;
- ggml_hexagon_session(int dev_id, ggml_backend_dev_t dev) noexcept(false);
+ ggml_hexagon_session(const ggml_hexagon_device_config & config, ggml_backend_dev_t dev = nullptr) noexcept(false);
~ggml_hexagon_session() noexcept(true);
const char* c_name() const { return name.c_str(); }
- void allocate(int dev_id) noexcept(false);
+ void allocate(const ggml_hexagon_device_config & config) noexcept(false);
void release() noexcept(true);
void enqueue_op(const htp_opnode & node);
@@ -430,14 +411,38 @@ struct ggml_hexagon_session {
// ** backend buffers
+struct ggml_backend_hexagon_device_context {
+ int dev_id;
+ ggml_hexagon_device_config config;
+ ggml_backend_dev_t dev = nullptr;
+ size_t max_bufsize = 0;
+
+ ggml_backend_buffer_type buffer_type = {};
+ ggml_backend_buffer_type host_buffer_type = {};
+
+ std::unique_ptr sess;
+
+ ggml_backend_hexagon_device_context(int dev_id, const ggml_hexagon_device_config & config, ggml_backend_dev_t dev);
+ ~ggml_backend_hexagon_device_context();
+
+ const char * c_name() const { return config.name.c_str(); }
+
+ ggml_hexagon_session * session() {
+ if (!sess) {
+ sess = std::make_unique(config, dev);
+ }
+ return sess.get();
+ }
+};
+
struct ggml_backend_hexagon_buffer_type_context {
- ggml_backend_hexagon_buffer_type_context(const std::string & name, ggml_hexagon_session * sess) {
- this->sess = sess;
- this->name = name;
+ ggml_backend_hexagon_buffer_type_context(const std::string & name, ggml_backend_hexagon_device_context * dev_ctx) {
+ this->dev_ctx = dev_ctx;
+ this->name = name;
}
- ggml_hexagon_session * sess;
- std::string name;
+ ggml_backend_hexagon_device_context * dev_ctx;
+ std::string name;
};
struct ggml_hexagon_rpcmem_block {
@@ -576,7 +581,8 @@ struct ggml_hexagon_shared_buffer {
};
static ggml_hexagon_session * ggml_backend_hexagon_buffer_get_sess(ggml_backend_buffer_t buffer) {
- return static_cast(buffer->buft->context)->sess;
+ auto sbuf = static_cast(buffer->context);
+ return sbuf->sess;
}
static void ggml_backend_hexagon_buffer_free_buffer(ggml_backend_buffer_t buffer) {
@@ -1494,24 +1500,26 @@ static const char * ggml_backend_hexagon_buffer_type_name(ggml_backend_buffer_ty
static ggml_backend_buffer_t ggml_backend_hexagon_buffer_type_alloc_buffer(
ggml_backend_buffer_type_t buffer_type, size_t size) {
- auto sess = static_cast(buffer_type->context)->sess;
+ auto dev_ctx = static_cast(buffer_type->context)->dev_ctx;
+ auto sess = dev_ctx->session();
try {
ggml_hexagon_shared_buffer * sbuf = new ggml_hexagon_shared_buffer(sess, size, false, GGML_HEXAGON_FENCE_BUFFER_SIZE);
return ggml_backend_buffer_init(buffer_type, ggml_backend_hexagon_buffer_interface, sbuf, size);
} catch (const std::exception & exc) {
- GGML_LOG_ERROR("ggml-hex: %s failed to allocate device buffer context: %s\n", sess->c_name(), exc.what());
+ GGML_LOG_ERROR("ggml-hex: %s failed to allocate device buffer context: %s\n", dev_ctx->c_name(), exc.what());
return nullptr;
}
}
static ggml_backend_buffer_t ggml_backend_hexagon_host_buffer_type_alloc_buffer(
ggml_backend_buffer_type_t buffer_type, size_t size) {
- auto sess = static_cast(buffer_type->context)->sess;
+ auto dev_ctx = static_cast(buffer_type->context)->dev_ctx;
+ auto sess = dev_ctx->session();
try {
ggml_hexagon_shared_buffer * sbuf = new ggml_hexagon_shared_buffer(sess, size, false, GGML_HEXAGON_FENCE_BUFFER_SIZE);
return ggml_backend_buffer_init(buffer_type, ggml_backend_hexagon_host_buffer_interface, sbuf, size);
} catch (const std::exception & exc) {
- GGML_LOG_ERROR("ggml-hex: %s failed to allocate host buffer context: %s\n", sess->c_name(), exc.what());
+ GGML_LOG_ERROR("ggml-hex: %s failed to allocate host buffer context: %s\n", dev_ctx->c_name(), exc.what());
return nullptr;
}
}
@@ -1536,7 +1544,7 @@ static size_t ggml_backend_hexagon_buffer_type_get_alloc_size(ggml_backend_buffe
static size_t ggml_backend_hexagon_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) {
auto * context = static_cast(buft->context);
- return context->sess->max_bufsize;
+ return context->dev_ctx->max_bufsize;
}
static bool ggml_backend_hexagon_buffer_type_is_host(ggml_backend_buffer_type_t buft) {
@@ -1567,6 +1575,22 @@ static ggml_backend_buffer_type_i ggml_backend_hexagon_host_buffer_type_interfac
/* .is_host = */ ggml_backend_hexagon_host_buffer_type_is_host,
};
+ggml_backend_hexagon_device_context::ggml_backend_hexagon_device_context(int dev_id, const ggml_hexagon_device_config & config, ggml_backend_dev_t dev)
+ : dev_id(dev_id), config(config), dev(dev), max_bufsize(opt_mbuf) {
+ buffer_type.device = dev;
+ buffer_type.iface = ggml_backend_hexagon_buffer_type_interface;
+ buffer_type.context = new ggml_backend_hexagon_buffer_type_context(config.name, this);
+
+ host_buffer_type.device = dev;
+ host_buffer_type.iface = ggml_backend_hexagon_host_buffer_type_interface;
+ host_buffer_type.context = new ggml_backend_hexagon_buffer_type_context(config.name + "-HOST", this);
+}
+
+ggml_backend_hexagon_device_context::~ggml_backend_hexagon_device_context() {
+ delete static_cast(buffer_type.context);
+ delete static_cast(host_buffer_type.context);
+}
+
static bool ggml_backend_buffer_is_hexagon(const struct ggml_backend_buffer * b) {
return b->buft->iface.get_alignment == ggml_backend_hexagon_buffer_type_get_alignment;
}
@@ -2811,8 +2835,7 @@ static size_t ggml_hexagon_measure_max_vmem(ggml_hexagon_session *sess) {
return vmem - step; // backoff to account for overhead from internal mappings
}
-void ggml_hexagon_session::allocate(int dev_id) noexcept(false) {
- const auto & config = opt_device_configs[dev_id];
+void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) noexcept(false) {
int phys_idx = config.physical_idx;
int virt_idx = config.virtual_idx;
@@ -2823,21 +2846,31 @@ void ggml_hexagon_session::allocate(int dev_id) noexcept(false) {
this->phys_idx = phys_idx;
this->virt_idx = virt_idx;
- this->domain_id = get_domain_id(phys_idx);
+ this->domain_id = config.domain_id;
this->session_id = 0;
- this->dev_id = dev_id;
this->name = config.name;
this->op_pending = 0;
GGML_LOG_DEBUG("ggml-hex: %s allocating new session\n", this->name.c_str());
- domain * my_domain = htpdrv_get_domain(this->domain_id);
- if (my_domain == NULL) {
- GGML_LOG_ERROR("ggml-hex: unable to get domain struct for CDSP (domain_id %d)\n", this->domain_id);
- throw std::runtime_error("ggml-hex: failed to get CDSP domain (see log for details)");
+ if (config.domain_id < 0 || config.domain_name.empty()) {
+ GGML_LOG_ERROR("ggml-hex: %s: invalid physical CDSP core %d\n", config.name.c_str(), config.physical_idx);
+ throw std::runtime_error("ggml-hex: invalid physical CDSP core");
}
- std::string dom_name = get_domain_name(phys_idx);
+ const std::string & dom_name = config.domain_name;
+
+ // Enable Unsigned PD for all domains
+ {
+ struct remote_rpc_control_unsigned_module u;
+ u.domain = -1;
+ u.enable = 1;
+ int err = remote_session_control(DSPRPC_CONTROL_UNSIGNED_MODULE, (void *) &u, sizeof(u));
+ if (err != AEE_SUCCESS) {
+ GGML_LOG_ERROR("ggml-hex: %s failed to enable unsigned PD : error 0x%x\n", this->c_name(), err);
+ throw std::runtime_error("ggml-hex: remote_session_control(unsign) failed (see log for details)");
+ }
+ }
// Create new session if virtual_idx > 0
if (virt_idx > 0) {
@@ -2849,7 +2882,8 @@ void ggml_hexagon_session::allocate(int dev_id) noexcept(false) {
int err = remote_session_control(FASTRPC_RESERVE_NEW_SESSION, (void *) &n, sizeof(n));
if (err != AEE_SUCCESS) {
- GGML_LOG_ERROR("ggml-hex: failed to reserve new session %d (physical %d, virtual %d) : error 0x%x\n", dev_id, phys_idx, virt_idx, err);
+ GGML_LOG_ERROR("ggml-hex: %s failed to reserve new session (physical %d, virtual %d) : error 0x%x\n",
+ this->c_name(), phys_idx, virt_idx, err);
throw std::runtime_error("ggml-hex: remote_session_control(new-sess) failed (see log for details)");
}
@@ -2857,10 +2891,21 @@ void ggml_hexagon_session::allocate(int dev_id) noexcept(false) {
this->session_id = n.session_id;
this->domain_id = n.effective_domain_id;
this->valid_session = true;
+ } else {
+ struct remote_rpc_effective_domain_id eff = {};
+ eff.domain_name = const_cast(dom_name.c_str());
+ eff.domain_name_len = dom_name.size();
+ eff.session_id = 0;
+
+ int err = remote_session_control(FASTRPC_GET_EFFECTIVE_DOMAIN_ID, (void *) &eff, sizeof(eff));
+ if (err == AEE_SUCCESS) {
+ this->domain_id = eff.effective_domain_id;
+ } else {
+ GGML_LOG_DEBUG("ggml-hex: %s FASTRPC_GET_EFFECTIVE_DOMAIN_ID returned 0x%x, using domain_id %d\n",
+ this->name.c_str(), err, this->domain_id);
+ }
}
- // Get session URI
-
char session_uri[256];
{
char htp_uri[256];
@@ -2877,31 +2922,18 @@ void ggml_hexagon_session::allocate(int dev_id) noexcept(false) {
int err = remote_session_control(FASTRPC_GET_URI, (void *) &u, sizeof(u));
if (err != AEE_SUCCESS) {
- // fallback to single session uris
- int htp_URI_domain_len = strlen(htp_uri) + MAX_DOMAIN_NAMELEN;
+ snprintf(session_uri, sizeof(session_uri), "%s&_dom=%s&_session=%u",
+ htp_uri, dom_name.c_str(), this->session_id);
- snprintf(session_uri, htp_URI_domain_len, "%s%s", htp_uri, my_domain->uri);
-
- GGML_LOG_WARN("ggml-hex: failed to get URI for session %d (physical %d, virtual %d) : error 0x%x. Falling back to single session URI: %s\n", dev_id, phys_idx, virt_idx, err, session_uri);
- }
- }
-
- // Enable Unsigned PD
- {
- struct remote_rpc_control_unsigned_module u;
- u.domain = this->domain_id;
- u.enable = 1;
- int err = remote_session_control(DSPRPC_CONTROL_UNSIGNED_MODULE, (void *) &u, sizeof(u));
- if (err != AEE_SUCCESS) {
- GGML_LOG_ERROR("ggml-hex: failed to enable unsigned PD for session %d : error 0x%x\n", dev_id, err);
- throw std::runtime_error("ggml-hex: remote_session_control(unsign) failed (see log for details)");
+ GGML_LOG_WARN("ggml-hex: %s failed to get URI (physical %d, virtual %d) : error 0x%x. Falling back to single session URI: %s\n",
+ this->c_name(), phys_idx, virt_idx, err, session_uri);
}
}
// Open session
int err = htp_iface_open(session_uri, &this->handle);
if (err != AEE_SUCCESS) {
- GGML_LOG_ERROR("ggml-hex: failed to open session %d : error 0x%x\n", dev_id, err);
+ GGML_LOG_ERROR("ggml-hex: %s failed to open session : error 0x%x\n", this->c_name(), err);
throw std::runtime_error("ggml-hex: failed to open session (see log for details)");
}
@@ -2991,7 +3023,7 @@ void ggml_hexagon_session::allocate(int dev_id) noexcept(false) {
this->op_batch = new ggml_hexagon_opbatch(this, opt_opbatch, this->max_vmem);
// Start dspqueue/opbatch processing
- err = htp_iface_start(this->handle, dev_id, this->queue_id, opt_nhvx, opt_nhmx, this->max_vmem);
+ err = htp_iface_start(this->handle, this->session_id, this->queue_id, opt_nhvx, opt_nhmx, this->max_vmem);
if (err != 0) {
GGML_LOG_ERROR("ggml-hex: %s failed to start session: 0x%08x\n", this->c_name(), (unsigned) err);
throw std::runtime_error("ggml-hex: iface start failed (see log for details)");
@@ -3054,33 +3086,23 @@ void ggml_hexagon_session::release() noexcept(true) {
this->cloned_buffers.clear();
}
-ggml_hexagon_session::ggml_hexagon_session(int dev_id, ggml_backend_dev_t dev) noexcept(false) {
- buffer_type.device = dev;
- host_buffer_type.device = dev;
-
+ggml_hexagon_session::ggml_hexagon_session(const ggml_hexagon_device_config & config, ggml_backend_dev_t dev) noexcept(false) {
op_batch = nullptr;
op_queue = nullptr;
fence_seq = ((uintptr_t)this) & 0xFFFF;
try {
- allocate(dev_id);
-
- buffer_type.iface = ggml_backend_hexagon_buffer_type_interface;
- buffer_type.context = new ggml_backend_hexagon_buffer_type_context(this->name, this);
-
- host_buffer_type.iface = ggml_backend_hexagon_host_buffer_type_interface;
- host_buffer_type.context = new ggml_backend_hexagon_buffer_type_context(this->name + "-HOST", this);
+ allocate(config);
} catch (const std::exception & exc) {
release();
throw;
}
+
+ GGML_UNUSED(dev);
}
ggml_hexagon_session::~ggml_hexagon_session() noexcept(true) {
release();
-
- delete static_cast(buffer_type.context);
- delete static_cast(host_buffer_type.context);
}
// ** backend interface
@@ -3957,11 +3979,13 @@ static void ggml_hexagon_precompute_fused_mmnx_params(
}
static bool ggml_hexagon_tensor_is_host(const struct ggml_hexagon_session * sess, const struct ggml_tensor * t) {
- return t && t->buffer && t->buffer->buft == &sess->host_buffer_type;
+ return t && t->buffer && ggml_backend_buft_is_host(t->buffer->buft);
+ GGML_UNUSED(sess);
}
static bool ggml_hexagon_tensor_is_non_host(const struct ggml_hexagon_session * sess, const struct ggml_tensor * t) {
- return t && t->buffer && t->buffer->buft != &sess->host_buffer_type;
+ return t && t->buffer && !ggml_backend_buft_is_host(t->buffer->buft);
+ GGML_UNUSED(sess);
}
static bool ggml_hexagon_supported_mul_mat(const struct ggml_hexagon_session * sess, const struct ggml_tensor * dst) {
@@ -4643,6 +4667,7 @@ static htp_op_code op_remap_to_htp(const ggml_tensor * t) {
case GGML_OP_CLAMP: return HTP_OP_CLAMP;
case GGML_OP_SQR: return HTP_OP_SQR;
case GGML_OP_SQRT: return HTP_OP_SQRT;
+ case GGML_OP_LOG: return HTP_OP_UNARY_LOG;
case GGML_OP_SOFT_MAX: return HTP_OP_SOFTMAX;
case GGML_OP_SSM_CONV: return HTP_OP_SSM_CONV;
case GGML_OP_GATED_DELTA_NET: return HTP_OP_GATED_DELTA_NET;
@@ -4666,6 +4691,7 @@ static htp_op_code op_remap_to_htp(const ggml_tensor * t) {
case GGML_UNARY_OP_EXP: return HTP_OP_UNARY_EXP;
case GGML_UNARY_OP_SOFTPLUS: return HTP_OP_UNARY_SOFTPLUS;
case GGML_UNARY_OP_TANH: return HTP_OP_UNARY_TANH;
+ case GGML_UNARY_OP_ABS: return HTP_OP_UNARY_ABS;
default:
break;
}
@@ -4982,7 +5008,9 @@ static std::vector ggml_hexagon_graph_optimize_reorder(const std::vectorn_nodes;
constexpr int MAX_FUSE = 16;
@@ -5265,7 +5293,8 @@ bool ggml_backend_is_hexagon(ggml_backend_t backend) {
// device interface
static ggml_backend_t ggml_backend_hexagon_device_init(ggml_backend_dev_t dev, const char * params) {
- auto sess = static_cast(dev->context);
+ auto dev_ctx = static_cast(dev->context);
+ auto sess = dev_ctx->session();
return new ggml_backend{
/* .guid = */ ggml_backend_hexagon_guid(),
@@ -5278,8 +5307,8 @@ static ggml_backend_t ggml_backend_hexagon_device_init(ggml_backend_dev_t dev, c
}
static const char * ggml_backend_hexagon_device_get_name(ggml_backend_dev_t dev) {
- auto sess = static_cast(dev->context);
- return sess->c_name();
+ auto dev_ctx = static_cast(dev->context);
+ return dev_ctx->c_name();
GGML_UNUSED(dev);
}
@@ -5317,16 +5346,16 @@ static void ggml_backend_hexagon_device_get_props(ggml_backend_dev_t dev, struct
}
static ggml_backend_buffer_type_t ggml_backend_hexagon_device_get_buffer_type(ggml_backend_dev_t dev) {
- auto sess = static_cast(dev->context);
- return &sess->buffer_type;
+ auto dev_ctx = static_cast(dev->context);
+ return &dev_ctx->buffer_type;
}
static ggml_backend_buffer_type_t ggml_backend_hexagon_device_get_host_buffer_type(ggml_backend_dev_t dev) {
if (!opt_hostbuf) {
return NULL;
}
- auto sess = static_cast(dev->context);
- return &sess->host_buffer_type;
+ auto dev_ctx = static_cast(dev->context);
+ return &dev_ctx->host_buffer_type;
}
static bool ggml_hexagon_supported_cpy(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
@@ -5417,7 +5446,8 @@ static bool ggml_hexagon_supported_fill(const struct ggml_hexagon_session * sess
}
static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) {
- auto sess = static_cast(dev->context);
+ auto dev_ctx = static_cast(dev->context);
+ auto sess = dev_ctx->session();
// reject ops that match the filter
if (opt_opfilter && std::regex_match(ggml_op_desc(op), *opt_opfilter)) {
@@ -5463,6 +5493,7 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons
case GGML_OP_SQR:
case GGML_OP_SQRT:
+ case GGML_OP_LOG:
supp = ggml_hexagon_supported_unary(sess, op);
break;
@@ -5481,12 +5512,14 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons
case GGML_UNARY_OP_SIGMOID:
case GGML_UNARY_OP_SOFTPLUS:
case GGML_UNARY_OP_TANH:
+ case GGML_UNARY_OP_ABS:
case GGML_UNARY_OP_SILU:
case GGML_UNARY_OP_GELU:
case GGML_UNARY_OP_GELU_QUICK:
supp = ggml_hexagon_supported_unary(sess, op);
break;
default:
+ supp = false;
break;
}
break;
@@ -5499,6 +5532,7 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons
supp = ggml_hexagon_supported_activations(sess, op);
break;
default:
+ supp = false;
break;
}
break;
@@ -5584,17 +5618,17 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons
}
static bool ggml_backend_hexagon_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) {
- auto sess = static_cast(dev->context);
+ auto dev_ctx = static_cast(dev->context);
// Technically we can clone hexagon buffers from any session but for some reason the output is garbled with layer-split,
// tensor-split works correctly, so it needs mode debugging and investigation. For now accept only our own buffers.
#if 0
bool supp = (buft->iface.get_alignment == ggml_backend_hexagon_buffer_type_get_alignment);
#else
- bool supp = (buft == &sess->host_buffer_type) || (buft == &sess->buffer_type);
+ bool supp = (buft == &dev_ctx->host_buffer_type) || (buft == &dev_ctx->buffer_type);
#endif
- HEX_VERBOSE("ggml-hex: %s device-supports-buft %s %s\n", sess->name.c_str(), ggml_backend_buft_name(buft), supp ? "yes" : "no");
+ HEX_VERBOSE("ggml-hex: %s device-supports-buft %s %s\n", dev_ctx->c_name(), ggml_backend_buft_name(buft), supp ? "yes" : "no");
return supp;
}
@@ -5623,16 +5657,11 @@ ggml_hexagon_registry::ggml_hexagon_registry(ggml_backend_reg_t reg) {
GGML_LOG_INFO("ggml-hex: Hexagon Arch version v%d\n", opt_arch);
- // Create devices / sessions
+ // Create devices
for (size_t i = 0; i < opt_ndev; i++) {
- devices[i].iface = ggml_backend_hexagon_device_i;
- devices[i].reg = reg;
- try {
- devices[i].context = new ggml_hexagon_session(i, &devices[i]);
- } catch (const std::exception & exc) {
- GGML_LOG_ERROR("ggml-hex: failed to create device/session %zu\n", i);
- devices[i].context = nullptr;
- }
+ devices[i].iface = ggml_backend_hexagon_device_i;
+ devices[i].reg = reg;
+ devices[i].context = new ggml_backend_hexagon_device_context(i, opt_device_configs[i], &devices[i]);
}
}
@@ -5640,10 +5669,10 @@ ggml_hexagon_registry::ggml_hexagon_registry(ggml_backend_reg_t reg) {
ggml_hexagon_registry::~ggml_hexagon_registry() {
GGML_LOG_INFO("ggml-hex: releasing registry\n");
- // Release devices / sessions
+ // Release devices
for (size_t i = 0; i < opt_ndev; i++) {
- auto sess = static_cast(devices[i].context);
- delete sess;
+ auto dev_ctx = static_cast(devices[i].context);
+ delete dev_ctx;
}
}
@@ -5812,6 +5841,85 @@ template std::string vec_to_str(std::vector v) {
return str;
}
+// Enumerate NPU (aka CDSP) domains via FASTRPC_GET_DOMAINS if supported,
+// and populate domain_id and domain_name for all configured devices.
+static void ggml_hexagon_discover_devices() {
+ std::unordered_map cdsp_map;
+ bool discovery_supported = false;
+
+ system_req_payload domain_info = {};
+ domain_info.id = FASTRPC_GET_DOMAINS;
+ domain_info.sys.domains = nullptr;
+ domain_info.sys.max_domains = 0;
+ domain_info.sys.flags = DOMAINS_LIST_FLAGS_SET_TYPE(0, FASTRPC_NSP);
+
+ int err = remote_system_request(&domain_info);
+ if (err == AEE_SUCCESS && domain_info.sys.num_domains > 0) {
+ std::vector domains(domain_info.sys.num_domains);
+ domain_info.sys.domains = domains.data();
+ domain_info.sys.max_domains = (int) domains.size();
+
+ err = remote_system_request(&domain_info);
+ if (err == AEE_SUCCESS) {
+ discovery_supported = true;
+ const int n_domains = std::min(domain_info.sys.num_domains, (int) domains.size());
+ for (int i = 0; i < n_domains; i++) {
+ GGML_LOG_INFO("ggml-hex: FASTRPC_GET_DOMAINS[%d]: type %d id %d name '%s' status %d instance-id %d\n",
+ i, (int) domains[i].type, domains[i].id, domains[i].name, domains[i].status, domains[i].instance_id);
+ if (domains[i].type != FASTRPC_NSP) {
+ GGML_LOG_DEBUG("ggml-hex: skipping non-CDSP domain (type=%d)\n", (int) domains[i].type);
+ continue;
+ }
+ if (!domains[i].status) {
+ GGML_LOG_WARN("ggml-hex: skipping CDSP domain id=%d (status=down)\n", domains[i].id);
+ continue;
+ }
+ cdsp_map[domains[i].instance_id] = domains[i];
+ GGML_LOG_INFO("ggml-hex: using CDSP domain: instance-id %d id %d name '%s'\n",
+ domains[i].instance_id, domains[i].id, domains[i].name);
+ }
+ } else {
+ GGML_LOG_WARN("ggml-hex: FASTRPC_GET_DOMAINS fetch failed (0x%x), using static CDSP domains\n", (unsigned) err);
+ }
+ } else if (err != AEE_SUCCESS) {
+ GGML_LOG_DEBUG("ggml-hex: FASTRPC_GET_DOMAINS query failed (0x%x), using static CDSP domains\n", (unsigned) err);
+ }
+
+ // Populate domain IDs and names for all configured devices
+ for (size_t i = 0; i < opt_ndev; i++) {
+ auto & cfg = opt_device_configs[i];
+ if (discovery_supported) {
+ auto it = cdsp_map.find(cfg.physical_idx);
+ if (it != cdsp_map.end()) {
+ cfg.domain_id = it->second.id;
+ cfg.domain_name = it->second.name;
+ } else {
+ GGML_LOG_ERROR("ggml-hex: physical CDSP core %d not found on device (%zu CDSP core(s) available)\n",
+ cfg.physical_idx, cdsp_map.size());
+ cfg.domain_id = -1;
+ cfg.domain_name = "";
+ }
+ } else {
+ switch (cfg.physical_idx) {
+ case 0:
+ cfg.domain_id = 3;
+ cfg.domain_name = CDSP_DOMAIN_NAME;
+ break;
+ case 1:
+ cfg.domain_id = 4;
+ cfg.domain_name = "cdsp1";
+ break;
+ default:
+ GGML_LOG_ERROR("ggml-hex: physical CDSP core %d not supported without dynamic discovery\n",
+ cfg.physical_idx);
+ cfg.domain_id = -1;
+ cfg.domain_name = "";
+ break;
+ }
+ }
+ }
+}
+
static void ggml_hexagon_init(ggml_backend_reg * reg) {
// Basic sanity checks to make sure definitions match
static_assert((unsigned int) HTP_TYPE_Q4_0 == (unsigned int) GGML_TYPE_Q4_0,
@@ -5977,6 +6085,9 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) {
}
#endif
+ // Resolve domain info for all configured devices
+ ggml_hexagon_discover_devices();
+
if (str_profile) {
opt_pmu_evt = [&]() -> std::vector {
auto v = str_to_vec(str_profile);
diff --git a/ggml/src/ggml-hexagon/htp-drv.cpp b/ggml/src/ggml-hexagon/htp-drv.cpp
index 4f0790801731..437e367c9d35 100644
--- a/ggml/src/ggml-hexagon/htp-drv.cpp
+++ b/ggml/src/ggml-hexagon/htp-drv.cpp
@@ -73,6 +73,7 @@ typedef int (*remote_handle64_close_pfn_t)(remote_handle h);
typedef int (*remote_handle_control_pfn_t)(uint32_t req, void* data, uint32_t datalen);
typedef int (*remote_handle64_control_pfn_t)(remote_handle64 h, uint32_t req, void* data, uint32_t datalen);
typedef int (*remote_session_control_pfn_t)(uint32_t req, void *data, uint32_t datalen);
+typedef int (*remote_system_request_pfn_t)(system_req_payload * req);
//
// Driver API pfns
@@ -99,6 +100,7 @@ remote_handle64_close_pfn_t remote_handle64_close_pfn = nullptr;
remote_handle_control_pfn_t remote_handle_control_pfn = nullptr;
remote_handle64_control_pfn_t remote_handle64_control_pfn = nullptr;
remote_session_control_pfn_t remote_session_control_pfn = nullptr;
+remote_system_request_pfn_t remote_system_request_pfn = nullptr;
//
// Driver API
@@ -206,6 +208,13 @@ HTPDRV_API int remote_session_control(uint32_t req, void * data, uint32_t datale
return remote_session_control_pfn(req, data, datalen);
}
+HTPDRV_API int remote_system_request(system_req_payload * req) {
+ if (!remote_system_request_pfn) {
+ return AEE_EUNSUPPORTEDAPI;
+ }
+ return remote_system_request_pfn(req);
+}
+
#ifdef _WIN32
static std::string wstr_to_str(std::wstring_view wstr) {
@@ -367,6 +376,7 @@ int htpdrv_init() {
dlsym(handle.get(), remote_handle64_control_pfn_t, remote_handle64_control_pfn, remote_handle64_control, false);
dlsym(handle.get(), remote_session_control_pfn_t, remote_session_control_pfn, remote_session_control, false);
dlsym(handle.get(), remote_handle64_close_pfn_t, remote_handle64_close_pfn, remote_handle64_close, false);
+ dlsym(handle.get(), remote_system_request_pfn_t, remote_system_request_pfn, remote_system_request, true);
lib_cdsp_rpc_handle = std::move(handle);
initialized = true;
diff --git a/ggml/src/ggml-hexagon/htp-drv.h b/ggml/src/ggml-hexagon/htp-drv.h
index f3cc0da75c28..8232780e7fdc 100644
--- a/ggml/src/ggml-hexagon/htp-drv.h
+++ b/ggml/src/ggml-hexagon/htp-drv.h
@@ -116,6 +116,8 @@ HTPDRV_API domain * htpdrv_get_domain(int domain_id);
*/
HTPDRV_API int htpdrv_get_arch(int domain, int * arch);
+HTPDRV_API int remote_system_request(system_req_payload * req);
+
#ifdef __cplusplus
}
#endif
diff --git a/ggml/src/ggml-hexagon/htp/htp-ops.h b/ggml/src/ggml-hexagon/htp/htp-ops.h
index b4023b34d388..e804844d5996 100644
--- a/ggml/src/ggml-hexagon/htp/htp-ops.h
+++ b/ggml/src/ggml-hexagon/htp/htp-ops.h
@@ -62,6 +62,8 @@ enum htp_op_code {
HTP_OP_UNARY_NEG,
HTP_OP_UNARY_SOFTPLUS,
HTP_OP_UNARY_TANH,
+ HTP_OP_UNARY_ABS,
+ HTP_OP_UNARY_LOG,
HTP_OP_GLU_SWIGLU,
HTP_OP_GLU_SWIGLU_OAI,
HTP_OP_GLU_GEGLU,
diff --git a/ggml/src/ggml-hexagon/htp/hvx-arith.h b/ggml/src/ggml-hexagon/htp/hvx-arith.h
index c8d0003ab5c0..765c3577668e 100644
--- a/ggml/src/ggml-hexagon/htp/hvx-arith.h
+++ b/ggml/src/ggml-hexagon/htp/hvx-arith.h
@@ -358,6 +358,34 @@ static inline void hvx_clamp_scalar_f32(uint8_t * restrict dst, const uint8_t *
}
}
+//
+// Abs
+//
+
+static inline void hvx_abs_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
+ assert((unsigned long) dst % 128 == 0);
+ assert((unsigned long) src % 128 == 0);
+
+ HVX_Vector * restrict vdst = (HVX_Vector *) dst;
+ HVX_Vector * restrict vsrc = (HVX_Vector *) src;
+
+ const uint32_t elem_size = sizeof(float);
+ const uint32_t epv = 128 / elem_size;
+ const uint32_t nvec = n / epv;
+ const uint32_t nloe = n % epv;
+
+ uint32_t i = 0;
+
+ _Pragma("unroll(4)")
+ for (; i < nvec; i++) {
+ vdst[i] = hvx_vec_abs_f32(vsrc[i]);
+ }
+ if (nloe) {
+ HVX_Vector v = hvx_vec_abs_f32(vsrc[i]);
+ hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v);
+ }
+}
+
//
// Square
//
diff --git a/ggml/src/ggml-hexagon/htp/hvx-log.h b/ggml/src/ggml-hexagon/htp/hvx-log.h
index 7013dae785ac..a209f88d555c 100644
--- a/ggml/src/ggml-hexagon/htp/hvx-log.h
+++ b/ggml/src/ggml-hexagon/htp/hvx-log.h
@@ -62,4 +62,28 @@ static inline HVX_Vector hvx_vec_log_f32(HVX_Vector x) {
return hvx_vec_add_f32_f32(term_e, res);
}
+static inline void hvx_log_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
+ assert((unsigned long) dst % 128 == 0);
+ assert((unsigned long) src % 128 == 0);
+
+ HVX_Vector * restrict vdst = (HVX_Vector *) dst;
+ HVX_Vector * restrict vsrc = (HVX_Vector *) src;
+
+ const uint32_t elem_size = sizeof(float);
+ const uint32_t epv = 128 / elem_size;
+ const uint32_t nvec = n / epv;
+ const uint32_t nloe = n % epv;
+
+ uint32_t i = 0;
+
+ _Pragma("unroll(4)")
+ for (; i < nvec; i++) {
+ vdst[i] = hvx_vec_log_f32(vsrc[i]);
+ }
+ if (nloe) {
+ HVX_Vector v = hvx_vec_log_f32(vsrc[i]);
+ hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v);
+ }
+}
+
#endif /* HVX_LOG_H */
diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.c
index 975ba0c7af51..fe7d093a81c1 100644
--- a/ggml/src/ggml-hexagon/htp/main.c
+++ b/ggml/src/ggml-hexagon/htp/main.c
@@ -777,6 +777,8 @@ static int execute_op(struct htp_ops_context * octx) {
case HTP_OP_UNARY_NEG:
case HTP_OP_UNARY_EXP:
case HTP_OP_UNARY_TANH:
+ case HTP_OP_UNARY_ABS:
+ case HTP_OP_UNARY_LOG:
case HTP_OP_L2_NORM:
return op_unary(octx);
diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.c b/ggml/src/ggml-hexagon/htp/unary-ops.c
index b21415a67d64..1a632bf5631e 100644
--- a/ggml/src/ggml-hexagon/htp/unary-ops.c
+++ b/ggml/src/ggml-hexagon/htp/unary-ops.c
@@ -443,6 +443,34 @@ static void tanh_f32(const float * restrict src,
}
}
+static void abs_f32(const float * restrict src,
+ float * restrict dst,
+ const uint32_t num_rows,
+ const struct htp_unary_context * uctx) {
+ htp_unary_op_preamble;
+
+ for (uint32_t ir = 0; ir < num_rows; ir++) {
+ const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
+ uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);
+
+ hvx_abs_f32_aa(dst_local, src_local, ne0);
+ }
+}
+
+static void log_f32(const float * restrict src,
+ float * restrict dst,
+ const uint32_t num_rows,
+ const struct htp_unary_context * uctx) {
+ htp_unary_op_preamble;
+
+ for (uint32_t ir = 0; ir < num_rows; ir++) {
+ const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
+ uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);
+
+ hvx_log_f32_aa(dst_local, src_local, ne0);
+ }
+}
+
#define DEFINE_UNARY_TASK(NAME, IS_RMS_NORM_MUL, IS_TRI, CORE_EXPR) \
static void unary_task_f32_##NAME(unsigned int nth, unsigned int ith, void * data) { \
const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; \
@@ -478,6 +506,9 @@ static void unary_task_f32_##NAME(unsigned int nth, unsigned int ith, void * dat
const uint32_t nb11 = src1 ? src1->nb[1] : 0; \
const uint32_t nb12 = src1 ? src1->nb[2] : 0; \
const uint32_t nb13 = src1 ? src1->nb[3] : 0; \
+ const uint32_t nb11_bc = (src1 && src1->ne[1] > 1) ? nb11 : 0; \
+ const uint32_t nb12_bc = (src1 && src1->ne[2] > 1) ? nb12 : 0; \
+ const uint32_t nb13_bc = (src1 && src1->ne[3] > 1) ? nb13 : 0; \
const bool src1_contig = src1 ? ((nb12 == (size_t)ne01 * nb11) && (nb13 == (size_t)ne02 * nb12)) : false; \
\
uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread); \
@@ -497,8 +528,12 @@ static void unary_task_f32_##NAME(unsigned int nth, unsigned int ith, void * dat
const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02; \
const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012; \
\
- const uint32_t src0_max_block = src0_contig ? uctx->block : MIN((uint32_t)uctx->block, ne01); \
- const uint32_t dst_max_block = dst_contig ? uctx->block : MIN((uint32_t)uctx->block, ne1); \
+ const bool src1_needs_row_clip = (IS_RMS_NORM_MUL) && !uctx->broadcast_weight && !src1_contig; \
+ const bool block_src0_contig = src0_contig && !src1_needs_row_clip; \
+ const bool block_dst_contig = dst_contig && !src1_needs_row_clip; \
+ \
+ const uint32_t src0_max_block = block_src0_contig ? uctx->block : MIN((uint32_t)uctx->block, ne01); \
+ const uint32_t dst_max_block = block_dst_contig ? uctx->block : MIN((uint32_t)uctx->block, ne1); \
const uint32_t BLOCK = MIN(src0_max_block, dst_max_block); \
if (BLOCK == 0) { \
FARF(ERROR, "unary-f32 : current VTCM reservation %zu is too small, needed at least %zu\n", \
@@ -515,8 +550,8 @@ static void unary_task_f32_##NAME(unsigned int nth, unsigned int ith, void * dat
} \
\
for (uint32_t ir = src0_start_row, vtcm_idx = 0; ir < src0_end_row && vtcm_idx < 2; vtcm_idx++) { \
- const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, src0_contig, dst_contig, ne01, \
- div_ne01); \
+ const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, block_src0_contig, block_dst_contig, \
+ ne01, div_ne01); \
\
dma_queue_push(dma_queue, \
dma_make_ptr(data_dst, dst_vtcm_data + (vtcm_idx * dst_vtcm_half_size)), \
@@ -530,7 +565,7 @@ static void unary_task_f32_##NAME(unsigned int nth, unsigned int ith, void * dat
\
if ((IS_RMS_NORM_MUL) && !uctx->broadcast_weight) { \
const size_t src1_off = src1_contig ? (ir * nb11) : \
- unary_row_offset(ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb11, nb12, nb13); \
+ unary_row_offset(ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb11_bc, nb12_bc, nb13_bc); \
dma_queue_push(dma_queue, \
dma_make_ptr(src1_vtcm_data + (vtcm_idx * src1_vtcm_half_size), data_src1 + src1_off), \
uctx->src1_row_size_aligned, nb11, uctx->src1_data_row_size, block_size); \
@@ -540,8 +575,8 @@ static void unary_task_f32_##NAME(unsigned int nth, unsigned int ith, void * dat
} \
\
for (uint32_t ir = src0_start_row; ir < src0_end_row; ) { \
- const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, src0_contig, dst_contig, ne01, \
- div_ne01); \
+ const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, block_src0_contig, block_dst_contig, \
+ ne01, div_ne01); \
\
float * dst_vtcm = (float *) dma_queue_pop(dma_queue).src; \
float * src0_vtcm = (float *) dma_queue_pop(dma_queue).dst; \
@@ -562,12 +597,12 @@ static void unary_task_f32_##NAME(unsigned int nth, unsigned int ith, void * dat
\
const uint32_t next_ir = ir + block_size; \
if (next_ir < src0_end_row) { \
- const uint32_t next_block_size = unary_block_size(next_ir, src0_end_row, BLOCK, src0_contig, dst_contig,\
- ne01, div_ne01); \
+ const uint32_t next_block_size = unary_block_size(next_ir, src0_end_row, BLOCK, block_src0_contig, \
+ block_dst_contig, ne01, div_ne01); \
const uint32_t pref_ir = next_ir + next_block_size; \
if (pref_ir < src0_end_row) { \
- const uint32_t pref_block_size = unary_block_size(pref_ir, src0_end_row, BLOCK, src0_contig, \
- dst_contig, ne01, div_ne01); \
+ const uint32_t pref_block_size = unary_block_size(pref_ir, src0_end_row, BLOCK, block_src0_contig, \
+ block_dst_contig, ne01, div_ne01); \
const size_t src0_pref_off = src0_contig ? (pref_ir * nb01) : \
unary_row_offset(pref_ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03); \
dma_queue_push(dma_queue, \
@@ -576,7 +611,8 @@ static void unary_task_f32_##NAME(unsigned int nth, unsigned int ith, void * dat
\
if ((IS_RMS_NORM_MUL) && !uctx->broadcast_weight) { \
const size_t src1_pref_off = src1_contig ? (pref_ir * nb11) : \
- unary_row_offset(pref_ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb11, nb12, nb13); \
+ unary_row_offset(pref_ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb11_bc, nb12_bc, \
+ nb13_bc); \
dma_queue_push(dma_queue, \
dma_make_ptr(src1_vtcm, data_src1 + src1_pref_off), \
uctx->src1_row_size_aligned, nb11, uctx->src1_data_row_size, pref_block_size); \
@@ -603,6 +639,8 @@ DEFINE_UNARY_TASK(unary_silu, false, false, silu_f32(src0_vtcm, dst_vtcm, bl
DEFINE_UNARY_TASK(unary_gelu, false, false, gelu_f32(src0_vtcm, dst_vtcm, block_size, uctx))
DEFINE_UNARY_TASK(unary_softplus, false, false, softplus_f32(src0_vtcm, dst_vtcm, block_size, uctx))
DEFINE_UNARY_TASK(unary_tanh, false, false, tanh_f32(src0_vtcm, dst_vtcm, block_size, uctx))
+DEFINE_UNARY_TASK(unary_abs, false, false, abs_f32(src0_vtcm, dst_vtcm, block_size, uctx))
+DEFINE_UNARY_TASK(unary_log, false, false, log_f32(src0_vtcm, dst_vtcm, block_size, uctx))
DEFINE_UNARY_TASK(l2_norm, false, false, l2_norm_f32(src0_vtcm, dst_vtcm, block_size, uctx))
DEFINE_UNARY_TASK(tri, false, true, tri_f32(src0_vtcm, dst_vtcm, block_size, ir, uctx))
@@ -850,6 +888,8 @@ DEFINE_UNARY_TILED_TASK(unary_silu, false, tile_silu_f32(dst_vtcm, src_vtcm,
DEFINE_UNARY_TILED_TASK(unary_gelu, false, tile_gelu_f32(dst_vtcm, src_vtcm, tw))
DEFINE_UNARY_TILED_TASK(unary_softplus, false, tile_unary_softplus_f32(dst_vtcm, src_vtcm, tw))
DEFINE_UNARY_TILED_TASK(unary_tanh, false, hvx_tanh_f32_aa(dst_vtcm, src_vtcm, tw))
+DEFINE_UNARY_TILED_TASK(unary_abs, false, hvx_abs_f32_aa(dst_vtcm, src_vtcm, tw))
+DEFINE_UNARY_TILED_TASK(unary_log, false, hvx_log_f32_aa(dst_vtcm, src_vtcm, tw))
DEFINE_UNARY_TILED_TASK(tri, true, tri_apply_tile_f32(src_vtcm, dst_vtcm, tw, col, i01, ne0, tri_ttype))
static int execute_op_unary_f32(struct htp_ops_context * octx) {
@@ -875,6 +915,8 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) {
case HTP_OP_UNARY_GELU: op_type = "gelu-f32"; break;
case HTP_OP_UNARY_SOFTPLUS: op_type = "softplus-f32"; break;
case HTP_OP_UNARY_TANH: op_type = "tanh-f32"; break;
+ case HTP_OP_UNARY_ABS: op_type = "abs-f32"; break;
+ case HTP_OP_UNARY_LOG: op_type = "log-f32"; break;
case HTP_OP_L2_NORM: op_type = "l2norm-f32"; break;
case HTP_OP_TRI: op_type = "tri-f32"; break;
@@ -973,6 +1015,8 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) {
case HTP_OP_UNARY_GELU: task_func = unary_task_f32_tiled_unary_gelu; break;
case HTP_OP_UNARY_SOFTPLUS: task_func = unary_task_f32_tiled_unary_softplus; break;
case HTP_OP_UNARY_TANH: task_func = unary_task_f32_tiled_unary_tanh; break;
+ case HTP_OP_UNARY_ABS: task_func = unary_task_f32_tiled_unary_abs; break;
+ case HTP_OP_UNARY_LOG: task_func = unary_task_f32_tiled_unary_log; break;
case HTP_OP_TRI: task_func = unary_task_f32_tiled_tri; break;
default: break;
}
@@ -992,6 +1036,8 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) {
case HTP_OP_UNARY_GELU: task_func = unary_task_f32_unary_gelu; break;
case HTP_OP_UNARY_SOFTPLUS: task_func = unary_task_f32_unary_softplus; break;
case HTP_OP_UNARY_TANH: task_func = unary_task_f32_unary_tanh; break;
+ case HTP_OP_UNARY_ABS: task_func = unary_task_f32_unary_abs; break;
+ case HTP_OP_UNARY_LOG: task_func = unary_task_f32_unary_log; break;
case HTP_OP_L2_NORM: task_func = unary_task_f32_l2_norm; break;
case HTP_OP_TRI: task_func = unary_task_f32_tri; break;
default: break;
diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.h b/ggml/src/ggml-hexagon/htp/unary-ops.h
index 1f4c3a5c4d96..458218ff4431 100644
--- a/ggml/src/ggml-hexagon/htp/unary-ops.h
+++ b/ggml/src/ggml-hexagon/htp/unary-ops.h
@@ -55,6 +55,8 @@ static inline bool htp_op_is_unary(uint32_t opcode) {
case HTP_OP_UNARY_GELU:
case HTP_OP_UNARY_SOFTPLUS:
case HTP_OP_UNARY_TANH:
+ case HTP_OP_UNARY_ABS:
+ case HTP_OP_UNARY_LOG:
case HTP_OP_L2_NORM:
case HTP_OP_TRI:
return true;
diff --git a/ggml/src/ggml-metal/ggml-metal-common.cpp b/ggml/src/ggml-metal/ggml-metal-common.cpp
index 2eb9820bff91..6f1638a1147e 100644
--- a/ggml/src/ggml-metal/ggml-metal-common.cpp
+++ b/ggml/src/ggml-metal/ggml-metal-common.cpp
@@ -1,10 +1,27 @@
#include "ggml-metal-common.h"
+#include "ggml.h"
#include "ggml-impl.h"
#include "ggml-backend-impl.h"
#include
+bool ggml_metal_op_mul_mat_use_mm(const struct ggml_tensor * op, bool has_simdgroup_mm) {
+ const int64_t ne00 = op->src[0]->ne[0];
+ const int64_t ne11 = op->src[1]->ne[1];
+
+ return !ggml_is_transposed(op->src[0]) &&
+ !ggml_is_transposed(op->src[1]) &&
+ has_simdgroup_mm && ne00 >= 64 && ne11 > 8;
+}
+
+bool ggml_metal_op_mul_mat_id_use_mm(const struct ggml_tensor * op, bool has_simdgroup_mm) {
+ const int64_t ne00 = op->src[0]->ne[0];
+ const int64_t ne21 = op->src[2]->ne[1];
+
+ return has_simdgroup_mm && ne00 >= 64 && ne21 >= 32;
+}
+
// represents a memory range (i.e. an interval from a starting address p0 to an ending address p1 in a given buffer pb)
// the type indicates whether it is a source range (i.e. ops read data from it) or a destination range (i.e. ops write data to it)
struct ggml_mem_range {
diff --git a/ggml/src/ggml-metal/ggml-metal-common.h b/ggml/src/ggml-metal/ggml-metal-common.h
index 3acbc6ae174a..66abdb52efe3 100644
--- a/ggml/src/ggml-metal/ggml-metal-common.h
+++ b/ggml/src/ggml-metal/ggml-metal-common.h
@@ -47,6 +47,10 @@ bool ggml_mem_ranges_check(ggml_mem_ranges_t mrs, const struct ggml_tensor * ten
// if it proves to work well, we can start using it for other backends in the future
void ggml_graph_optimize(struct ggml_cgraph * gf);
+// mat-mat vs mat-vec dispatch; used by both supports_op and ggml_metal_op_mul_mat*
+bool ggml_metal_op_mul_mat_use_mm (const struct ggml_tensor * op, bool has_simdgroup_mm);
+bool ggml_metal_op_mul_mat_id_use_mm(const struct ggml_tensor * op, bool has_simdgroup_mm);
+
#ifdef __cplusplus
}
#endif
diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cpp
index a82caa5e4303..4e855be44670 100644
--- a/ggml/src/ggml-metal/ggml-metal-device.cpp
+++ b/ggml/src/ggml-metal/ggml-metal-device.cpp
@@ -593,7 +593,7 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_scan(ggml_me
// - sgptg floats for shared_x_dt (nsg)
// - sgptg floats for shared_dA (nsg)
// Total: nsg * (32 + 2) floats
- res.smem = (32 + 2)*sizeof(float)*nsg;
+ res.smem = GGML_PAD((32 + 2)*sizeof(float)*nsg, 16);
return res;
}
@@ -1029,6 +1029,7 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_map0(g
}
res.smem = (size_t) ne02*ne20*sizeof(uint16_t);
+ res.smem = GGML_PAD(res.smem, 16);
return res;
}
diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m
index 41ce90dc8a93..a053887a3370 100644
--- a/ggml/src/ggml-metal/ggml-metal-device.m
+++ b/ggml/src/ggml-metal/ggml-metal-device.m
@@ -3,6 +3,7 @@
#import "ggml-impl.h"
#import "ggml-backend-impl.h"
#import "ggml-metal-impl.h"
+#import "ggml-metal-common.h"
#include
@@ -788,6 +789,10 @@ void ggml_metal_encoder_debug_group_pop (ggml_metal_encoder_t encoder) {
}
void ggml_metal_encoder_set_pipeline(ggml_metal_encoder_t encoder, struct ggml_metal_pipeline_with_params pipeline) {
+ if (!pipeline.pipeline) {
+ GGML_ABORT("%s: nil Metal pipeline (missing kernel; see compile_pipeline log above)\n", __func__);
+ }
+
[encoder->obj setComputePipelineState:pipeline.pipeline->obj];
}
@@ -800,6 +805,9 @@ void ggml_metal_encoder_set_buffer(ggml_metal_encoder_t encoder, struct ggml_met
}
void ggml_metal_encoder_set_threadgroup_memory_size(ggml_metal_encoder_t encoder, size_t size, int idx) {
+ // ref: https://developer.apple.com/documentation/metal/mtlcomputecommandencoder/setthreadgroupmemorylength(_:index:)
+ GGML_ASSERT(size % 16 == 0);
+
[encoder->obj setThreadgroupMemoryLength:size atIndex:idx];
}
@@ -1407,6 +1415,30 @@ void ggml_metal_device_get_memory(ggml_metal_device_t dev, size_t * free, size_t
}
}
+static bool ggml_metal_supports_mul_mat_op(
+ bool has_simdgroup_reduction,
+ const struct ggml_tensor * op,
+ bool src0_f16_has_mv,
+ bool mm_path) {
+ if (!has_simdgroup_reduction || op->src[0]->type == GGML_TYPE_NVFP4) {
+ return false;
+ }
+
+ if (op->src[1]->type != GGML_TYPE_F16) {
+ return true;
+ }
+
+ if (op->src[0]->type == GGML_TYPE_BF16) {
+ return false;
+ }
+
+ if (src0_f16_has_mv && op->src[0]->type == GGML_TYPE_F16) {
+ return true;
+ }
+
+ return mm_path;
+}
+
bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_tensor * op) {
const bool has_simdgroup_mm = dev->props.has_simdgroup_mm;
const bool has_simdgroup_reduction = dev->props.has_simdgroup_reduction;
@@ -1710,9 +1742,15 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
case GGML_OP_GATED_DELTA_NET:
return has_simdgroup_reduction && op->src[2]->ne[0] % 32 == 0;
case GGML_OP_SOLVE_TRI:
+ return has_simdgroup_reduction && op->src[0]->type == GGML_TYPE_F32;
case GGML_OP_MUL_MAT:
+ return ggml_metal_supports_mul_mat_op(
+ has_simdgroup_reduction, op, true,
+ ggml_metal_op_mul_mat_use_mm(op, has_simdgroup_mm));
case GGML_OP_MUL_MAT_ID:
- return has_simdgroup_reduction && op->src[0]->type != GGML_TYPE_NVFP4;
+ return ggml_metal_supports_mul_mat_op(
+ has_simdgroup_reduction, op, false,
+ ggml_metal_op_mul_mat_id_use_mm(op, has_simdgroup_mm));
case GGML_OP_SET:
case GGML_OP_CPY:
case GGML_OP_DUP:
diff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.h
index 9becf04797ba..49102afe9c02 100644
--- a/ggml/src/ggml-metal/ggml-metal-impl.h
+++ b/ggml/src/ggml-metal/ggml-metal-impl.h
@@ -660,6 +660,7 @@ typedef struct {
uint64_t nb0;
uint64_t nb1;
uint64_t nb2;
+ uint64_t nb3;
} ggml_metal_kargs_conv_transpose_2d;
typedef struct {
diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp
index 75de0f6dd08a..7671d1d01564 100644
--- a/ggml/src/ggml-metal/ggml-metal-ops.cpp
+++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp
@@ -948,7 +948,7 @@ int ggml_metal_op_sum(ggml_metal_op_t ctx, int idx) {
ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);
ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2);
- ggml_metal_encoder_set_threadgroup_memory_size(enc, nsg * sizeof(float), 0);
+ ggml_metal_encoder_set_threadgroup_memory_size(enc, GGML_PAD(nsg * sizeof(float), 16), 0);
ggml_metal_encoder_dispatch_threadgroups(enc, 1, 1, 1, nth, 1, 1);
@@ -2362,10 +2362,6 @@ int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) {
const int16_t r2 = ne12/ne02;
const int16_t r3 = ne13/ne03;
- // find the break-even point where the matrix-matrix kernel becomes more efficient compared
- // to the matrix-vector kernel
- const int ne11_mm_min = 8;
-
// first try to use small-batch mat-mv kernels
// these should be efficient for BS [2, ~8]
if (op->src[1]->type == GGML_TYPE_F32 && (ne00%128 == 0) &&
@@ -2468,12 +2464,7 @@ int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) {
ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3);
ggml_metal_encoder_dispatch_threadgroups(enc, ((ne01 + r0ptg - 1)/r0ptg), ((ne11 + r1ptg - 1)/r1ptg), ne12*ne13, 32, nsg, 1);
- } else if (
- !ggml_is_transposed(op->src[0]) &&
- !ggml_is_transposed(op->src[1]) &&
- // for now the matrix-matrix multiplication kernel only works on A14+/M1+ SoCs
- // AMD GPU and older A-chips will reuse matrix-vector multiplication kernel
- props_dev->has_simdgroup_mm && ne00 >= 64 && ne11 > ne11_mm_min) {
+ } else if (ggml_metal_op_mul_mat_use_mm(op, props_dev->has_simdgroup_mm)) {
//GGML_LOG_INFO("matrix: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12);
// some Metal matrix data types require aligned pointers
@@ -2622,13 +2613,7 @@ int ggml_metal_op_mul_mat_id(ggml_metal_op_t ctx, int idx) {
const uint32_t r2 = 1;
const uint32_t r3 = 1;
- // find the break-even point where the matrix-matrix kernel becomes more efficient compared
- // to the matrix-vector kernel
- // ne20 = n_used_experts
- // ne21 = n_rows (batch size)
- const int ne21_mm_id_min = 32;
-
- if (props_dev->has_simdgroup_mm && ne00 >= 64 && (ne21 >= ne21_mm_id_min)) {
+ if (ggml_metal_op_mul_mat_id_use_mm(op, props_dev->has_simdgroup_mm)) {
// some Metal matrix data types require aligned pointers
// ref: https://developer.apple.com/metal/Metal-Shading-Language-Specification.pdf (Table 2.5)
//switch (op->src[0]->type) {
@@ -4645,6 +4630,7 @@ int ggml_metal_op_conv_transpose_2d(ggml_metal_op_t ctx, int idx) {
const int32_t OW = op->ne[0];
const int32_t OH = op->ne[1];
const int32_t OC = op->ne[2];
+ const int32_t N = op->src[1]->ne[3];
ggml_metal_kargs_conv_transpose_2d args = {
/*.IC =*/ IC,
@@ -4657,6 +4643,7 @@ int ggml_metal_op_conv_transpose_2d(ggml_metal_op_t ctx, int idx) {
/*.nb0 =*/ nb0,
/*.nb1 =*/ nb1,
/*.nb2 =*/ nb2,
+ /*.nb3 =*/ nb3,
};
auto pipeline = ggml_metal_library_get_pipeline_conv_transpose_2d(lib, op);
@@ -4671,7 +4658,7 @@ int ggml_metal_op_conv_transpose_2d(ggml_metal_op_t ctx, int idx) {
const size_t smem = GGML_PAD(KW * KH * sizeof(float), 16);
ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0);
- ggml_metal_encoder_dispatch_threadgroups(enc, OW, OH, OC, KW, KH, 1);
+ ggml_metal_encoder_dispatch_threadgroups(enc, OW, OH, OC * N, KW, KH, 1);
return 1;
}
diff --git a/ggml/src/ggml-metal/ggml-metal-tuning.cpp b/ggml/src/ggml-metal/ggml-metal-tuning.cpp
index 6d8c18e6a6a2..9114ab7424de 100644
--- a/ggml/src/ggml-metal/ggml-metal-tuning.cpp
+++ b/ggml/src/ggml-metal/ggml-metal-tuning.cpp
@@ -66,6 +66,7 @@ fa_vec_cfg_t fa_vec_baseline_cfg(int dk, int dv) {
// One row per kept bucket, plus per-(dtype,dk,dv) ne11-collapsed domain defaults
// (ne11_b = FA_VEC_NE11_DEFAULT, ne01_b = domain). To retune or add a device, re-run the
// sweep and paste its output. See ggml-metal-tuning.h for the row/lookup semantics.
+// ref: https://github.com/ggml-org/llama.cpp/pull/27824
constexpr fa_vec_entry_t fa_vec_tuned_table[] = {
{ { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 32, 32, 3, 3 }, { 2, 4 } },
{ { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 64, 64, -1, 0 }, { 1, 4 } },
@@ -278,6 +279,276 @@ constexpr fa_vec_entry_t fa_vec_tuned_table[] = {
{ { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } },
{ { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_F16, 64, 64, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_F16, 64, 64, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_F16, 128, 128, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_F16, 128, 128, 3, 3 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_F16, 128, 128, 3, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_F16, 192, 128, 1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_F16, 192, 128, 2, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_F16, 192, 128, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_F16, 320, 256, 2, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_F16, 320, 256, 3, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_F16, 320, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 32, 32, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 32, 32, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 32, 32, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 32, 32, 3, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 32, 32, 3, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 64, 64, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 64, 64, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 64, 64, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 96, 96, 1, 3 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 96, 96, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 96, 96, 2, 3 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 96, 96, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 96, 96, 3, 3 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 96, 96, 3, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 320, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 320, 256, 1, 3 }, { 4, 2 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 576, 512, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_0, 576, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 32, 32, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 32, 32, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 32, 32, 1, 4 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 32, 32, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 32, 32, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 32, 32, 3, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 64, 64, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 96, 96, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 96, 96, 2, 3 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 96, 96, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 96, 96, 3, 3 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 128, 128, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 128, 128, 1, 4 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 128, 128, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 128, 128, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 192, 192, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 192, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 256, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 256, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 320, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 320, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 320, 256, 1, 3 }, { 4, 2 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 512, 512, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 512, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 576, 512, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q4_1, 576, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 32, 32, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 32, 32, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 32, 32, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 128, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 128, 128, 1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 128, 128, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 128, 128, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 128, 128, 2, 4 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 128, 128, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 192, 192, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 192, 192, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 192, 192, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 192, 192, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 192, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 192, 128, 1, 3 }, { 4, 2 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 192, 128, 2, 3 }, { 4, 2 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 192, 128, 3, 3 }, { 4, 2 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 256, 256, 1, 3 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 320, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 320, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 320, 256, 1, 3 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 512, 512, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 512, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 576, 512, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_0, 576, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 32, 32, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 32, 32, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 32, 32, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 64, 64, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 96, 96, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 128, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 128, 128, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 128, 128, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 128, 128, 2, 3 }, { 4, 2 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 128, 128, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 192, 192, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 192, 192, 1, 3 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 192, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 192, 128, 1, 3 }, { 4, 2 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 192, 128, 2, 3 }, { 4, 2 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 192, 128, 3, 3 }, { 4, 2 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 256, 256, 1, 3 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 256, 256, 2, 3 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 320, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 320, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 320, 256, 1, 3 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 320, 256, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 320, 256, 2, 3 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 320, 256, 2, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 320, 256, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 320, 256, 3, 3 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 512, 512, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 576, 512, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q5_1, 576, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 32, 32, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 32, 32, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 32, 32, 2, 4 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 32, 32, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 32, 32, 3, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 64, 64, 1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 64, 64, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 64, 64, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 64, 64, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 96, 96, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 96, 96, 2, 3 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 96, 96, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 96, 96, 3, 3 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 96, 96, 3, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 320, 256, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 320, 256, 1, 3 }, { 4, 2 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 320, 256, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 320, 256, 2, 3 }, { 4, 2 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 320, 256, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 320, 256, 3, 3 }, { 4, 2 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } },
+
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_F16, 32, 32, 1, 3 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_F16, 64, 64, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_F16, 64, 64, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_F16, 128, 128, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_F16, 128, 128, 1, 1 }, { 1, 1 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_F16, 128, 128, 1, 2 }, { 1, 1 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_F16, 128, 128, 1, 3 }, { 1, 1 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_F16, 128, 128, 1, 4 }, { 1, 1 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_F16, 192, 128, 1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_F16, 192, 128, 1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_F16, 192, 128, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_F16, 192, 128, 1, 3 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_F16, 192, 128, 1, 4 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_F16, 320, 256, 2, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_F16, 320, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_F16, 320, 256, 1, 1 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_F16, 320, 256, 1, 3 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_F16, 320, 256, 1, 4 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 32, 32, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 32, 32, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 32, 32, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 32, 32, 2, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 32, 32, 3, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 32, 32, 3, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 64, 64, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 96, 96, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 96, 96, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 96, 96, 1, 4 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 96, 96, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 96, 96, 2, 4 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 96, 96, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 320, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 512, 512, 2, 3 }, { 1, 1 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 512, 512, 3, 3 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 576, 512, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q4_0, 576, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 32, 32, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 32, 32, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 32, 32, 2, 4 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 32, 32, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 32, 32, 3, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 64, 64, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 64, 64, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 64, 64, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 96, 96, 1, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 96, 96, 2, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 96, 96, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 96, 96, 3, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 320, 256, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 320, 256, 1, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 320, 256, 1, 4 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 320, 256, 2, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 320, 256, 3, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } },
+
{ { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 64, 64, -1, 0 }, { 1, 4 } },
{ { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 64, 64, -1, 1 }, { 1, 4 } },
{ { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 128, 128, 2, 1 }, { 1, 4 } },
@@ -449,6 +720,945 @@ constexpr fa_vec_entry_t fa_vec_tuned_table[] = {
{ { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } },
{ { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 32, 32, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 32, 32, 1, 3 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 32, 32, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 32, 32, 2, 3 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 32, 32, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 32, 32, 3, 3 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 64, 64, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 64, 64, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 64, 64, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 64, 64, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 96, 96, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 96, 96, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 96, 96, 1, 4 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 96, 96, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 96, 96, 2, 4 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 128, 128, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 128, 128, 1, 2 }, { 1, 1 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 128, 128, 1, 4 }, { 1, 1 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 128, 128, 2, 2 }, { 1, 1 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 128, 128, 2, 4 }, { 1, 1 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 192, 192, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 192, 192, 1, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 192, 192, 1, 4 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 192, 128, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 192, 128, 1, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 192, 128, 1, 4 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 192, 128, 2, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 256, 256, -1, 0 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 256, 256, 3, 0 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 256, 256, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 320, 256, 3, 0 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 320, 256, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 512, 512, 3, 0 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 32, 32, -1, 1 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 32, 32, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 32, 32, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 32, 32, 2, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 32, 32, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 64, 64, 3, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 64, 64, 3, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 96, 96, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 96, 96, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 96, 96, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 128, 128, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 128, 128, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 128, 128, 2, 4 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 128, 128, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 192, 128, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 192, 128, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 192, 128, 2, 4 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 192, 128, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 320, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 576, 512, 2, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 576, 512, 3, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 576, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 576, 512, 1, 1 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 576, 512, 1, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 576, 512, 1, 3 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_0, 576, 512, 1, 4 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 32, 32, -1, 1 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 32, 32, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 32, 32, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 32, 32, 2, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 32, 32, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 64, 64, 3, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 64, 64, 3, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 96, 96, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 96, 96, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 96, 96, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 128, 128, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 128, 128, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 128, 128, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 192, 192, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 192, 192, 2, 3 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 192, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 192, 128, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 192, 128, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 192, 128, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 256, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 256, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 320, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 576, 512, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q4_1, 576, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 32, 32, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 32, 32, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 32, 32, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 96, 96, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 96, 96, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 96, 96, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 128, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 128, 128, 2, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 128, 128, 2, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 128, 128, 3, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 128, 128, 3, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 192, 192, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 192, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 256, 256, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 256, 256, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 256, 256, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 320, 256, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 320, 256, 1, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 320, 256, 2, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 320, 256, 3, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 512, 512, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 512, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 576, 512, 2, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 576, 512, 2, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 576, 512, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 576, 512, 2, 3 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_0, 576, 512, 2, 4 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 32, 32, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 32, 32, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 32, 32, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 64, 64, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 96, 96, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 96, 96, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 96, 96, 2, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 96, 96, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 96, 96, 3, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 128, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 128, 128, 1, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 128, 128, 2, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 128, 128, 2, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 128, 128, 3, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 128, 128, 3, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 192, 192, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 192, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 256, 256, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 256, 256, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 256, 256, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 320, 256, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 320, 256, 1, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 320, 256, 2, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 320, 256, 3, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 512, 512, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 576, 512, 2, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 576, 512, 2, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 576, 512, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 576, 512, 2, 3 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q5_1, 576, 512, 2, 4 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 32, 32, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 32, 32, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 32, 32, 2, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 32, 32, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 64, 64, 3, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 64, 64, 3, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 96, 96, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 128, 128, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 128, 128, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 192, 128, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 192, 128, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 320, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 320, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } },
+
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 32, 32, 1, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 32, 32, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 32, 32, 2, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 32, 32, 2, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 32, 32, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 32, 32, 3, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 64, 64, 2, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 64, 64, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 64, 64, 1, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 64, 64, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 64, 64, 3, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 64, 64, 3, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 96, 96, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 96, 96, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 96, 96, 2, 4 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 96, 96, 3, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 96, 96, 3, 3 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 96, 96, 3, 4 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 128, 128, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 128, 128, 1, 4 }, { 1, 1 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 128, 128, 2, 4 }, { 1, 1 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 192, 192, 1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 192, 192, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 192, 192, 1, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 192, 128, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 192, 128, 1, 3 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 192, 128, 1, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 192, 128, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 256, 256, -1, 0 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 256, 256, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 256, 256, 2, 3 }, { 1, 1 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 320, 256, 1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 320, 256, 3, 0 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 320, 256, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 512, 512, 2, 0 }, { 4, 2 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 512, 512, 3, 0 }, { 4, 1 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 512, 512, 1, 3 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 512, 512, 2, 1 }, { 4, 2 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 512, 512, 2, 3 }, { 4, 2 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 512, 512, 3, 1 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 512, 512, 3, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 576, 512, 2, 0 }, { 4, 2 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 576, 512, 2, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 576, 512, 2, 3 }, { 4, 2 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_F16, 576, 512, 3, 1 }, { 4, 2 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 32, 32, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 32, 32, 1, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 32, 32, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 32, 32, 2, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 32, 32, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 64, 64, 2, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 64, 64, 3, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 96, 96, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 96, 96, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 96, 96, 1, 4 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 96, 96, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 96, 96, 2, 4 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 96, 96, 3, 4 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 128, 128, 1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 128, 128, 3, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 128, 128, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 128, 128, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 128, 128, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 192, 192, 3, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 192, 192, 1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 192, 192, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 192, 192, 1, 4 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 192, 192, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 192, 192, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 192, 128, 1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 192, 128, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 192, 128, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 192, 128, 2, 4 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 192, 128, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 320, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 320, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 576, 512, 2, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 576, 512, 3, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 576, 512, 1, 1 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_MAX, GGML_TYPE_Q8_0, 576, 512, 1, 4 }, { 1, 2 } },
+
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 32, 32, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 32, 32, 3, 3 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 64, 64, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 64, 64, 1, 1 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 64, 64, 1, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 64, 64, 2, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 96, 96, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 96, 96, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 96, 96, 1, 3 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 96, 96, 1, 4 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 96, 96, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 128, 128, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 128, 128, 1, 1 }, { 1, 1 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 128, 128, 1, 2 }, { 1, 1 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 128, 128, 1, 4 }, { 1, 1 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 192, 192, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 192, 192, 1, 1 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 192, 192, 1, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 192, 128, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 192, 128, 1, 1 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 192, 128, 1, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 256, 256, -1, 0 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 256, 256, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 256, 256, 1, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 320, 256, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 512, 512, 3, 3 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_F16, 512, 512, 3, 4 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 32, 32, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 32, 32, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 32, 32, 3, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 64, 64, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 96, 96, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 96, 96, 3, 3 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 96, 96, 3, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 192, 128, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 320, 256, 2, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 320, 256, 3, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 576, 512, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 576, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 576, 512, 1, 1 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 576, 512, 1, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 576, 512, 1, 3 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_0, 576, 512, 1, 4 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 32, 32, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 32, 32, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 32, 32, 3, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 64, 64, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 64, 64, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 96, 96, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 96, 96, 3, 3 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 96, 96, 3, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 128, 128, 1, 2 }, { 1, 1 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 128, 128, 1, 4 }, { 1, 1 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 128, 128, 2, 2 }, { 1, 1 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 128, 128, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 192, 192, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 192, 192, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 192, 192, 2, 3 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 192, 192, 2, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 192, 128, 2, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 192, 128, 3, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 192, 128, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 256, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 256, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 320, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 320, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 576, 512, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q4_1, 576, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 32, 32, 3, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 32, 32, 3, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 32, 32, 3, 4 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 96, 96, 1, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 96, 96, 1, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 128, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 128, 128, 1, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 128, 128, 1, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 192, 192, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 192, 192, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 192, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 256, 256, 1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 256, 256, 2, 3 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 256, 256, 2, 4 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 320, 256, 1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 512, 512, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 512, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 576, 512, 2, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 576, 512, 3, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 576, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 576, 512, 1, 1 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 576, 512, 1, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 576, 512, 1, 3 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_0, 576, 512, 1, 4 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 32, 32, 3, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 32, 32, 3, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 32, 32, 3, 4 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 64, 64, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 96, 96, 1, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 96, 96, 1, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 128, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 128, 128, 1, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 128, 128, 1, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 192, 192, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 192, 192, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 192, 192, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 192, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 256, 256, 2, 3 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 256, 256, 2, 4 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 320, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 320, 256, 1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 320, 256, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 320, 256, 1, 3 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 512, 512, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 512, 512, 1, 3 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 576, 512, 2, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 576, 512, 3, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 576, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 576, 512, 1, 1 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 576, 512, 1, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 576, 512, 1, 3 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q5_1, 576, 512, 1, 4 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 32, 32, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 32, 32, 3, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 64, 64, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 96, 96, 2, 3 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 96, 96, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 96, 96, 3, 3 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 96, 96, 3, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 128, 128, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 128, 128, 3, 3 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 128, 128, 3, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 192, 128, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 192, 128, 2, 3 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 192, 128, 2, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 192, 128, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 192, 128, 3, 3 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 192, 128, 3, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 320, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 320, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 512, 512, 1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M3_ULTRA, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } },
+
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 32, 32, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 32, 32, 1, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 32, 32, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 32, 32, 2, 3 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 32, 32, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 32, 32, 3, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 64, 64, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 64, 64, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 64, 64, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 64, 64, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 96, 96, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 96, 96, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 96, 96, 1, 4 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 96, 96, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 96, 96, 2, 4 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 128, 128, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 128, 128, 1, 2 }, { 1, 1 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 128, 128, 1, 4 }, { 1, 1 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 128, 128, 2, 2 }, { 1, 1 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 128, 128, 2, 4 }, { 1, 1 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 192, 192, 3, 0 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 192, 192, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 192, 192, 1, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 192, 128, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 192, 128, 1, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 192, 128, 1, 4 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 256, 256, -1, 0 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 256, 256, 3, 0 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 256, 256, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 320, 256, 3, 0 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 320, 256, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 512, 512, 3, 0 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 512, 512, 3, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 512, 512, 3, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_F16, 512, 512, 3, 3 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 32, 32, -1, 1 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 32, 32, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 32, 32, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 32, 32, 2, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 32, 32, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 64, 64, 1, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 64, 64, 2, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 64, 64, 2, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 64, 64, 3, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 64, 64, 3, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 96, 96, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 96, 96, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 128, 128, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 128, 128, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 128, 128, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 192, 128, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 192, 128, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 192, 128, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 320, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 576, 512, 2, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 576, 512, 3, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 576, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 576, 512, 1, 1 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 576, 512, 1, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 576, 512, 1, 3 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_0, 576, 512, 1, 4 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 32, 32, -1, 1 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 32, 32, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 32, 32, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 32, 32, 2, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 32, 32, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 64, 64, 1, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 64, 64, 2, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 64, 64, 2, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 64, 64, 3, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 64, 64, 3, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 96, 96, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 96, 96, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 96, 96, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 128, 128, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 128, 128, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 128, 128, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 192, 192, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 192, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 192, 128, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 192, 128, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 192, 128, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 256, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 256, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 320, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 320, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 576, 512, 2, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 576, 512, 3, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 576, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 576, 512, 1, 1 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 576, 512, 1, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 576, 512, 1, 3 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q4_1, 576, 512, 1, 4 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 32, 32, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 32, 32, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 32, 32, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 96, 96, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 96, 96, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 96, 96, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 128, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 128, 128, 1, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 128, 128, 1, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 128, 128, 2, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 128, 128, 2, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 128, 128, 3, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 128, 128, 3, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 192, 192, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 192, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 256, 256, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 256, 256, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 320, 256, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 320, 256, 1, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 320, 256, 2, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 320, 256, 3, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 512, 512, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 512, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 576, 512, 2, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 576, 512, 2, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 576, 512, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 576, 512, 2, 3 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_0, 576, 512, 2, 4 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 32, 32, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 32, 32, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 32, 32, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 64, 64, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 96, 96, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 96, 96, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 96, 96, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 128, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 128, 128, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 128, 128, 1, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 128, 128, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 128, 128, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 128, 128, 3, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 192, 192, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 192, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 256, 256, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 256, 256, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 320, 256, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 320, 256, 1, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 320, 256, 1, 4 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 320, 256, 2, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 320, 256, 3, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 512, 512, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 576, 512, 2, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 576, 512, 2, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 576, 512, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 576, 512, 2, 3 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q5_1, 576, 512, 2, 4 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 32, 32, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 32, 32, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 32, 32, 2, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 32, 32, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 64, 64, 1, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 64, 64, 2, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 64, 64, 3, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 64, 64, 3, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 96, 96, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 128, 128, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 128, 128, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 192, 192, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 192, 192, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 192, 192, 2, 4 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 192, 192, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 320, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 320, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 576, 512, 2, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 576, 512, 3, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } },
+
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 32, 32, 1, 3 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 32, 32, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 32, 32, 2, 3 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 32, 32, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 32, 32, 3, 3 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 64, 64, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 64, 64, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 64, 64, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 96, 96, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 96, 96, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 96, 96, 1, 4 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 96, 96, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 96, 96, 2, 4 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 128, 128, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 128, 128, 1, 2 }, { 1, 1 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 128, 128, 1, 4 }, { 1, 1 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 192, 192, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 192, 128, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 192, 128, 1, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 256, 256, -1, 0 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 256, 256, 3, 0 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 256, 256, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 320, 256, 3, 0 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 320, 256, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 512, 512, 3, 0 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 32, 32, -1, 1 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 32, 32, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 32, 32, 1, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 32, 32, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 32, 32, 2, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 32, 32, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 64, 64, 3, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 64, 64, 3, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 96, 96, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 96, 96, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 96, 96, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 128, 128, 3, 3 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 192, 128, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 192, 128, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 192, 128, 2, 4 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 192, 128, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 320, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 576, 512, 2, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 576, 512, 3, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 576, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 576, 512, 1, 1 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 576, 512, 1, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 576, 512, 1, 3 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_0, 576, 512, 1, 4 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 32, 32, -1, 1 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 32, 32, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 32, 32, 1, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 32, 32, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 32, 32, 2, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 32, 32, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 64, 64, 3, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 96, 96, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 96, 96, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 96, 96, 1, 4 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 96, 96, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 96, 96, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 128, 128, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 128, 128, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 128, 128, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 192, 192, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 192, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 192, 128, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 192, 128, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 192, 128, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 256, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 256, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 320, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 512, 512, 3, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 576, 512, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q4_1, 576, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 32, 32, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 32, 32, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 32, 32, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 96, 96, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 96, 96, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 96, 96, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 128, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 128, 128, 2, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 128, 128, 2, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 192, 192, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 192, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 256, 256, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 256, 256, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 256, 256, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 320, 256, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 320, 256, 1, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 320, 256, 2, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 320, 256, 2, 4 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 320, 256, 3, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 320, 256, 3, 4 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 512, 512, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 512, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 576, 512, 2, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 576, 512, 2, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 576, 512, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 576, 512, 2, 3 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_0, 576, 512, 2, 4 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 32, 32, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 32, 32, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 32, 32, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 64, 64, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 96, 96, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 96, 96, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 96, 96, 2, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 96, 96, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 96, 96, 3, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 128, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 128, 128, 2, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 128, 128, 2, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 128, 128, 3, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 128, 128, 3, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 192, 192, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 192, 192, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 192, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 256, 256, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 256, 256, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 256, 256, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 320, 256, 1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 320, 256, 1, 3 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 320, 256, 1, 4 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 320, 256, 3, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 512, 512, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 576, 512, 2, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 576, 512, 2, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 576, 512, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 576, 512, 2, 3 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q5_1, 576, 512, 2, 4 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 32, 32, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 32, 32, 1, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 32, 32, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 32, 32, 2, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 32, 32, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 64, 64, 3, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 96, 96, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 96, 96, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 128, 128, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 128, 128, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 128, 128, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 192, 128, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 192, 128, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 192, 128, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 320, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 320, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 576, 512, 2, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 576, 512, 3, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 576, 512, 1, 1 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 576, 512, 1, 3 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 576, 512, 1, 4 }, { 1, 2 } },
+
{ { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 32, 32, 2, 1 }, { 2, 4 } },
{ { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 32, 32, 2, 3 }, { 2, 4 } },
{ { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 32, 32, 3, 1 }, { 2, 4 } },
@@ -640,7 +1850,215 @@ constexpr fa_vec_entry_t fa_vec_tuned_table[] = {
{ { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } },
{ { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } },
- { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 32, 32, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 32, 32, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 32, 32, 1, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 32, 32, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 32, 32, 2, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 32, 32, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 32, 32, 3, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 32, 32, 3, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 32, 32, 3, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 64, 64, 1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 64, 64, 2, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 64, 64, -1, 1 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 64, 64, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 64, 64, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 64, 64, 1, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 64, 64, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 64, 64, 2, 4 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 64, 64, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 96, 96, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 96, 96, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 96, 96, 1, 4 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 96, 96, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 96, 96, 3, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 128, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 128, 128, 3, 0 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 128, 128, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 128, 128, 1, 2 }, { 1, 1 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 128, 128, 1, 4 }, { 1, 1 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 128, 128, 2, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 128, 128, 3, 3 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 128, 128, 3, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 192, 192, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 192, 192, 1, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 192, 192, 1, 4 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 192, 128, 1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 192, 128, 2, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 192, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 192, 128, 1, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 192, 128, 1, 4 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 192, 128, 3, 2 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 192, 128, 3, 3 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 256, 256, -1, 0 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 256, 256, 1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 256, 256, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 256, 256, 1, 4 }, { 1, 1 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 320, 256, 2, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 320, 256, 3, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 320, 256, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 320, 256, 1, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 320, 256, 1, 4 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 512, 512, 2, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 512, 512, 1, 1 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 512, 512, 2, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 512, 512, 2, 3 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 512, 512, 3, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 512, 512, 3, 2 }, { 4, 1 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 512, 512, 3, 3 }, { 4, 2 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 512, 512, 3, 4 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 576, 512, 2, 0 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 576, 512, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 576, 512, 1, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 576, 512, 1, 4 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_F16, 576, 512, 2, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 32, 32, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 32, 32, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 32, 32, 2, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 32, 32, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 32, 32, 3, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 64, 64, 1, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 64, 64, 1, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 64, 64, 2, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 64, 64, 3, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 64, 64, 3, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 96, 96, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 96, 96, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 96, 96, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 96, 96, 2, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 96, 96, 3, 4 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 128, 128, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 192, 192, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 192, 192, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 192, 192, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 192, 128, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 192, 128, 2, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 192, 128, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 320, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 320, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 320, 256, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 512, 512, 3, 0 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 512, 512, 2, 1 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 512, 512, 3, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } },
+
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 32, 32, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 32, 32, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 32, 32, 1, 4 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 32, 32, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 32, 32, 2, 4 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 32, 32, 3, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 32, 32, 3, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 64, 64, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 64, 64, 1, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 64, 64, 3, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 64, 64, 3, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 64, 64, 3, 4 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 96, 96, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 128, 128, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 128, 128, 1, 4 }, { 1, 1 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 192, 192, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 192, 192, 1, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 192, 128, 2, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 192, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 192, 128, 3, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 256, 256, -1, 0 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 256, 256, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 256, 256, 1, 4 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 320, 256, 3, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 320, 256, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 320, 256, 1, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 512, 512, 2, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 512, 512, 2, 3 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 512, 512, 3, 3 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 576, 512, -1, 1 }, { 2, 2 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 576, 512, 1, 1 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 576, 512, 1, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 576, 512, 1, 3 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 576, 512, 1, 4 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_F16, 576, 512, 2, 2 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 32, 32, -1, 1 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 32, 32, 1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 32, 32, 1, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 32, 32, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 32, 32, 3, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 64, 64, 1, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 96, 96, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 96, 96, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 96, 96, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 96, 96, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 128, 128, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 192, 192, 1, 3 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 192, 192, 2, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 192, 192, 2, 3 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 192, 192, 2, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 192, 192, 3, 4 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 192, 128, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 192, 128, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 320, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 512, 512, 1, 0 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 2 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 512, 512, 2, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q4_0, 512, 512, 2, 3 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 32, 32, 1, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 32, 32, 1, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 32, 32, 2, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 32, 32, 2, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 32, 32, 3, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 32, 32, 3, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 64, 64, 3, 2 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 64, 64, 3, 3 }, { 4, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 96, 96, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 96, 96, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 96, 96, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 128, 128, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 128, 128, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 192, 192, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 192, 192, 2, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 192, 192, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 2, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 192, 128, 1, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 192, 128, 3, 2 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 320, 256, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 320, 256, -1, 1 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } },
+ { { GGML_METAL_DEVICE_M5_PRO, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } },
+
+ { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 32, 32, -1, 1 }, { 2, 4 } },
{ { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 32, 32, 1, 2 }, { 1, 4 } },
{ { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 32, 32, 1, 4 }, { 1, 4 } },
{ { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 32, 32, 2, 2 }, { 4, 4 } },
diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal.cpp
index 9756d47050c3..4d58dc821cf4 100644
--- a/ggml/src/ggml-metal/ggml-metal.cpp
+++ b/ggml/src/ggml-metal/ggml-metal.cpp
@@ -558,7 +558,9 @@ static void ggml_backend_metal_event_wait(ggml_backend_t backend, ggml_backend_e
ggml_metal_event_wait(ctx, ev);
}
-static void ggml_backend_metal_graph_optimize(ggml_backend_t backend, ggml_cgraph * cgraph) {
+static void ggml_backend_metal_graph_optimize(ggml_backend_t backend, ggml_cgraph * cgraph, ggml_backend_graph_optimize_params * params) {
+ GGML_UNUSED(params);
+
ggml_metal_t ctx = (ggml_metal_t)backend->context;
ggml_metal_graph_optimize(ctx, cgraph);
diff --git a/ggml/src/ggml-metal/kernels/conv.metal b/ggml/src/ggml-metal/kernels/conv.metal
index 5685b5cd4915..a5d5aa9d9293 100644
--- a/ggml/src/ggml-metal/kernels/conv.metal
+++ b/ggml/src/ggml-metal/kernels/conv.metal
@@ -366,7 +366,8 @@ kernel void kernel_conv_transpose_2d(
const int64_t out_x = tgpig[0];
const int64_t out_y = tgpig[1];
- const int64_t out_c = tgpig[2];
+ const int64_t batch = tgpig[2] / args.OC;
+ const int64_t out_c = tgpig[2] % args.OC;
const int64_t kw = tpitg[0];
const int64_t kh = tpitg[1];
@@ -390,7 +391,7 @@ kernel void kernel_conv_transpose_2d(
if (in_x >= args.IW) continue;
- const int64_t input_idx = (args.IW * args.IH) * in_c + (args.IW) * in_y + in_x;
+ const int64_t input_idx = (args.IW * args.IH) * (args.IC * batch + in_c) + (args.IW) * in_y + in_x;
const int64_t kernel_idx = (args.KH * args.KW * args.OC) * in_c + (args.KH * args.KW) * out_c + (args.KW) * kh + kw;
v += (float)src0[kernel_idx] * src1[input_idx];
@@ -408,7 +409,7 @@ kernel void kernel_conv_transpose_2d(
total += shared_sum[i];
}
- device float * dst_ptr = (device float *) (dst + out_x*args.nb0 + out_y * args.nb1 + out_c*args.nb2);
+ device float * dst_ptr = (device float *) (dst + batch*args.nb3 + out_c*args.nb2 + out_y * args.nb1 + out_x*args.nb0);
dst_ptr[0] = total;
}
}
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp
index 6ae83449b082..426aac523164 100644
--- a/ggml/src/ggml-opencl/ggml-opencl.cpp
+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp
@@ -6059,9 +6059,13 @@ static ggml_backend_opencl_context * ggml_cl_init(ggml_backend_dev_t dev) {
}
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
- // determine whether to use Adreno xmem GEMM
- backend_ctx->adreno_xmem_gemm_enabled = getenv("GGML_OPENCL_ADRENO_XMEM_GEMM") != nullptr &&
- backend_ctx->gpu_family == GPU_FAMILY::ADRENO;
+ // Adreno xmem F16xF32 GEMM, default on adreno, opt out with GGML_OPENCL_ADRENO_XMEM_GEMM=0.
+ // This helps models with f16 attention weights, e.g., gpt-oss-20b-f16
+ {
+ const char * xmem_env = getenv("GGML_OPENCL_ADRENO_XMEM_GEMM");
+ backend_ctx->adreno_xmem_gemm_enabled = backend_ctx->gpu_family == GPU_FAMILY::ADRENO &&
+ (xmem_env ? atoi(xmem_env) != 0 : true);
+ }
#endif
// determine whether to use large buffer for Adreno
@@ -19534,9 +19538,18 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
// GEMM using local memory
// Current BK = 16, so ne00 % 16 == 0
+ //
+ // Certain A7X compiler (E031.41) executes kernel_mul_mm_f32_f32_l4_lm poorly;
+ // matrices with ne11 <= 8 appears OK.
+ // Fallback to the MV style kernels for A7x and ne11 > 8.
+ // Override with GGML_OPENCL_A7X_F32_LM_BYPASS=0.
+ static const char * a7x_f32lm_env = getenv("GGML_OPENCL_A7X_F32_LM_BYPASS");
+ static const bool a7x_f32lm_bypass = (a7x_f32lm_env == nullptr || a7x_f32lm_env[0] != '0');
if (src1t == GGML_TYPE_F32 &&
ne00 % 16 == 0 &&
- ne11 > 1) {
+ ne11 > 1 &&
+ !(a7x_f32lm_bypass && src0t == GGML_TYPE_F32 && ne11 > 8 &&
+ backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X)) {
switch(src0t) {
case GGML_TYPE_F32: {
kernel = backend_ctx->kernel_mul_mm_f32_f32_l4_lm;
diff --git a/ggml/src/ggml-openvino/CMakeLists.txt b/ggml/src/ggml-openvino/CMakeLists.txt
index cc089b721fc3..af3e0758ca2f 100644
--- a/ggml/src/ggml-openvino/CMakeLists.txt
+++ b/ggml/src/ggml-openvino/CMakeLists.txt
@@ -1,6 +1,8 @@
find_package(OpenVINO REQUIRED COMPONENTS Runtime Threading)
find_package(OpenCL REQUIRED)
+message(STATUS "Found OpenVINO: ${OpenVINO_DIR} (found version \"${OpenVINO_VERSION}\")")
+
file(GLOB_RECURSE GGML_HEADERS_OPENVINO "*.h" "*.hpp")
file(GLOB_RECURSE GGML_SOURCES_OPENVINO "*.cpp")
diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp
index 599f41aebbdc..006e005cb7aa 100644
--- a/ggml/src/ggml-openvino/ggml-decoder.cpp
+++ b/ggml/src/ggml-openvino/ggml-decoder.cpp
@@ -357,6 +357,18 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const {
break;
}
case GGML_OP_VIEW: {
+ if (m_is_static && node->src[0] != nullptr &&
+ (node->src[0]->op == GGML_OP_GATED_DELTA_NET || node->src[0]->op == GGML_OP_CONCAT)) {
+ // VIEW slicing a GATED_DELTA_NET combined [attn|state] output, or the conv_input
+ // CONCAT. The consuming CPY/RMS_NORM op recovers the true window at runtime via
+ // ssm_state_size / the fixed conv kernel width, so this VIEW must stay an identity
+ // pass-through of the full source here too (it already is on the dynamic path);
+ // otherwise the generic static-mode Slice below would bake in the *captured*
+ // cgraph's token count, which is wrong once the compiled static model runs with a
+ // different token count (prefill chunk size or 1).
+ op_case = 1;
+ break;
+ }
if (node->src[0]->op == GGML_OP_VIEW) {
auto * src = node->src[0];
if (ggml_nelements(node) != ggml_nelements(src)) {
@@ -408,6 +420,23 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const {
}
break;
}
+ case GGML_OP_POOL_2D: {
+ const ggml_op_pool pool_mode = static_cast(node->op_params[0]);
+ switch (pool_mode) {
+ case GGML_OP_POOL_MAX: {
+ op_case = 1;
+ break;
+ }
+ case GGML_OP_POOL_AVG: {
+ op_case = 2;
+ break;
+ }
+ default:
+ op_case = 0;
+ break;
+ }
+ break;
+ }
case GGML_OP_CPY: {
if (node->src[0]->op == GGML_OP_VIEW) {
if (node->src[0]->src[0]->op == GGML_OP_GATED_DELTA_NET) {
@@ -425,6 +454,31 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const {
is_kvcache(node->src[1]->view_src, nullptr)) {
// s_copy defrag remainder writeback: gathered extra state rows copied back into the cache
op_case = 3;
+ } else if (node->src[1] != nullptr && node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src != nullptr) {
+ // op_case 5: KV write for decoder self-attention (dynamic write offset)
+ // op_case 6: KV write for encoder self-attn or cross-attn (static offset)
+ const ggml_tensor * kv_buf = node->src[1]->view_src;
+ if (kv_buf->ne[1] == 1 && kv_buf->ne[2] == 1 && kv_buf->ne[3] == 1) {
+ op_case = 6;
+ // Forward-scan the graph for a FLASH_ATTN_EXT that reads from
+ // the same buffer. Having a mask (src[3] != nullptr) implies
+ // decoder self-attention and the write offset is dynamic.
+ for (int i = 0; i < m_cgraph->n_nodes; i++) {
+ const ggml_tensor * n = m_cgraph->nodes[i];
+ if (n->op != GGML_OP_FLASH_ATTN_EXT) {
+ continue;
+ }
+ // K (src[1]) and V (src[2]) are 3-D views whose view_src is
+ // the flat KV buffer we are writing to.
+ if ((n->src[1] != nullptr && n->src[1]->view_src == kv_buf) ||
+ (n->src[2] != nullptr && n->src[2]->view_src == kv_buf)) {
+ if (n->src[3] != nullptr) {
+ op_case = 5; // decoder self-attention: mask present
+ }
+ break;
+ }
+ }
+ }
}
break;
}
@@ -448,6 +502,15 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const {
}
break;
}
+ case GGML_OP_FLASH_ATTN_EXT: {
+ if (node->src[1] != nullptr && node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src != nullptr) {
+ const ggml_tensor * kv_buf = node->src[1]->view_src;
+ if (kv_buf->ne[1] == 1 && kv_buf->ne[2] == 1 && kv_buf->ne[3] == 1) {
+ op_case = (node->src[3] != nullptr) ? 1 : 2;
+ }
+ }
+ break;
+ }
default:
break;
}
@@ -479,23 +542,35 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr
switch (node->op) {
case GGML_OP_FLASH_ATTN_EXT:
- if (node->src[0] == nullptr || node->src[1] == nullptr || node->src[3] == nullptr) {
+ if (node->src[0] == nullptr || node->src[1] == nullptr) {
return -1;
}
switch (node->src[1]->op) {
case GGML_OP_PERMUTE:
- // case 0: node op is FLASH_ATTN_EXT, src 1 not null & op is PERMUTE & the permuted tensor src is the view of cache k
- if (node->src[1]->src[0] != nullptr && node->src[1]->src[0]->op == GGML_OP_VIEW) {
+ // case 0: src[1] is PERMUTE of a cache VIEW, mask required
+ if (node->src[3] != nullptr && node->src[1]->src[0] != nullptr &&
+ node->src[1]->src[0]->op == GGML_OP_VIEW) {
return 0;
}
break;
case GGML_OP_CPY:
- // case 1: node op is FLASH_ATTN_EXT, src 1 not null & op is CPY & the copied tensor src is PERMUTE & the permuted tensor src is the view of cache k
- if (node->src[1]->src[0] != nullptr && node->src[1]->src[0]->op == GGML_OP_PERMUTE &&
- node->src[1]->src[0]->src[0] != nullptr && node->src[1]->src[0]->src[0]->op == GGML_OP_VIEW) {
+ // case 1: src[1] is CPY of a PERMUTE(VIEW), mask required
+ if (node->src[3] != nullptr && node->src[1]->src[0] != nullptr &&
+ node->src[1]->src[0]->op == GGML_OP_PERMUTE && node->src[1]->src[0]->src[0] != nullptr &&
+ node->src[1]->src[0]->src[0]->op == GGML_OP_VIEW) {
return 1;
}
break;
+ case GGML_OP_VIEW:
+ // cases 4/5/6: whisper - K is a direct non-contiguous VIEW_3D of a KV cache
+ if (node->src[1]->view_src != nullptr) {
+ if (node->src[3] != nullptr) {
+ return 4; // decoder self-attention
+ } else {
+ return 5; // cross-attention or encoder self-attention
+ };
+ }
+ break;
default:
break;
}
@@ -548,6 +623,18 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr
cache_k_permute = node->src[0]->src[0]->src[0];
mask = node->src[1];
break;
+ case 4:
+ case 5: {
+ // whisper: K is a direct VIEW_3D of the KV buffer, no PERMUTE node
+ auto * cache_k_view = node->src[1]; // VIEW_3D of kv_self.k or kv_cross.k`
+ compute_params.token_len_per_seq = node->src[0]->ne[1];
+ if (attention_pattern_case == 4) {
+ compute_params.attention_size = cache_k_view->ne[1];
+ } else {
+ compute_params.attention_size_static = cache_k_view->ne[1];
+ }
+ continue;
+ }
default:
break;
}
@@ -654,10 +741,8 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr
ComputeParams::RsWriteback writeback;
writeback.slot_begin = (int) (dest_view->view_offs / row_bytes);
if (is_conv) {
- // conv_input column the copied window starts at
writeback.src_begin = (int) (node->src[0]->view_offs / node->src[0]->view_src->nb[0]);
} else if (is_gdn) {
- // first row of the state part of the gated-delta-net output
writeback.src_begin = (int) (node->src[0]->view_offs / node->src[0]->view_src->nb[1]);
}
compute_params.rs_writebacks[get_tensor_ov_name(cgraph, node)] = writeback;
@@ -718,11 +803,15 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op,
} else if (is_kvcache(input, op)) {
// kvcache
input_shape = ov::PartialShape{get_shape(input)};
- if (!m_is_static) {
+ // Whisper.cpp uses a fixed size 1D KV buffer [N, 1, 1, 1] (GGML) or [1, 1, 1, N] (OV).
+ // the token fill level is handled by token_len_per_seq + dynamic mask input.
+ // skip dynamic dim and stateful reshape for this layout.
+ const bool is_flat_kv = (input->ne[1] == 1 && input->ne[2] == 1 && input->ne[3] == 1);
+ if (!m_is_static && !is_flat_kv) {
// do not fix ctx size to make llama-bench work across test params
input_shape[2] = -1;
}
- if (is_stateful()) {
+ if (is_stateful() && !is_flat_kv) {
// Convert stateless KV cache layout [1, 1, seq, n_heads_kv * head_size]
// to stateful layout [1, seq, n_heads_kv, head_size].
assert(input_shape.size() == 4 && input_shape[0] == 1 && input_shape[1] == 1 &&
@@ -738,7 +827,9 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op,
input_shape = ov::PartialShape{1, 1, 1, len};
} else if (is_inp_s_copy(input, op) || is_s_copy_leaf(input)) {
- input_shape = ov::PartialShape{1, 1, 1, -1};
+ // On NPU the total slot count (n_seq_max) is fixed at translation time, so the s_copy
+ // index list has a static length; on CPU/GPU it may change across compiles (defrag).
+ input_shape = m_is_static ? ov::PartialShape{get_shape(input)} : ov::PartialShape{1, 1, 1, -1};
} else {
input_shape = ov::PartialShape{get_shape(input)};
@@ -790,13 +881,16 @@ void GgmlOvDecoder::add_extra_inputs() {
// see llama_kv_cache_unified::get_n_kv and llama_kv_cache_unified::get_padding.
// 2. `n_seq_active` and `seq_active_start`, used in FLASH_ATTN_EXT to indicate the active sequences in the batch
- auto create_1d_input = [this](const std::string & name, int64_t value) {
- m_model_extra_inputs[name] = {ov::element::i64, ov::Shape{1}, value, !m_is_static};
+ auto create_1d_input = [this](const std::string & name, int64_t value, bool force_parameter = false) {
+ m_model_extra_inputs[name] = {ov::element::i64, ov::Shape{1}, value, force_parameter || !m_is_static};
};
if (m_compute_params.attention_size != -1) {
create_1d_input("attention_size", m_compute_params.attention_size);
}
+ if (m_compute_params.attention_size_static != -1) {
+ create_1d_input("attention_size_static", m_compute_params.attention_size_static);
+ }
if (m_compute_params.attention_size_swa != -1) {
create_1d_input("attention_size_swa", m_compute_params.attention_size_swa);
}
@@ -809,17 +903,32 @@ void GgmlOvDecoder::add_extra_inputs() {
// create_1d_input("token_len", m_compute_params.token_len_per_seq * m_compute_params.n_seq_active);
if (m_compute_params.cache_rs_reset_idx != -1) {
- create_1d_input("cache_rs_reset_idx", m_compute_params.cache_rs_reset_idx);
- create_1d_input("cache_rs_reset_len", m_compute_params.cache_rs_reset_len);
+ // Whether/which cache slot to reset varies per compute call (e.g. a new sequence starting
+ // vs. continued decoding). can_reuse_statically() does not invalidate the cached static
+ // model on ComputeParams changes, so these must stay runtime Parameters even when static
+ // (scale.cpp op_case 1 only uses them in value comparisons, never as Slice bounds, so this
+ // does not reintroduce dynamic shapes).
+ create_1d_input("cache_rs_reset_idx", m_compute_params.cache_rs_reset_idx, /*force_parameter=*/true);
+ create_1d_input("cache_rs_reset_len", m_compute_params.cache_rs_reset_len, /*force_parameter=*/true);
}
if (m_compute_params.s_copy_active_slot_len != -1) {
create_1d_input("s_copy_active_slot_len", m_compute_params.s_copy_active_slot_len);
+ if (m_is_static) {
+ // Number of real tokens in the current prefill chunk. The last chunk is padded with
+ // fabricated token ids; attention masks them out, but the recurrent (GDN/conv) path
+ // would otherwise fold them into cache_r/cache_s permanently. Varies per chunk, so it
+ // must stay a runtime Parameter; it is only compared against a Range or used as Gather
+ // indices, so it does not make any shape dynamic.
+ create_1d_input("chunk_valid_len", get_static_n_tokens(), /*force_parameter=*/true);
+ }
}
for (const auto & [node_name, writeback] : m_compute_params.rs_writebacks) {
create_1d_input("rs_slot_begin_" + node_name, writeback.slot_begin);
- create_1d_input("rs_src_begin_" + node_name, writeback.src_begin);
+ if (!m_is_static) {
+ create_1d_input("rs_src_begin_" + node_name, writeback.src_begin);
+ }
}
}
@@ -1785,13 +1894,23 @@ void GgmlOvDecoder::compute_node_dynamic_dims() {
auto dynamic_dim_stride = src_logical_nb[dynamic_dim_idx] / ggml_type_size(node->src[0]->type) *
ggml_type_size(node->type);
int matched_dim_count = 0;
+ int first_matched_dim = -1;
for (int i = 0; i < GGML_MAX_DIMS; i++) {
if (node->nb[i] == dynamic_dim_stride && node->ne[i] == node->src[0]->ne[dynamic_dim_idx]) {
+ if (first_matched_dim == -1) {
+ first_matched_dim = i;
+ }
m_node_dynamic_dims[node] = i;
matched_dim_count++;
}
}
- if (matched_dim_count != 1) {
+ if (matched_dim_count > 1 && node->src[0]->ne[dynamic_dim_idx] == 1) {
+ // Single-token capture: every trailing dim is size 1 with the same stride, so
+ // the match is ambiguous. The lowest index is the real axis; the rest are
+ // ggml's size-1 padding. Bailing out here would bake the captured token count
+ // into the static prefill model, which then runs with a different one.
+ m_node_dynamic_dims[node] = first_matched_dim;
+ } else if (matched_dim_count != 1) {
m_node_dynamic_dims[node] = -1;
GGML_LOG_WARN("ggml-openvino: cannot determine dynamic dim for CONT node '%s', src[0]: '%s'\n",
node->name, node->src[0]->name);
diff --git a/ggml/src/ggml-openvino/ggml-decoder.h b/ggml/src/ggml-openvino/ggml-decoder.h
index 8e39a26c8b79..74cb7385029a 100644
--- a/ggml/src/ggml-openvino/ggml-decoder.h
+++ b/ggml/src/ggml-openvino/ggml-decoder.h
@@ -47,6 +47,7 @@ struct ComputeParams {
int seq_active_start = 0;
int attention_size = -1;
int attention_size_swa = -1;
+ int attention_size_static = -1; // encoder/cross-attn KV fill level (whisper)
int input_len = -1;
int token_len_per_seq = -1;
int past_kv_len = -1;
@@ -84,14 +85,15 @@ struct ComputeParams {
struct RsWriteback {
int slot_begin = 0; // first cache slot written by the CPY
- int src_begin = 0; // where the copied data starts in the source tensor (in rows of it)
+ int src_begin = 0; // first source row or column copied by the CPY
};
std::map rs_writebacks;
- // Offsets of the state cache writeback CPY nodes, keyed by node name. They change with the
- // batch (kv head, active sequence count, token count) and, with rollback enabled
- // (cparams.n_rs_seq > 0), the conv state is written back once per snapshot slot, each snapshot
- // taking a different conv_input window. Passed to the cached model as runtime inputs.
+ // Destination slot offset of each state cache writeback CPY node, keyed by node name. It
+ // changes with the batch (kv head, active sequence count) and, with rollback enabled
+ // (cparams.n_rs_seq > 0), the conv state is written back once per snapshot slot. Passed to the
+ // cached model as a runtime input. Dynamic models also receive the source-side offset; static
+ // models use a fixed end-anchored offset in the translator.
};
class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder {
diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp
index 36c749244f83..36dfa4d9471b 100644
--- a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp
+++ b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp
@@ -32,6 +32,8 @@ void ggml_openvino_device_config::init() {
"GGML_OPENVINO_DEVICE",
"GGML_OPENVINO_CACHE_DIR",
"GGML_OPENVINO_DEBUG_NODE",
+ "GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR",
+ "GGML_OPENVINO_NPU_COMPILE_CONFIG",
// Integer values (use ggml_openvino_getenv_int)
"GGML_OPENVINO_PREFILL_CHUNK_SIZE",
// Boolean toggles (treated as int flags via ggml_openvino_getenv_int)
@@ -41,6 +43,9 @@ void ggml_openvino_device_config::init() {
"GGML_OPENVINO_DUMP_IR",
"GGML_OPENVINO_DEBUG_INPUT",
"GGML_OPENVINO_DEBUG_OUTPUT",
+ // Force the static (NPU-shape) compute path on any device, e.g. GGML_OPENVINO_DEVICE=CPU,
+ // to test the static-shape translation without NPUW/real NPU hardware in the loop.
+ "GGML_OPENVINO_FORCE_STATIC",
"GGML_OPENVINO_PRINT_CGRAPH_TENSOR_ADDRESS",
"GGML_OPENVINO_ENABLE_CACHE",
"GGML_OPENVINO_DISABLE_CACHE",
@@ -50,7 +55,7 @@ void ggml_openvino_device_config::init() {
"GGML_OPENVINO_MEMORY_OPTIMIZE",
"GGML_OPENVINO_RELEASE_WEIGHTS",
"GGML_OPENVINO_REDUCE_COMPILE_MEM",
- "GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR",
+ "GGML_OPENVINO_LOG_UNSUPPORTED_OPS",
};
for (const char * const & env_var : env_var_names) {
@@ -85,6 +90,11 @@ void ggml_openvino_device_config::init() {
compile_config["NPUW_CACHE_DIR"] = cache_dir;
compile_config.insert(ov::cache_mode(ov::CacheMode::OPTIMIZE_SIZE));
}
+ const char * compilation_mode_params =
+ ggml_openvino_getenv_str("GGML_OPENVINO_NPU_COMPILE_CONFIG");
+ if (compilation_mode_params && strlen(compilation_mode_params) > 0) {
+ compile_config["NPU_COMPILATION_MODE_PARAMS"] = compilation_mode_params;
+ }
} else if (cache_dir && strlen(cache_dir) > 0) {
compile_config.insert(ov::cache_dir(cache_dir));
compile_config.insert(ov::cache_mode(ov::CacheMode::OPTIMIZE_SIZE));
diff --git a/ggml/src/ggml-openvino/ggml-openvino.cpp b/ggml/src/ggml-openvino/ggml-openvino.cpp
index e299e16c778a..4b1789713d1d 100644
--- a/ggml/src/ggml-openvino/ggml-openvino.cpp
+++ b/ggml/src/ggml-openvino/ggml-openvino.cpp
@@ -908,11 +908,27 @@ static bool has_non_contiguous_view_input(const ggml_tensor * op) {
}
static bool is_supported_flash_attn_pattern(const ggml_tensor * op) {
- // pattern of q,k,v should be q->op==PERMUTE, q->src[0]->op==VIEW, q->src[0]->src[0]->view_src==nullptr
+ // Each Q/K/V input must follow one of:
+ // PERMUTE -> VIEW -> base (view_src==nullptr) (llama KV-cache path)
+ // PERMUTE -> RESHAPE -> base (view_src==nullptr) (whisper Q)
+ // VIEW -> base (view_src==nullptr) (whisper K/V from kv_pad)
for (int i = 0; i < 3; i++) {
const ggml_tensor * src = op->src[i];
- if (src->op != GGML_OP_PERMUTE || src->src[0] == nullptr || src->src[0]->op != GGML_OP_VIEW ||
- src->src[0]->src[0] == nullptr || src->src[0]->src[0]->view_src != nullptr) {
+ if (src->op == GGML_OP_PERMUTE) {
+ if (src->src[0] == nullptr) {
+ return false;
+ }
+ if (src->src[0]->op != GGML_OP_VIEW && src->src[0]->op != GGML_OP_RESHAPE) {
+ return false;
+ }
+ if (src->src[0]->src[0] == nullptr || src->src[0]->src[0]->view_src != nullptr) {
+ return false;
+ }
+ } else if (src->op == GGML_OP_VIEW) {
+ if (src->src[0] == nullptr || src->src[0]->view_src != nullptr) {
+ return false;
+ }
+ } else {
return false;
}
}
@@ -1030,18 +1046,29 @@ static bool is_msa_block_mask_expansion(const ggml_tensor * op) {
return tensor_name_starts_with(src, "msa_block_mask");
}
-static bool is_op_unsupported_case(const ggml_tensor * op) {
+namespace {
+struct ggml_openvino_op_support {
+ bool is_supported = true;
+ std::string reason;
+
+ operator bool() const {
+ return is_supported;
+ }
+};
+} // namespace
+
+static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) {
if (is_msa_block_mask_expansion(op)) {
- return true;
+ return {false, "MSA block mask expansion is not supported"};
}
switch (op->op) {
case GGML_OP_CONCAT: {
if (op->type == GGML_TYPE_I64) {
- return true;
+ return {false, "CONCAT with I64 type is not supported"};
}
if (ggml_openvino_get_device_name() == "GPU" && op->type == GGML_TYPE_BF16 && has_view_op_input(op)) {
- return true;
+ return {false, "CONCAT with BF16 type and VIEW input is not supported on GPU"};
}
break;
}
@@ -1052,24 +1079,21 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
// OpenVINO SET translation currently supports dst layouts that match src0 strides.
if (op->src[0] == nullptr || nb1 != op->src[0]->nb[1] || nb2 != op->src[0]->nb[2] || nb3 != op->src[0]->nb[3]) {
- // std::cout << "Unsupported SET op with dst nb1=" << nb1 << ", nb2=" << nb2 << ", nb3=" << nb3
- // << " that does not match src0 strides nb[1]="
- // << (op->src[0] != nullptr ? std::to_string(op->src[0]->nb[1]) : "null")
- // << ", nb[2]=" << (op->src[0] != nullptr ? std::to_string(op->src[0]->nb[2]) : "null")
- // << ", nb[3]=" << (op->src[0] != nullptr ? std::to_string(op->src[0]->nb[3]) : "null")
- // << std::endl;
- return true;
+ return {false, "SET op with dst nb1=" + std::to_string(nb1) + ", nb2=" + std::to_string(nb2) + ", nb3=" + std::to_string(nb3) +
+ " that does not match src0 strides nb[1]=" + (op->src[0] != nullptr ? std::to_string(op->src[0]->nb[1]) : "null") +
+ ", nb[2]=" + (op->src[0] != nullptr ? std::to_string(op->src[0]->nb[2]) : "null") +
+ ", nb[3]=" + (op->src[0] != nullptr ? std::to_string(op->src[0]->nb[3]) : "null")};
}
break;
}
case GGML_OP_GET_ROWS:
case GGML_OP_SET_ROWS: {
if (op->ne[3] != 1) {
- return true;
+ return {false, "GET_ROWS/SET_ROWS with ne[3] != 1 (ne[3]=" + std::to_string(op->ne[3]) + ") is not supported"};
}
if (op->op == GGML_OP_GET_ROWS && ggml_openvino_get_device_name() == "GPU" &&
op->src[0]->type == GGML_TYPE_BF16) {
- return true;
+ return {false, "GET_ROWS with BF16 src0 is not supported on GPU"};
}
if (op->ne[0] == 256 && (op->src[0]->type == GGML_TYPE_Q4_K || op->src[0]->type == GGML_TYPE_Q5_K ||
op->src[0]->type == GGML_TYPE_Q4_1 || op->src[0]->type == GGML_TYPE_Q5_1)) {
@@ -1078,14 +1102,14 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
// make_int8_weights/make_int4_weights: dequant is done in f16, not f32, to keep the
// Convert/Subtract/Multiply chain fusable into GatherMatmulCompressed/FullyConnectedCompressed
// for the shared non-test code paths).
- return true;
+ return {false, "GET_ROWS/SET_ROWS with ne[0] == 256 and type " + std::string(ggml_type_name(op->src[0]->type)) +
+ " rejected due to f16-arithmetic dequant rounding errors that intermittently exceed 1e-7 NMSE threshold"};
}
-
break;
}
case GGML_OP_RESHAPE: {
if (strncmp(op->name, "ffn_norm_exps", sizeof("ffn_norm_exps") - 1) == 0) {
- return true;
+ return {false, "RESHAPE for ffn_norm_exps is not supported"};
}
break;
}
@@ -1093,11 +1117,13 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
case GGML_OP_MUL:
case GGML_OP_SUB: {
if (op->src[1]->op == GGML_OP_PERMUTE) {
- return true;
+ return {false, "ADD/MUL/SUB with PERMUTE src1 is not supported"};
}
for (int i = 0; i < 4; i++) {
if (op->src[0]->ne[i] != op->src[1]->ne[i] && (op->src[0]->ne[i] != 1 && op->src[1]->ne[i] != 1)) {
- return true;
+ return {false, "ADD/MUL/SUB with incompatible broadcast shapes: src0->ne[" + std::to_string(i) + "]=" +
+ std::to_string(op->src[0]->ne[i]) + ", src1->ne[" + std::to_string(i) + "]=" +
+ std::to_string(op->src[1]->ne[i])};
}
}
break;
@@ -1106,7 +1132,7 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
// Keep support aligned with the CPU backend implementation, which only handles f32 inputs/output and i32 ids.
if (op->type != GGML_TYPE_F32 || op->src[0]->type != GGML_TYPE_F32 || op->src[1]->type != GGML_TYPE_F32 ||
op->src[2]->type != GGML_TYPE_I32) {
- return true;
+ return {false, "ADD_ID only supports F32 inputs/output and I32 ids"};
}
break;
}
@@ -1116,14 +1142,27 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
// until the fused GPU kernel is reliable. (falied case llama-arch-test mpt)
if (ggml_openvino_get_device_name() == "GPU" && op->src[1]->ne[0] == op->ne[0] &&
op->src[1]->ne[1] == 1 && op->src[1]->ne[2] == 1 && op->src[1]->ne[3] == 1) {
- return true;
+ return {false, "DIV per-channel scale broadcast is not supported on GPU"};
+ }
+ break;
+ }
+ case GGML_OP_POOL_2D: {
+ const auto& name = ggml_openvino_get_device_name();
+ if (name == "GPU") {
+ const int32_t * params = op->op_params;
+ const int k0 = params[1];
+ const int k1 = params[2];
+ const int p0 = params[5];
+ const int p1 = params[6];
+ if ((p0 > 0 || p1 > 0) && (k0 < 3 || k1 < 3)) {
+ return {false, "POOL_2D with padding and kernel size < 3 is not supported on " + name};
+ }
}
break;
}
case GGML_OP_SUM_ROWS: {
- // if the input is PERMUTE skip
if (op->src[0]->op == GGML_OP_PERMUTE) {
- return true;
+ return {false, "SUM_ROWS with PERMUTE input is not supported"};
}
break;
}
@@ -1140,54 +1179,51 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
// accuracy drift in the OpenVINO path. Restrict by scale=1.0 to avoid
// affecting non-gemma3n models such as Llama-3.2.
if (fabsf(scale - 1.0f) < 1e-6f && is_gemma3n_flash_attn_pattern(op)) {
- return true;
+ return {false, "FLASH_ATTN_EXT gemma3n pattern on GPU is not supported"};
}
if (op->src[4] != nullptr) {
- // GGML_LOG_WARN("OpenVINO backend does not support FLASH_ATTN_EXT with sinks\n");
- return true;
+ return {false, "FLASH_ATTN_EXT with sinks is not supported"};
}
if (!is_supported_flash_attn_pattern(op)) {
- return true;
+ return {false, "FLASH_ATTN_EXT unsupported attention pattern"};
}
if (max_bias > 0) {
- // GGML_LOG_WARN("OpenVINO backend does not support FLASH_ATTN_EXT with max_bias > 0\n");
- return true;
+ return {false, "FLASH_ATTN_EXT with max_bias > 0 (max_bias=" + std::to_string(max_bias) + ") is not supported"};
}
if (logit_softcap != 0) {
- // GGML_LOG_WARN("OpenVINO backend does not support FLASH_ATTN_EXT with logit_softcap != 0\n");
- return true;
+ return {false, "FLASH_ATTN_EXT with logit_softcap != 0 (logit_softcap=" + std::to_string(logit_softcap) + ") is not supported"};
}
break;
}
case GGML_OP_PERMUTE: {
- if (op->type == GGML_TYPE_BF16) {
- // err msg: [GPU] Could not find a suitable kernel for transpose
- // GGML_LOG_WARN("OpenVINO backend does not support PERMUTE with BF16 type\n");
- return true;
+ if (op->type == GGML_TYPE_BF16 && ggml_openvino_get_device_name() == "GPU") {
+ return {false, "PERMUTE with BF16 type is not supported on GPU"};
}
break;
}
case GGML_OP_CPY: {
if (op->src[0]->type == GGML_TYPE_BF16 || op->src[1]->type == GGML_TYPE_BF16) {
- // GGML_LOG_WARN("OpenVINO backend does not support CPY with non-contiguous data or bf16 types\n");
- return true;
+ return {false, "CPY with BF16 src type is not supported"};
}
// CPY to a quantized destination (e.g. f32 -> q4_0) is numerically unstable with OpenVINO backend.
if (ggml_is_quantized(op->type)) {
- return true;
+ return {false, "CPY to quantized destination (e.g. f32 -> q4_0) is numerically unstable"};
}
if (ggml_nelements(op->src[0]) != ggml_nelements(op->src[1])) {
- return true;
+ return {false, "CPY with mismatched element counts is not supported: src0=" + std::to_string(ggml_nelements(op->src[0])) +
+ " != src1=" + std::to_string(ggml_nelements(op->src[1]))};
}
// op test case with non-contiguous src or dst
if ((op->ne[0] == 3 && op->ne[1] == 4 && op->ne[2] == 3 && op->ne[3] == 2) ||
(op->ne[0] == 1 && op->ne[1] == 4 && op->ne[2] == 3 && op->ne[3] == 2) ||
(op->ne[0] == 2 && op->ne[1] == 4 && op->ne[2] == 3 && op->ne[3] == 2)) {
- return true;
+ return {false, "CPY with non-contiguous shape [" + std::to_string(op->ne[0]) + ", " +
+ std::to_string(op->ne[1]) + ", " + std::to_string(op->ne[2]) + ", " +
+ std::to_string(op->ne[3]) + "] is not supported"};
}
if (!cpy_output_view_is_supported(op)) {
- return true;
+ return {false, "CPY with non-contiguous output view is not supported"};
}
break;
}
@@ -1196,13 +1232,14 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
ggml_is_quantized(op->src[0]->type) && strcmp(op->src[0]->name, "a") == 0 &&
strcmp(op->src[1]->name, "b") == 0 && op->src[0]->ne[1] == 1 && op->src[1]->ne[1] == 64 &&
op->src[0]->ne[0] == 256 && op->src[1]->ne[0] == 256) {
- return true;
+ return {false, "MUL_MAT quantized benchmark test case on GPU is not supported"};
}
if (op->src[0]->ne[3] != op->src[1]->ne[3] && op->src[0]->ne[3] != 1 && op->src[1]->ne[3] != 1) {
- return true;
+ return {false, "MUL_MAT with incompatible broadcast on ne[3]: src0->ne[3]=" + std::to_string(op->src[0]->ne[3]) +
+ ", src1->ne[3]=" + std::to_string(op->src[1]->ne[3])};
}
if (op->src[0]->op == GGML_OP_VIEW && op->src[1]->op == GGML_OP_VIEW) {
- return true;
+ return {false, "MUL_MAT with both inputs as VIEW is not supported"};
}
break;
}
@@ -1210,16 +1247,17 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
// Single-expert (or empty) MUL_MAT_ID is a degenerate shape that stresses GatherMatmul edge
// cases and never occurs in real MoE; let it fall back to CPU.
if (op->src[0] != nullptr && op->src[0]->ne[2] <= 1) {
- return true;
+ return {false, "MUL_MAT_ID with single-expert or empty ne[2] <= 1 (ne[2]=" +
+ std::to_string(op->src[0]->ne[2]) + ") is not supported"};
}
if (ggml_openvino_get_device_name() == "GPU" && op->src[0] != nullptr && op->src[0]->type == GGML_TYPE_BF16) {
- return true;
+ return {false, "MUL_MAT_ID with BF16 weights on GPU is not supported"};
}
// GPU MUL_MAT_ID uses a Gather+MatMul fallback because the GPU plugin rejects internal
// GatherMatmul for these test shapes. Skip cases that would materialize a large selected
// expert-weight temporary.
if (ggml_openvino_get_device_name() == "GPU" && mul_mat_id_requires_large_tmp(op)) {
- return true;
+ return {false, "MUL_MAT_ID requires large temporary on GPU"};
}
break;
}
@@ -1229,51 +1267,46 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
const int mode = op_params[2];
if (op_params[15] != 0) {
// FIXME: support ggml_rope_set_offset
- return true;
+ return {false, "ggml_rope_set_offset is not supported"};
}
if (mode != GGML_ROPE_TYPE_NORMAL && mode != GGML_ROPE_TYPE_NEOX && mode != GGML_ROPE_TYPE_IMROPE) {
- // GGML_LOG_WARN("OpenVINO backend does not support ROPE with mode %d\n", mode);
- return true;
+ return {false, "ROPE with mode " + std::to_string(mode) + " is not supported"};
}
const int64_t head_dim = op->src[0]->ne[0];
const int64_t rope_dims = n_dims == 0 ? head_dim : n_dims;
if (rope_dims <= 0 || rope_dims > head_dim || (rope_dims % 2) != 0) {
- // GGML_LOG_WARN("OpenVINO backend does not support ROPE with n_dims %d and src[0]->ne[0] %ld\n", n_dims,
- // op->src[0]->ne[0]);
- return true;
+ return {false, "ROPE with n_dims=" + std::to_string(n_dims) + ", head_dim=" + std::to_string(head_dim) + " is not supported"};
}
if (op->type != GGML_TYPE_F32 && op->type != GGML_TYPE_F16) {
- // GGML_LOG_WARN("OpenVINO backend does not support ROPE with type %s\n", ggml_type_name(op->type));
- return true;
+ return {false, "ROPE with type " + std::string(ggml_type_name(op->type)) + " is not supported"};
}
if (op->src[0]->op == GGML_OP_VIEW) {
- if (op->src[0]->view_src->ne[1] != op->src[0]->ne[2]) {
- // GGML_LOG_WARN(
- // "OpenVINO backend does not support ROPE with src[0]->view_src->ne[1] %ld != src[0]->ne[2] "
- // "%ld\n",
- // op->src[0]->view_src->ne[1], op->src[0]->ne[2]);
- return true;
+ const struct ggml_tensor * view = op->src[0];
+ const struct ggml_tensor * view_src = view->view_src;
+ if (view_src->ne[1] != view->ne[1] || view_src->ne[2] != view->ne[2] || view_src->ne[3] != view->ne[3]) {
+ return {false, "ROPE with view_src->ne [" + std::to_string(view_src->ne[1]) + ", " +
+ std::to_string(view_src->ne[2]) + ", " + std::to_string(view_src->ne[3]) +
+ "] != view->ne [" + std::to_string(view->ne[1]) + ", " +
+ std::to_string(view->ne[2]) + ", " + std::to_string(view->ne[3]) +
+ "] is not supported"};
}
}
if (mode == GGML_ROPE_TYPE_IMROPE &&
(op->src[2] != 0 || ((const float *) op_params)[6] != 1 || ((const float *) op_params)[7] != 0 ||
((const float *) op_params)[8] != 1)) {
- // GGML_LOG_WARN("OpenVINO backend does not support IMROPE with freq_factors, freq_scale, ext_factor, and attn_factor\n");
- return true;
+ return {false, "IMROPE with freq_factors, freq_scale, ext_factor, and attn_factor is not supported"};
}
break;
}
case GGML_OP_TRANSPOSE: {
- // if the type is bf16, will return true
if (op->type == GGML_TYPE_BF16) {
- // GGML_LOG_WARN("OpenVINO backend does not support CONT with BF16 type\n");
- return true;
+ return {false, "TRANSPOSE with BF16 type is not supported"};
}
break;
}
case GGML_OP_REPEAT: {
if (ggml_openvino_get_device_name() == "GPU" && op->type == GGML_TYPE_BF16) {
- return true;
+ return {false, "REPEAT with BF16 type is not supported on GPU"};
}
break;
}
@@ -1285,15 +1318,15 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
// return true;
// }
if (op->src[2]->op == GGML_OP_PERMUTE) {
- return true;
+ return {false, "GATED_DELTA_NET with PERMUTE src2 is not supported"};
}
// kda (per-key-dimension gating) not supported by fused GatedDeltaNet op
if (op->src[3]->ne[0] != 1) {
- return true;
+ return {false, "GATED_DELTA_NET with kda (per-key-dimension gating) is not supported"};
}
// K > 1 (multiple state snapshots) not supported by fused op
if (((const int32_t *) op->op_params)[0] > 1) {
- return true;
+ return {false, "GATED_DELTA_NET with K > 1 (multiple state snapshots) is not supported"};
}
break;
}
@@ -1307,17 +1340,17 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
// Skip TOPK_MOE fused tests until it is fully supported.
// The argsort_top_k VIEW wrapping ARGSORT is named "selected_experts" in test_topk_moe.
if (strcmp(op->name, "selected_experts") == 0) {
- return true;
+ return {false, "VIEW for selected_experts (argsort_top_k) is not supported"};
}
break;
}
default:
break;
}
- return false;
+ return {true, ""};
}
-static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) {
+static ggml_openvino_op_support ggml_backend_openvino_device_supports_op_impl(ggml_backend_dev_t dev, const ggml_tensor * op) {
GGML_ASSERT(dev->reg != nullptr);
static std::unordered_set supported_types{
@@ -1367,48 +1400,41 @@ static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, con
case GGML_OP_UNARY: {
auto supported = supported_unary_ops.find(ggml_get_unary_op(op)) != supported_unary_ops.end();
if (!supported) {
- // GGML_LOG_WARN("OpenVINO backend does not support unary op %s\n", ggml_unary_op_name(ggml_get_unary_op(op)));
- return false;
+ return {false, "unary op " + std::string(ggml_unary_op_name(ggml_get_unary_op(op))) + " has no op translator"};
}
if (ggml_get_unary_op(op) == GGML_UNARY_OP_EXP && op->type == GGML_TYPE_F32) {
- return false;
+ return {false, "UNARY_EXP with F32 type is not supported"};
}
break;
}
case GGML_OP_GLU: {
auto supported = supported_glu_ops.find(ggml_get_glu_op(op)) != supported_glu_ops.end();
if (!supported) {
- // GGML_LOG_WARN("OpenVINO backend does not support GLU op %s\n", ggml_glu_op_name(ggml_get_glu_op(op)));
- return false;
+ return {false, "GLU op " + std::string(ggml_glu_op_name(ggml_get_glu_op(op))) + " has no op translator"};
}
// if (has_view_op_input(op)) {
- // // GGML_LOG_WARN("OpenVINO backend does not support unary op %s with view input\n",
- // // ggml_glu_op_name(ggml_get_glu_op(op)));
- // return false;
+ // return {false, "GLU op " + std::string(ggml_glu_op_name(ggml_get_glu_op(op))) + " with view input is not supported"};
// }
if (op->src[1] == nullptr && op->src[0]->ne[0] % 2 != 0) {
// triggers bug in ov gpu
- return false;
+ return {false, "GLU op with odd src0 ne[0] and null src1 is not supported"};
}
break;
}
default: {
auto supported = supported_ops.find(op->op) != supported_ops.end();
if (!supported) {
- // GGML_LOG_WARN("OpenVINO backend does not support op %s\n", ggml_op_name(op->op));
- return false;
+ return {false, "op " + std::string(ggml_op_name(op->op)) + " has no op translator"};
}
static std::set ops_not_support_view_input{};
if (ops_not_support_view_input.find(op->op) != ops_not_support_view_input.end() && has_view_op_input(op)) {
- // GGML_LOG_WARN("OpenVINO backend does not support op %s with view input\n", ggml_op_name(op->op));
- return false;
+ return {false, "op " + std::string(ggml_op_name(op->op)) + " with VIEW input is not supported"};
}
}
}
if (supported_types.find(op->type) == supported_types.end()) {
- // GGML_LOG_WARN("OpenVINO backend does not support tensor type %s\n", ggml_type_name(op->type));
- return false;
+ return {false, "tensor type " + std::string(ggml_type_name(op->type)) + " is not supported"};
}
for (int i = 0; i < GGML_MAX_SRC; i++) {
auto * src = op->src[i];
@@ -1416,21 +1442,32 @@ static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, con
break;
}
if (supported_types.find(src->type) == supported_types.end()) {
- // GGML_LOG_WARN("OpenVINO backend does not support tensor type %s\n", ggml_type_name(src->type));
- return false;
+ return {false, "src[" + std::to_string(i) + "] type " + std::string(ggml_type_name(src->type)) + " is not supported"};
}
const bool is_supported_3d_moe_expert =
op->op == GGML_OP_MUL_MAT_ID && i == 0 && (src->type == GGML_TYPE_MXFP4 || src->ne[3] == 1);
if (ggml_is_quantized(src->type) && src->ne[2] != 1 && !is_supported_3d_moe_expert) {
- // GGML_LOG_WARN("OpenVINO backend does not support 3D quantized tensors\n");
- return false;
+ return {false, "3D quantized tensor for src[" + std::to_string(i) + "] is not supported"};
}
}
- if (is_op_unsupported_case(op)) {
- return false;
+ auto op_support_case = is_op_supported_case(op);
+ if (!op_support_case.is_supported) {
+ return op_support_case;
}
- return true;
+ return {true, ""};
+}
+
+static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) {
+ auto res = ggml_backend_openvino_device_supports_op_impl(dev, op);
+ if (!res.is_supported) {
+ static const bool log_unsupported = ggml_openvino_getenv_int("GGML_OPENVINO_LOG_UNSUPPORTED_OPS") != 0;
+ if (log_unsupported) {
+ GGML_LOG_WARN("OpenVINO op unsupported: op '%s' (%s), type %s: %s\n",
+ op->name, ggml_op_name(op->op), ggml_type_name(op->type), res.reason.c_str());
+ }
+ }
+ return res.is_supported;
}
static bool ggml_backend_openvino_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) {
diff --git a/ggml/src/ggml-openvino/openvino/op/cpy.cpp b/ggml/src/ggml-openvino/openvino/op/cpy.cpp
index 5b387fc50d38..6f1e34779ac4 100644
--- a/ggml/src/ggml-openvino/openvino/op/cpy.cpp
+++ b/ggml/src/ggml-openvino/openvino/op/cpy.cpp
@@ -3,8 +3,11 @@
#include "../utils.h"
#include
+#include
+#include
#include
-#include
+#include
+#include
#include
#include
#include
@@ -12,9 +15,14 @@
#include
#include
#include
+#include
#include
+#include
#include
#include
+#include
+#include
+#include
namespace ov {
namespace frontend {
@@ -61,10 +69,27 @@ OutputVector translate_cpy(const NodeContext & context) {
return rename_outputs_with_suffix({res}, context.get_name());
}
- // Recurrent state cache writeback into a slot block of the cache. Where the block starts and
- // where the copied data starts in the source are runtime inputs, so the cached model works for
- // any kv head, active sequence count and token count. The result is the full updated cache.
+ // Recurrent state cache writeback into a slot block of the cache. Where the block starts is a
+ // runtime input, so the cached model works for any kv head and active sequence count. The
+ // result is the full updated cache.
// op_case 1: gated-delta-net state, op_case 2: conv state, op_case 3: defrag remainder.
+ if (op_case == 3) {
+ // With -np 1 (and generally whenever there is no defrag remainder) this GET_ROWS gathers
+ // zero rows: nothing to write back, and the cache is unchanged. NPU rejects zero-size
+ // tensors, so short-circuit instead of building a degenerate Slice/Concat chain.
+ bool is_empty = false;
+ if (input_shape.rank().is_static()) {
+ for (const auto & d : input_shape) {
+ if (d.is_static() && d.get_length() == 0) {
+ is_empty = true;
+ break;
+ }
+ }
+ }
+ if (is_empty) {
+ return {context.get_input(1)};
+ }
+ }
const std::string slot_begin_name = "rs_slot_begin_" + context.get_name();
const bool slice_assign =
context.has_input(slot_begin_name) && !context.is_stateful() && (op_case >= 1 && op_case <= 3);
@@ -81,19 +106,49 @@ OutputVector translate_cpy(const NodeContext & context) {
ov::Output begin = context.get_input(slot_begin_name);
auto base = context.get_input(1);
if (op_case == 1) {
- // GDN packs [attn | state snapshots]; the state part runs from src_begin to the end.
- auto src_begin = context.get_input("rs_src_begin_" + context.get_name());
- auto state_part = std::make_shared(context.get_input(0), src_begin, int_max, one, axis);
+ ov::Output state_begin;
+ const std::string src_begin_name = "rs_src_begin_" + context.get_name();
+ if (context.has_input(src_begin_name)) {
+ state_begin = context.get_input(src_begin_name);
+ } else {
+ auto ssm_state_size = context.get_ssm_state_size();
+ if (context.has_input("s_copy_active_slot_len")) {
+ auto len = context.get_input("s_copy_active_slot_len");
+ auto state_rows = std::make_shared(
+ ov::op::v0::Constant::create(ov::element::i64, {1}, {ssm_state_size}), len);
+ state_begin = std::make_shared(state_rows);
+ } else {
+ state_begin = ov::op::v0::Constant::create(ov::element::i64, {1}, {-ssm_state_size});
+ }
+ }
+ auto state_part =
+ std::make_shared(context.get_input(0), state_begin, int_max, one, axis);
src = std::make_shared(state_part, feature, false);
} else if (op_case == 2) {
- // conv_input is [previous conv state | new tokens]; copy the conv_kernel_size - 1 wide
- // window starting at src_begin, which is the snapshot this writeback corresponds to.
+ // conv_input is [previous conv state | new tokens]; the snapshot is the conv_kernel_size - 1
+ // columns ending at the last *valid* token. Gather (rather than Slice) keeps the output
+ // shape static even though the window start is a runtime value.
auto window_size = (int64_t) input_shape[3].get_length();
- auto src_begin = context.get_input("rs_src_begin_" + context.get_name());
- auto src_end = std::make_shared(
- src_begin, ov::op::v0::Constant::create(ov::element::i64, {1}, {window_size}));
- auto window = std::make_shared(context.get_input(0), src_begin, src_end, one,
- ov::op::v0::Constant::create(ov::element::i64, {1}, {3}));
+ ov::Output window;
+ auto col_axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {3});
+ const std::string src_begin_name = "rs_src_begin_" + context.get_name();
+ if (context.has_input(src_begin_name)) {
+ auto src_begin = context.get_input(src_begin_name);
+ auto src_end = std::make_shared(
+ src_begin, ov::op::v0::Constant::create(ov::element::i64, {1}, {window_size}));
+ window = std::make_shared(context.get_input(0), src_begin, src_end, one, col_axis);
+ } else if (context.has_input("chunk_valid_len")) {
+ std::vector offsets(window_size);
+ std::iota(offsets.begin(), offsets.end(), 0);
+ auto indices = std::make_shared(
+ ov::op::v0::Constant::create(ov::element::i64, {(size_t) window_size}, offsets),
+ context.get_input("chunk_valid_len"));
+ window = std::make_shared(context.get_input(0), indices, col_axis);
+ } else {
+ auto window_begin = ov::op::v0::Constant::create(ov::element::i64, {1}, {-window_size});
+ window =
+ std::make_shared(context.get_input(0), window_begin, int_max, one, col_axis);
+ }
const auto base_shape = base.get_partial_shape();
FRONT_END_OP_CONVERSION_CHECK(base_shape.rank().is_static() && base_shape.rank().get_length() == 4,
"CPY conv state cache update requires rank-4 base cache");
@@ -157,6 +212,63 @@ OutputVector translate_cpy(const NodeContext & context) {
auto input = process_view_input_new(context, 0);
+ if (op_case == 5 || op_case == 6) {
+ auto input_shape = context.get_input_shape(0);
+ auto output_shape = context.get_output_shape();
+ auto dst_ggml_shape = context.get_view_input_ggml_shape(1, 0);
+ auto dst_stride = context.get_view_input_stride(1, 0);
+ size_t offset_bytes = context.get_view_input_offset(1, 0);
+ auto n_state = (int64_t) context.get_input_shape(0)[3].get_length();
+ auto n_state_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_state});
+ auto kv_buf = context.get_input(1); // shape {1,1,1,N}
+
+ Output token_len_per_seq;
+ Output n_write_dyn;
+ if (context.has_input("token_len_per_seq")) {
+ token_len_per_seq = context.get_input("token_len_per_seq");
+ n_write_dyn = std::make_shared(token_len_per_seq, n_state_c);
+ } else {
+ n_write_dyn = ov::op::v0::Constant::create(ov::element::i64, {1}, {(int64_t) dst_ggml_shape[3]});
+ }
+ size_t elem_size = dst_stride[3];
+ FRONT_END_OP_CONVERSION_CHECK(elem_size > 0, "CPY KV cache view update has invalid element size");
+ int64_t start_elem = (int64_t) (offset_bytes / elem_size);
+ // op_case 5: decoder self-attention – write offset advances each step.
+ // op_case 6: encoder self-attn or cross-attn – offset fixed at compile time.
+ const bool is_decoder_self_attn = (op_case == 5);
+ auto ones_c = ov::op::v0::Constant::create(ov::element::i64, {3}, std::vector{1, 1, 1});
+ auto new_shape = std::make_shared(ov::OutputVector{ones_c, n_write_dyn}, 0);
+
+ auto reshaped = std::make_shared(input, new_shape, false);
+ auto data = std::make_shared(reshaped, context.get_output_type());
+ // Indices [start_elem .. start_elem + n_write) on axis 3 of {1,1,1,N}
+ // For decoder self-attention the write offset advances each step, so compute it
+ // dynamically from the model inputs: start = (attention_size - token_len_per_seq) * n_state.
+ // For encoder self-attn and cross-attn the offset is fixed at graph-compile time.
+ ov::Output start;
+ if (is_decoder_self_attn && context.has_input("attention_size") && context.has_input("token_len_per_seq")) {
+ auto attention_size_in = context.get_input("attention_size");
+ auto token_len_in = context.get_input("token_len_per_seq");
+ auto past_tokens = std::make_shared(attention_size_in, token_len_in);
+ auto new_start = std::make_shared(past_tokens, n_state_c);
+ start = std::make_shared(
+ new_start, ov::op::v0::Constant::create(ov::element::i64, {1}, {start_elem}));
+ } else {
+ start = ov::op::v0::Constant::create(ov::element::i64, {1}, {start_elem});
+ }
+ auto start_squeezed = std::make_shared(start);
+ auto end = std::make_shared(start_squeezed, n_write_dyn);
+ auto end_squeezed = std::make_shared(end);
+ auto step = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
+ auto step_squeezed = std::make_shared(step);
+ auto indices =
+ std::make_shared(start_squeezed, end_squeezed, step_squeezed, ov::element::i64);
+ auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {3});
+
+ auto kv_updated = std::make_shared(kv_buf, indices, data, axis);
+ return rename_outputs_with_suffix({kv_updated}, context.get_name());
+ }
+
if (input_shape != output_shape) {
auto new_shape = ov::op::v0::Constant::create(
ov::element::i64, {static_cast(output_shape.rank().get_length())}, output_shape.to_shape());
diff --git a/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp b/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp
index 582df0130b59..06547f3d2968 100644
--- a/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp
+++ b/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp
@@ -3,8 +3,8 @@
#include "../utils.h"
#include "ggml-openvino/ggml-openvino-extra.h"
+#include
#include
-#include
#include
#include
#include
@@ -15,6 +15,7 @@
#include
#include
#include
+#include
#include
#include
#include
@@ -24,13 +25,62 @@ namespace ov {
namespace frontend {
namespace ggml {
namespace op {
+static ov::Output reshape_flat_kv(const ov::Output & kv_flat,
+ size_t view_offset_bytes,
+ size_t nb1_bytes,
+ int64_t n_head,
+ int64_t head_size,
+ const ov::Output & attention_size) {
+ int64_t n_state = n_head * head_size;
+ int64_t layer_start_elem = (int64_t) (view_offset_bytes / (nb1_bytes / n_state));
+ // Dynamic slice: [layer_start_elem, layer_start_elem + n_kv * n_state)
+ auto start_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {layer_start_elem});
+ auto n_state_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_state});
+ // end = start + attention_size * n_state (both static + dynamic)
+ auto kv_len_elems = std::make_shared(attention_size, n_state_c);
+ auto end_c = std::make_shared(start_c, kv_len_elems);
+ auto step_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
+ auto axis_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {3});
+ auto sliced = std::make_shared(kv_flat, start_c, end_c, step_c, axis_c);
+
+ // KV cache is laid out as {n_kv, n_head, head_size} in memory
+ // Reshape to {1, n_kv, n_head, head_size}, then transpose to {1, n_head, n_kv, head_size}
+ // as required by SDPA.
+ auto one_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
+ auto n_head_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_head});
+ auto head_size_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {head_size});
+ // reshape: {n_kv*n_state} -> {1, n_kv, n_head, head_size}
+ auto new_shape =
+ std::make_shared(ov::OutputVector{one_c, attention_size, n_head_c, head_size_c}, 0);
+ auto reshaped = std::make_shared(sliced, new_shape, false);
+ // transpose: {1, n_kv, n_head, head_size} -> {1, n_head, n_kv, head_size}
+ auto perm = ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3});
+ auto ret = std::make_shared(reshaped, perm);
+ return ret;
+}
OutputVector translate_flash_attn_ext(const NodeContext & context) {
- num_inputs_check(context, 4, 4);
+ num_inputs_check(context, 3, 4);
+ const bool has_mask = context.get_input_size() == 4;
auto q_f32 = context.get_input(0);
auto k = context.get_input(1);
auto v = context.get_input(2);
- auto mask = context.get_input(3);
+ const int op_case = context.get_op_case();
+
+ if (op_case == 1 || op_case == 2) {
+ int64_t n_state_head = (int64_t) context.get_view_input_ggml_shape(1, 0)[3];
+ int64_t n_head = (int64_t) context.get_view_input_ggml_shape(1, 0)[1];
+ size_t nb1 = context.get_view_input_stride(1, 0)[2];
+ size_t offset = context.get_view_input_offset(1, 0);
+ ov::Output attention_size;
+ if (op_case == 1) {
+ attention_size = context.get_input("attention_size");
+ } else {
+ attention_size = context.get_input("attention_size_static");
+ }
+ k = reshape_flat_kv(k, offset, nb1, n_head, n_state_head, attention_size);
+ v = reshape_flat_kv(v, offset, nb1, n_head, n_state_head, attention_size);
+ }
float * params = reinterpret_cast(context.get_output_op_params());
float scale = params[0];
@@ -43,16 +93,19 @@ OutputVector translate_flash_attn_ext(const NodeContext & context) {
ov::Output res;
// For stateful
- std::string mask_name = "KQ_mask_sliced";
- if (context.get_input_names()[3].find("swa") != std::string::npos) {
- mask_name = "KQ_mask_swa_sliced";
- }
- if (context.has_input(mask_name)) {
- mask = context.get_input(mask_name);
- }
-
- if (mask.get_element_type() != ov::element::f16) {
- mask = std::make_shared(mask, ov::element::f16);
+ ov::Output mask;
+ if (has_mask) {
+ mask = context.get_input(3);
+ std::string mask_name = "KQ_mask_sliced";
+ if (context.get_input_names()[3].find("swa") != std::string::npos) {
+ mask_name = "KQ_mask_swa_sliced";
+ }
+ if (context.has_input(mask_name)) {
+ mask = context.get_input(mask_name);
+ }
+ if (mask.get_element_type() != ov::element::f16) {
+ mask = std::make_shared(mask, ov::element::f16);
+ }
}
//auto tile_kv = [&](int64_t num_heads, int64_t num_heads_kv, int64_t head_size, ov::Output kv) {
@@ -108,10 +161,14 @@ OutputVector translate_flash_attn_ext(const NodeContext & context) {
// get [B, 1, 1, S_q, S_k], which NUMPY-broadcasts cleanly against the
// [B, num_heads_kv, factor, S_q, S_k] scores: B==B, then 1→num_heads_kv and
// 1→factor on the head dims.
- auto mask_unsq1 =
- std::make_shared(mask, ov::op::v0::Constant::create(ov::element::i64, {1}, {2}));
- // mask_unsq1: [B, 1, 1, S_q, S_k] (rank 5)
- ov::Output qk_masked = std::make_shared(qk_scaled, mask_unsq1);
+ ov::Output qk_masked;
+ if (has_mask) {
+ auto mask_unsq1 =
+ std::make_shared(mask, ov::op::v0::Constant::create(ov::element::i64, {1}, {2}));
+ qk_masked = std::make_shared(qk_scaled, mask_unsq1);
+ } else {
+ qk_masked = qk_scaled;
+ }
auto softmax = std::make_shared(qk_masked, /*axis=*/-1);
@@ -164,9 +221,16 @@ OutputVector translate_flash_attn_ext(const NodeContext & context) {
k = tile_kv(num_heads, num_heads_kv, head_size, k);
v = tile_kv(num_heads, num_heads_kv, head_size, v);
- auto sdpa = std::make_shared(q, k, v, mask, scale_node, false);
- res = std::make_shared(sdpa,
- ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3}));
+ constexpr auto causal = false;
+ if (has_mask) {
+ auto sdpa = std::make_shared(q, k, v, mask, scale_node, causal);
+ res = std::make_shared(
+ sdpa, ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3}));
+ } else {
+ auto sdpa = std::make_shared(q, k, v, scale_node, causal);
+ res = std::make_shared(
+ sdpa, ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3}));
+ }
res = std::make_shared(res, ov::element::f32);
return rename_outputs_with_suffix({res}, context.get_name());
}
diff --git a/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp b/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp
index 66c748283311..07eeb3c8fd6d 100644
--- a/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp
+++ b/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp
@@ -7,12 +7,15 @@
#include
#include
#include
+#include
#include
#include
#include
#include
+#include
#include
#include
+#include
#include
#include
#include
@@ -80,6 +83,28 @@ OutputVector translate_gated_delta_net(const NodeContext & context) {
g = std::make_shared(g, ov::op::v0::Constant::create(ov::element::i64, {1}, {3}));
beta = std::make_shared(beta, ov::op::v0::Constant::create(ov::element::i64, {1}, {3}));
+ if (context.has_input("chunk_valid_len")) {
+ // The last prefill chunk is padded with fabricated tokens. The recurrence is
+ // S_t = S_{t-1} * exp(g_t) + k_t (x) ((v_t - S_{t-1}^T k_t) * beta_t)
+ // so forcing g = 0 and beta = 0 makes a padded step an exact identity and keeps the final
+ // state equal to the state after the last real token. Attention output at those positions
+ // is garbage but never read.
+ const auto & g_ps = g.get_partial_shape();
+ FRONT_END_OP_CONVERSION_CHECK(g_ps.rank().is_static() && g_ps.rank().get_length() == 3 && g_ps[1].is_static(),
+ "GATED_DELTA_NET pad masking requires a static token dimension");
+ const int64_t n_tokens = g_ps[1].get_length();
+ std::vector positions(n_tokens);
+ std::iota(positions.begin(), positions.end(), 0);
+ auto valid = std::make_shared(
+ ov::op::v0::Constant::create(ov::element::i64, {(size_t) n_tokens}, positions),
+ context.get_input("chunk_valid_len"));
+ auto mask = std::make_shared(
+ std::make_shared(valid, g.get_element_type()),
+ ov::op::v0::Constant::create(ov::element::i64, {2}, std::vector{0, 2}));
+ g = std::make_shared(g, mask);
+ beta = std::make_shared(beta, mask);
+ }
+
// std::cout << "GatedDeltaNet input shapes: q=" << q.get_partial_shape() << ", k=" << k.get_partial_shape()
// << ", v=" << v.get_partial_shape() << ", g=" << g.get_partial_shape()
// << ", beta=" << beta.get_partial_shape() << ", state=" << state.get_partial_shape() << std::endl;
diff --git a/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp b/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp
new file mode 100644
index 000000000000..c6d64aed43aa
--- /dev/null
+++ b/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp
@@ -0,0 +1,64 @@
+#include "../node_context.h"
+#include "../op_table.h"
+#include "../utils.h"
+
+#include
+#include
+#include
+#include
+#include
+#include
+
+namespace ov {
+namespace frontend {
+namespace ggml {
+namespace op {
+
+OutputVector translate_glu_geglu_quick(const NodeContext & context) {
+ num_inputs_check(context, 1, 2);
+
+ ov::Output src0;
+ ov::Output src1;
+ if (context.get_input_size() == 2) {
+ src0 = process_view_input_new(context, 0);
+ src1 = process_view_input_new(context, 1);
+ } else {
+ // split along last axis, nc = ne[0] / 2
+ auto combined = process_view_input_new(context, 0);
+ auto combined_shape = combined.get_partial_shape();
+ int64_t last_dim_val = combined_shape[combined_shape.rank().get_length() - 1].get_length();
+ int64_t nc = last_dim_val / 2;
+
+ auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1});
+ auto step = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
+ auto start0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
+ auto stop0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {nc});
+ auto start1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {nc});
+ auto stop1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {2 * nc});
+
+ src0 = std::make_shared(combined, start0, stop0, step, axis);
+ src1 = std::make_shared(combined, start1, stop1, step, axis);
+ }
+
+ int32_t * params = context.get_output_op_params();
+ const int32_t swapped = params[1];
+ if (swapped) {
+ std::swap(src0, src1);
+ }
+
+ // GELU_QUICK(x) = x * sigmoid(1.702 * x)
+ // Create the constant in the same type as src0 to avoid f16/f32 mismatch.
+ auto input_type = src0.get_element_type();
+ auto coef = ov::op::v0::Constant::create(input_type, ov::Shape{}, {1.702f});
+ auto scaled = std::make_shared(src0, coef);
+ auto sigmoid = std::make_shared(scaled);
+ auto gated = std::make_shared(src0, sigmoid);
+ auto res = std::make_shared(gated, src1);
+
+ return rename_outputs_with_suffix({res}, context.get_name());
+}
+
+} // namespace op
+} // namespace ggml
+} // namespace frontend
+} // namespace ov
diff --git a/ggml/src/ggml-openvino/openvino/op/pool_2d.cpp b/ggml/src/ggml-openvino/openvino/op/pool_2d.cpp
new file mode 100644
index 000000000000..fb6333175f02
--- /dev/null
+++ b/ggml/src/ggml-openvino/openvino/op/pool_2d.cpp
@@ -0,0 +1,53 @@
+#include "../node_context.h"
+#include "../op_table.h"
+#include "../utils.h"
+
+#include
+#include
+#include
+
+namespace ov {
+namespace frontend {
+namespace ggml {
+namespace op {
+
+OutputVector translate_pool_2d(const NodeContext & context) {
+ num_inputs_check(context, 1, 1);
+ const int32_t * params = context.get_output_op_params();
+
+ const int k0 = params[1];
+ const int k1 = params[2];
+ const int s0 = params[3];
+ const int s1 = params[4];
+ const int p0 = params[5];
+ const int p1 = params[6];
+
+ const int op_case = context.get_op_case();
+ ov::Output input = context.get_input(0);
+ ov::Strides strides{static_cast(s1), static_cast(s0)};
+ ov::Shape pads_begin{static_cast(p1), static_cast(p0)};
+ ov::Shape pads_end{static_cast(p1), static_cast(p0)};
+ ov::Shape kernel{static_cast(k1), static_cast(k0)};
+ ov::Output res;
+
+ switch (op_case) {
+ case 1: // GGML_OP_POOL_MAX
+ {
+ res = std::make_shared(input, strides, pads_begin, pads_end, kernel);
+ break;
+ }
+ case 2: // GGML_OP_POOL_AVG
+ {
+ res = std::make_shared(input, strides, pads_begin, pads_end, kernel, false);
+ break;
+ }
+ default:
+ break;
+ }
+ return rename_outputs_with_suffix({res}, context.get_name());
+}
+
+} // namespace op
+} // namespace ggml
+} // namespace frontend
+} // namespace ov
diff --git a/ggml/src/ggml-openvino/openvino/op/roll.cpp b/ggml/src/ggml-openvino/openvino/op/roll.cpp
new file mode 100644
index 000000000000..e8d1b8e50b34
--- /dev/null
+++ b/ggml/src/ggml-openvino/openvino/op/roll.cpp
@@ -0,0 +1,36 @@
+#include "../node_context.h"
+#include "../op_table.h"
+#include "../utils.h"
+
+#include
+#include
+
+namespace ov {
+namespace frontend {
+namespace ggml {
+namespace op {
+
+OutputVector translate_roll(const NodeContext & context) {
+ num_inputs_check(context, 1, 1);
+ const int32_t * params = context.get_output_op_params();
+
+ int64_t s0 = params[0];
+ int64_t s1 = params[1];
+ int64_t s2 = params[2];
+ int64_t s3 = params[3];
+
+ auto input = context.get_input(0);
+
+ auto shift = ov::op::v0::Constant::create(
+ ov::element::i64, ov::Shape{4}, std::vector{s3, s2, s1, s0});
+ auto axes = ov::op::v0::Constant::create(
+ ov::element::i64, ov::Shape{4}, std::vector{0, 1, 2, 3});
+
+ auto roll = std::make_shared(input, shift, axes);
+ return rename_outputs_with_suffix({roll}, context.get_name());
+}
+
+} // namespace op
+} // namespace ggml
+} // namespace frontend
+} // namespace ov
diff --git a/ggml/src/ggml-openvino/openvino/op/view.cpp b/ggml/src/ggml-openvino/openvino/op/view.cpp
index 138526cb49c6..56f5ceec9bb0 100644
--- a/ggml/src/ggml-openvino/openvino/op/view.cpp
+++ b/ggml/src/ggml-openvino/openvino/op/view.cpp
@@ -17,6 +17,13 @@ namespace op {
OutputVector translate_view(const NodeContext & context) {
num_inputs_check(context, 1, 1);
+ if (context.get_op_case() == 1) {
+ // Static-mode identity pass-through for VIEWs over a GATED_DELTA_NET combined output or
+ // the conv_input CONCAT; the consuming op (CPY/RMS_NORM) does its own runtime-correct
+ // slicing on the full tensor (see ggml-decoder.cpp compute_op_case, GGML_OP_VIEW).
+ return {context.get_input(0)};
+ }
+
if (!context.is_static()) {
// On the stateless/non-static path VIEW is normally a no-op (consumers re-slice).
// EXCEPTION: the MoE expert aggregation slices each expert plane out of
diff --git a/ggml/src/ggml-openvino/openvino/op_table.cpp b/ggml/src/ggml-openvino/openvino/op_table.cpp
index 3c26fe83b1ad..9c9d8eeac781 100644
--- a/ggml/src/ggml-openvino/openvino/op_table.cpp
+++ b/ggml/src/ggml-openvino/openvino/op_table.cpp
@@ -10,6 +10,7 @@
#include
#include
#include
+#include
#include
#include
#include
@@ -55,10 +56,12 @@ std::unordered_map get_supported_ops() {
{"GGML_UNARY_OP_SIGMOID", op::translate_1to1_match_1_input },
{"GGML_UNARY_OP_EXP", op::translate_1to1_match_1_input },
{"GGML_UNARY_OP_NEG", op::translate_1to1_match_1_input },
+ {"GGML_UNARY_OP_RELU", op::translate_1to1_match_1_input },
{"GGML_OP_VIEW", op::translate_view },
{"GGML_GLU_OP_SWIGLU", op::translate_glu_swiglu },
{"GGML_GLU_OP_SWIGLU_OAI", op::translate_glu_swiglu_oai },
{"GGML_GLU_OP_GEGLU", op::translate_glu_geglu },
+ {"GGML_GLU_OP_GEGLU_QUICK", op::translate_glu_geglu_quick },
{"GGML_OP_SET_ROWS", op::translate_set_rows },
{"GGML_OP_CPY", op::translate_cpy },
{"GGML_OP_FLASH_ATTN_EXT", op::translate_flash_attn_ext },
@@ -72,6 +75,8 @@ std::unordered_map get_supported_ops() {
{"GGML_OP_DIAG", op::translate_diag },
{"GGML_OP_TRI", op::translate_tri },
{"GGML_OP_SET", op::translate_set },
+ {"GGML_OP_POOL_2D", op::translate_pool_2d },
+ {"GGML_OP_ROLL", op::translate_roll },
// solve_tri has accuracy issues on GPU
// {"GGML_OP_SOLVE_TRI", op::translate_solve_tri },
};
diff --git a/ggml/src/ggml-openvino/openvino/op_table.h b/ggml/src/ggml-openvino/openvino/op_table.h
index d4b9292d6377..0a81a57a6677 100644
--- a/ggml/src/ggml-openvino/openvino/op_table.h
+++ b/ggml/src/ggml-openvino/openvino/op_table.h
@@ -38,6 +38,7 @@ GGML_OP_CONVERTER(translate_view);
GGML_OP_CONVERTER(translate_glu_swiglu);
GGML_OP_CONVERTER(translate_glu_swiglu_oai);
GGML_OP_CONVERTER(translate_glu_geglu);
+GGML_OP_CONVERTER(translate_glu_geglu_quick);
GGML_OP_CONVERTER(translate_set_rows);
GGML_OP_CONVERTER(translate_cpy);
GGML_OP_CONVERTER(translate_argsort);
@@ -53,6 +54,8 @@ GGML_OP_CONVERTER(translate_set);
GGML_OP_CONVERTER(translate_diag);
GGML_OP_CONVERTER(translate_tri);
GGML_OP_CONVERTER(translate_solve_tri);
+GGML_OP_CONVERTER(translate_pool_2d);
+GGML_OP_CONVERTER(translate_roll);
} // namespace op
diff --git a/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.cpp b/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.cpp
new file mode 100644
index 000000000000..21801c0f3992
--- /dev/null
+++ b/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.cpp
@@ -0,0 +1,212 @@
+#include "fuse_to_conv.h"
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+namespace opp = ov::pass::pattern;
+
+namespace ov {
+namespace frontend {
+namespace ggml {
+namespace pass {
+
+// This pass fuses an IM2COL + MatMul convolution into OpenVINO's Convolution op for performance gains.
+// Reference the im2col.cpp translator for reference on the pattern being matched.
+
+FuseToConv::FuseToConv() {
+ const auto m_wei = opp::any_input();
+ const auto m_act = opp::any_input();
+ const auto m_matmul = opp::wrap_type({m_wei, m_act});
+
+ const auto callback = [=](ov::pass::pattern::Matcher & m) {
+ const auto & pm = m.get_pattern_value_map();
+
+ auto matmul_node = ov::as_type_ptr(pm.at(m_matmul).get_node_shared_ptr());
+ if (!matmul_node || matmul_node->get_transpose_a() || !matmul_node->get_transpose_b()) {
+ return false;
+ }
+
+ auto trace = matmul_node->input_value(1);
+
+ // Optional Convert
+ if (auto n = ov::as_type_ptr(trace.get_node_shared_ptr())) {
+ trace = n->input_value(0);
+ }
+
+ for (int i = 0; i < 2; ++i) {
+ auto n = ov::as_type_ptr(trace.get_node_shared_ptr());
+ if (!n) {
+ return false;
+ }
+ trace = n->input_value(0);
+ }
+
+ if (auto n = ov::as_type_ptr(trace.get_node_shared_ptr())) {
+ trace = n->input_value(0);
+ } else {
+ return false;
+ }
+
+ if (auto n = ov::as_type_ptr(trace.get_node_shared_ptr())) {
+ trace = n->input_value(0);
+ } else {
+ return false;
+ }
+
+ if (auto n = ov::as_type_ptr(trace.get_node_shared_ptr())) {
+ trace = n->input_value(0);
+ } else {
+ return false;
+ }
+
+ auto eip = ov::as_type_ptr(trace.get_node_shared_ptr());
+ if (!eip) {
+ return false;
+ }
+ const auto eip_strides = eip->get_strides(); // {stride_h, stride_w}
+ const auto eip_rates = eip->get_rates(); // {dil_h, dil_w}
+
+ auto pad = ov::as_type_ptr