From d73917ff801c280d6fdb7b7f7680fb8624e3514d Mon Sep 17 00:00:00 2001 From: YiChen Lv <63285796+forforever73@users.noreply.github.com> Date: Wed, 29 Jul 2026 21:46:01 +0800 Subject: [PATCH 01/14] metal : per-device tuned (Q, NE) for flash-attn vec (#25750) * rebase Q-generic FA vec body from 01dc93607 (#23114) * add 53 f16 (Q,NE) flash-attn vec instantiations (vec 80 -> 133) * add FA vec (Q,NE) tuning table + dispatch wiring + SMEM cap fallback * add FA vec (Q,NE) perf sweep * fill tuning result * fold family table into a per-family representative SKU * refactor tuning result format * extend FA vec tuning to quantized KV caches * sync fa vec tuner bucketing with runtime, use pointwise tuning regret * update tuned table * format and cleanup * prefix fa_vec tuning procs with ggml_backend_metal_tuning_, drop unused fa_vec_override_active --- ggml/src/ggml-metal/CMakeLists.txt | 1 + ggml/src/ggml-metal/ggml-metal-device.cpp | 13 +- ggml/src/ggml-metal/ggml-metal-device.h | 3 + ggml/src/ggml-metal/ggml-metal-device.m | 3 +- ggml/src/ggml-metal/ggml-metal-ops.cpp | 21 +- ggml/src/ggml-metal/ggml-metal-tuning.cpp | 347 ++++++++++ ggml/src/ggml-metal/ggml-metal-tuning.h | 63 ++ ggml/src/ggml-metal/ggml-metal.cpp | 37 ++ ggml/src/ggml-metal/kernels/fa.metal | 730 +++++++++++++++++----- tests/test-backend-ops.cpp | 511 ++++++++++++++- 10 files changed, 1555 insertions(+), 174 deletions(-) create mode 100644 ggml/src/ggml-metal/ggml-metal-tuning.cpp create mode 100644 ggml/src/ggml-metal/ggml-metal-tuning.h diff --git a/ggml/src/ggml-metal/CMakeLists.txt b/ggml/src/ggml-metal/CMakeLists.txt index f02654a2e7a..140c5d809e0 100644 --- a/ggml/src/ggml-metal/CMakeLists.txt +++ b/ggml/src/ggml-metal/CMakeLists.txt @@ -11,6 +11,7 @@ ggml_add_backend_library(ggml-metal ggml-metal-common.cpp ggml-metal-context.m ggml-metal-ops.cpp + ggml-metal-tuning.cpp ) target_link_libraries(ggml-metal PRIVATE diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cpp index 24caad7ffa9..4036cc21daa 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.cpp +++ b/ggml/src/ggml-metal/ggml-metal-device.cpp @@ -1,6 +1,7 @@ #include "ggml-metal-device.h" #include "ggml-metal-impl.h" +#include "ggml-metal-tuning.h" #include "ggml-impl.h" @@ -1544,6 +1545,8 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_v bool has_bias, bool has_scap, bool has_kvpad, + int32_t nqpsg, + int32_t ne, int32_t nsg, int32_t nwg, bool use_kv_f16, @@ -1559,11 +1562,17 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_v const char * type = use_kv_f16 ? "f16" : ggml_type_name(op->src[1]->type); - snprintf(base, 256, "kernel_%s_%s_dk%d_dv%d", + char qne_suffix[16] = {0}; + if (!(nqpsg == 1 && ne == ggml_metal_tuning::fa_vec_baseline_ne(dk, dv))) { + snprintf(qne_suffix, sizeof(qne_suffix), "_q%d_ne%d", nqpsg, ne); + } + + snprintf(base, 256, "kernel_%s_%s_dk%d_dv%d%s", "flash_attn_ext_vec", type, dk, - dv); + dv, + qne_suffix); snprintf(name, 256, "%s_mask=%d_sink=%d_bias=%d_scap=%d_kvpad=%d_ns10=%d_ns20=%d_nsg=%d_nwg=%d", base, diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.h index 5f5410a03a4..8846f7e9a47 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.h +++ b/ggml/src/ggml-metal/ggml-metal-device.h @@ -207,6 +207,8 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_att bool has_bias, bool has_scap, bool has_kvpad, + int32_t nqpsg, + int32_t ne, int32_t nsg, int32_t nwg, bool use_kv_f16, @@ -279,6 +281,7 @@ struct ggml_metal_device_props { bool supports_gpu_family_apple7; enum ggml_metal_device_id device_id; + int gpu_family; int op_offload_min_batch_size; }; diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m index 6647cacd617..e049c9e5edf 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.m +++ b/ggml/src/ggml-metal/ggml-metal-device.m @@ -1220,7 +1220,8 @@ ggml_metal_device_t ggml_metal_device_init(int device, int n_devices) { { for (int i = MTLGPUFamilyApple1 + 20; i >= MTLGPUFamilyApple1; --i) { if ([dev->mtl_device supportsFamily:i]) { - GGML_LOG_INFO("%s: GPU family: MTLGPUFamilyApple%d (%d)\n", __func__, i - (int) MTLGPUFamilyApple1 + 1, i); + dev->props.gpu_family = i - (int) MTLGPUFamilyApple1 + 1; + GGML_LOG_INFO("%s: GPU family: MTLGPUFamilyApple%d (%d)\n", __func__, dev->props.gpu_family, i); break; } } diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp index 8311544b397..1c3bb936b90 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -7,6 +7,7 @@ #include "ggml-metal-impl.h" #include "ggml-metal-common.h" #include "ggml-metal-device.h" +#include "ggml-metal-tuning.h" #include #include @@ -3346,12 +3347,18 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { #undef FATTN_SMEM } else { // half4x4 kernel - const int nqptg = OP_FLASH_ATTN_EXT_VEC_NQPSG; // queries per threadgroup + auto cfg = ggml_metal_tuning::fa_vec_pick( + props_dev->device_id, + props_dev->gpu_family, + (int) op->src[1]->type, + (int) ne00, (int) ne20, // dk, dv (ne00 == dk for FA) + ne11, ne01); + int nqptg = cfg.Q; // queries per threadgroup const int ncpsg = OP_FLASH_ATTN_EXT_VEC_NCPSG; // cache values per simdgroup !! sync with kernel template arguments !! const int nhptg = 1; // heads per threadgroup GGML_ASSERT(nqptg <= 32); - GGML_ASSERT(nqptg % 1 == 0); + GGML_ASSERT(nqptg == 1 || nqptg == 2 || nqptg == 4); // only instantiated Q values GGML_ASSERT(ncpsg % 32 == 0); bool need_sync = false; @@ -3410,7 +3417,7 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { // ne20*(nsg) // each simdgroup has a full f32 head vector in shared mem to accumulate results // -#define FATTN_SMEM(nsg) (GGML_PAD(((GGML_PAD(ne00, 128) + 4*ncpsg + 2*GGML_PAD(ne20, 128))*(nsg))*(sizeof(float)/2), 16)) +#define FATTN_SMEM(nsg) (GGML_PAD(((GGML_PAD(ne00, 128) + 4*ncpsg + 2*GGML_PAD(ne20, 128))*(nsg)*nqptg)*(sizeof(float)/2), 16)) int64_t nsg = 1; @@ -3430,6 +3437,12 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { } } + // fall back to baseline (Q=1) if the tuned config exceeds threadgroup memory + if ((size_t) FATTN_SMEM(nsg) > props_dev->max_theadgroup_memory_size) { + cfg = ggml_metal_tuning::fa_vec_baseline_cfg((int) ne00, (int) ne20); + nqptg = cfg.Q; // = 1 + } + const int32_t ns10 = nb11_attn/nb10_attn; const int32_t ns20 = nb21_attn/nb20_attn; @@ -3468,7 +3481,7 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { /*.logit_softcap =*/ logit_softcap, }; - auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext_vec(lib, op, has_mask, has_sinks, has_bias, has_scap, has_kvpad, nsg, nwg, use_kv_f16, ns10, ns20); + auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext_vec(lib, op, has_mask, has_sinks, has_bias, has_scap, has_kvpad, nqptg, cfg.NE, nsg, nwg, use_kv_f16, ns10, ns20); GGML_ASSERT(nsg*32 <= ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); diff --git a/ggml/src/ggml-metal/ggml-metal-tuning.cpp b/ggml/src/ggml-metal/ggml-metal-tuning.cpp new file mode 100644 index 00000000000..2bf9bb5b48d --- /dev/null +++ b/ggml/src/ggml-metal/ggml-metal-tuning.cpp @@ -0,0 +1,347 @@ +#include "ggml-metal-tuning.h" + +#include +#include +#include + +namespace ggml_metal_tuning { + +int fa_vec_ne11_bucket(int64_t ne11) { + for (int i = 0; i < (int) std::size(FA_VEC_NE11_BUCKETS); ++i) { + if (ne11 < FA_VEC_NE11_BUCKETS[i]) { + return i; + } + } + return (int) std::size(FA_VEC_NE11_BUCKETS); +} + +int fa_vec_ne01_bucket(int64_t ne01) { + for (int i = 0; i < (int) std::size(FA_VEC_NE01_BUCKETS); ++i) { + if (ne01 < FA_VEC_NE01_BUCKETS[i]) { + return i; + } + } + return (int) std::size(FA_VEC_NE01_BUCKETS); +} + +int fa_vec_baseline_ne(int dk, int dv) { + if (dk == 32 && dv == 32) { + return 4; + } + if (dk == 64 && dv == 64) { + return 2; + } + if (dk == 96 && dv == 96) { + return 4; + } + if (dk == 128 && dv == 128) { + return 1; + } + if (dk == 192 && dv == 192) { + return 2; + } + if (dk == 192 && dv == 128) { + return 2; + } + if (dk == 256 && dv == 256) { + return 1; + } + if (dk == 320 && dv == 256) { + return 2; + } + if (dk == 512 && dv == 512) { + return 1; + } + if (dk == 576 && dv == 512) { + return 2; + } + return 4; // template default +} + +fa_vec_cfg_t fa_vec_baseline_cfg(int dk, int dv) { + return { 1, (int8_t) fa_vec_baseline_ne(dk, dv) }; +} + +// Generated by `test-backend-ops tune --tune-perf`; do not hand-edit. +// One row per kept bucket, plus per-(dtype,dk,dv) ne11-collapsed domain defaults +// (ne11_b = FA_VEC_NE11_DEFAULT, ne01_b = domain). To retune or add a device, re-run the +// sweep and paste its block. See ggml-metal-tuning.h for the row/lookup semantics. +constexpr fa_vec_entry_t fa_vec_tuned_table[] = { + // ---- f16: 13 rows ---- + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 32, 32, 1, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 32, 32, 2, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 32, 32, 2, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 32, 32, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 32, 32, 3, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 64, 64, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 96, 96, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 128, 128, -1, 1 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 192, 192, -1, 1 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 192, 128, -1, 1 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 256, 256, -1, 0 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 256, 256, -1, 1 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 320, 256, -1, 1 }, { 2, 2 } }, + // ---- q4_0: 29 rows ---- + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 32, 32, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 32, 32, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 32, 32, 3, 2 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 32, 32, 3, 3 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 32, 32, 3, 4 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 96, 96, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 96, 96, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 96, 96, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 192, 128, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 192, 128, 3, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 320, 256, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 576, 512, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 576, 512, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 576, 512, 1, 1 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 576, 512, 1, 2 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 576, 512, 1, 4 }, { 1, 2 } }, + // ---- q4_1: 28 rows ---- + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 32, 32, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 32, 32, 3, 2 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 32, 32, 3, 3 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 32, 32, 3, 4 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 96, 96, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 96, 96, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 96, 96, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 128, 128, 2, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 128, 128, 2, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 128, 128, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 128, 128, 3, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 128, 128, 3, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 192, 192, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 192, 128, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 192, 128, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 192, 128, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 192, 128, 3, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 256, 256, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 256, 256, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 320, 256, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 576, 512, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 576, 512, -1, 1 }, { 1, 4 } }, + // ---- q5_0: 45 rows ---- + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 32, 32, 1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 32, 32, 2, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 32, 32, 2, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 32, 32, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 96, 96, 1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 96, 96, 2, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 96, 96, 2, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 96, 96, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 96, 96, 3, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 128, 128, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 192, 192, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 192, 192, 1, 1 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 192, 192, 1, 2 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 192, 192, 1, 3 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 192, 128, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 256, 256, 1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 256, 256, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 256, 256, 1, 4 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 256, 256, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 256, 256, 3, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 320, 256, -1, 1 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 320, 256, 1, 2 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 320, 256, 1, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 320, 256, 1, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 320, 256, 2, 2 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 320, 256, 3, 2 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 512, 512, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 512, 512, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 576, 512, 2, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 576, 512, 2, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 576, 512, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 576, 512, 2, 3 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 576, 512, 2, 4 }, { 1, 4 } }, + // ---- q5_1: 49 rows ---- + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 32, 32, 1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 32, 32, 2, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 32, 32, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 64, 64, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 96, 96, 1, 2 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 96, 96, 1, 3 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 96, 96, 2, 2 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 96, 96, 2, 3 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 128, 128, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 192, 192, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 192, 192, 1, 1 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 192, 192, 1, 2 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 192, 192, 1, 3 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 192, 192, 1, 4 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 192, 128, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 256, 256, 1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 256, 256, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 256, 256, 1, 4 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 256, 256, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 256, 256, 3, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 320, 256, -1, 1 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 320, 256, 1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 320, 256, 1, 2 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 320, 256, 1, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 320, 256, 1, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 320, 256, 2, 2 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 320, 256, 3, 2 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 512, 512, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 512, 512, 1, 1 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 512, 512, 1, 2 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 512, 512, 1, 3 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 512, 512, 1, 4 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 576, 512, 2, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 576, 512, 2, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 576, 512, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 576, 512, 2, 3 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 576, 512, 2, 4 }, { 1, 4 } }, + // ---- q8_0: 29 rows ---- + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 32, 32, 3, 2 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 32, 32, 3, 3 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 32, 32, 3, 4 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 96, 96, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 128, 128, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 128, 128, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 128, 128, 3, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 192, 128, 1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 192, 128, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 192, 128, 1, 3 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 192, 128, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 192, 128, 3, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 320, 256, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 320, 256, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } }, +}; + +static enum ggml_metal_device_id fa_vec_family_representative(int gpu_family) { + switch (gpu_family) { + case 9: return GGML_METAL_DEVICE_M4_MAX; + default: return GGML_METAL_DEVICE_GENERIC; + } +} + +static bool g_override_set = false; +static fa_vec_cfg_t g_override_cfg = { 1, 4 }; + +void fa_vec_set_override(fa_vec_cfg_t cfg) { + g_override_cfg = cfg; + g_override_set = true; +} + +void fa_vec_clear_override() { + g_override_set = false; +} + +static const fa_vec_cfg_t * find_cfg(const fa_vec_entry_t * tbl, size_t n, const fa_vec_key_t & k) { + for (size_t i = 0; i < n; ++i) { + if (memcmp(&tbl[i].key, &k, sizeof(k)) == 0) { + return &tbl[i].cfg; + } + } + return nullptr; +} + +fa_vec_cfg_t fa_vec_pick(enum ggml_metal_device_id device_id, int gpu_family, int dtype, int dk, int dv, int64_t ne11, int64_t ne01) { + if (g_override_set) { + return g_override_cfg; + } + + const fa_vec_cfg_t baseline = fa_vec_baseline_cfg(dk, dv); + + const int ne11_b = fa_vec_ne11_bucket(ne11); + if (ne11_b == 0) { + return baseline; // short KV: attention is a small slice of the step, left to baseline + } + const int ne01_b = fa_vec_ne01_bucket(ne01); + + fa_vec_key_t k{}; + k.dtype = (int8_t) dtype; + k.dk = (int16_t) dk; + k.dv = (int16_t) dv; + + // exact bucket, then the ne01 domain default (ne11 collapsed); tried under each device tier + auto lookup = [&](enum ggml_metal_device_id dev) -> const fa_vec_cfg_t * { + k.device_id = (int8_t) dev; + k.ne11_b = (int8_t) ne11_b; + k.ne01_b = (int8_t) ne01_b; + if (auto * c = find_cfg(fa_vec_tuned_table, std::size(fa_vec_tuned_table), k)) { + return c; + } + k.ne11_b = FA_VEC_NE11_DEFAULT; + k.ne01_b = (ne01_b == 0) ? FA_VEC_DOMAIN_DECODE : FA_VEC_DOMAIN_BATCH; + return find_cfg(fa_vec_tuned_table, std::size(fa_vec_tuned_table), k); + }; + + if (auto * c = lookup(device_id)) { + return *c; + } + + // family fallback: retry under the family's representative SKU; none -> baseline + if (gpu_family > 0) { + const enum ggml_metal_device_id rep = fa_vec_family_representative(gpu_family); + if (rep != GGML_METAL_DEVICE_GENERIC) { + if (auto * c = lookup(rep)) { + return *c; + } + } + } + + return baseline; +} + +} // namespace ggml_metal_tuning diff --git a/ggml/src/ggml-metal/ggml-metal-tuning.h b/ggml/src/ggml-metal/ggml-metal-tuning.h new file mode 100644 index 00000000000..6ea948b4077 --- /dev/null +++ b/ggml/src/ggml-metal/ggml-metal-tuning.h @@ -0,0 +1,63 @@ +#pragma once + +#include "ggml-metal-device.h" // enum ggml_metal_device_id +#include "ggml.h" + +#include + +namespace ggml_metal_tuning { + +// FA vec selection buckets. ne01 (query rows) splits decode (==1) from batch (>=2), the +// batch side refined into {2,3,4,5}: Q>1 reuses one K/V load across rows, so it only pays +// off once ne01 aligns with Q. ne11 (KV length) is bucketed too, as the Q>1 crossover is +// head-size dependent (small dk crosses late, large dk wins even at short KV). +constexpr int FA_VEC_NE11_BUCKETS[] = { 1024, 4096, 16384 }; +constexpr int FA_VEC_NE01_BUCKETS[] = { 2, 3, 4, 5 }; + +int fa_vec_ne11_bucket(int64_t ne11); +int fa_vec_ne01_bucket(int64_t ne01); + +// NE baked into each (dk,dv) baseline instantiation in kernels/fa.metal. +// Hand-maintained mirror; keep in sync with those instantiations (run_fa_vec_tune_check +// exercises every (Q,NE), so a missing instantiation surfaces there). +int fa_vec_baseline_ne(int dk, int dv); + +// Tuned table has two row kinds. Exact rows key a (ne11_b, ne01_b) bucket. Default rows +// collapse ne11 over one ne01 domain: ne11_b == FA_VEC_NE11_DEFAULT and ne01_b holds the +// domain. fa_vec_pick tries exact bucket -> domain default -> baseline; short KV +// (ne11 < FA_VEC_NE11_BUCKETS[0]) always uses baseline. +constexpr int8_t FA_VEC_NE11_DEFAULT = -1; +constexpr int8_t FA_VEC_DOMAIN_DECODE = 0; // ne01 == 1 +constexpr int8_t FA_VEC_DOMAIN_BATCH = 1; // ne01 >= 2 + +struct fa_vec_key_t { + int8_t device_id; + int8_t dtype; + int16_t dk; + int16_t dv; + int8_t ne11_b; + int8_t ne01_b; +}; + +static_assert(sizeof(fa_vec_key_t) == 8, "fa_vec_key_t must be tightly packed for memcmp"); + +struct fa_vec_cfg_t { + int8_t Q; + int8_t NE; +}; + +struct fa_vec_entry_t { + fa_vec_key_t key; + fa_vec_cfg_t cfg; +}; + +// test/tune-only override; when set, fa_vec_pick returns it directly. +void fa_vec_set_override(fa_vec_cfg_t cfg); +void fa_vec_clear_override(); +fa_vec_cfg_t fa_vec_baseline_cfg(int dk, int dv); + +// device_id selects a per-SKU row; on a miss, gpu_family (0 if unknown) maps to a representative +// SKU and the table is retried. No match -> baseline. +fa_vec_cfg_t fa_vec_pick(enum ggml_metal_device_id device_id, int gpu_family, int dtype, int dk, int dv, int64_t ne11, int64_t ne01); + +} // namespace ggml_metal_tuning diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal.cpp index 31aa61e32d1..fe7796c41db 100644 --- a/ggml/src/ggml-metal/ggml-metal.cpp +++ b/ggml/src/ggml-metal/ggml-metal.cpp @@ -6,6 +6,7 @@ #include "ggml-metal-device.h" #include "ggml-metal-context.h" #include "ggml-metal-ops.h" +#include "ggml-metal-tuning.h" #include #include @@ -870,10 +871,46 @@ static ggml_backend_feature * ggml_backend_metal_get_features(ggml_backend_reg_t GGML_UNUSED(reg); } +// test/tune-only override for the FA vec (Q, NE) selection, reached via proc_address. +static void ggml_backend_metal_tuning_set_fa_vec_override(int Q, int NE) { + ggml_metal_tuning::fa_vec_set_override({ (int8_t) Q, (int8_t) NE }); +} + +static void ggml_backend_metal_tuning_clear_fa_vec_override(void) { + ggml_metal_tuning::fa_vec_clear_override(); +} + +static int ggml_backend_metal_tuning_fa_vec_ne11_bucket(int64_t ne11) { + return ggml_metal_tuning::fa_vec_ne11_bucket(ne11); +} + +static int ggml_backend_metal_tuning_fa_vec_ne01_bucket(int64_t ne01) { + return ggml_metal_tuning::fa_vec_ne01_bucket(ne01); +} + +static int ggml_backend_metal_tuning_fa_vec_baseline_ne(int dk, int dv) { + return ggml_metal_tuning::fa_vec_baseline_ne(dk, dv); +} + static void * ggml_backend_metal_get_proc_address(ggml_backend_reg_t reg, const char * name) { if (strcmp(name, "ggml_backend_get_features") == 0) { return (void *)ggml_backend_metal_get_features; } + if (strcmp(name, "ggml_backend_metal_tuning_set_fa_vec_override") == 0) { + return (void *)ggml_backend_metal_tuning_set_fa_vec_override; + } + if (strcmp(name, "ggml_backend_metal_tuning_clear_fa_vec_override") == 0) { + return (void *)ggml_backend_metal_tuning_clear_fa_vec_override; + } + if (strcmp(name, "ggml_backend_metal_tuning_fa_vec_ne11_bucket") == 0) { + return (void *)ggml_backend_metal_tuning_fa_vec_ne11_bucket; + } + if (strcmp(name, "ggml_backend_metal_tuning_fa_vec_ne01_bucket") == 0) { + return (void *)ggml_backend_metal_tuning_fa_vec_ne01_bucket; + } + if (strcmp(name, "ggml_backend_metal_tuning_fa_vec_baseline_ne") == 0) { + return (void *)ggml_backend_metal_tuning_fa_vec_baseline_ne; + } return NULL; diff --git a/ggml/src/ggml-metal/kernels/fa.metal b/ggml/src/ggml-metal/kernels/fa.metal index 60f4bb4a74a..e95dec258a3 100644 --- a/ggml/src/ggml-metal/kernels/fa.metal +++ b/ggml/src/ggml-metal/kernels/fa.metal @@ -1130,25 +1130,25 @@ kernel void kernel_flash_attn_ext_vec( constexpr short NW = N_SIMDWIDTH; constexpr short NL = NW/NE; // note: this can be adjusted to support different head sizes and simdgroup work loads - constexpr short SH = 4*C; // shared memory per simdgroup + constexpr short SH = 4*Q*C; // shared memory per simdgroup static_assert(DK4 % NL == 0, "DK4 must be divisible by NL"); static_assert(DV4 % NL == 0, "DV4 must be divisible by NL"); //const short T = PK + NSG*SH; // shared memory size per query in (half) - //threadgroup q_t * sq = (threadgroup q_t *) (shmem_f16 + 0*PK); // holds the query data - threadgroup q4_t * sq4 = (threadgroup q4_t *) (shmem_f16 + 0*PK); // same as above but in q4_t - threadgroup s_t * ss = (threadgroup s_t *) (shmem_f16 + sgitg*SH + NSG*PK); // scratch buffer for attention - threadgroup s4_t * ss4 = (threadgroup s4_t *) (shmem_f16 + sgitg*SH + NSG*PK); // same as above but in s4_t - threadgroup half * sm = (threadgroup half *) (shmem_f16 + sgitg*SH + 2*C + NSG*PK); // scratch buffer for mask - threadgroup o4_t * so4 = (threadgroup o4_t *) (shmem_f16 + 2*sgitg*PV + NSG*PK + NSG*SH); // scratch buffer for the results + //threadgroup q_t * sq = (threadgroup q_t *) (shmem_f16 + 0*PK); // holds the query data + threadgroup q4_t * sq4 = (threadgroup q4_t *) (shmem_f16 + 0*PK); // same as above but in q4_t + threadgroup s_t * ss = (threadgroup s_t *) (shmem_f16 + sgitg*SH + Q*NSG*PK); // scratch buffer for attention + threadgroup s4_t * ss4 = (threadgroup s4_t *) (shmem_f16 + sgitg*SH + Q*NSG*PK); // same as above but in s4_t + threadgroup half * sm = (threadgroup half *) (shmem_f16 + sgitg*SH + 2*Q*C + Q*NSG*PK); // scratch buffer for mask + threadgroup o4_t * so4 = (threadgroup o4_t *) (shmem_f16 + 2*sgitg*Q*PV + Q*NSG*PK + NSG*SH); // scratch buffer for the results // store the result for all queries in shared memory (the O matrix from the paper) so4 += tiisg; { - q += iq1*args.nb01 + iq2*args.nb02 + iq3*args.nb03; + q += iq1*Q*args.nb01 + iq2*args.nb02 + iq3*args.nb03; const short ikv2 = iq2/(args.ne02/args.ne_12_2); const short ikv3 = iq3/(args.ne03/args.ne_12_3); @@ -1157,22 +1157,32 @@ kernel void kernel_flash_attn_ext_vec( v += ikv2*args.nb22 + ikv3*args.nb23; } - // load heads from Q to shared memory - device const float4 * q4 = (device const float4 *) ((device const char *) q); - - if (iq1 < args.ne01) { - for (short i = tiisg; i < PK4; i += NW) { - if (i < DK4) { - sq4[i] = (q4_t) q4[i]; + // load Q query rows to shared memory + { + for (short qq = 0; qq < Q; ++qq) { + const int iq1_q = iq1*Q + qq; + device const float4 * q4 = (device const float4 *) ((device const char *) q + qq*args.nb01); + if (iq1_q < args.ne01) { + for (short i = tiisg; i < PK4; i += NW) { + if (i < DK4) { + sq4[qq*PK4 + i] = (q4_t) q4[i]; + } else { + sq4[qq*PK4 + i] = (q4_t) 0.0f; + } + } } else { - sq4[i] = (q4_t) 0.0f; + for (short i = tiisg; i < PK4; i += NW) { + sq4[qq*PK4 + i] = (q4_t) 0.0f; + } } } } // zero out so - for (short i = 0; i < DV4/NL; ++i) { - so4[i*NL] = (o4_t) 0.0f; + for (short qq = 0; qq < Q; ++qq) { + for (short i = 0; i < DV4/NL; ++i) { + so4[qq*DV4 + i*NL] = (o4_t) 0.0f; + } } // zero out shared memory SH @@ -1183,15 +1193,19 @@ kernel void kernel_flash_attn_ext_vec( threadgroup_barrier(mem_flags::mem_threadgroup); { - float S = 0.0f; - float M = -FLT_MAX/2; + float S[Q]; + float M[Q]; + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + S[qq] = 0.0f; + M[qq] = -FLT_MAX/2; + } // thread indices inside the simdgroup const short tx = tiisg%NL; const short ty = tiisg/NL; // pointer to the mask - device const half * pm = (device const half *) (mask + iq1*args.nb31 + (iq2%args.ne32)*args.nb32 + (iq3%args.ne33)*args.nb33); + device const half * pm_base = (device const half *) (mask + iq1*Q*args.nb31 + (iq2%args.ne32)*args.nb32 + (iq3%args.ne33)*args.nb33); float slope = 1.0f; @@ -1213,6 +1227,13 @@ kernel void kernel_flash_attn_ext_vec( break; } + device const half * pm[Q]; + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + // padded query rows clamp to row 0 of the mask to avoid OOB; their scores + // are forced to -inf below, so the values never affect the result. + pm[qq] = pm_base + ((iq1*Q + qq) < args.ne01 ? qq*(args.nb31/sizeof(half)) : -iq1*Q*(args.nb31/sizeof(half))); + } + // the last partial chunk uses the pad buffer as source if (FC_flash_attn_ext_vec_has_kvpad && ic + C > args.ne11) { k = pad; @@ -1226,43 +1247,72 @@ kernel void kernel_flash_attn_ext_vec( v += (ikv2 + ikv3*args.ne_12_2)*args.nb21*C; if (!FC_flash_attn_ext_vec_has_mask) { - if (ic + tiisg >= args.ne11) { - sm[tiisg] = -MAXHALF; + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + if (ic + tiisg >= args.ne11) { + sm[qq*C + tiisg] = -MAXHALF; + } } } else { - pm = (device const half *) (mask) + - iq1*C + - (iq2%args.ne32)*(C*args.ne31) + - (iq3%args.ne33)*(C*args.ne31*args.ne32); + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + pm[qq] = (device const half *) (mask) + + (iq1*Q + qq)*C + + (iq2%args.ne32)*(C*args.ne31) + + (iq3%args.ne33)*(C*args.ne31*args.ne32); + } } ic = 0; } if (FC_flash_attn_ext_vec_has_mask) { - sm[tiisg] = pm[ic + tiisg]; + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + if ((iq1*Q + qq) < args.ne01) { + sm[qq*C + tiisg] = pm[qq][ic + tiisg]; + } else { + sm[qq*C + tiisg] = -MAXHALF; + } + } + } else { + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + if ((iq1*Q + qq) >= args.ne01) { + sm[qq*C + tiisg] = -MAXHALF; + } + } } - // skip -INF blocks - if (simd_max(sm[tiisg]) <= -MAXHALF) { - continue; + { + bool any_finite = false; + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + if (simd_max(sm[qq*C + tiisg]) > -MAXHALF) { + any_finite = true; + } + } + if (!any_finite) { + continue; + } } // Q*K^T { device const k4_t * pk4 = (device const k4_t *) (k + ic*args.nb11); - threadgroup const q4_t * pq4 = sq4; pk4 += ty*NS10/4 + tx; - pq4 += tx; - qk_t mqk[C/NE] = { [ 0 ... C/NE - 1] = 0.0f }; + qk_t mqk[Q][C/NE]; + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + FOR_UNROLL (short cc = 0; cc < C/NE; ++cc) { + mqk[qq][cc] = 0.0f; + } + } - // each simdgroup processes 1 query and NE (NW/NL) cache elements + // each simdgroup processes Q queries and NE (NW/NL) cache elements FOR_UNROLL (short cc = 0; cc < C/NE; ++cc) { if (is_same::value) { FOR_UNROLL (short ii = 0; ii < DK4/NL; ++ii) { - mqk[cc] += dot((float4) pk4[cc*NE*NS10/4 + ii*NL], (float4) pq4[ii*NL]); + const k4_t k_elem = pk4[cc*NE*NS10/4 + ii*NL]; + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + mqk[qq][cc] += dot((float4) k_elem, (float4) sq4[qq*PK4 + ii*NL + tx]); + } } } else { device const kd4_t * pk = (device const kd4_t *) (k + ((ic + NE*cc + ty)*args.nb11)); @@ -1274,57 +1324,63 @@ kernel void kernel_flash_attn_ext_vec( deq_k_t4(pk + i/nl_k, i%nl_k, mk); - mqk[cc] += dot((float4) mk, (float4) sq4[i]); + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + mqk[qq][cc] += dot((float4) mk, (float4) sq4[qq*PK4 + i]); + } } } - if (NE == 1) { - mqk[cc] = simd_sum(mqk[cc]); - } else { - // simdgroup reduce (NE = 4) - // [ 0 .. 7] -> [ 0] - // [ 8 .. 15] -> [ 8] - // [16 .. 23] -> [16] - // [24 .. 31] -> [24] - if (NE <= 1) { - mqk[cc] += simd_shuffle_down(mqk[cc], 16); - } - if (NE <= 2) { - mqk[cc] += simd_shuffle_down(mqk[cc], 8); - } - if (NE <= 4) { - mqk[cc] += simd_shuffle_down(mqk[cc], 4); - } - if (NE <= 8) { - mqk[cc] += simd_shuffle_down(mqk[cc], 2); - } - if (NE <= 16) { - mqk[cc] += simd_shuffle_down(mqk[cc], 1); - } + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + if (NE == 1) { + mqk[qq][cc] = simd_sum(mqk[qq][cc]); + } else { + // simdgroup reduce (NE = 4) + // [ 0 .. 7] -> [ 0] + // [ 8 .. 15] -> [ 8] + // [16 .. 23] -> [16] + // [24 .. 31] -> [24] + if (NE <= 1) { + mqk[qq][cc] += simd_shuffle_down(mqk[qq][cc], 16); + } + if (NE <= 2) { + mqk[qq][cc] += simd_shuffle_down(mqk[qq][cc], 8); + } + if (NE <= 4) { + mqk[qq][cc] += simd_shuffle_down(mqk[qq][cc], 4); + } + if (NE <= 8) { + mqk[qq][cc] += simd_shuffle_down(mqk[qq][cc], 2); + } + if (NE <= 16) { + mqk[qq][cc] += simd_shuffle_down(mqk[qq][cc], 1); + } - // broadcast - mqk[cc] = simd_shuffle(mqk[cc], NL*ty); + // broadcast + mqk[qq][cc] = simd_shuffle(mqk[qq][cc], NL*ty); + } } } - if (FC_flash_attn_ext_vec_has_mask && - !FC_flash_attn_ext_vec_has_scap && - !FC_flash_attn_ext_vec_has_bias) { - ss[NE*tx + ty] = fma(mqk[tx], args.scale, (qk_t) sm[NE*tx + ty]); - } else { - mqk[tx] *= args.scale; + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + if (FC_flash_attn_ext_vec_has_mask && + !FC_flash_attn_ext_vec_has_scap && + !FC_flash_attn_ext_vec_has_bias) { + ss[qq*C + NE*tx + ty] = fma(mqk[qq][tx], args.scale, (qk_t) sm[qq*C + NE*tx + ty]); + } else { + mqk[qq][tx] *= args.scale; - if (FC_flash_attn_ext_vec_has_scap) { - mqk[tx] = args.logit_softcap*precise::tanh(mqk[tx]); - } + if (FC_flash_attn_ext_vec_has_scap) { + mqk[qq][tx] = args.logit_softcap*precise::tanh(mqk[qq][tx]); + } - if (FC_flash_attn_ext_vec_has_bias) { - mqk[tx] += (qk_t) sm[NE*tx + ty]*slope; - } else { - mqk[tx] += (qk_t) sm[NE*tx + ty]; - } + if (FC_flash_attn_ext_vec_has_bias) { + mqk[qq][tx] += (qk_t) sm[qq*C + NE*tx + ty]*slope; + } else { + mqk[qq][tx] += (qk_t) sm[qq*C + NE*tx + ty]; + } - ss[NE*tx + ty] = mqk[tx]; + ss[qq*C + NE*tx + ty] = mqk[qq][tx]; + } } } @@ -1332,23 +1388,25 @@ kernel void kernel_flash_attn_ext_vec( // online softmax { - const float m = M; - const float s = ss[tiisg]; + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + const float m = M[qq]; + const float s = ss[qq*C + tiisg]; - M = simd_max(max(M, s)); + M[qq] = simd_max(max(M[qq], s)); - const float ms = exp(m - M); - const float vs = exp(s - M); + const float ms = exp(m - M[qq]); + const float vs = exp(s - M[qq]); - S = S*ms + simd_sum(vs); + S[qq] = S[qq]*ms + simd_sum(vs); - // the P matrix from the paper (Q rows, C columns) - ss[tiisg] = vs; + // the P matrix from the paper (Q rows, C columns) + ss[qq*C + tiisg] = vs; - // O = diag(ms)*O - if ((DV4/NL % NW == 0) || ty == 0) { - FOR_UNROLL (short ii = 0; ii < DV4/NL; ++ii) { - so4[ii*NL] *= ms; + // O = diag(ms)*O + if ((DV4/NL % NW == 0) || ty == 0) { + FOR_UNROLL (short ii = 0; ii < DV4/NL; ++ii) { + so4[qq*DV4 + ii*NL] *= ms; + } } } } @@ -1357,9 +1415,11 @@ kernel void kernel_flash_attn_ext_vec( // O = O + (Q*K^T)*V { - o4_t lo[DV4/NL]; - FOR_UNROLL (short ii = 0; ii < DV4/NL; ++ii) { - lo[ii] = 0.0f; + o4_t lo[Q][DV4/NL]; + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + FOR_UNROLL (short ii = 0; ii < DV4/NL; ++ii) { + lo[qq][ii] = 0.0f; + } } if (is_same::value) { @@ -1367,11 +1427,12 @@ kernel void kernel_flash_attn_ext_vec( pv4 += ty*NS20/4 + tx; - const auto sst = ss + ty; - FOR_UNROLL (short cc = 0; cc < C/NE; ++cc) { FOR_UNROLL (short ii = 0; ii < DV4/NL; ++ii) { - lo[ii] += o4_t(float4(pv4[cc*NE*NS20/4 + ii*NL])*float4(sst[cc*NE])); + const v4_t v_elem = pv4[cc*NE*NS20/4 + ii*NL]; + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + lo[qq][ii] += o4_t(float4(v_elem)*float4(ss[qq*C + cc*NE + ty])); + } } } } else { @@ -1384,78 +1445,88 @@ kernel void kernel_flash_attn_ext_vec( v4_t mv; deq_v_t4(pv4 + i/nl_v, i%nl_v, mv); - lo[ii] += o4_t(float4(mv)*float4(ss[NE*cc + ty])); + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + lo[qq][ii] += o4_t(float4(mv)*float4(ss[qq*C + NE*cc + ty])); + } } } } - FOR_UNROLL (short ii = 0; ii < DV4/NL; ++ii) { - if (NE > 1) { - lo[ii][0] += simd_shuffle_down(lo[ii][0], 16); - lo[ii][1] += simd_shuffle_down(lo[ii][1], 16); - lo[ii][2] += simd_shuffle_down(lo[ii][2], 16); - lo[ii][3] += simd_shuffle_down(lo[ii][3], 16); - } + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + FOR_UNROLL (short ii = 0; ii < DV4/NL; ++ii) { + if (NE > 1) { + lo[qq][ii][0] += simd_shuffle_down(lo[qq][ii][0], 16); + lo[qq][ii][1] += simd_shuffle_down(lo[qq][ii][1], 16); + lo[qq][ii][2] += simd_shuffle_down(lo[qq][ii][2], 16); + lo[qq][ii][3] += simd_shuffle_down(lo[qq][ii][3], 16); + } - if (NE > 2) { - lo[ii][0] += simd_shuffle_down(lo[ii][0], 8); - lo[ii][1] += simd_shuffle_down(lo[ii][1], 8); - lo[ii][2] += simd_shuffle_down(lo[ii][2], 8); - lo[ii][3] += simd_shuffle_down(lo[ii][3], 8); - } + if (NE > 2) { + lo[qq][ii][0] += simd_shuffle_down(lo[qq][ii][0], 8); + lo[qq][ii][1] += simd_shuffle_down(lo[qq][ii][1], 8); + lo[qq][ii][2] += simd_shuffle_down(lo[qq][ii][2], 8); + lo[qq][ii][3] += simd_shuffle_down(lo[qq][ii][3], 8); + } - if (NE > 4) { - lo[ii][0] += simd_shuffle_down(lo[ii][0], 4); - lo[ii][1] += simd_shuffle_down(lo[ii][1], 4); - lo[ii][2] += simd_shuffle_down(lo[ii][2], 4); - lo[ii][3] += simd_shuffle_down(lo[ii][3], 4); - } + if (NE > 4) { + lo[qq][ii][0] += simd_shuffle_down(lo[qq][ii][0], 4); + lo[qq][ii][1] += simd_shuffle_down(lo[qq][ii][1], 4); + lo[qq][ii][2] += simd_shuffle_down(lo[qq][ii][2], 4); + lo[qq][ii][3] += simd_shuffle_down(lo[qq][ii][3], 4); + } - if (NE > 8) { - lo[ii][0] += simd_shuffle_down(lo[ii][0], 2); - lo[ii][1] += simd_shuffle_down(lo[ii][1], 2); - lo[ii][2] += simd_shuffle_down(lo[ii][2], 2); - lo[ii][3] += simd_shuffle_down(lo[ii][3], 2); - } + if (NE > 8) { + lo[qq][ii][0] += simd_shuffle_down(lo[qq][ii][0], 2); + lo[qq][ii][1] += simd_shuffle_down(lo[qq][ii][1], 2); + lo[qq][ii][2] += simd_shuffle_down(lo[qq][ii][2], 2); + lo[qq][ii][3] += simd_shuffle_down(lo[qq][ii][3], 2); + } - if (NE > 16) { - lo[ii][0] += simd_shuffle_down(lo[ii][0], 1); - lo[ii][1] += simd_shuffle_down(lo[ii][1], 1); - lo[ii][2] += simd_shuffle_down(lo[ii][2], 1); - lo[ii][3] += simd_shuffle_down(lo[ii][3], 1); + if (NE > 16) { + lo[qq][ii][0] += simd_shuffle_down(lo[qq][ii][0], 1); + lo[qq][ii][1] += simd_shuffle_down(lo[qq][ii][1], 1); + lo[qq][ii][2] += simd_shuffle_down(lo[qq][ii][2], 1); + lo[qq][ii][3] += simd_shuffle_down(lo[qq][ii][3], 1); + } } } if ((DV4/NL % NW == 0) || ty == 0) { - FOR_UNROLL (short ii = 0; ii < DV4/NL; ++ii) { - so4[ii*NL] += lo[ii]; + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + FOR_UNROLL (short ii = 0; ii < DV4/NL; ++ii) { + so4[qq*DV4 + ii*NL] += lo[qq][ii]; + } } } } } if (FC_flash_attn_ext_vec_has_sinks && sgitg == 0 && iwg == 0) { - const float m = M; - const float s = tiisg == 0 ? ((device const float *) sinks)[iq2] : -FLT_MAX/2; + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + const float m = M[qq]; + const float s = tiisg == 0 ? ((device const float *) sinks)[iq2] : -FLT_MAX/2; - M = simd_max(max(M, s)); + M[qq] = simd_max(max(M[qq], s)); - const float ms = exp(m - M); - const float vs = exp(s - M); + const float ms = exp(m - M[qq]); + const float vs = exp(s - M[qq]); - S = S*ms + simd_sum(vs); + S[qq] = S[qq]*ms + simd_sum(vs); - if ((DV4/NL % NW == 0) || ty == 0) { - FOR_UNROLL (short ii = 0; ii < DV4/NL; ++ii) { - so4[ii*NL] *= ms; + if ((DV4/NL % NW == 0) || ty == 0) { + FOR_UNROLL (short ii = 0; ii < DV4/NL; ++ii) { + so4[qq*DV4 + ii*NL] *= ms; + } } } } // these are needed for reducing the results from the simdgroups (reuse the ss buffer) if (tiisg == 0) { - ss[0] = (s_t) S; - ss[1] = (s_t) M; + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + ss[2*qq + 0] = (s_t) S[qq]; + ss[2*qq + 1] = (s_t) M[qq]; + } } } @@ -1466,27 +1537,29 @@ kernel void kernel_flash_attn_ext_vec( // parallel reduce for (short r = NSG/2; r > 0; r >>= 1) { if (sgitg < r) { - const float S0 = ss[ 0]; - const float S1 = ss[r*(SH/2) + 0]; + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + const float S0 = ss[ 2*qq + 0]; + const float S1 = ss[r*(SH/2) + 2*qq + 0]; - const float M0 = ss[ 1]; - const float M1 = ss[r*(SH/2) + 1]; + const float M0 = ss[ 2*qq + 1]; + const float M1 = ss[r*(SH/2) + 2*qq + 1]; - const float M = max(M0, M1); + const float Mx = max(M0, M1); - const float ms0 = exp(M0 - M); - const float ms1 = exp(M1 - M); + const float ms0 = exp(M0 - Mx); + const float ms1 = exp(M1 - Mx); - const float S = S0*ms0 + S1*ms1; + const float Sx = S0*ms0 + S1*ms1; - if (tiisg == 0) { - ss[0] = S; - ss[1] = M; - } + if (tiisg == 0) { + ss[2*qq + 0] = Sx; + ss[2*qq + 1] = Mx; + } - // O_0 = diag(ms0)*O_0 + diag(ms1)*O_1 - for (short i = tiisg; i < DV4; i += NW) { - so4[i] = so4[i]*ms0 + so4[i + r*PV4]*ms1; + // O_0 = diag(ms0)*O_0 + diag(ms1)*O_1 + for (short i = tiisg; i < DV4; i += NW) { + so4[qq*DV4 + i] = so4[qq*DV4 + i]*ms0 + so4[qq*DV4 + i + r*Q*PV4]*ms1; + } } } @@ -1496,23 +1569,31 @@ kernel void kernel_flash_attn_ext_vec( // final rescale with 1/S and store to global memory if (sgitg == 0) { const int64_t nrows = args.ne3*args.ne2*args.ne1; - const int64_t rid = iq3*args.ne2*args.ne1 + iq2 + iq1*args.ne1; device float4 * dst4 = (device float4 *) dst; device float * dst1 = (device float *) dst + nrows*DV*NWG; // the S and M are stored after the results - const float S = NWG == 1 ? (ss[0] == 0.0f ? 0.0f : 1.0f/ss[0]) : 1.0f; + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + const int iq1_q = iq1*Q + qq; + if (iq1_q >= args.ne01) { + continue; + } - // interleave the workgroup data - for (short i = tiisg; i < DV4; i += NW) { - dst4[rid*DV4*NWG + NWG*i + iwg] = (float4) so4[i]*S; - } + const int64_t rid = iq3*args.ne2*args.ne1 + iq2 + iq1_q*args.ne1; - // store S and M - if (NWG > 1) { - if (tiisg == 0) { - dst1[rid*(2*NWG) + 2*iwg + 0] = ss[0]; - dst1[rid*(2*NWG) + 2*iwg + 1] = ss[1]; + const float Sval = NWG == 1 ? (ss[2*qq + 0] == 0.0f ? 0.0f : 1.0f/ss[2*qq + 0]) : 1.0f; + + // interleave the workgroup data + for (short i = tiisg; i < DV4; i += NW) { + dst4[rid*DV4*NWG + NWG*i + iwg] = (float4) so4[qq*DV4 + i]*Sval; + } + + // store S and M + if (NWG > 1) { + if (tiisg == 0) { + dst1[rid*(2*NWG) + 2*iwg + 0] = ss[2*qq + 0]; + dst1[rid*(2*NWG) + 2*iwg + 1] = ss[2*qq + 1]; + } } } } @@ -1546,113 +1627,432 @@ typedef decltype(kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_f16_dk32_dv32")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk32_dv32_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk32_dv32_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; #if defined(GGML_METAL_HAS_BF16) template [[host_name("kernel_flash_attn_ext_vec_bf16_dk32_dv32")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; #endif template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk32_dv32")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk32_dv32_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk32_dv32_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk32_dv32")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk32_dv32_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk32_dv32_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk32_dv32")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk32_dv32_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk32_dv32_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk32_dv32")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk32_dv32_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk32_dv32_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk32_dv32")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk32_dv32_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk32_dv32_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_f32_dk64_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_f16_dk64_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk64_dv64_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk64_dv64_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk64_dv64_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk64_dv64_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk64_dv64_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; #if defined(GGML_METAL_HAS_BF16) template [[host_name("kernel_flash_attn_ext_vec_bf16_dk64_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; #endif template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk64_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk64_dv64_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk64_dv64_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk64_dv64_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk64_dv64_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk64_dv64_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk64_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk64_dv64_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk64_dv64_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk64_dv64_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk64_dv64_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk64_dv64_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk64_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk64_dv64_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk64_dv64_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk64_dv64_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk64_dv64_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk64_dv64_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk64_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk64_dv64_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk64_dv64_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk64_dv64_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk64_dv64_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk64_dv64_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk64_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk64_dv64_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk64_dv64_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk64_dv64_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk64_dv64_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk64_dv64_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_f32_dk96_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_f16_dk96_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk96_dv96_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk96_dv96_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; #if defined(GGML_METAL_HAS_BF16) template [[host_name("kernel_flash_attn_ext_vec_bf16_dk96_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; #endif template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk96_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk96_dv96_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk96_dv96_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk96_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk96_dv96_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk96_dv96_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk96_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk96_dv96_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk96_dv96_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk96_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk96_dv96_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk96_dv96_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk96_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk96_dv96_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk96_dv96_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_f32_dk128_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv128_q1_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv128_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv128_q2_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv128_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv128_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv128_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv128_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv128_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; #if defined(GGML_METAL_HAS_BF16) template [[host_name("kernel_flash_attn_ext_vec_bf16_dk128_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; #endif template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk128_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk128_dv128_q1_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk128_dv128_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk128_dv128_q2_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk128_dv128_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk128_dv128_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk128_dv128_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk128_dv128_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk128_dv128_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk128_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk128_dv128_q1_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk128_dv128_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk128_dv128_q2_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk128_dv128_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk128_dv128_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk128_dv128_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk128_dv128_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk128_dv128_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk128_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk128_dv128_q1_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk128_dv128_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk128_dv128_q2_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk128_dv128_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk128_dv128_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk128_dv128_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk128_dv128_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk128_dv128_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk128_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk128_dv128_q1_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk128_dv128_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk128_dv128_q2_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk128_dv128_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk128_dv128_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk128_dv128_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk128_dv128_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk128_dv128_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk128_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk128_dv128_q1_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk128_dv128_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk128_dv128_q2_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk128_dv128_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk128_dv128_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk128_dv128_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk128_dv128_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk128_dv128_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_f32_dk192_dv192")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_f16_dk192_dv192")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk192_dv192_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk192_dv192_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk192_dv192_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk192_dv192_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk192_dv192_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; #if defined(GGML_METAL_HAS_BF16) template [[host_name("kernel_flash_attn_ext_vec_bf16_dk192_dv192")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; #endif template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk192_dv192")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk192_dv192_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk192_dv192_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk192_dv192_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk192_dv192_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk192_dv192_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk192_dv192")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk192_dv192_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk192_dv192_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk192_dv192_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk192_dv192_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk192_dv192_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk192_dv192")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk192_dv192_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk192_dv192_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk192_dv192_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk192_dv192_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk192_dv192_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk192_dv192")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk192_dv192_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk192_dv192_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk192_dv192_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk192_dv192_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk192_dv192_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk192_dv192")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk192_dv192_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk192_dv192_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk192_dv192_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk192_dv192_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk192_dv192_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_f32_dk192_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_f16_dk192_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk192_dv128_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk192_dv128_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk192_dv128_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk192_dv128_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk192_dv128_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; #if defined(GGML_METAL_HAS_BF16) template [[host_name("kernel_flash_attn_ext_vec_bf16_dk192_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; #endif template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk192_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk192_dv128_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk192_dv128_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk192_dv128_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk192_dv128_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk192_dv128_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk192_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk192_dv128_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk192_dv128_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk192_dv128_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk192_dv128_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk192_dv128_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk192_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk192_dv128_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk192_dv128_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk192_dv128_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk192_dv128_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk192_dv128_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk192_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk192_dv128_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk192_dv128_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk192_dv128_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk192_dv128_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk192_dv128_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk192_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk192_dv128_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk192_dv128_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk192_dv128_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk192_dv128_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk192_dv128_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_f32_dk256_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_f16_dk256_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk256_dv256_q1_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk256_dv256_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk256_dv256_q2_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk256_dv256_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk256_dv256_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk256_dv256_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk256_dv256_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk256_dv256_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; #if defined(GGML_METAL_HAS_BF16) template [[host_name("kernel_flash_attn_ext_vec_bf16_dk256_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; #endif template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk256_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk256_dv256_q1_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk256_dv256_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk256_dv256_q2_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk256_dv256_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk256_dv256_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk256_dv256_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk256_dv256_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk256_dv256_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk256_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk256_dv256_q1_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk256_dv256_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk256_dv256_q2_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk256_dv256_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk256_dv256_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk256_dv256_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk256_dv256_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk256_dv256_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk256_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk256_dv256_q1_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk256_dv256_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk256_dv256_q2_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk256_dv256_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk256_dv256_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk256_dv256_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk256_dv256_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk256_dv256_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk256_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk256_dv256_q1_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk256_dv256_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk256_dv256_q2_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk256_dv256_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk256_dv256_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk256_dv256_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk256_dv256_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk256_dv256_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk256_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk256_dv256_q1_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk256_dv256_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk256_dv256_q2_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk256_dv256_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk256_dv256_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk256_dv256_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk256_dv256_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk256_dv256_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_f32_dk320_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_f16_dk320_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk320_dv256_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk320_dv256_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk320_dv256_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk320_dv256_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk320_dv256_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; #if defined(GGML_METAL_HAS_BF16) template [[host_name("kernel_flash_attn_ext_vec_bf16_dk320_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; #endif template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk320_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk320_dv256_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk320_dv256_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk320_dv256_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk320_dv256_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk320_dv256_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk320_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk320_dv256_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk320_dv256_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk320_dv256_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk320_dv256_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk320_dv256_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk320_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk320_dv256_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk320_dv256_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk320_dv256_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk320_dv256_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk320_dv256_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk320_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk320_dv256_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk320_dv256_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk320_dv256_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk320_dv256_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk320_dv256_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk320_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk320_dv256_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk320_dv256_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk320_dv256_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk320_dv256_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk320_dv256_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_f32_dk512_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_f16_dk512_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk512_dv512_q1_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk512_dv512_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk512_dv512_q2_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk512_dv512_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk512_dv512_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk512_dv512_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk512_dv512_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk512_dv512_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; #if defined(GGML_METAL_HAS_BF16) template [[host_name("kernel_flash_attn_ext_vec_bf16_dk512_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; #endif template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk512_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk512_dv512_q1_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk512_dv512_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk512_dv512_q2_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk512_dv512_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk512_dv512_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk512_dv512_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk512_dv512_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk512_dv512_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk512_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk512_dv512_q1_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk512_dv512_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk512_dv512_q2_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk512_dv512_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk512_dv512_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk512_dv512_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk512_dv512_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk512_dv512_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk512_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk512_dv512_q1_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk512_dv512_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk512_dv512_q2_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk512_dv512_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk512_dv512_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk512_dv512_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk512_dv512_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk512_dv512_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk512_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk512_dv512_q1_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk512_dv512_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk512_dv512_q2_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk512_dv512_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk512_dv512_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk512_dv512_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk512_dv512_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk512_dv512_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk512_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk512_dv512_q1_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk512_dv512_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk512_dv512_q2_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk512_dv512_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk512_dv512_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk512_dv512_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk512_dv512_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk512_dv512_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_f32_dk576_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_f16_dk576_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk576_dv512_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk576_dv512_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk576_dv512_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk576_dv512_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk576_dv512_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; #if defined(GGML_METAL_HAS_BF16) template [[host_name("kernel_flash_attn_ext_vec_bf16_dk576_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; #endif template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk576_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk576_dv512_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk576_dv512_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk576_dv512_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk576_dv512_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk576_dv512_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk576_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk576_dv512_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk576_dv512_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk576_dv512_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk576_dv512_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk576_dv512_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk576_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk576_dv512_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk576_dv512_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk576_dv512_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk576_dv512_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk576_dv512_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk576_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk576_dv512_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk576_dv512_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk576_dv512_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk576_dv512_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk576_dv512_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk576_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk576_dv512_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk576_dv512_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk576_dv512_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk576_dv512_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk576_dv512_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; + #undef FA_TYPES #undef FA_TYPES_F32 diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 8e3b273a1e4..c6a27aa6178 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -478,6 +478,7 @@ enum test_mode { MODE_PERF, MODE_GRAD, MODE_SUPPORT, + MODE_TUNE, }; // Output format support similar to llama-bench @@ -10573,8 +10574,482 @@ static std::vector> make_test_cases_from_file(const c return test_cases; } +// ---- FA vec (Q,NE) tuning: numerical correctness check + drift guard ---- +// metal proc_address bridges (resolved by string, not symbol linkage) +using set_fa_vec_override_t = void (*)(int, int); +using clear_fa_vec_override_t = void (*)(void); +using fa_vec_bucket_t = int (*)(int64_t); // runtime ne11/ne01 bucketers, shared via proc bridge +using fa_vec_baseline_ne_t = int (*)(int, int); // runtime (dk,dv) -> baseline NE + +// legal NE for a (dk,dv): NL = 32/NE, require (dk/4)%NL==0 && (dv/4)%NL==0 +static std::vector fa_vec_legal_ne(int dk, int dv) { + std::vector r; + for (int ne : {1, 2, 4}) { + const int nl = 32 / ne; + if ((dk/4) % nl == 0 && (dv/4) % nl == 0) { + r.push_back(ne); + } + } + return r; +} + +// Baseline-path smoke test: with no override, fa_vec_pick returns (1, baseline_ne) and +// dispatches the no-suffix baseline kernel. Checks baseline correctness across all 10 shapes +// (NE is numerically transparent; the (Q,NE) kernels are covered by run_fa_vec_tune_check). +static bool run_fa_vec_drift_guard(ggml_backend_t backend_metal, ggml_backend_t backend_cpu) { + struct shape_t { int dk, dv; }; + const shape_t shapes[] = { {32,32},{64,64},{96,96},{128,128},{192,192}, + {192,128},{256,256},{320,256},{512,512},{576,512} }; + bool ok = true; + for (auto s : shapes) { + // no override + short KV (ne11 < FA_VEC_NE11_BUCKETS[0]) -> fa_vec_pick returns baseline; + // compare metal vs CPU-ref + test_flash_attn_ext tc(s.dk, s.dv, /*nh=*/4, {1,1}, /*kv=*/512, /*nb=*/8, + /*mask=*/true, /*sinks=*/false, 0.0f, 0.0f, GGML_PREC_F32, + GGML_TYPE_F16, GGML_TYPE_F16); + auto st = tc.eval(backend_metal, backend_cpu, "FLASH_ATTN_EXT", nullptr); + if (st == test_status_t::FAIL) { + printf("DRIFT/baseline FAIL dk=%d dv=%d\n", s.dk, s.dv); + ok = false; + } + } + return ok; +} + +// Numerical gate: for each (dk,dv) x legal (Q,NE), force the override and compare metal +// vs CPU-ref. The ne01/ne11/mask/sinks points exercise the rebased body's padded rows +// (ne01 not a multiple of Q), per-qq sinks, skip-INF, kvpad, and the nsg-dependent shmem +// offsets / parallel-reduce stride (ne11 -> nsg 1/2/4). +static bool run_fa_vec_tune_check(ggml_backend_t backend_metal, ggml_backend_t backend_cpu) { + auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_metal)); + auto set_ov = (set_fa_vec_override_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_set_fa_vec_override"); + auto clear_ov = (clear_fa_vec_override_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_clear_fa_vec_override"); + if (!set_ov || !clear_ov) { printf("metal fa_vec override proc unavailable\n"); return false; } + + struct shape_t { int dk, dv; }; + const shape_t shapes[] = { {32,32},{64,64},{96,96},{128,128},{192,192}, + {192,128},{256,256},{320,256},{512,512},{576,512} }; + const int ne01_pts[] = { 1, 2, 3, 8, 17 }; // 1, Q+/-1, 2Q-1, prime; vec upper bound < 20 + const int ne11_pts[] = { 512, 4096, 8192 }; // drives adaptive nsg 1/2/4 + const int ne11_kvpad[] = { 4097, 8193 }; // has_kvpad (non-32-multiple kv) + const bool mask_pts[] = { true, false }; + const bool sinks_pts[] = { false, true }; + + bool ok = true; + int n_run = 0, n_fail = 0; + for (auto s : shapes) { + for (int ne : fa_vec_legal_ne(s.dk, s.dv)) { + for (int Q : {1, 2, 4}) { + for (bool mask : mask_pts) { + for (bool sinks : sinks_pts) { + for (int ne01 : ne01_pts) { + for (int ne11 : ne11_pts) { + set_ov(Q, ne); + test_flash_attn_ext tc(s.dk, s.dv, /*nh=*/4, {1,1}, /*kv=*/ne11, /*nb=*/ne01, + mask, sinks, 0.0f, 0.0f, GGML_PREC_F32, + GGML_TYPE_F16, GGML_TYPE_F16); + auto st = tc.eval(backend_metal, backend_cpu, "FLASH_ATTN_EXT", nullptr); + clear_ov(); + if (st == test_status_t::FAIL) { + printf("FAIL dk=%d dv=%d Q=%d ne=%d ne01=%d ne11=%d mask=%d sinks=%d\n", + s.dk, s.dv, Q, ne, ne01, ne11, (int) mask, (int) sinks); + ok = false; n_fail++; + } + n_run++; + } + } + } + } + } + } + } + // kvpad pass (non-32-multiple kv); dk128/256 only to keep the case count down + for (auto s : shapes) { + if (!((s.dk == 128 && s.dv == 128) || (s.dk == 256 && s.dv == 256))) { + continue; + } + for (int ne : fa_vec_legal_ne(s.dk, s.dv)) { + for (int Q : {1, 2, 4}) { + for (int ne11 : ne11_kvpad) { + set_ov(Q, ne); + test_flash_attn_ext tc(s.dk, s.dv, /*nh=*/4, {1,1}, /*kv=*/ne11, /*nb=*/8, + /*mask=*/true, /*sinks=*/false, 0.0f, 0.0f, GGML_PREC_F32, + GGML_TYPE_F16, GGML_TYPE_F16); + auto st = tc.eval(backend_metal, backend_cpu, "FLASH_ATTN_EXT", nullptr); + clear_ov(); + if (st == test_status_t::FAIL) { + printf("FAIL(kvpad) dk=%d dv=%d Q=%d ne=%d ne11=%d\n", s.dk, s.dv, Q, ne, ne11); + ok = false; n_fail++; + } + n_run++; + } + } + } + } + // quantized K/V numerical check: the rebased body's dequant path is Q-generic + // (dequant once, reuse across Q rows); confirm it stays correct for every quant precision. + const ggml_type qtypes[] = { GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, GGML_TYPE_Q8_0 }; + const int q_ne01[] = { 1, 2, 3, 8 }; + const int q_ne11[] = { 512, 4096, 8192 }; + for (ggml_type qt : qtypes) { + for (auto s : shapes) { + for (int ne : fa_vec_legal_ne(s.dk, s.dv)) { + for (int Q : { 1, 2, 4 }) { + for (bool sinks : { false, true }) { + for (int ne01 : q_ne01) { + for (int ne11 : q_ne11) { + set_ov(Q, ne); + test_flash_attn_ext tc(s.dk, s.dv, /*nh=*/4, {1, 1}, /*kv=*/ne11, /*nb=*/ne01, + /*mask=*/true, sinks, 0.0f, 0.0f, GGML_PREC_F32, qt, qt); + auto st = tc.eval(backend_metal, backend_cpu, "FLASH_ATTN_EXT", nullptr); + clear_ov(); + if (st == test_status_t::FAIL) { + printf("FAIL(quant) type=%s dk=%d dv=%d Q=%d ne=%d ne01=%d ne11=%d sinks=%d\n", + ggml_type_name(qt), s.dk, s.dv, Q, ne, ne01, ne11, (int) sinks); + ok = false; n_fail++; + } + n_run++; + } + } + } + } + } + } + } + printf("fa_vec tune-check: %d cases run, %d failed\n", n_run, n_fail); + return ok; +} + +// A prebuilt FA op graph for one (dk,dv,ne01,ne11) cell. The op is replicated n_runs +// times so a single graph_compute amortizes dispatch/sync overhead. The graph is reused +// across (Q,NE) overrides (the override only changes the pipeline picked at encode time), +// which avoids re-allocating/re-initializing the (large) K/V tensors per candidate. +struct fa_perf_cell { + ggml_context_ptr ctx; + ggml_backend_buffer_ptr buf; + ggml_cgraph * gf = nullptr; + int n_runs = 0; + bool ok = false; +}; + +static fa_perf_cell fa_build_perf_cell(ggml_backend_t backend, int dk, int dv, int ne01, int ne11, + ggml_type type_kv = GGML_TYPE_F16) { + fa_perf_cell cell; + + // GQA shape (nr23=[8,1]) matching real spec-decode / verify workloads: enough query + // heads to keep the GPU busy so the Q>1 K/V-reuse benefit is visible (and comparable + // to the documented #23114 numbers). nh here is the number of KV heads. + test_flash_attn_ext tc(dk, dv, /*nh=*/4, { 8, 1 }, /*kv=*/ne11, /*nb=*/ne01, + /*mask=*/true, /*sinks=*/false, 0.0f, 0.0f, GGML_PREC_F32, + type_kv, type_kv); + + const size_t graph_nodes = 1024; + ggml_init_params params = { + /* .mem_size = */ ggml_tensor_overhead() * 128 + ggml_graph_overhead_custom(graph_nodes, false), + /* .mem_base = */ NULL, + /* .no_alloc = */ true, + }; + cell.ctx.reset(ggml_init(params)); + GGML_ASSERT(cell.ctx); + + ggml_tensor * out = tc.build_graph(cell.ctx.get()); + if (!ggml_backend_supports_op(backend, out)) { + return cell; + } + + cell.buf.reset(ggml_backend_alloc_ctx_tensors(cell.ctx.get(), backend)); + if (cell.buf == NULL) { + return cell; + } + + tc.initialize_tensors(cell.ctx.get()); + + cell.gf = ggml_new_graph_custom(cell.ctx.get(), graph_nodes, false); + ggml_build_forward_expand(cell.gf, out); + + // replicate the op to amortize overhead (target ~50 GFLOP/compute, capped to bound graph size) + cell.n_runs = 1; + if (tc.op_flops(out) > 0) { + const uint64_t target_flops = 50ULL * 1000 * 1000 * 1000; + const int cap = 512; + const int by_flops = (int) std::min(cap, (int64_t) (target_flops / tc.op_flops(out))); + cell.n_runs = std::max(1, std::min(by_flops, (int) (ggml_graph_size(cell.gf) - ggml_graph_n_nodes(cell.gf)))); + } + for (int i = 1; i < cell.n_runs; ++i) { + ggml_graph_add_node(cell.gf, out); + } + cell.ok = true; + return cell; +} + +// Median per-op GPU time (us) for the currently-set override over the prebuilt cell graph. +static double time_fa_cell_median(ggml_backend_t backend, const fa_perf_cell & cell, int reps) { + if (!cell.ok) { + return -1.0; + } + ggml_backend_graph_compute(backend, cell.gf); // warmup (compiles the pipeline for the override) + ggml_backend_synchronize(backend); + + std::vector samples; + samples.reserve(reps); + for (int r = 0; r < reps; ++r) { + const int64_t t0 = ggml_time_us(); + ggml_backend_graph_compute(backend, cell.gf); + ggml_backend_synchronize(backend); + samples.push_back((double) (ggml_time_us() - t0)); + } + std::nth_element(samples.begin(), samples.begin() + samples.size() / 2, samples.end()); + return samples[samples.size() / 2] / cell.n_runs; +} + +// Perf sweep over the (Q,NE) grid, emitting pasteable fa_vec_tuned_table rows. +// nsg/nwg are left to the ops.cpp adaptive heuristic (not part of the table). +static bool run_fa_vec_tune_perf(ggml_backend_t backend_metal) { + auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_metal)); + auto set_ov = (set_fa_vec_override_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_set_fa_vec_override"); + auto clr_ov = (clear_fa_vec_override_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_clear_fa_vec_override"); + // Share the runtime's bucketers + baseline NE via read-only proc bridges, so the emitted + // (ne11_b, ne01_b) keys match fa_vec_pick and can't silently desync from FA_VEC_*_BUCKETS. + auto ne11_bucket = (fa_vec_bucket_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_fa_vec_ne11_bucket"); + auto ne01_bucket = (fa_vec_bucket_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_fa_vec_ne01_bucket"); + auto baseline_ne = (fa_vec_baseline_ne_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_fa_vec_baseline_ne"); + if (!set_ov || !clr_ov || !ne11_bucket || !ne01_bucket || !baseline_ne) { + printf("metal fa_vec tuning procs unavailable\n"); + return false; + } + + struct shape_t { int dk, dv; }; + const shape_t shapes[] = { { 32, 32 }, { 64, 64 }, { 96, 96 }, { 128, 128 }, { 192, 192 }, + { 192, 128 }, { 256, 256 }, { 320, 256 }, { 512, 512 }, { 576, 512 } }; + const int ne11_rep[] = { 512, 2048, 8192, 32768 }; // ne11 bucket representatives + const int ne01_rep[] = { 1, 2, 3, 4, 5, 6, 7, 8, 16 }; // PR grid: point-bucket reps (1-4) + tail mod-4 cycle (5-8) + large anchor (16); vec serves ne01<20 + const int REPS = 7; // odd -> exact median + + printf("# fa_vec perf sweep — replace GGML_METAL_DEVICE_M4_MAX with this machine's device\n"); + printf("# [1] per-bucket timings (us, winner *): '=> cfg Nx' beats baseline, else '=> baseline'\n"); + printf("# [2] a pasteable fa_vec_tuned_table block (domain defaults + exceptions) is printed after [1]\n"); + + struct cand_t { int Q, NE; double t; }; // one timed (Q,NE) candidate + struct pt_t { int dk, dv, ne11, ne01; // one swept grid point with its candidate times + std::vector cs; double base_t; }; + + // group_split compression knobs (see ggml-metal-tuning.h for the row / lookup semantics) + const double TUNE_TAU = 0.05; // max POINTWISE regret (worst point in a bucket) to ride a domain + // default instead of emitting the bucket's own exception row + const double TUNE_THETA = 1.05; // min AGGREGATE bucket speedup vs baseline to tune the bucket at all + + auto type_token = [](ggml_type t) -> const char * { + switch (t) { + case GGML_TYPE_Q4_0: return "GGML_TYPE_Q4_0"; + case GGML_TYPE_Q4_1: return "GGML_TYPE_Q4_1"; + case GGML_TYPE_Q5_0: return "GGML_TYPE_Q5_0"; + case GGML_TYPE_Q5_1: return "GGML_TYPE_Q5_1"; + case GGML_TYPE_Q8_0: return "GGML_TYPE_Q8_0"; + default: return "GGML_TYPE_F16"; + } + }; + + const ggml_type types[] = { GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, + GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, GGML_TYPE_Q8_0 }; + for (ggml_type type_kv : types) { + printf("\n### dtype=%s\n", ggml_type_name(type_kv)); + std::vector pts; // one per swept (shape, ne11, ne01); bucketed + compressed below + + for (auto s : shapes) { + const int base_ne = baseline_ne(s.dk, s.dv); + const std::vector legal = fa_vec_legal_ne(s.dk, s.dv); + for (int ne11 : ne11_rep) { + for (int ne01 : ne01_rep) { + fa_perf_cell cell = fa_build_perf_cell(backend_metal, s.dk, s.dv, ne01, ne11, type_kv); + if (!cell.ok) { + continue; + } + + std::vector cs; + for (int ne : legal) { + for (int Q : { 1, 2, 4 }) { + cs.push_back({ Q, ne, 0.0 }); + } + } + // randomize config order to decorrelate thermal throttling across the cell + std::shuffle(cs.begin(), cs.end(), std::mt19937(1234)); + + double anchor = 0.0; // periodic baseline re-measure -> throttling detection + for (size_t i = 0; i < cs.size(); ++i) { + set_ov(cs[i].Q, cs[i].NE); + cs[i].t = time_fa_cell_median(backend_metal, cell, REPS); + clr_ov(); + + if (i % 4 == 0) { + const double a = time_fa_cell_median(backend_metal, cell, REPS); + if (anchor > 0.0) { + double drift = (a - anchor) / anchor; + if (drift < 0) { + drift = -drift; + } + if (drift > 0.10) { + printf("# WARN throttling? anchor drift %.1f%% dk=%d ne11=%d\n", 100.0 * drift, s.dk, ne11); + } + } + anchor = (anchor > 0.0) ? std::min(anchor, a) : a; + } + } + + std::sort(cs.begin(), cs.end(), [](const cand_t & a, const cand_t & b) { + return a.Q != b.Q ? a.Q < b.Q : a.NE < b.NE; + }); + cand_t best = cs[0]; + for (const auto & c : cs) { + if (c.t > 0.0 && (best.t <= 0.0 || c.t < best.t)) { + best = c; + } + } + double base_t = 0.0; // the swept (Q=1, base_ne) candidate == baseline kernel + for (const auto & c : cs) { + if (c.Q == 1 && c.NE == base_ne) { base_t = c.t; break; } + } + const bool keep = best.t > 0.0 && base_t > 0.0 && best.t < base_t * 0.98; + + printf("# dtype=%s dk=%d dv=%d ne11=%d ne01=%d:", ggml_type_name(type_kv), s.dk, s.dv, ne11, ne01); + for (const auto & c : cs) { + printf(" Q%dNE%d=%.1f%s", c.Q, c.NE, c.t, (c.Q == best.Q && c.NE == best.NE) ? "*" : ""); + } + if (keep) { + printf(" => Q%d,NE%d %.2fx\n", best.Q, best.NE, base_t / best.t); + } else { + printf(" => baseline\n"); + } + pts.push_back({ s.dk, s.dv, ne11, ne01, cs, base_t }); + } + } + } + + // [2] Compress into pasteable rows. Per (dk,dv) and ne01 domain {decode==1, batch>=2}, emit + // one ne11-collapsed default cfg (fewest rows, ne11_b=-1) plus a per-bucket exception wherever + // the default's pointwise regret vs the bucket's min-max-regret target, or its aggregate + // slowdown vs baseline, exceeds TUNE_TAU. The target still carries an intrinsic per-point regret + // from lumping ne01 into one tail bucket — TUNE_TAU bounds resolved-vs-target, not vs-oracle. + std::vector rows_out; + char rbuf[192]; + for (auto s : shapes) { + const int base_ne = baseline_ne(s.dk, s.dv); + std::vector cfgs; // candidate list (identical for every grid point of this shape) + int base_i = 0; + for (int ne : fa_vec_legal_ne(s.dk, s.dv)) { + for (int Q : { 1, 2, 4 }) { + if (Q == 1 && ne == base_ne) { base_i = (int) cfgs.size(); } + cfgs.push_back({ Q, ne, 0.0 }); + } + } + auto cfg_time = [&](const pt_t & p, int i) { + for (const auto & c : p.cs) { if (c.Q == cfgs[i].Q && c.NE == cfgs[i].NE) { return c.t; } } + return 0.0; + }; + + // Bucket the grid points using the runtime's bucketers (via proc), so keys match fa_vec_pick. + // Short-KV points (ne11 bucket 0) are dropped — the runtime serves those from baseline. Each + // kept bucket picks a min-max-regret target (gated by the aggregate TUNE_THETA benefit gate). + struct bkt_t { int b11, b01, Ti; std::vector agg; double base_agg; + std::vector bp; }; // bp kept for the pointwise ride test below + std::set> seen; + for (const auto & p : pts) { + if (p.dk != s.dk || p.dv != s.dv) { continue; } + const int b11 = ne11_bucket(p.ne11); + if (b11 == 0) { continue; } + seen.insert({ b11, ne01_bucket(p.ne01) }); + } + std::vector bks; + for (const auto & bb : seen) { + const int b11 = bb.first, b01 = bb.second; + std::vector bp; + for (const auto & p : pts) { + if (p.dk == s.dk && p.dv == s.dv && ne11_bucket(p.ne11) == b11 && ne01_bucket(p.ne01) == b01) { + bp.push_back(&p); + } + } + std::vector agg(cfgs.size(), 0.0), worst(cfgs.size(), 0.0); + for (const auto * p : bp) { + double bestt = 0.0; + for (size_t i = 0; i < cfgs.size(); ++i) { + double t = cfg_time(*p, (int) i); + if (t > 0.0 && (bestt == 0.0 || t < bestt)) { bestt = t; } + } + for (size_t i = 0; i < cfgs.size(); ++i) { + double t = cfg_time(*p, (int) i); + agg[i] += t; + if (t > 0.0 && bestt > 0.0) { worst[i] = std::max(worst[i], t / bestt); } + } + } + int robust = 0; + for (size_t i = 1; i < cfgs.size(); ++i) { + if (worst[i] < worst[robust] || + (worst[i] == worst[robust] && (cfgs[i].Q < cfgs[robust].Q || + (cfgs[i].Q == cfgs[robust].Q && cfgs[i].NE < cfgs[robust].NE)))) { + robust = (int) i; + } + } + const bool tune = robust != base_i && agg[base_i] / agg[robust] >= TUNE_THETA; + bks.push_back({ b11, b01, tune ? robust : base_i, agg, agg[base_i], bp }); + } + + // Pointwise regret of default cfg d vs the bucket target Ti. Per-point (not aggregate): + // a ratio-of-sums lets a default that wins on aligned ne01 (8, 16) hide a large penalty + // on a misaligned point (ne01=5), so the ride/exception decision must be per point. + auto reg_pointwise = [&](const bkt_t * b, int d) { + double r = 0.0; + for (const auto * p : b->bp) { + const double td = cfg_time(*p, d), tT = cfg_time(*p, b->Ti); + if (td > 0.0 && tT > 0.0) { r = std::max(r, td / tT - 1.0); } + } + return r; + }; + + for (int dom = 0; dom <= 1; ++dom) { // 0 = decode (ne01==1), 1 = batch (ne01>=2) + std::vector db; + for (const auto & b : bks) { if ((dom == 0) == (b.b01 == 0)) { db.push_back(&b); } } + if (db.empty()) { continue; } + + // default cfg = the one minimizing (#rows, total achieved time, Q, NE) + int bestD = -1, bestRows = 1 << 30; double bestTot = 0.0; + for (size_t d = 0; d < cfgs.size(); ++d) { + int rows = ((int) d != base_i) ? 1 : 0; double tot = 0.0; + for (const auto * b : db) { + const double reg = reg_pointwise(b, (int) d); // pointwise vs bucket target + const double slow = b->agg[d] / b->base_agg - 1.0; // aggregate vs baseline (safety net) + if (reg > TUNE_TAU || slow > TUNE_TAU) { rows++; tot += b->agg[b->Ti]; } + else { tot += b->agg[d]; } + } + const bool better = bestD < 0 || rows < bestRows || + (rows == bestRows && (tot < bestTot || + (tot == bestTot && (cfgs[d].Q < cfgs[bestD].Q || + (cfgs[d].Q == cfgs[bestD].Q && cfgs[d].NE < cfgs[bestD].NE))))); + if (better) { bestD = (int) d; bestRows = rows; bestTot = tot; } + } + + const int dom_id = (dom == 0) ? 0 : 1; // FA_VEC_DOMAIN_DECODE / FA_VEC_DOMAIN_BATCH + if (bestD != base_i) { + snprintf(rbuf, sizeof(rbuf), " { { GGML_METAL_DEVICE_M4_MAX, %s, %d, %d, -1, %d }, { %d, %d } },", + type_token(type_kv), s.dk, s.dv, dom_id, cfgs[bestD].Q, cfgs[bestD].NE); + rows_out.emplace_back(rbuf); + } + for (const auto * b : db) { + const double reg = reg_pointwise(b, bestD); // pointwise vs bucket target + const double slow = b->agg[bestD] / b->base_agg - 1.0; // aggregate vs baseline + if (reg <= TUNE_TAU && slow <= TUNE_TAU) { continue; } // rides the default / baseline + snprintf(rbuf, sizeof(rbuf), " { { GGML_METAL_DEVICE_M4_MAX, %s, %d, %d, %d, %d }, { %d, %d } },", + type_token(type_kv), s.dk, s.dv, b->b11, b->b01, cfgs[b->Ti].Q, cfgs[b->Ti].NE); + rows_out.emplace_back(rbuf); + } + } + } + printf("\n // ---- %s: %zu rows ----\n", ggml_type_name(type_kv), rows_out.size()); + for (const auto & r : rows_out) { printf("%s\n", r.c_str()); } + } // for type_kv + return true; +} + static bool test_backend(ggml_backend_t backend, ggml_backend_dev_t dev, test_mode mode, const char * op_names_filter, const char * params_filter, - printer * output_printer, const char * test_file_path, int parallel_workers) { + printer * output_printer, const char * test_file_path, int parallel_workers, bool tune_perf = false) { auto filter_test_cases = [](std::vector> & test_cases, const char * params_filter) { if (params_filter == nullptr) { return; @@ -10604,6 +11079,9 @@ static bool test_backend(ggml_backend_t backend, ggml_backend_dev_t dev, test_mo case MODE_PERF: test_cases = make_test_cases_perf(); break; + case MODE_TUNE: + // MODE_TUNE routes to its own dispatch below; no generic test_cases needed + break; } } else { test_cases = make_test_cases_from_file(test_file_path); @@ -10739,6 +11217,29 @@ static bool test_backend(ggml_backend_t backend, ggml_backend_dev_t dev, test_mo return true; } + if (mode == MODE_TUNE) { + // self-create a CPU backend with the reference implementation as golden. + // (backend_cpu in the MODE_TEST block above is out of scope here.) + ggml_backend_t backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, NULL); + GGML_ASSERT(backend_cpu != NULL); + { + using ggml_backend_cpu_set_use_ref_t = void (*)(ggml_backend_t, bool); + auto * cpu_reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_cpu)); + auto * set_use_ref = (ggml_backend_cpu_set_use_ref_t) ggml_backend_reg_get_proc_address(cpu_reg, "ggml_backend_cpu_set_use_ref"); + if (set_use_ref) { + set_use_ref(backend_cpu, true); + } + } + + // backend here is the MODE_TUNE-selected metal backend (-b MTL0) + const bool ok = tune_perf + ? run_fa_vec_tune_perf(backend) + : (run_fa_vec_drift_guard(backend, backend_cpu) && run_fa_vec_tune_check(backend, backend_cpu)); + + ggml_backend_free(backend_cpu); + return ok; + } + if (mode == MODE_SUPPORT) { // Filter out fusion cases test_cases.erase( @@ -10864,6 +11365,7 @@ static void usage(char ** argv) { printf(" - grad (compare gradients from backpropagation with method of finite differences)\n"); printf(" - perf (performance evaluation)\n"); printf(" - support (probe backend operation support)\n"); + printf(" - tune (FA vec (Q,NE) numerical correctness check vs CPU reference; --tune-perf for the perf sweep)\n"); printf(" op names for -o are as given by ggml_op_desc() (e.g. ADD, MUL_MAT, etc),\n"); printf(" optionally including the full test case string (e.g. \"ADD(type=f16,ne=[1,1,8,1],nr=[1,1,1,1],nf=1)\")\n"); printf(" --output specifies output format (default: console, options: console, sql, csv)\n"); @@ -10881,6 +11383,7 @@ int main(int argc, char ** argv) { const char * params_filter = nullptr; const char * test_file_path = nullptr; int parallel_workers = 1; + bool tune_perf = false; for (int i = 1; i < argc; i++) { if (strcmp(argv[i], "test") == 0) { @@ -10891,6 +11394,10 @@ int main(int argc, char ** argv) { mode = MODE_GRAD; } else if (strcmp(argv[i], "support") == 0) { mode = MODE_SUPPORT; + } else if (strcmp(argv[i], "tune") == 0) { + mode = MODE_TUNE; + } else if (strcmp(argv[i], "--tune-perf") == 0) { + tune_perf = true; } else if (strcmp(argv[i], "-o") == 0) { if (i + 1 < argc) { op_names_filter = argv[++i]; @@ -10998,7 +11505,7 @@ int main(int argc, char ** argv) { false, "", ggml_backend_dev_description(dev), total / 1024 / 1024, free / 1024 / 1024, true)); - bool ok = test_backend(backend.get(), dev, mode, op_names_filter, params_filter, output_printer.get(), test_file_path, parallel_workers); + bool ok = test_backend(backend.get(), dev, mode, op_names_filter, params_filter, output_printer.get(), test_file_path, parallel_workers, tune_perf); if (ok) { n_ok++; From 88f905fe8661696e922f9a589f14ec2989c9afae Mon Sep 17 00:00:00 2001 From: forforever73 <690105611@qq.com> Date: Fri, 31 Jul 2026 11:25:03 +0800 Subject: [PATCH 02/14] add device id -> token lookup for the offline tuning tool --- ggml/src/ggml-metal/ggml-metal-device.h | 2 + ggml/src/ggml-metal/ggml-metal-device.m | 68 +++++++++++++++---------- ggml/src/ggml-metal/ggml-metal.cpp | 9 ++++ 3 files changed, 51 insertions(+), 28 deletions(-) diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.h index 8846f7e9a47..6c39428c777 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.h +++ b/ggml/src/ggml-metal/ggml-metal-device.h @@ -259,6 +259,8 @@ enum ggml_metal_device_id { GGML_METAL_DEVICE_M5_ULTRA, }; +const char * ggml_metal_device_id_token(enum ggml_metal_device_id id); + struct ggml_metal_device_props { int device; int device_phys; diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m index e049c9e5edf..19c57820e85 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.m +++ b/ggml/src/ggml-metal/ggml-metal-device.m @@ -962,6 +962,34 @@ void ggml_metal_rsets_free(ggml_metal_rsets_t rsets) { free(rsets); } +static const struct { + const char * name; + const char * token; + enum ggml_metal_device_id id; +} k_metal_devices[] = { +#define DEV(name, id) { name, #id, id } + DEV("M1", GGML_METAL_DEVICE_M1), + DEV("M1 Pro", GGML_METAL_DEVICE_M1_PRO), + DEV("M1 Max", GGML_METAL_DEVICE_M1_MAX), + DEV("M1 Ultra", GGML_METAL_DEVICE_M1_ULTRA), + DEV("M2", GGML_METAL_DEVICE_M2), + DEV("M2 Pro", GGML_METAL_DEVICE_M2_PRO), + DEV("M2 Max", GGML_METAL_DEVICE_M2_MAX), + DEV("M2 Ultra", GGML_METAL_DEVICE_M2_ULTRA), + DEV("M3", GGML_METAL_DEVICE_M3), + DEV("M3 Pro", GGML_METAL_DEVICE_M3_PRO), + DEV("M3 Max", GGML_METAL_DEVICE_M3_MAX), + DEV("M3 Ultra", GGML_METAL_DEVICE_M3_ULTRA), + DEV("M4", GGML_METAL_DEVICE_M4), + DEV("M4 Pro", GGML_METAL_DEVICE_M4_PRO), + DEV("M4 Max", GGML_METAL_DEVICE_M4_MAX), + DEV("M5", GGML_METAL_DEVICE_M5), + DEV("M5 Pro", GGML_METAL_DEVICE_M5_PRO), + DEV("M5 Max", GGML_METAL_DEVICE_M5_MAX), + DEV("M5 Ultra", GGML_METAL_DEVICE_M5_ULTRA), +#undef DEV +}; + static enum ggml_metal_device_id ggml_metal_device_id_parse(const char * name) { if (!name) { return GGML_METAL_DEVICE_GENERIC; @@ -973,39 +1001,23 @@ static enum ggml_metal_device_id ggml_metal_device_id_parse(const char * name) { } const char * suffix = name + sizeof(prefix) - 1; - static const struct { - const char * name; - enum ggml_metal_device_id id; - } table[] = { - {"M1", GGML_METAL_DEVICE_M1}, - {"M1 Pro", GGML_METAL_DEVICE_M1_PRO}, - {"M1 Max", GGML_METAL_DEVICE_M1_MAX}, - {"M1 Ultra", GGML_METAL_DEVICE_M1_ULTRA}, - {"M2", GGML_METAL_DEVICE_M2}, - {"M2 Pro", GGML_METAL_DEVICE_M2_PRO}, - {"M2 Max", GGML_METAL_DEVICE_M2_MAX}, - {"M2 Ultra", GGML_METAL_DEVICE_M2_ULTRA}, - {"M3", GGML_METAL_DEVICE_M3}, - {"M3 Pro", GGML_METAL_DEVICE_M3_PRO}, - {"M3 Max", GGML_METAL_DEVICE_M3_MAX}, - {"M3 Ultra", GGML_METAL_DEVICE_M3_ULTRA}, - {"M4", GGML_METAL_DEVICE_M4}, - {"M4 Pro", GGML_METAL_DEVICE_M4_PRO}, - {"M4 Max", GGML_METAL_DEVICE_M4_MAX}, - {"M5", GGML_METAL_DEVICE_M5}, - {"M5 Pro", GGML_METAL_DEVICE_M5_PRO}, - {"M5 Max", GGML_METAL_DEVICE_M5_MAX}, - {"M5 Ultra", GGML_METAL_DEVICE_M5_ULTRA}, - }; - - for (size_t i = 0; i < sizeof(table)/sizeof(table[0]); ++i) { - if (strcmp(suffix, table[i].name) == 0) { - return table[i].id; + for (size_t i = 0; i < sizeof(k_metal_devices)/sizeof(k_metal_devices[0]); ++i) { + if (strcmp(suffix, k_metal_devices[i].name) == 0) { + return k_metal_devices[i].id; } } return GGML_METAL_DEVICE_GENERIC; } +const char * ggml_metal_device_id_token(enum ggml_metal_device_id id) { + for (size_t i = 0; i < sizeof(k_metal_devices)/sizeof(k_metal_devices[0]); ++i) { + if (k_metal_devices[i].id == id) { + return k_metal_devices[i].token; + } + } + return "GGML_METAL_DEVICE_GENERIC"; +} + ggml_metal_device_t ggml_metal_device_init(int device, int n_devices) { ggml_metal_device_t dev = calloc(1, sizeof(struct ggml_metal_device)); diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal.cpp index fe7796c41db..e5b2ee8a55b 100644 --- a/ggml/src/ggml-metal/ggml-metal.cpp +++ b/ggml/src/ggml-metal/ggml-metal.cpp @@ -892,6 +892,12 @@ static int ggml_backend_metal_tuning_fa_vec_baseline_ne(int dk, int dv) { return ggml_metal_tuning::fa_vec_baseline_ne(dk, dv); } +static const char * ggml_backend_metal_tuning_device_token(ggml_backend_dev_t dev) { + ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context; + + return ggml_metal_device_id_token(ggml_metal_device_get_props(ctx_dev)->device_id); +} + static void * ggml_backend_metal_get_proc_address(ggml_backend_reg_t reg, const char * name) { if (strcmp(name, "ggml_backend_get_features") == 0) { return (void *)ggml_backend_metal_get_features; @@ -911,6 +917,9 @@ static void * ggml_backend_metal_get_proc_address(ggml_backend_reg_t reg, const if (strcmp(name, "ggml_backend_metal_tuning_fa_vec_baseline_ne") == 0) { return (void *)ggml_backend_metal_tuning_fa_vec_baseline_ne; } + if (strcmp(name, "ggml_backend_metal_tuning_device_token") == 0) { + return (void *)ggml_backend_metal_tuning_device_token; + } return NULL; From 1f9a360b009d73d01ab40ec763ea3ebe6a034bff Mon Sep 17 00:00:00 2001 From: forforever73 <690105611@qq.com> Date: Fri, 31 Jul 2026 14:26:04 +0800 Subject: [PATCH 03/14] add ggml-metal-tuning skeleton --- tools/CMakeLists.txt | 3 + tools/tuning/CMakeLists.txt | 9 +++ tools/tuning/bench.cpp | 1 + tools/tuning/bench.h | 1 + tools/tuning/fa-vec.cpp | 13 ++++ tools/tuning/fa-vec.h | 20 ++++++ tools/tuning/main.cpp | 140 ++++++++++++++++++++++++++++++++++++ 7 files changed, 187 insertions(+) create mode 100644 tools/tuning/CMakeLists.txt create mode 100644 tools/tuning/bench.cpp create mode 100644 tools/tuning/bench.h create mode 100644 tools/tuning/fa-vec.cpp create mode 100644 tools/tuning/fa-vec.h create mode 100644 tools/tuning/main.cpp diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 37561563f65..c8ad1db4362 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -38,5 +38,8 @@ else() add_subdirectory(export-lora) endif() add_subdirectory(fit-params) + if (GGML_METAL) + add_subdirectory(tuning) + endif() add_subdirectory(results) endif() diff --git a/tools/tuning/CMakeLists.txt b/tools/tuning/CMakeLists.txt new file mode 100644 index 00000000000..a6909c2307f --- /dev/null +++ b/tools/tuning/CMakeLists.txt @@ -0,0 +1,9 @@ +set(TARGET ggml-metal-tuning) + +add_executable(${TARGET} main.cpp bench.cpp fa-vec.cpp) +target_link_libraries(${TARGET} PRIVATE ggml ${CMAKE_THREAD_LIBS_INIT}) +target_compile_features(${TARGET} PRIVATE cxx_std_17) + +if(LLAMA_TOOLS_INSTALL) + install(TARGETS ${TARGET} RUNTIME) +endif() diff --git a/tools/tuning/bench.cpp b/tools/tuning/bench.cpp new file mode 100644 index 00000000000..9adf305c654 --- /dev/null +++ b/tools/tuning/bench.cpp @@ -0,0 +1 @@ +#include "bench.h" diff --git a/tools/tuning/bench.h b/tools/tuning/bench.h new file mode 100644 index 00000000000..6f70f09beec --- /dev/null +++ b/tools/tuning/bench.h @@ -0,0 +1 @@ +#pragma once diff --git a/tools/tuning/fa-vec.cpp b/tools/tuning/fa-vec.cpp new file mode 100644 index 00000000000..e21e17a4d02 --- /dev/null +++ b/tools/tuning/fa-vec.cpp @@ -0,0 +1,13 @@ +#include "fa-vec.h" + +#include + +bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tuner_opts & opts) { + fprintf(stderr, "fa-vec tuner: not implemented yet\n"); + + (void) backend; + (void) dev; + (void) opts; + + return true; +} diff --git a/tools/tuning/fa-vec.h b/tools/tuning/fa-vec.h new file mode 100644 index 00000000000..b015a8051f4 --- /dev/null +++ b/tools/tuning/fa-vec.h @@ -0,0 +1,20 @@ +#pragma once + +#include "ggml-backend.h" + +// options shared by all tuners; parsed in main.cpp +struct tuner_opts { + const char * dtype_filter = nullptr; // comma-separated, e.g. "f16,q4_0"; null = all + const char * dk_filter = nullptr; // comma-separated dk values, e.g. "128,192"; null = all + int reps = 7; + unsigned seed = 1234; + bool cooldown = true; + double cool_drift = 0.10; + double cool_eps = 0.03; + int cool_max_wait = 120; + int cool_max_retry = 2; +}; + +// runs the FA-vec (Q,NE) sweep and prints a pasteable table block on stdout. +// returns false only on environment failure (missing procs), never on perf results. +bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tuner_opts & opts); diff --git a/tools/tuning/main.cpp b/tools/tuning/main.cpp new file mode 100644 index 00000000000..ce9eb0a9939 --- /dev/null +++ b/tools/tuning/main.cpp @@ -0,0 +1,140 @@ +#include "fa-vec.h" + +#include "ggml.h" +#include "ggml-backend.h" + +#include +#include +#include + +struct tuner_def { + const char * name; + bool (*run)(ggml_backend_t, ggml_backend_dev_t, const tuner_opts &); +}; + +static const tuner_def k_tuners[] = { + { "fa-vec", tuner_fa_vec_run }, +}; + +static void usage(const char * argv0) { + printf("usage: %s [options]\n", argv0); + printf("\n"); + printf(" offline kernel tuner for the Metal backend: sweeps a kernel's config grid and\n"); + printf(" prints pasteable table rows for the machine it runs on. never a pass/fail test.\n"); + printf("\n"); + printf(" tuners:\n"); + printf(" fa-vec flash-attn vec (Q,NE) for ggml-metal-tuning.cpp\n"); + printf("\n"); + printf(" options:\n"); + printf(" -b backend device (default: first Metal device)\n"); + printf(" --dtype restrict KV dtypes, e.g. f16,q4_0 (default: all)\n"); + printf(" --dk restrict head sizes, e.g. 128,192 (default: all)\n"); + printf(" --reps timed reps per candidate, odd for an exact median (default: 7)\n"); + printf(" --seed RNG seed; per-cell seeds mix it with the shape (default: 1234)\n"); + printf(" --no-cooldown do not pause/re-measure on thermal drift, only warn\n"); + printf(" --cool-drift anchor drift that triggers a cooldown (default: 0.10)\n"); + printf(" --cool-eps anchor tolerance to consider the GPU cool again (default: 0.03)\n"); + printf(" --cool-max-wait give up cooling a cell after this many seconds (default: 120)\n"); + printf(" --cool-max-retry re-measure rounds per cell before giving up (default: 2)\n"); + printf("\n"); + printf(" the table goes to stdout, all diagnostics to stderr:\n"); + printf(" %s fa-vec > rows.txt 2> sweep.log\n", argv0); +} + +int main(int argc, char ** argv) { + const char * tuner = nullptr; + const char * bname = nullptr; + tuner_opts opts; + + for (int i = 1; i < argc; i++) { + const char * a = argv[i]; + if (strcmp(a, "-h") == 0 || strcmp(a, "--help") == 0) { + usage(argv[0]); + return 0; + } else if (strcmp(a, "-b") == 0 && i + 1 < argc) { + bname = argv[++i]; + } else if (strcmp(a, "--dtype") == 0 && i + 1 < argc) { + opts.dtype_filter = argv[++i]; + } else if (strcmp(a, "--dk") == 0 && i + 1 < argc) { + opts.dk_filter = argv[++i]; + } else if (strcmp(a, "--reps") == 0 && i + 1 < argc) { + opts.reps = atoi(argv[++i]); + } else if (strcmp(a, "--seed") == 0 && i + 1 < argc) { + opts.seed = (unsigned) strtoul(argv[++i], nullptr, 10); + } else if (strcmp(a, "--no-cooldown") == 0) { + opts.cooldown = false; + } else if (strcmp(a, "--cool-drift") == 0 && i + 1 < argc) { + opts.cool_drift = atof(argv[++i]); + } else if (strcmp(a, "--cool-eps") == 0 && i + 1 < argc) { + opts.cool_eps = atof(argv[++i]); + } else if (strcmp(a, "--cool-max-wait") == 0 && i + 1 < argc) { + opts.cool_max_wait = atoi(argv[++i]); + } else if (strcmp(a, "--cool-max-retry") == 0 && i + 1 < argc) { + opts.cool_max_retry = atoi(argv[++i]); + } else if (a[0] != '-' && tuner == nullptr) { + tuner = a; + } else { + fprintf(stderr, "error: unrecognized or incomplete argument: %s\n\n", a); + usage(argv[0]); + return 1; + } + } + + if (tuner == nullptr) { + usage(argv[0]); + return 1; + } + if (opts.reps < 1) { + fprintf(stderr, "error: --reps must be >= 1\n"); + return 1; + } + + const tuner_def * t = nullptr; + for (const auto & cand : k_tuners) { + if (strcmp(tuner, cand.name) == 0) { + t = &cand; + break; + } + } + if (t == nullptr) { + fprintf(stderr, "error: unknown tuner: %s\n\n", tuner); + usage(argv[0]); + return 1; + } + + ggml_backend_load_all(); + + ggml_backend_dev_t dev = nullptr; + for (size_t i = 0; i < ggml_backend_dev_count(); i++) { + ggml_backend_dev_t d = ggml_backend_dev_get(i); + if (bname) { + if (strcmp(ggml_backend_dev_name(d), bname) == 0) { + dev = d; + break; + } + } else if (strncmp(ggml_backend_dev_name(d), "MTL", 3) == 0) { + dev = d; + break; + } + } + + if (dev == nullptr) { + fprintf(stderr, "error: no %s device found\n", bname ? bname : "Metal"); + return 1; + } + + ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr); + if (backend == nullptr) { + fprintf(stderr, "error: failed to init backend %s\n", ggml_backend_dev_name(dev)); + return 1; + } + + fprintf(stderr, "device: %s (%s)\n", ggml_backend_dev_name(dev), ggml_backend_dev_description(dev)); + + const bool ok = t->run(backend, dev, opts); + + ggml_backend_free(backend); + ggml_quantize_free(); + + return ok ? 0 : 1; +} From 0a540a5e89be3727c0ba2331c8a6ac30a561ed1a Mon Sep 17 00:00:00 2001 From: forforever73 <690105611@qq.com> Date: Fri, 31 Jul 2026 14:56:55 +0800 Subject: [PATCH 04/14] add op-agnostic perf cell + median timing for the tuner --- tools/tuning/bench.cpp | 118 +++++++++++++++++++++++++++++++++++++++++ tools/tuning/bench.h | 63 ++++++++++++++++++++++ 2 files changed, 181 insertions(+) diff --git a/tools/tuning/bench.cpp b/tools/tuning/bench.cpp index 9adf305c654..4e369b56666 100644 --- a/tools/tuning/bench.cpp +++ b/tools/tuning/bench.cpp @@ -1 +1,119 @@ #include "bench.h" + +#include +#include +#include + +perf_cell build_perf_cell(ggml_backend_t backend, const build_graph_fn & build, + const init_tensors_fn & init, const op_flops_fn & flops) { + perf_cell cell; + + const size_t graph_nodes = 1024; + + ggml_init_params params = { + /* .mem_size = */ ggml_tensor_overhead()*128 + ggml_graph_overhead_custom(graph_nodes, false), + /* .mem_base = */ NULL, + /* .no_alloc = */ true, + }; + + cell.ctx.reset(ggml_init(params)); + GGML_ASSERT(cell.ctx); + + ggml_tensor * out = build(cell.ctx.get()); + if (!out || !ggml_backend_supports_op(backend, out)) { + return cell; + } + + cell.buf.reset(ggml_backend_alloc_ctx_tensors(cell.ctx.get(), backend)); + if (cell.buf == NULL) { + return cell; + } + + init(cell.ctx.get()); + + cell.gf = ggml_new_graph_custom(cell.ctx.get(), graph_nodes, false); + ggml_build_forward_expand(cell.gf, out); + + // replicate the op to amortize overhead (target ~50 GFLOP/compute, capped to bound graph size) + cell.n_runs = 1; + if (flops(out) > 0) { + const uint64_t target_flops = 50ULL * 1000 * 1000 * 1000; + const int cap = 512; + const int by_flops = (int) std::min(cap, (int64_t) (target_flops / flops(out))); + cell.n_runs = std::max(1, std::min(by_flops, (int) (ggml_graph_size(cell.gf) - ggml_graph_n_nodes(cell.gf)))); + } + for (int i = 1; i < cell.n_runs; ++i) { + ggml_graph_add_node(cell.gf, out); + } + + cell.ok = true; + + return cell; +} + +double time_cell_median(ggml_backend_t backend, const perf_cell & cell, int reps) { + if (!cell.ok) { + return -1.0; + } + + ggml_backend_graph_compute(backend, cell.gf); // warmup (compiles the pipeline for this config) + ggml_backend_synchronize(backend); + + std::vector samples; + samples.reserve(reps); + for (int r = 0; r < reps; ++r) { + const int64_t t0 = ggml_time_us(); + ggml_backend_graph_compute(backend, cell.gf); + ggml_backend_synchronize(backend); + samples.push_back((double) (ggml_time_us() - t0)); + } + std::nth_element(samples.begin(), samples.begin() + samples.size()/2, samples.end()); + + return samples[samples.size()/2] / cell.n_runs; +} + +cell_result measure_cell(ggml_backend_t backend, const perf_cell & cell, int reps, + int n_cands, const std::vector & order, + const set_candidate_fn & set_cand, + const clear_candidate_fn & clear_cand, + int baseline_cand, const cooldown_opts & cool, + const char * cell_label) { + cell_result res; + res.t.assign(n_cands, 0.0); + + double anchor_ref = 0.0; + + for (size_t i = 0; i < order.size(); ++i) { + set_cand(order[i]); + res.t[order[i]] = time_cell_median(backend, cell, reps); + clear_cand(); + + if (i % 4 != 0) { + continue; + } + + // re-measure the baseline config as an anchor: same config every time, so any + // change is the machine, not the kernel + set_cand(baseline_cand); + const double a = time_cell_median(backend, cell, reps); + clear_cand(); + + if (a <= 0.0) { + continue; + } + + res.anchor_min = res.anchor_min > 0.0 ? std::min(res.anchor_min, a) : a; + res.anchor_max = std::max(res.anchor_max, a); + + if (anchor_ref > 0.0) { + const double drift = std::fabs(a - anchor_ref) / anchor_ref; + if (drift > cool.drift) { + fprintf(stderr, "# WARN throttling? anchor drift %.1f%% %s\n", 100.0*drift, cell_label); + } + } + + anchor_ref = anchor_ref > 0.0 ? std::min(anchor_ref, a) : a; + } + + return res; +} diff --git a/tools/tuning/bench.h b/tools/tuning/bench.h index 6f70f09beec..14761cf3ff1 100644 --- a/tools/tuning/bench.h +++ b/tools/tuning/bench.h @@ -1 +1,64 @@ #pragma once + +#include "ggml.h" +#include "ggml-backend.h" +#include "ggml-cpp.h" + +#include +#include +#include + +// one prebuilt op graph, replicated n_runs times so a single graph_compute amortizes +// dispatch/sync overhead. reused across candidates: an override only changes which +// pipeline is picked at encode time, so the (large) input tensors stay allocated. +struct perf_cell { + ggml_context_ptr ctx; + ggml_backend_buffer_ptr buf; + ggml_cgraph * gf = nullptr; + int n_runs = 0; + bool ok = false; +}; + +// builds the op graph for one shape. returns the output tensor, or null if unsupported. +using build_graph_fn = std::function; +// fills the allocated tensors of ctx with input data +using init_tensors_fn = std::function; +// flops of one op instance, used to size n_runs +using op_flops_fn = std::function; + +perf_cell build_perf_cell(ggml_backend_t backend, const build_graph_fn & build, + const init_tensors_fn & init, const op_flops_fn & flops); + +// median per-op time (us) over the prebuilt cell for whatever config is currently set +double time_cell_median(ggml_backend_t backend, const perf_cell & cell, int reps); + +struct cooldown_opts { + bool enabled = true; + double drift = 0.10; // anchor drift that triggers a cooldown + double eps = 0.03; // anchor tolerance to call the GPU cool again + int max_wait = 120; // seconds of cooling per cell before giving up + int max_retry = 2; // re-measure rounds per cell before giving up +}; + +// applies candidate i (an index into the tuner's own candidate list) +using set_candidate_fn = std::function; +// undoes the last set_candidate +using clear_candidate_fn = std::function; + +struct cell_result { + std::vector t; // time (us) per candidate index, <= 0 if not measured + bool trusted = true; // false -> caller must drop this cell + double anchor_min = 0.0; + double anchor_max = 0.0; + int n_cooldowns = 0; +}; + +// times every candidate over the prebuilt cell, re-measuring a periodic baseline anchor +// to watch for thermal drift. order[] gives the (shuffled) visiting order; baseline_cand is +// the candidate the anchor forces, so drift is measured against a config the tuner controls. +cell_result measure_cell(ggml_backend_t backend, const perf_cell & cell, int reps, + int n_cands, const std::vector & order, + const set_candidate_fn & set_cand, + const clear_candidate_fn & clear_cand, + int baseline_cand, const cooldown_opts & cool, + const char * cell_label); From f09bef3184114461444a23ff7922c85bb05245a9 Mon Sep 17 00:00:00 2001 From: forforever73 <690105611@qq.com> Date: Fri, 31 Jul 2026 15:41:18 +0800 Subject: [PATCH 05/14] add FA-vec graph build + tensor init to the tuner --- tools/tuning/fa-vec.cpp | 187 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 186 insertions(+), 1 deletion(-) diff --git a/tools/tuning/fa-vec.cpp b/tools/tuning/fa-vec.cpp index e21e17a4d02..38060576ab2 100644 --- a/tools/tuning/fa-vec.cpp +++ b/tools/tuning/fa-vec.cpp @@ -1,9 +1,194 @@ #include "fa-vec.h" +#include "bench.h" +#include "ggml.h" +#include "ggml-backend.h" + +#include +#include #include +#include +#include +#include +#include +#include + +// GQA spec-decode shape: enough query heads to keep the GPU busy so the Q>1 K/V-reuse +// benefit is visible. nh KV heads, nr2 query heads each, nr3 batches. +static const int FA_NH = 4; +static const int FA_NR2 = 8; +static const int FA_NR3 = 1; + +struct fa_shape { + int dk; + int dv; + int ne01; // query rows + int ne11; // KV length + ggml_type type_kv; +}; + +// mirrors test_flash_attn_ext::build_graph for the subset this tuner sweeps +// (mask=true, sinks=false, prec=F32, type_K==type_V, no permute) +static ggml_tensor * fa_build_graph(ggml_context * ctx, const fa_shape & s) { + const int64_t dk_padded = GGML_PAD(s.dk, ggml_blck_size(s.type_kv)); + const int64_t dv_padded = GGML_PAD(s.dv, ggml_blck_size(s.type_kv)); + + ggml_tensor * q = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, dk_padded, s.ne01, FA_NH*FA_NR2, FA_NR3); + ggml_set_name(q, "q"); + + // K/V are views of a 2x-tall parent, as they are of the KV cache in production + ggml_tensor * k0 = ggml_new_tensor_4d(ctx, s.type_kv, dk_padded, 2*s.ne11, FA_NH, FA_NR3); + ggml_tensor * k = ggml_view_4d(ctx, k0, dk_padded, s.ne11, FA_NH, FA_NR3, + k0->nb[1], k0->nb[2], k0->nb[3], 0); + ggml_set_name(k, "k"); + + ggml_tensor * v = nullptr; + if (dk_padded == 576 && dv_padded == 512) { + // MLA: the V cache is a sub-view of the K cache + v = ggml_view_4d(ctx, k, dv_padded, s.ne11, FA_NH, FA_NR3, k->nb[1], k->nb[2], k->nb[3], 0); + } else { + ggml_tensor * v0 = ggml_new_tensor_4d(ctx, s.type_kv, dv_padded, 2*s.ne11, FA_NH, FA_NR3); + v = ggml_view_4d(ctx, v0, dv_padded, s.ne11, FA_NH, FA_NR3, + v0->nb[1], v0->nb[2], v0->nb[3], 0); + } + ggml_set_name(v, "v"); + + ggml_tensor * m = ggml_new_tensor_4d(ctx, GGML_TYPE_F16, s.ne11, s.ne01, 1, FA_NR3); + ggml_set_name(m, "m"); + + ggml_tensor * out = ggml_flash_attn_ext(ctx, q, k, v, m, 1.0f/sqrtf((float) s.dk), 0.0f, 0.0f); + ggml_flash_attn_ext_set_prec(out, GGML_PREC_F32); + ggml_set_name(out, "out"); + + return out; +} + +static uint64_t fa_op_flops(const fa_shape & s) { + // Q*K^T is ne01 x dk x ne11, P*V is ne01 x ne11 x dv, per head + return (uint64_t) 2*FA_NH*FA_NR2*s.ne01*(s.dk + s.dv)*s.ne11*FA_NR3; +} + +// mirrors init_tensor_uniform: uniform f32 data, quantized in place for quantized types +static void fa_init_uniform(ggml_tensor * t, std::mt19937 & rng, float min, float max) { + const size_t nels = ggml_nelements(t); + + std::vector data(nels); + std::uniform_real_distribution dist(min, max); + for (size_t i = 0; i < nels; i++) { + data[i] = dist(rng); + } + + if (t->type == GGML_TYPE_F32) { + ggml_backend_tensor_set(t, data.data(), 0, nels*sizeof(float)); + return; + } + + GGML_ASSERT(ggml_is_quantized(t->type) || t->type == GGML_TYPE_F16 || t->type == GGML_TYPE_BF16); + GGML_ASSERT(nels % ggml_blck_size(t->type) == 0); + + std::vector imatrix(t->ne[0], 1.0f); + const float * im = imatrix.data(); + if (!ggml_quantize_requires_imatrix(t->type)) { + // when the imatrix is optional, exercise both paths; pick via one of the random numbers + if (data[0] > 0.5f*(min + max)) { + im = nullptr; + } + } + + const size_t blck_size = ggml_blck_size(t->type); + const size_t n_blocks = nels / blck_size; + + std::vector dataq(ggml_row_size(t->type, nels)); + ggml_quantize_chunk(t->type, data.data(), dataq.data(), 0, n_blocks, blck_size, im); + + ggml_backend_tensor_set(t, dataq.data(), 0, dataq.size()); +} + +// mirrors init_tensor_kq_mask: f16 mask with ~20% of its blocks set to -INF or zero. +// the -INF blocks are what drives the kernel's skip-INF path, so this pattern is +// load-bearing for the timings, not just for numerics. +static void fa_init_kq_mask(ggml_tensor * t, std::mt19937 & rng, float min, float max) { + GGML_ASSERT(t->type == GGML_TYPE_F16); + + const int32_t ne0 = (int32_t) t->ne[0]; + const int32_t ne1 = (int32_t) t->ne[1]; + const int32_t ne2 = (int32_t) t->ne[2]; + const int32_t ne3 = (int32_t) t->ne[3]; + + std::vector data_f32(size_t(ne0)*ne1*ne2*ne3); + std::vector data_f16(size_t(ne0)*ne1*ne2*ne3); + + std::uniform_real_distribution dis(min, max); + for (size_t i = 0; i < data_f32.size(); i++) { + data_f32[i] = dis(rng); + } + + const int blck0 = 128; + const int blck1 = 64; + + const int n_inf_zero_blocks = 0.2*(ne0*ne1*ne2*ne3)/(blck0*blck1); + + for (int b = 0; b < n_inf_zero_blocks; b++) { + const int p3 = (int) (rng() % ne3); + const int p2 = (int) (rng() % ne2); + const int p1 = (int) (rng() % ne1); + const int p0 = (int) (rng() % ne0); + + const bool inf = rng() & 1; + + for (int i1 = 0; i1 < blck1 && p1 + i1 < ne1; i1++) { + const int idx = p3*ne2*ne1*ne0 + p2*ne1*ne0 + (p1 + i1)*ne0 + p0; + + for (int i0 = 0; i0 < blck0 && p0 + i0 < ne0; i0++) { + data_f32[idx + i0] = inf ? -INFINITY : 0.0f; + } + } + } + + ggml_fp32_to_fp16_row(data_f32.data(), data_f16.data(), ne0*ne1*ne2*ne3); + + ggml_backend_tensor_set(t, data_f16.data(), 0, data_f16.size()*sizeof(ggml_fp16_t)); +} + +// per-cell deterministic seed: the shape decides it, so a cell is reproducible +// regardless of what else the sweep visited before it +static unsigned fa_cell_seed(const fa_shape & s, unsigned base) { + unsigned h = base; + for (int v : { s.dk, s.dv, s.ne01, s.ne11, (int) s.type_kv }) { + h = h*1000003u + (unsigned) v; + } + return h; +} + +static void fa_init_tensors(ggml_context * ctx, const fa_shape & s, unsigned base_seed) { + std::mt19937 rng(fa_cell_seed(s, base_seed)); + + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (t->view_src != NULL) { + continue; // views share their parent's data + } + if (strcmp(t->name, "m") == 0) { + fa_init_kq_mask(t, rng, -1.0f, 1.0f); + } else { + fa_init_uniform(t, rng, -1.0f, 1.0f); + } + } +} + +// legal NE for a (dk,dv): NL = 32/NE, require (dk/4)%NL==0 && (dv/4)%NL==0 +static std::vector fa_legal_ne(int dk, int dv) { + std::vector r; + for (int ne : { 1, 2, 4 }) { + const int nl = 32 / ne; + if ((dk/4) % nl == 0 && (dv/4) % nl == 0) { + r.push_back(ne); + } + } + return r; +} bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tuner_opts & opts) { - fprintf(stderr, "fa-vec tuner: not implemented yet\n"); + fprintf(stderr, "fa-vec tuner: sweep not implemented yet\n"); (void) backend; (void) dev; From 9c99dbcb102c17532d457e3c1e006b26218e9eb3 Mon Sep 17 00:00:00 2001 From: forforever73 <690105611@qq.com> Date: Fri, 31 Jul 2026 16:33:06 +0800 Subject: [PATCH 06/14] tools : add FA-vec (Q,NE) sweep, compression and table emit --- tools/tuning/fa-vec.cpp | 380 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 376 insertions(+), 4 deletions(-) diff --git a/tools/tuning/fa-vec.cpp b/tools/tuning/fa-vec.cpp index 38060576ab2..ac235920b1f 100644 --- a/tools/tuning/fa-vec.cpp +++ b/tools/tuning/fa-vec.cpp @@ -187,12 +187,384 @@ static std::vector fa_legal_ne(int dk, int dv) { return r; } +using set_override_t = void (*)(int, int); +using clear_override_t = void (*)(void); +using bucket_t = int (*)(int64_t); +using baseline_ne_t = int (*)(int, int); +using device_token_t = const char * (*)(ggml_backend_dev_t); + +struct fa_procs { + set_override_t set_ov = nullptr; + clear_override_t clr_ov = nullptr; + bucket_t ne11_bucket = nullptr; + bucket_t ne01_bucket = nullptr; + baseline_ne_t baseline_ne = nullptr; + device_token_t dev_token = nullptr; + + bool ok() const { + return set_ov && clr_ov && ne11_bucket && ne01_bucket && baseline_ne && dev_token; + } +}; + +static fa_procs fa_resolve_procs(ggml_backend_dev_t dev) { + ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev); + + fa_procs p; + p.set_ov = (set_override_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_set_fa_vec_override"); + p.clr_ov = (clear_override_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_clear_fa_vec_override"); + p.ne11_bucket = (bucket_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_fa_vec_ne11_bucket"); + p.ne01_bucket = (bucket_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_fa_vec_ne01_bucket"); + p.baseline_ne = (baseline_ne_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_fa_vec_baseline_ne"); + p.dev_token = (device_token_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_device_token"); + + return p; +} + +static const char * fa_type_token(ggml_type t) { + switch (t) { + case GGML_TYPE_Q4_0: return "GGML_TYPE_Q4_0"; + case GGML_TYPE_Q4_1: return "GGML_TYPE_Q4_1"; + case GGML_TYPE_Q5_0: return "GGML_TYPE_Q5_0"; + case GGML_TYPE_Q5_1: return "GGML_TYPE_Q5_1"; + case GGML_TYPE_Q8_0: return "GGML_TYPE_Q8_0"; + default: return "GGML_TYPE_F16"; + } +} + +// "f16,q4_0" -> does it contain ggml_type_name(t)? null filter accepts everything +static bool fa_filter_has(const char * filter, const char * name) { + if (!filter) { + return true; + } + + const std::string f = std::string(",") + filter + ","; + + return f.find(std::string(",") + name + ",") != std::string::npos; +} + +struct fa_cand { int Q, NE; }; + +struct fa_point { // one swept grid point with its candidate times + int dk, dv, ne11, ne01; + std::vector t; // indexed like the shape's candidate list +}; + +// candidate list for one shape, identical for every grid point of it. base_i is the index of +// the (Q=1, baseline NE) candidate: the anchor config, and what the tuning gates compare to. +static std::vector fa_build_cands(const fa_procs & procs, int dk, int dv, int & base_i) { + const int base_ne = procs.baseline_ne(dk, dv); + + std::vector cands; + base_i = -1; + for (int ne : fa_legal_ne(dk, dv)) { + for (int Q : { 1, 2, 4 }) { + if (Q == 1 && ne == base_ne) { + base_i = (int) cands.size(); + } + cands.push_back({ Q, ne }); + } + } + GGML_ASSERT(base_i >= 0); + + return cands; +} + bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tuner_opts & opts) { - fprintf(stderr, "fa-vec tuner: sweep not implemented yet\n"); + const fa_procs procs = fa_resolve_procs(dev); + if (!procs.ok()) { + fprintf(stderr, "error: metal fa_vec tuning procs unavailable\n"); + return false; + } + + const char * dev_token = procs.dev_token(dev); + + struct shape_t { int dk, dv; }; + const shape_t shapes[] = { { 32, 32 }, { 64, 64 }, { 96, 96 }, { 128, 128 }, { 192, 192 }, + { 192, 128 }, { 256, 256 }, { 320, 256 }, { 512, 512 }, { 576, 512 } }; + const int ne11_rep[] = { 512, 2048, 8192, 32768 }; // ne11 bucket representatives + const int ne01_rep[] = { 1, 2, 3, 4, 5, 6, 7, 8, 16 }; // point buckets (1-4) + tail mod-4 cycle + anchor + const ggml_type types[] = { GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, + GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, GGML_TYPE_Q8_0 }; + + const double TUNE_TAU = 0.05; // max POINTWISE regret to ride a domain default + const double TUNE_THETA = 1.05; // min AGGREGATE bucket speedup vs baseline to tune at all + + cooldown_opts cool; + cool.enabled = opts.cooldown; + cool.drift = opts.cool_drift; + cool.eps = opts.cool_eps; + cool.max_wait = opts.cool_max_wait; + cool.max_retry = opts.cool_max_retry; - (void) backend; - (void) dev; - (void) opts; + fprintf(stderr, "seed=%u reps=%d cooldown=%s (drift=%.2f eps=%.2f max_wait=%ds max_retry=%d)\n", + opts.seed, opts.reps, cool.enabled ? "on" : "off", + cool.drift, cool.eps, cool.max_wait, cool.max_retry); + fprintf(stderr, "device token: %s\n", dev_token); + + int n_untrusted = 0; + + printf("// ==== BEGIN fa_vec_tuned_table rows (%s) ====\n", dev_token); + + for (ggml_type type_kv : types) { + if (!fa_filter_has(opts.dtype_filter, ggml_type_name(type_kv))) { + continue; + } + + fprintf(stderr, "\n### dtype=%s\n", ggml_type_name(type_kv)); + + std::vector pts; + + for (auto s : shapes) { + if (!fa_filter_has(opts.dk_filter, std::to_string(s.dk).c_str())) { + continue; + } + + int base_i = 0; + std::vector cands = fa_build_cands(procs, s.dk, s.dv, base_i); + + for (int ne11 : ne11_rep) { + for (int ne01 : ne01_rep) { + const fa_shape sh = { s.dk, s.dv, ne01, ne11, type_kv }; + + perf_cell cell = build_perf_cell(backend, + [&](ggml_context * ctx) { return fa_build_graph(ctx, sh); }, + [&](ggml_context * ctx) { fa_init_tensors(ctx, sh, opts.seed); }, + [&](ggml_tensor *) { return fa_op_flops(sh); }); + + if (!cell.ok) { + continue; + } + + // randomize candidate order to decorrelate thermal drift across the cell + std::vector order((size_t) cands.size()); + for (size_t i = 0; i < order.size(); ++i) { + order[i] = (int) i; + } + std::shuffle(order.begin(), order.end(), std::mt19937(opts.seed)); + + char label[128]; + snprintf(label, sizeof(label), "dk=%d ne11=%d", s.dk, ne11); + + cell_result r = measure_cell(backend, cell, opts.reps, + (int) cands.size(), order, + [&](int i) { procs.set_ov(cands[i].Q, cands[i].NE); }, + [&]() { procs.clr_ov(); }, + base_i, cool, label); + + // per-cell noise floor: spread of the repeated same-config anchor + if (r.anchor_min > 0.0) { + fprintf(stderr, "# noise dk=%d dv=%d ne11=%d ne01=%d spread=%.1f%%\n", + s.dk, s.dv, ne11, ne01, 100.0*(r.anchor_max - r.anchor_min)/r.anchor_min); + } + + if (!r.trusted) { + n_untrusted++; + fprintf(stderr, "# DROP untrusted cell dk=%d dv=%d ne11=%d ne01=%d\n", + s.dk, s.dv, ne11, ne01); + continue; + } + + int best_i = -1; + for (size_t i = 0; i < cands.size(); ++i) { + if (r.t[i] > 0.0 && (best_i < 0 || r.t[i] < r.t[best_i])) { + best_i = (int) i; + } + } + const double base_t = r.t[base_i]; + const bool keep = best_i >= 0 && base_t > 0.0 && r.t[best_i] < base_t*0.98; + + fprintf(stderr, "# dtype=%s dk=%d dv=%d ne11=%d ne01=%d:", + ggml_type_name(type_kv), s.dk, s.dv, ne11, ne01); + for (size_t i = 0; i < cands.size(); ++i) { + fprintf(stderr, " Q%dNE%d=%.1f%s", cands[i].Q, cands[i].NE, r.t[i], + (int) i == best_i ? "*" : ""); + } + if (keep) { + fprintf(stderr, " => Q%d,NE%d %.2fx\n", + cands[best_i].Q, cands[best_i].NE, base_t/r.t[best_i]); + } else { + fprintf(stderr, " => baseline\n"); + } + + pts.push_back({ s.dk, s.dv, ne11, ne01, r.t }); + } + } + } + + // compress into pasteable rows. per (dk,dv) and ne01 domain {decode==1, batch>=2}, + // emit one ne11-collapsed default cfg (ne11_b=-1) plus a per-bucket exception wherever + // the default's pointwise regret vs the bucket target, or its aggregate slowdown vs + // baseline, exceeds TUNE_TAU. + std::vector rows_out; + char rbuf[192]; + + for (auto s : shapes) { + if (!fa_filter_has(opts.dk_filter, std::to_string(s.dk).c_str())) { + continue; + } + + int base_i = 0; + std::vector cands = fa_build_cands(procs, s.dk, s.dv, base_i); + + struct bkt_t { + int b11, b01, Ti; + std::vector agg; + double base_agg; + std::vector bp; + }; + + // bucket the grid points with the runtime's bucketers, so keys match fa_vec_pick. + // short-KV points (ne11 bucket 0) are dropped: the runtime serves those from baseline. + std::set> seen; + for (const auto & p : pts) { + if (p.dk != s.dk || p.dv != s.dv) { + continue; + } + const int b11 = procs.ne11_bucket(p.ne11); + if (b11 == 0) { + continue; + } + seen.insert({ b11, procs.ne01_bucket(p.ne01) }); + } + + std::vector bks; + for (const auto & bb : seen) { + const int b11 = bb.first, b01 = bb.second; + + std::vector bp; + for (const auto & p : pts) { + if (p.dk == s.dk && p.dv == s.dv && + procs.ne11_bucket(p.ne11) == b11 && procs.ne01_bucket(p.ne01) == b01) { + bp.push_back(&p); + } + } + + std::vector agg(cands.size(), 0.0), worst(cands.size(), 0.0); + for (const auto * p : bp) { + double bestt = 0.0; + for (size_t i = 0; i < cands.size(); ++i) { + if (p->t[i] > 0.0 && (bestt == 0.0 || p->t[i] < bestt)) { + bestt = p->t[i]; + } + } + for (size_t i = 0; i < cands.size(); ++i) { + agg[i] += p->t[i]; + if (p->t[i] > 0.0 && bestt > 0.0) { + worst[i] = std::max(worst[i], p->t[i]/bestt); + } + } + } + + int robust = 0; + for (size_t i = 1; i < cands.size(); ++i) { + if (worst[i] < worst[robust] || + (worst[i] == worst[robust] && (cands[i].Q < cands[robust].Q || + (cands[i].Q == cands[robust].Q && cands[i].NE < cands[robust].NE)))) { + robust = (int) i; + } + } + + const bool tune = robust != base_i && agg[base_i] > 0.0 && agg[robust] > 0.0 && + agg[base_i]/agg[robust] >= TUNE_THETA; + + bks.push_back({ b11, b01, tune ? robust : base_i, agg, agg[base_i], bp }); + } + + // bucket coverage: a hardcoded sampling grid can't produce a wrong key, only miss + // a bucket, so report what each bucket actually got + for (const auto & b : bks) { + fprintf(stderr, "# bucket dk=%d dv=%d ne11_b=%d ne01_b=%d samples=%zu\n", + s.dk, s.dv, b.b11, b.b01, b.bp.size()); + if (b.bp.empty()) { + fprintf(stderr, "# WARN empty bucket dk=%d dv=%d ne11_b=%d ne01_b=%d\n", + s.dk, s.dv, b.b11, b.b01); + } + } + + // pointwise regret of default cfg d vs the bucket target: a ratio-of-sums lets a + // default that wins on aligned ne01 hide a large penalty on a misaligned point + auto reg_pointwise = [&](const bkt_t * b, int d) { + double r = 0.0; + for (const auto * p : b->bp) { + const double td = p->t[d], tT = p->t[b->Ti]; + if (td > 0.0 && tT > 0.0) { + r = std::max(r, td/tT - 1.0); + } + } + return r; + }; + + for (int dom = 0; dom <= 1; ++dom) { // 0 = decode (ne01==1), 1 = batch (ne01>=2) + std::vector db; + for (const auto & b : bks) { + if ((dom == 0) == (b.b01 == 0)) { + db.push_back(&b); + } + } + if (db.empty()) { + continue; + } + + // default cfg = the one minimizing (#rows, total achieved time, Q, NE) + int bestD = -1, bestRows = 1 << 30; + double bestTot = 0.0; + for (size_t d = 0; d < cands.size(); ++d) { + int rows = ((int) d != base_i) ? 1 : 0; + double tot = 0.0; + for (const auto * b : db) { + const double reg = reg_pointwise(b, (int) d); + const double slow = b->base_agg > 0.0 ? b->agg[d]/b->base_agg - 1.0 : 0.0; + if (reg > TUNE_TAU || slow > TUNE_TAU) { + rows++; + tot += b->agg[b->Ti]; + } else { + tot += b->agg[d]; + } + } + const bool better = bestD < 0 || rows < bestRows || + (rows == bestRows && (tot < bestTot || + (tot == bestTot && (cands[d].Q < cands[bestD].Q || + (cands[d].Q == cands[bestD].Q && cands[d].NE < cands[bestD].NE))))); + if (better) { + bestD = (int) d; + bestRows = rows; + bestTot = tot; + } + } + + const int dom_id = (dom == 0) ? 0 : 1; // FA_VEC_DOMAIN_DECODE / FA_VEC_DOMAIN_BATCH + if (bestD != base_i) { + snprintf(rbuf, sizeof(rbuf), " { { %s, %s, %d, %d, -1, %d }, { %d, %d } },", + dev_token, fa_type_token(type_kv), s.dk, s.dv, dom_id, + cands[bestD].Q, cands[bestD].NE); + rows_out.emplace_back(rbuf); + } + for (const auto * b : db) { + const double reg = reg_pointwise(b, bestD); + const double slow = b->base_agg > 0.0 ? b->agg[bestD]/b->base_agg - 1.0 : 0.0; + if (reg <= TUNE_TAU && slow <= TUNE_TAU) { + continue; // rides the default / baseline + } + snprintf(rbuf, sizeof(rbuf), " { { %s, %s, %d, %d, %d, %d }, { %d, %d } },", + dev_token, fa_type_token(type_kv), s.dk, s.dv, b->b11, b->b01, + cands[b->Ti].Q, cands[b->Ti].NE); + rows_out.emplace_back(rbuf); + } + } + } + + printf("\n // ---- %s: %zu rows ----\n", ggml_type_name(type_kv), rows_out.size()); + for (const auto & r : rows_out) { + printf("%s\n", r.c_str()); + } + fflush(stdout); + } + + printf("// ==== END fa_vec_tuned_table rows (%s) ====\n", dev_token); + + if (n_untrusted > 0) { + fprintf(stderr, "\n%d cells excluded as untrusted (see DROP lines above)\n", n_untrusted); + } return true; } From 48c64c6abf2b0057f5e0eba78ee2c0d40a3cdb35 Mon Sep 17 00:00:00 2001 From: forforever73 <690105611@qq.com> Date: Fri, 31 Jul 2026 16:58:56 +0800 Subject: [PATCH 07/14] cool down and re-measure the dirty window on thermal drift --- tools/tuning/bench.cpp | 136 ++++++++++++++++++++++++++++++++++++----- tools/tuning/bench.h | 3 + 2 files changed, 124 insertions(+), 15 deletions(-) diff --git a/tools/tuning/bench.cpp b/tools/tuning/bench.cpp index 4e369b56666..fa51d29e89e 100644 --- a/tools/tuning/bench.cpp +++ b/tools/tuning/bench.cpp @@ -1,8 +1,11 @@ #include "bench.h" #include +#include #include #include +#include +#include perf_cell build_perf_cell(ggml_backend_t backend, const build_graph_fn & build, const init_tensors_fn & init, const op_flops_fn & flops) { @@ -72,6 +75,54 @@ double time_cell_median(ggml_backend_t backend, const perf_cell & cell, int reps return samples[samples.size()/2] / cell.n_runs; } +// times one candidate and returns its time; -1 on failure +static double measure_one(ggml_backend_t backend, const perf_cell & cell, int reps, + const set_candidate_fn & set_cand, const clear_candidate_fn & clear_cand, + int cand) { + set_cand(cand); + const double t = time_cell_median(backend, cell, reps); + clear_cand(); + + return t; +} + +// waits for the anchor to come back within eps of anchor_ref, with exponential backoff. +// returns the converged anchor, or -1 if it never converged within max_wait. +static double cool_until_steady(ggml_backend_t backend, const perf_cell & cell, int reps, + const set_candidate_fn & set_cand, const clear_candidate_fn & clear_cand, + int baseline_cand, double & anchor_ref, const cooldown_opts & cool, + const char * cell_label) { + int total_wait = 0; + + for (int sleep_s = 2; total_wait < cool.max_wait; sleep_s = std::min(sleep_s*2, 32)) { + const int this_wait = std::min(sleep_s, cool.max_wait - total_wait); + + fprintf(stderr, "# COOL sleeping %ds (%ds/%ds) %s\n", + this_wait, total_wait + this_wait, cool.max_wait, cell_label); + std::this_thread::sleep_for(std::chrono::seconds(this_wait)); + total_wait += this_wait; + + const double a = measure_one(backend, cell, reps, set_cand, clear_cand, baseline_cand); + if (a <= 0.0) { + continue; + } + + // a faster anchor means the machine got cooler than anything seen so far: adopt it + if (a < anchor_ref) { + anchor_ref = a; + } + + if (a <= anchor_ref*(1.0 + cool.eps)) { + fprintf(stderr, "# COOL steady after %ds %s\n", total_wait, cell_label); + return a; + } + } + + fprintf(stderr, "# COOL gave up after %ds %s\n", total_wait, cell_label); + + return -1.0; +} + cell_result measure_cell(ggml_backend_t backend, const perf_cell & cell, int reps, int n_cands, const std::vector & order, const set_candidate_fn & set_cand, @@ -83,21 +134,30 @@ cell_result measure_cell(ggml_backend_t backend, const perf_cell & cell, int rep double anchor_ref = 0.0; + // anchors accepted as clean, as (value, position in order[]). the dirty window starts + // at the position of the last anchor still within eps of anchor_ref, so a downward + // drift (anchor_ref dropping) naturally widens the window to the whole cell. + std::vector> anchors; + + auto window_start = [&]() -> size_t { + for (size_t i = anchors.size(); i-- > 0; ) { + if (anchors[i].first <= anchor_ref*(1.0 + cool.eps)) { + return anchors[i].second; + } + } + return 0; // no clean anchor left -> the whole cell is suspect + }; + + int retries_left = cool.max_retry; + for (size_t i = 0; i < order.size(); ++i) { - set_cand(order[i]); - res.t[order[i]] = time_cell_median(backend, cell, reps); - clear_cand(); + res.t[order[i]] = measure_one(backend, cell, reps, set_cand, clear_cand, order[i]); if (i % 4 != 0) { continue; } - // re-measure the baseline config as an anchor: same config every time, so any - // change is the machine, not the kernel - set_cand(baseline_cand); - const double a = time_cell_median(backend, cell, reps); - clear_cand(); - + const double a = measure_one(backend, cell, reps, set_cand, clear_cand, baseline_cand); if (a <= 0.0) { continue; } @@ -105,14 +165,60 @@ cell_result measure_cell(ggml_backend_t backend, const perf_cell & cell, int rep res.anchor_min = res.anchor_min > 0.0 ? std::min(res.anchor_min, a) : a; res.anchor_max = std::max(res.anchor_max, a); - if (anchor_ref > 0.0) { - const double drift = std::fabs(a - anchor_ref) / anchor_ref; - if (drift > cool.drift) { - fprintf(stderr, "# WARN throttling? anchor drift %.1f%% %s\n", 100.0*drift, cell_label); - } + if (anchor_ref == 0.0) { + anchor_ref = a; + anchors.push_back({ a, i }); + continue; + } + + const double drift = std::fabs(a - anchor_ref)/anchor_ref; + + // a cooler anchor than any so far becomes the reference: whatever was measured + // before it was measured on a hotter machine + if (a < anchor_ref) { + anchor_ref = a; + } + + if (drift <= cool.drift) { + anchors.push_back({ a, i }); + continue; } - anchor_ref = anchor_ref > 0.0 ? std::min(anchor_ref, a) : a; + fprintf(stderr, "# WARN throttling? anchor drift %.1f%% %s\n", 100.0*drift, cell_label); + + if (!cool.enabled) { + anchors.push_back({ a, i }); + continue; + } + + if (retries_left <= 0) { + fprintf(stderr, "# DIRTY retries exhausted %s\n", cell_label); + res.trusted = false; + return res; + } + + const size_t dirty_from = window_start(); + + res.n_cooldowns++; + + const double a_cool = cool_until_steady(backend, cell, reps, set_cand, clear_cand, + baseline_cand, anchor_ref, cool, cell_label); + if (a_cool <= 0.0) { + res.trusted = false; + return res; + } + + // the converged anchor is the only clean one now; re-measure the dirty window from it + anchors.clear(); + anchors.push_back({ a_cool, dirty_from }); + + retries_left--; + + fprintf(stderr, "# REDO candidates %zu..%zu %s\n", dirty_from, i, cell_label); + for (size_t j = dirty_from; j <= i; ++j) { + res.t[order[j]] = measure_one(backend, cell, reps, set_cand, clear_cand, order[j]); + res.n_remeasures++; + } } return res; diff --git a/tools/tuning/bench.h b/tools/tuning/bench.h index 14761cf3ff1..6349d970bd5 100644 --- a/tools/tuning/bench.h +++ b/tools/tuning/bench.h @@ -45,12 +45,15 @@ using set_candidate_fn = std::function; // undoes the last set_candidate using clear_candidate_fn = std::function; +// giving up on a cell returns early with trusted == false, so a trusted cell is one every +// candidate of was measured; callers may still see a non-positive t[] from a failed measure. struct cell_result { std::vector t; // time (us) per candidate index, <= 0 if not measured bool trusted = true; // false -> caller must drop this cell double anchor_min = 0.0; double anchor_max = 0.0; int n_cooldowns = 0; + int n_remeasures = 0; }; // times every candidate over the prebuilt cell, re-measuring a periodic baseline anchor From 6c65b7d13bb1dbeccaafbae95e436f4771137055 Mon Sep 17 00:00:00 2001 From: forforever73 <690105611@qq.com> Date: Fri, 31 Jul 2026 19:37:12 +0800 Subject: [PATCH 08/14] test-backend-ops : replace the FA vec tune mode with a bounded (Q,NE) slice --- tests/test-backend-ops.cpp | 504 +++---------------------------------- 1 file changed, 36 insertions(+), 468 deletions(-) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index c6a27aa6178..f26863a57a0 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -478,7 +478,6 @@ enum test_mode { MODE_PERF, MODE_GRAD, MODE_SUPPORT, - MODE_TUNE, }; // Output format support similar to llama-bench @@ -10574,12 +10573,10 @@ static std::vector> make_test_cases_from_file(const c return test_cases; } -// ---- FA vec (Q,NE) tuning: numerical correctness check + drift guard ---- +// ---- FA vec (Q,NE): forced-config numerical slice (Metal only) ---- // metal proc_address bridges (resolved by string, not symbol linkage) using set_fa_vec_override_t = void (*)(int, int); using clear_fa_vec_override_t = void (*)(void); -using fa_vec_bucket_t = int (*)(int64_t); // runtime ne11/ne01 bucketers, shared via proc bridge -using fa_vec_baseline_ne_t = int (*)(int, int); // runtime (dk,dv) -> baseline NE // legal NE for a (dk,dv): NL = 32/NE, require (dk/4)%NL==0 && (dv/4)%NL==0 static std::vector fa_vec_legal_ne(int dk, int dv) { @@ -10593,120 +10590,45 @@ static std::vector fa_vec_legal_ne(int dk, int dv) { return r; } -// Baseline-path smoke test: with no override, fa_vec_pick returns (1, baseline_ne) and -// dispatches the no-suffix baseline kernel. Checks baseline correctness across all 10 shapes -// (NE is numerically transparent; the (Q,NE) kernels are covered by run_fa_vec_tune_check). -static bool run_fa_vec_drift_guard(ggml_backend_t backend_metal, ggml_backend_t backend_cpu) { - struct shape_t { int dk, dv; }; - const shape_t shapes[] = { {32,32},{64,64},{96,96},{128,128},{192,192}, - {192,128},{256,256},{320,256},{512,512},{576,512} }; - bool ok = true; - for (auto s : shapes) { - // no override + short KV (ne11 < FA_VEC_NE11_BUCKETS[0]) -> fa_vec_pick returns baseline; - // compare metal vs CPU-ref - test_flash_attn_ext tc(s.dk, s.dv, /*nh=*/4, {1,1}, /*kv=*/512, /*nb=*/8, - /*mask=*/true, /*sinks=*/false, 0.0f, 0.0f, GGML_PREC_F32, - GGML_TYPE_F16, GGML_TYPE_F16); - auto st = tc.eval(backend_metal, backend_cpu, "FLASH_ATTN_EXT", nullptr); - if (st == test_status_t::FAIL) { - printf("DRIFT/baseline FAIL dk=%d dv=%d\n", s.dk, s.dv); - ok = false; - } - } - return ok; -} +// Forces every legal (Q,NE) on two representative shapes and compares against the CPU +// reference. Metal-only: the override is a backend-global switch, so it cannot be expressed +// per test case in the backend-agnostic case list. Covers padded rows (ne01 % Q != 0), +// per-qq sinks, kvpad, the nsg-dependent shmem offsets / parallel-reduce stride +// (ne11 -> nsg 1/2/4) and the quantized dequant-once path. +static bool run_fa_vec_slice(ggml_backend_t backend, ggml_backend_t backend_cpu) { + auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend)); -// Numerical gate: for each (dk,dv) x legal (Q,NE), force the override and compare metal -// vs CPU-ref. The ne01/ne11/mask/sinks points exercise the rebased body's padded rows -// (ne01 not a multiple of Q), per-qq sinks, skip-INF, kvpad, and the nsg-dependent shmem -// offsets / parallel-reduce stride (ne11 -> nsg 1/2/4). -static bool run_fa_vec_tune_check(ggml_backend_t backend_metal, ggml_backend_t backend_cpu) { - auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_metal)); auto set_ov = (set_fa_vec_override_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_set_fa_vec_override"); auto clear_ov = (clear_fa_vec_override_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_clear_fa_vec_override"); - if (!set_ov || !clear_ov) { printf("metal fa_vec override proc unavailable\n"); return false; } + if (!set_ov || !clear_ov) { + return true; // not the Metal backend: nothing to force + } struct shape_t { int dk, dv; }; - const shape_t shapes[] = { {32,32},{64,64},{96,96},{128,128},{192,192}, - {192,128},{256,256},{320,256},{512,512},{576,512} }; - const int ne01_pts[] = { 1, 2, 3, 8, 17 }; // 1, Q+/-1, 2Q-1, prime; vec upper bound < 20 - const int ne11_pts[] = { 512, 4096, 8192 }; // drives adaptive nsg 1/2/4 - const int ne11_kvpad[] = { 4097, 8193 }; // has_kvpad (non-32-multiple kv) - const bool mask_pts[] = { true, false }; - const bool sinks_pts[] = { false, true }; - - bool ok = true; + const shape_t shapes[] = { { 128, 128 }, { 576, 512 } }; // mainstream head size + MLA shared K/V view + const int ne01_pts[] = { 1, 3 }; // decode, and padded rows for Q=2 and Q=4 + const int ne11_pts[] = { 512, 4097 }; // nsg=1, and nsg>=2 together with kvpad + const ggml_type types[] = { GGML_TYPE_F16, GGML_TYPE_Q4_0 }; + int n_run = 0, n_fail = 0; for (auto s : shapes) { for (int ne : fa_vec_legal_ne(s.dk, s.dv)) { - for (int Q : {1, 2, 4}) { - for (bool mask : mask_pts) { - for (bool sinks : sinks_pts) { + for (int Q : { 1, 2, 4 }) { + for (ggml_type type_kv : types) { + for (bool sinks : { false, true }) { for (int ne01 : ne01_pts) { for (int ne11 : ne11_pts) { set_ov(Q, ne); - test_flash_attn_ext tc(s.dk, s.dv, /*nh=*/4, {1,1}, /*kv=*/ne11, /*nb=*/ne01, - mask, sinks, 0.0f, 0.0f, GGML_PREC_F32, - GGML_TYPE_F16, GGML_TYPE_F16); - auto st = tc.eval(backend_metal, backend_cpu, "FLASH_ATTN_EXT", nullptr); - clear_ov(); - if (st == test_status_t::FAIL) { - printf("FAIL dk=%d dv=%d Q=%d ne=%d ne01=%d ne11=%d mask=%d sinks=%d\n", - s.dk, s.dv, Q, ne, ne01, ne11, (int) mask, (int) sinks); - ok = false; n_fail++; - } - n_run++; - } - } - } - } - } - } - } - // kvpad pass (non-32-multiple kv); dk128/256 only to keep the case count down - for (auto s : shapes) { - if (!((s.dk == 128 && s.dv == 128) || (s.dk == 256 && s.dv == 256))) { - continue; - } - for (int ne : fa_vec_legal_ne(s.dk, s.dv)) { - for (int Q : {1, 2, 4}) { - for (int ne11 : ne11_kvpad) { - set_ov(Q, ne); - test_flash_attn_ext tc(s.dk, s.dv, /*nh=*/4, {1,1}, /*kv=*/ne11, /*nb=*/8, - /*mask=*/true, /*sinks=*/false, 0.0f, 0.0f, GGML_PREC_F32, - GGML_TYPE_F16, GGML_TYPE_F16); - auto st = tc.eval(backend_metal, backend_cpu, "FLASH_ATTN_EXT", nullptr); - clear_ov(); - if (st == test_status_t::FAIL) { - printf("FAIL(kvpad) dk=%d dv=%d Q=%d ne=%d ne11=%d\n", s.dk, s.dv, Q, ne, ne11); - ok = false; n_fail++; - } - n_run++; - } - } - } - } - // quantized K/V numerical check: the rebased body's dequant path is Q-generic - // (dequant once, reuse across Q rows); confirm it stays correct for every quant precision. - const ggml_type qtypes[] = { GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, GGML_TYPE_Q8_0 }; - const int q_ne01[] = { 1, 2, 3, 8 }; - const int q_ne11[] = { 512, 4096, 8192 }; - for (ggml_type qt : qtypes) { - for (auto s : shapes) { - for (int ne : fa_vec_legal_ne(s.dk, s.dv)) { - for (int Q : { 1, 2, 4 }) { - for (bool sinks : { false, true }) { - for (int ne01 : q_ne01) { - for (int ne11 : q_ne11) { - set_ov(Q, ne); - test_flash_attn_ext tc(s.dk, s.dv, /*nh=*/4, {1, 1}, /*kv=*/ne11, /*nb=*/ne01, - /*mask=*/true, sinks, 0.0f, 0.0f, GGML_PREC_F32, qt, qt); - auto st = tc.eval(backend_metal, backend_cpu, "FLASH_ATTN_EXT", nullptr); + test_flash_attn_ext tc(s.dk, s.dv, /*nh=*/4, { 1, 1 }, /*kv=*/ne11, /*nb=*/ne01, + /*mask=*/true, sinks, 0.0f, 0.0f, GGML_PREC_F32, + type_kv, type_kv); + auto st = tc.eval(backend, backend_cpu, "FLASH_ATTN_EXT", nullptr); clear_ov(); + if (st == test_status_t::FAIL) { - printf("FAIL(quant) type=%s dk=%d dv=%d Q=%d ne=%d ne01=%d ne11=%d sinks=%d\n", - ggml_type_name(qt), s.dk, s.dv, Q, ne, ne01, ne11, (int) sinks); - ok = false; n_fail++; + printf(" FAIL fa_vec slice: dk=%d dv=%d Q=%d ne=%d type=%s ne01=%d ne11=%d sinks=%d\n", + s.dk, s.dv, Q, ne, ggml_type_name(type_kv), ne01, ne11, (int) sinks); + n_fail++; } n_run++; } @@ -10716,340 +10638,14 @@ static bool run_fa_vec_tune_check(ggml_backend_t backend_metal, ggml_backend_t b } } } - printf("fa_vec tune-check: %d cases run, %d failed\n", n_run, n_fail); - return ok; -} - -// A prebuilt FA op graph for one (dk,dv,ne01,ne11) cell. The op is replicated n_runs -// times so a single graph_compute amortizes dispatch/sync overhead. The graph is reused -// across (Q,NE) overrides (the override only changes the pipeline picked at encode time), -// which avoids re-allocating/re-initializing the (large) K/V tensors per candidate. -struct fa_perf_cell { - ggml_context_ptr ctx; - ggml_backend_buffer_ptr buf; - ggml_cgraph * gf = nullptr; - int n_runs = 0; - bool ok = false; -}; - -static fa_perf_cell fa_build_perf_cell(ggml_backend_t backend, int dk, int dv, int ne01, int ne11, - ggml_type type_kv = GGML_TYPE_F16) { - fa_perf_cell cell; - - // GQA shape (nr23=[8,1]) matching real spec-decode / verify workloads: enough query - // heads to keep the GPU busy so the Q>1 K/V-reuse benefit is visible (and comparable - // to the documented #23114 numbers). nh here is the number of KV heads. - test_flash_attn_ext tc(dk, dv, /*nh=*/4, { 8, 1 }, /*kv=*/ne11, /*nb=*/ne01, - /*mask=*/true, /*sinks=*/false, 0.0f, 0.0f, GGML_PREC_F32, - type_kv, type_kv); - - const size_t graph_nodes = 1024; - ggml_init_params params = { - /* .mem_size = */ ggml_tensor_overhead() * 128 + ggml_graph_overhead_custom(graph_nodes, false), - /* .mem_base = */ NULL, - /* .no_alloc = */ true, - }; - cell.ctx.reset(ggml_init(params)); - GGML_ASSERT(cell.ctx); - - ggml_tensor * out = tc.build_graph(cell.ctx.get()); - if (!ggml_backend_supports_op(backend, out)) { - return cell; - } - - cell.buf.reset(ggml_backend_alloc_ctx_tensors(cell.ctx.get(), backend)); - if (cell.buf == NULL) { - return cell; - } - - tc.initialize_tensors(cell.ctx.get()); - cell.gf = ggml_new_graph_custom(cell.ctx.get(), graph_nodes, false); - ggml_build_forward_expand(cell.gf, out); + printf(" fa_vec (Q,NE) slice: %d cases run, %d failed\n", n_run, n_fail); - // replicate the op to amortize overhead (target ~50 GFLOP/compute, capped to bound graph size) - cell.n_runs = 1; - if (tc.op_flops(out) > 0) { - const uint64_t target_flops = 50ULL * 1000 * 1000 * 1000; - const int cap = 512; - const int by_flops = (int) std::min(cap, (int64_t) (target_flops / tc.op_flops(out))); - cell.n_runs = std::max(1, std::min(by_flops, (int) (ggml_graph_size(cell.gf) - ggml_graph_n_nodes(cell.gf)))); - } - for (int i = 1; i < cell.n_runs; ++i) { - ggml_graph_add_node(cell.gf, out); - } - cell.ok = true; - return cell; -} - -// Median per-op GPU time (us) for the currently-set override over the prebuilt cell graph. -static double time_fa_cell_median(ggml_backend_t backend, const fa_perf_cell & cell, int reps) { - if (!cell.ok) { - return -1.0; - } - ggml_backend_graph_compute(backend, cell.gf); // warmup (compiles the pipeline for the override) - ggml_backend_synchronize(backend); - - std::vector samples; - samples.reserve(reps); - for (int r = 0; r < reps; ++r) { - const int64_t t0 = ggml_time_us(); - ggml_backend_graph_compute(backend, cell.gf); - ggml_backend_synchronize(backend); - samples.push_back((double) (ggml_time_us() - t0)); - } - std::nth_element(samples.begin(), samples.begin() + samples.size() / 2, samples.end()); - return samples[samples.size() / 2] / cell.n_runs; -} - -// Perf sweep over the (Q,NE) grid, emitting pasteable fa_vec_tuned_table rows. -// nsg/nwg are left to the ops.cpp adaptive heuristic (not part of the table). -static bool run_fa_vec_tune_perf(ggml_backend_t backend_metal) { - auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_metal)); - auto set_ov = (set_fa_vec_override_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_set_fa_vec_override"); - auto clr_ov = (clear_fa_vec_override_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_clear_fa_vec_override"); - // Share the runtime's bucketers + baseline NE via read-only proc bridges, so the emitted - // (ne11_b, ne01_b) keys match fa_vec_pick and can't silently desync from FA_VEC_*_BUCKETS. - auto ne11_bucket = (fa_vec_bucket_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_fa_vec_ne11_bucket"); - auto ne01_bucket = (fa_vec_bucket_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_fa_vec_ne01_bucket"); - auto baseline_ne = (fa_vec_baseline_ne_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_fa_vec_baseline_ne"); - if (!set_ov || !clr_ov || !ne11_bucket || !ne01_bucket || !baseline_ne) { - printf("metal fa_vec tuning procs unavailable\n"); - return false; - } - - struct shape_t { int dk, dv; }; - const shape_t shapes[] = { { 32, 32 }, { 64, 64 }, { 96, 96 }, { 128, 128 }, { 192, 192 }, - { 192, 128 }, { 256, 256 }, { 320, 256 }, { 512, 512 }, { 576, 512 } }; - const int ne11_rep[] = { 512, 2048, 8192, 32768 }; // ne11 bucket representatives - const int ne01_rep[] = { 1, 2, 3, 4, 5, 6, 7, 8, 16 }; // PR grid: point-bucket reps (1-4) + tail mod-4 cycle (5-8) + large anchor (16); vec serves ne01<20 - const int REPS = 7; // odd -> exact median - - printf("# fa_vec perf sweep — replace GGML_METAL_DEVICE_M4_MAX with this machine's device\n"); - printf("# [1] per-bucket timings (us, winner *): '=> cfg Nx' beats baseline, else '=> baseline'\n"); - printf("# [2] a pasteable fa_vec_tuned_table block (domain defaults + exceptions) is printed after [1]\n"); - - struct cand_t { int Q, NE; double t; }; // one timed (Q,NE) candidate - struct pt_t { int dk, dv, ne11, ne01; // one swept grid point with its candidate times - std::vector cs; double base_t; }; - - // group_split compression knobs (see ggml-metal-tuning.h for the row / lookup semantics) - const double TUNE_TAU = 0.05; // max POINTWISE regret (worst point in a bucket) to ride a domain - // default instead of emitting the bucket's own exception row - const double TUNE_THETA = 1.05; // min AGGREGATE bucket speedup vs baseline to tune the bucket at all - - auto type_token = [](ggml_type t) -> const char * { - switch (t) { - case GGML_TYPE_Q4_0: return "GGML_TYPE_Q4_0"; - case GGML_TYPE_Q4_1: return "GGML_TYPE_Q4_1"; - case GGML_TYPE_Q5_0: return "GGML_TYPE_Q5_0"; - case GGML_TYPE_Q5_1: return "GGML_TYPE_Q5_1"; - case GGML_TYPE_Q8_0: return "GGML_TYPE_Q8_0"; - default: return "GGML_TYPE_F16"; - } - }; - - const ggml_type types[] = { GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, - GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, GGML_TYPE_Q8_0 }; - for (ggml_type type_kv : types) { - printf("\n### dtype=%s\n", ggml_type_name(type_kv)); - std::vector pts; // one per swept (shape, ne11, ne01); bucketed + compressed below - - for (auto s : shapes) { - const int base_ne = baseline_ne(s.dk, s.dv); - const std::vector legal = fa_vec_legal_ne(s.dk, s.dv); - for (int ne11 : ne11_rep) { - for (int ne01 : ne01_rep) { - fa_perf_cell cell = fa_build_perf_cell(backend_metal, s.dk, s.dv, ne01, ne11, type_kv); - if (!cell.ok) { - continue; - } - - std::vector cs; - for (int ne : legal) { - for (int Q : { 1, 2, 4 }) { - cs.push_back({ Q, ne, 0.0 }); - } - } - // randomize config order to decorrelate thermal throttling across the cell - std::shuffle(cs.begin(), cs.end(), std::mt19937(1234)); - - double anchor = 0.0; // periodic baseline re-measure -> throttling detection - for (size_t i = 0; i < cs.size(); ++i) { - set_ov(cs[i].Q, cs[i].NE); - cs[i].t = time_fa_cell_median(backend_metal, cell, REPS); - clr_ov(); - - if (i % 4 == 0) { - const double a = time_fa_cell_median(backend_metal, cell, REPS); - if (anchor > 0.0) { - double drift = (a - anchor) / anchor; - if (drift < 0) { - drift = -drift; - } - if (drift > 0.10) { - printf("# WARN throttling? anchor drift %.1f%% dk=%d ne11=%d\n", 100.0 * drift, s.dk, ne11); - } - } - anchor = (anchor > 0.0) ? std::min(anchor, a) : a; - } - } - - std::sort(cs.begin(), cs.end(), [](const cand_t & a, const cand_t & b) { - return a.Q != b.Q ? a.Q < b.Q : a.NE < b.NE; - }); - cand_t best = cs[0]; - for (const auto & c : cs) { - if (c.t > 0.0 && (best.t <= 0.0 || c.t < best.t)) { - best = c; - } - } - double base_t = 0.0; // the swept (Q=1, base_ne) candidate == baseline kernel - for (const auto & c : cs) { - if (c.Q == 1 && c.NE == base_ne) { base_t = c.t; break; } - } - const bool keep = best.t > 0.0 && base_t > 0.0 && best.t < base_t * 0.98; - - printf("# dtype=%s dk=%d dv=%d ne11=%d ne01=%d:", ggml_type_name(type_kv), s.dk, s.dv, ne11, ne01); - for (const auto & c : cs) { - printf(" Q%dNE%d=%.1f%s", c.Q, c.NE, c.t, (c.Q == best.Q && c.NE == best.NE) ? "*" : ""); - } - if (keep) { - printf(" => Q%d,NE%d %.2fx\n", best.Q, best.NE, base_t / best.t); - } else { - printf(" => baseline\n"); - } - pts.push_back({ s.dk, s.dv, ne11, ne01, cs, base_t }); - } - } - } - - // [2] Compress into pasteable rows. Per (dk,dv) and ne01 domain {decode==1, batch>=2}, emit - // one ne11-collapsed default cfg (fewest rows, ne11_b=-1) plus a per-bucket exception wherever - // the default's pointwise regret vs the bucket's min-max-regret target, or its aggregate - // slowdown vs baseline, exceeds TUNE_TAU. The target still carries an intrinsic per-point regret - // from lumping ne01 into one tail bucket — TUNE_TAU bounds resolved-vs-target, not vs-oracle. - std::vector rows_out; - char rbuf[192]; - for (auto s : shapes) { - const int base_ne = baseline_ne(s.dk, s.dv); - std::vector cfgs; // candidate list (identical for every grid point of this shape) - int base_i = 0; - for (int ne : fa_vec_legal_ne(s.dk, s.dv)) { - for (int Q : { 1, 2, 4 }) { - if (Q == 1 && ne == base_ne) { base_i = (int) cfgs.size(); } - cfgs.push_back({ Q, ne, 0.0 }); - } - } - auto cfg_time = [&](const pt_t & p, int i) { - for (const auto & c : p.cs) { if (c.Q == cfgs[i].Q && c.NE == cfgs[i].NE) { return c.t; } } - return 0.0; - }; - - // Bucket the grid points using the runtime's bucketers (via proc), so keys match fa_vec_pick. - // Short-KV points (ne11 bucket 0) are dropped — the runtime serves those from baseline. Each - // kept bucket picks a min-max-regret target (gated by the aggregate TUNE_THETA benefit gate). - struct bkt_t { int b11, b01, Ti; std::vector agg; double base_agg; - std::vector bp; }; // bp kept for the pointwise ride test below - std::set> seen; - for (const auto & p : pts) { - if (p.dk != s.dk || p.dv != s.dv) { continue; } - const int b11 = ne11_bucket(p.ne11); - if (b11 == 0) { continue; } - seen.insert({ b11, ne01_bucket(p.ne01) }); - } - std::vector bks; - for (const auto & bb : seen) { - const int b11 = bb.first, b01 = bb.second; - std::vector bp; - for (const auto & p : pts) { - if (p.dk == s.dk && p.dv == s.dv && ne11_bucket(p.ne11) == b11 && ne01_bucket(p.ne01) == b01) { - bp.push_back(&p); - } - } - std::vector agg(cfgs.size(), 0.0), worst(cfgs.size(), 0.0); - for (const auto * p : bp) { - double bestt = 0.0; - for (size_t i = 0; i < cfgs.size(); ++i) { - double t = cfg_time(*p, (int) i); - if (t > 0.0 && (bestt == 0.0 || t < bestt)) { bestt = t; } - } - for (size_t i = 0; i < cfgs.size(); ++i) { - double t = cfg_time(*p, (int) i); - agg[i] += t; - if (t > 0.0 && bestt > 0.0) { worst[i] = std::max(worst[i], t / bestt); } - } - } - int robust = 0; - for (size_t i = 1; i < cfgs.size(); ++i) { - if (worst[i] < worst[robust] || - (worst[i] == worst[robust] && (cfgs[i].Q < cfgs[robust].Q || - (cfgs[i].Q == cfgs[robust].Q && cfgs[i].NE < cfgs[robust].NE)))) { - robust = (int) i; - } - } - const bool tune = robust != base_i && agg[base_i] / agg[robust] >= TUNE_THETA; - bks.push_back({ b11, b01, tune ? robust : base_i, agg, agg[base_i], bp }); - } - - // Pointwise regret of default cfg d vs the bucket target Ti. Per-point (not aggregate): - // a ratio-of-sums lets a default that wins on aligned ne01 (8, 16) hide a large penalty - // on a misaligned point (ne01=5), so the ride/exception decision must be per point. - auto reg_pointwise = [&](const bkt_t * b, int d) { - double r = 0.0; - for (const auto * p : b->bp) { - const double td = cfg_time(*p, d), tT = cfg_time(*p, b->Ti); - if (td > 0.0 && tT > 0.0) { r = std::max(r, td / tT - 1.0); } - } - return r; - }; - - for (int dom = 0; dom <= 1; ++dom) { // 0 = decode (ne01==1), 1 = batch (ne01>=2) - std::vector db; - for (const auto & b : bks) { if ((dom == 0) == (b.b01 == 0)) { db.push_back(&b); } } - if (db.empty()) { continue; } - - // default cfg = the one minimizing (#rows, total achieved time, Q, NE) - int bestD = -1, bestRows = 1 << 30; double bestTot = 0.0; - for (size_t d = 0; d < cfgs.size(); ++d) { - int rows = ((int) d != base_i) ? 1 : 0; double tot = 0.0; - for (const auto * b : db) { - const double reg = reg_pointwise(b, (int) d); // pointwise vs bucket target - const double slow = b->agg[d] / b->base_agg - 1.0; // aggregate vs baseline (safety net) - if (reg > TUNE_TAU || slow > TUNE_TAU) { rows++; tot += b->agg[b->Ti]; } - else { tot += b->agg[d]; } - } - const bool better = bestD < 0 || rows < bestRows || - (rows == bestRows && (tot < bestTot || - (tot == bestTot && (cfgs[d].Q < cfgs[bestD].Q || - (cfgs[d].Q == cfgs[bestD].Q && cfgs[d].NE < cfgs[bestD].NE))))); - if (better) { bestD = (int) d; bestRows = rows; bestTot = tot; } - } - - const int dom_id = (dom == 0) ? 0 : 1; // FA_VEC_DOMAIN_DECODE / FA_VEC_DOMAIN_BATCH - if (bestD != base_i) { - snprintf(rbuf, sizeof(rbuf), " { { GGML_METAL_DEVICE_M4_MAX, %s, %d, %d, -1, %d }, { %d, %d } },", - type_token(type_kv), s.dk, s.dv, dom_id, cfgs[bestD].Q, cfgs[bestD].NE); - rows_out.emplace_back(rbuf); - } - for (const auto * b : db) { - const double reg = reg_pointwise(b, bestD); // pointwise vs bucket target - const double slow = b->agg[bestD] / b->base_agg - 1.0; // aggregate vs baseline - if (reg <= TUNE_TAU && slow <= TUNE_TAU) { continue; } // rides the default / baseline - snprintf(rbuf, sizeof(rbuf), " { { GGML_METAL_DEVICE_M4_MAX, %s, %d, %d, %d, %d }, { %d, %d } },", - type_token(type_kv), s.dk, s.dv, b->b11, b->b01, cfgs[b->Ti].Q, cfgs[b->Ti].NE); - rows_out.emplace_back(rbuf); - } - } - } - printf("\n // ---- %s: %zu rows ----\n", ggml_type_name(type_kv), rows_out.size()); - for (const auto & r : rows_out) { printf("%s\n", r.c_str()); } - } // for type_kv - return true; + return n_fail == 0; } static bool test_backend(ggml_backend_t backend, ggml_backend_dev_t dev, test_mode mode, const char * op_names_filter, const char * params_filter, - printer * output_printer, const char * test_file_path, int parallel_workers, bool tune_perf = false) { + printer * output_printer, const char * test_file_path, int parallel_workers) { auto filter_test_cases = [](std::vector> & test_cases, const char * params_filter) { if (params_filter == nullptr) { return; @@ -11079,9 +10675,6 @@ static bool test_backend(ggml_backend_t backend, ggml_backend_dev_t dev, test_mo case MODE_PERF: test_cases = make_test_cases_perf(); break; - case MODE_TUNE: - // MODE_TUNE routes to its own dispatch below; no generic test_cases needed - break; } } else { test_cases = make_test_cases_from_file(test_file_path); @@ -11188,7 +10781,11 @@ static bool test_backend(ggml_backend_t backend, ggml_backend_dev_t dev, test_mo output_printer->print_summary(test_summary_info(n_ok, tests_run, false)); output_printer->print_failed_tests(failed_tests); - return n_ok == tests_run; + // Metal-only: force every legal (Q,NE) on a bounded slice of shapes. Reuses the + // reference CPU backend above; a no-op on backends without the override proc. + const bool slice_ok = run_fa_vec_slice(backend, backend_cpu.get()); + + return n_ok == tests_run && slice_ok; } if (mode == MODE_GRAD) { @@ -11217,29 +10814,6 @@ static bool test_backend(ggml_backend_t backend, ggml_backend_dev_t dev, test_mo return true; } - if (mode == MODE_TUNE) { - // self-create a CPU backend with the reference implementation as golden. - // (backend_cpu in the MODE_TEST block above is out of scope here.) - ggml_backend_t backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, NULL); - GGML_ASSERT(backend_cpu != NULL); - { - using ggml_backend_cpu_set_use_ref_t = void (*)(ggml_backend_t, bool); - auto * cpu_reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_cpu)); - auto * set_use_ref = (ggml_backend_cpu_set_use_ref_t) ggml_backend_reg_get_proc_address(cpu_reg, "ggml_backend_cpu_set_use_ref"); - if (set_use_ref) { - set_use_ref(backend_cpu, true); - } - } - - // backend here is the MODE_TUNE-selected metal backend (-b MTL0) - const bool ok = tune_perf - ? run_fa_vec_tune_perf(backend) - : (run_fa_vec_drift_guard(backend, backend_cpu) && run_fa_vec_tune_check(backend, backend_cpu)); - - ggml_backend_free(backend_cpu); - return ok; - } - if (mode == MODE_SUPPORT) { // Filter out fusion cases test_cases.erase( @@ -11365,7 +10939,6 @@ static void usage(char ** argv) { printf(" - grad (compare gradients from backpropagation with method of finite differences)\n"); printf(" - perf (performance evaluation)\n"); printf(" - support (probe backend operation support)\n"); - printf(" - tune (FA vec (Q,NE) numerical correctness check vs CPU reference; --tune-perf for the perf sweep)\n"); printf(" op names for -o are as given by ggml_op_desc() (e.g. ADD, MUL_MAT, etc),\n"); printf(" optionally including the full test case string (e.g. \"ADD(type=f16,ne=[1,1,8,1],nr=[1,1,1,1],nf=1)\")\n"); printf(" --output specifies output format (default: console, options: console, sql, csv)\n"); @@ -11383,7 +10956,6 @@ int main(int argc, char ** argv) { const char * params_filter = nullptr; const char * test_file_path = nullptr; int parallel_workers = 1; - bool tune_perf = false; for (int i = 1; i < argc; i++) { if (strcmp(argv[i], "test") == 0) { @@ -11394,10 +10966,6 @@ int main(int argc, char ** argv) { mode = MODE_GRAD; } else if (strcmp(argv[i], "support") == 0) { mode = MODE_SUPPORT; - } else if (strcmp(argv[i], "tune") == 0) { - mode = MODE_TUNE; - } else if (strcmp(argv[i], "--tune-perf") == 0) { - tune_perf = true; } else if (strcmp(argv[i], "-o") == 0) { if (i + 1 < argc) { op_names_filter = argv[++i]; @@ -11505,7 +11073,7 @@ int main(int argc, char ** argv) { false, "", ggml_backend_dev_description(dev), total / 1024 / 1024, free / 1024 / 1024, true)); - bool ok = test_backend(backend.get(), dev, mode, op_names_filter, params_filter, output_printer.get(), test_file_path, parallel_workers, tune_perf); + bool ok = test_backend(backend.get(), dev, mode, op_names_filter, params_filter, output_printer.get(), test_file_path, parallel_workers); if (ok) { n_ok++; From ea337d0cda6114cc40cb776d4261d0c979ad058d Mon Sep 17 00:00:00 2001 From: forforever73 <690105611@qq.com> Date: Sun, 2 Aug 2026 00:15:20 +0800 Subject: [PATCH 09/14] tools : document the Metal tuner, point the table comment at it --- ggml/src/ggml-metal/ggml-metal-tuning.cpp | 2 +- ggml/src/ggml-metal/ggml-metal-tuning.h | 5 +- tools/tuning/README.md | 61 +++++++++++++++++++++++ 3 files changed, 65 insertions(+), 3 deletions(-) create mode 100644 tools/tuning/README.md diff --git a/ggml/src/ggml-metal/ggml-metal-tuning.cpp b/ggml/src/ggml-metal/ggml-metal-tuning.cpp index 2bf9bb5b48d..6eb56a7ef2d 100644 --- a/ggml/src/ggml-metal/ggml-metal-tuning.cpp +++ b/ggml/src/ggml-metal/ggml-metal-tuning.cpp @@ -62,7 +62,7 @@ fa_vec_cfg_t fa_vec_baseline_cfg(int dk, int dv) { return { 1, (int8_t) fa_vec_baseline_ne(dk, dv) }; } -// Generated by `test-backend-ops tune --tune-perf`; do not hand-edit. +// Generated by `ggml-metal-tuning fa-vec`; do not hand-edit. // One row per kept bucket, plus per-(dtype,dk,dv) ne11-collapsed domain defaults // (ne11_b = FA_VEC_NE11_DEFAULT, ne01_b = domain). To retune or add a device, re-run the // sweep and paste its block. See ggml-metal-tuning.h for the row/lookup semantics. diff --git a/ggml/src/ggml-metal/ggml-metal-tuning.h b/ggml/src/ggml-metal/ggml-metal-tuning.h index 6ea948b4077..8f1c2b238ca 100644 --- a/ggml/src/ggml-metal/ggml-metal-tuning.h +++ b/ggml/src/ggml-metal/ggml-metal-tuning.h @@ -18,8 +18,9 @@ int fa_vec_ne11_bucket(int64_t ne11); int fa_vec_ne01_bucket(int64_t ne01); // NE baked into each (dk,dv) baseline instantiation in kernels/fa.metal. -// Hand-maintained mirror; keep in sync with those instantiations (run_fa_vec_tune_check -// exercises every (Q,NE), so a missing instantiation surfaces there). +// Hand-maintained mirror; keep in sync with those instantiations. test-backend-ops forces +// every legal (Q,NE) on dk=128 and dk=576 under Metal, so a missing instantiation for +// those two surfaces there; the other head sizes are only covered by the offline tuner. int fa_vec_baseline_ne(int dk, int dv); // Tuned table has two row kinds. Exact rows key a (ne11_b, ne01_b) bucket. Default rows diff --git a/tools/tuning/README.md b/tools/tuning/README.md new file mode 100644 index 00000000000..e22f2842bf4 --- /dev/null +++ b/tools/tuning/README.md @@ -0,0 +1,61 @@ +# ggml-metal-tuning + +Offline kernel tuner for the Metal backend. +It sweeps a kernel's config grid on the machine it runs on and prints pasteable table rows for `ggml/src/ggml-metal/ggml-metal-tuning.cpp`. + +This is not a test: it never reports pass/fail on performance. +A non-zero exit code means bad arguments or a wrong environment (no Metal device, missing proc bridges), never a perf result. + +| tuner | tunes | table | +|---|---|---| +| `fa-vec` | flash-attn vec `(Q, NE)` per `(dtype, head size, KV depth, batch width)` | `fa_vec_tuned_table` | + +## Adding a device to the FA-vec table + +Build on the target machine: + +```bash +cmake -B build -DGGML_METAL=ON +cmake --build build --target ggml-metal-tuning -j +cmake --build build --target test-backend-ops -j +``` + +Sweep the grid (6 dtypes x 10 head sizes x 4 KV depths x 9 batch widths; a few hours): + +```bash +./build/bin/ggml-metal-tuning fa-vec > fa_vec_rows.txt 2> fa_vec_sweep.log +``` + +`fa_vec_rows.txt` is the finished table block, ready to paste: the min-max-regret target, the aggregate benefit gate, the short-KV drop and the pointwise compression are already applied. +`fa_vec_sweep.log` holds the per-cell timings, bucket coverage, noise floor and any cooldown activity. +Post both: the log is what makes the rows reviewable. + +Long sweeps can be split. +`--dtype f16,q4_0` and `--dk 128,192` restrict the grid, and the emitted rows for one `(dtype, head size)` do not depend on the others. + +Then validate the numerics, where Metal is compared against the CPU reference: + +```bash +./build/bin/test-backend-ops test -o FLASH_ATTN_EXT -b MTL0 +``` + +This forces every legal `(Q, NE)` on `dk=128` and `dk=576`. +The tuner itself does no numerical checks, so the other head sizes have no automated numerical coverage. + +If the device is not in `enum ggml_metal_device_id` yet, register it in `ggml/src/ggml-metal/ggml-metal-device.{h,m}` first. +The tuner emits whatever token the runtime reports for the machine, so an unregistered device emits `GGML_METAL_DEVICE_GENERIC` and its rows would apply to every unknown device. + +## Thermal throttling + +Long sweeps heat the GPU, and a throttled measurement is indistinguishable from a slow kernel. +The tuner re-measures a fixed baseline config every four candidates as an anchor. +When the anchor drifts more than `--cool-drift` (10% by default) from the coolest anchor seen in that cell, the tuner: + +1. discards every candidate measured since the last clean anchor, +2. sleeps with exponential backoff until the anchor comes back within `--cool-eps` (3%), +3. re-measures the discarded candidates. + +If it cannot cool down within `--cool-max-wait` seconds, or a cell needs more than `--cool-max-retry` rounds, that cell is dropped from the table and reported on stderr. + +`--no-cooldown` only warns on drift and keeps the measurement. +Use it to reproduce a sweep taken without cooling. From f801d5db31282ff2ac8ed9a2b848aa79790f832c Mon Sep 17 00:00:00 2001 From: forforever73 <690105611@qq.com> Date: Sun, 2 Aug 2026 12:02:07 +0800 Subject: [PATCH 10/14] abort on unknown KV type, single-source fa_vec_legal_ne --- ggml/src/ggml-metal/ggml-metal-tuning.h | 14 ++++++++++++++ tests/test-backend-ops.cpp | 4 +++- tools/tuning/CMakeLists.txt | 1 + tools/tuning/bench.cpp | 2 +- tools/tuning/fa-vec.cpp | 19 ++++--------------- 5 files changed, 23 insertions(+), 17 deletions(-) diff --git a/ggml/src/ggml-metal/ggml-metal-tuning.h b/ggml/src/ggml-metal/ggml-metal-tuning.h index 8f1c2b238ca..fd9a7ffd948 100644 --- a/ggml/src/ggml-metal/ggml-metal-tuning.h +++ b/ggml/src/ggml-metal/ggml-metal-tuning.h @@ -4,6 +4,7 @@ #include "ggml.h" #include +#include namespace ggml_metal_tuning { @@ -52,6 +53,19 @@ struct fa_vec_entry_t { fa_vec_cfg_t cfg; }; +// legal NE values for a (dk,dv): NL = 32/NE, require (dk/4)%NL==0 && (dv/4)%NL==0. +// single source shared by the offline tuner and test-backend-ops. +inline std::vector fa_vec_legal_ne(int dk, int dv) { + std::vector r; + for (int ne : { 1, 2, 4 }) { + const int nl = 32 / ne; + if ((dk/4) % nl == 0 && (dv/4) % nl == 0) { + r.push_back(ne); + } + } + return r; +} + // test/tune-only override; when set, fa_vec_pick returns it directly. void fa_vec_set_override(fa_vec_cfg_t cfg); void fa_vec_clear_override(); diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index f26863a57a0..0a546535318 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -10578,7 +10578,8 @@ static std::vector> make_test_cases_from_file(const c using set_fa_vec_override_t = void (*)(int, int); using clear_fa_vec_override_t = void (*)(void); -// legal NE for a (dk,dv): NL = 32/NE, require (dk/4)%NL==0 && (dv/4)%NL==0 +// legal NE for a (dk,dv): NL = 32/NE, require (dk/4)%NL==0 && (dv/4)%NL==0. +// keep in sync with ggml_metal_tuning::fa_vec_legal_ne in ggml-metal-tuning.h (used by the tool) static std::vector fa_vec_legal_ne(int dk, int dv) { std::vector r; for (int ne : {1, 2, 4}) { @@ -10595,6 +10596,7 @@ static std::vector fa_vec_legal_ne(int dk, int dv) { // per test case in the backend-agnostic case list. Covers padded rows (ne01 % Q != 0), // per-qq sinks, kvpad, the nsg-dependent shmem offsets / parallel-reduce stride // (ne11 -> nsg 1/2/4) and the quantized dequant-once path. +// single-threaded; g_override_set is backend-global. called only after all parallel workers have joined. static bool run_fa_vec_slice(ggml_backend_t backend, ggml_backend_t backend_cpu) { auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend)); diff --git a/tools/tuning/CMakeLists.txt b/tools/tuning/CMakeLists.txt index a6909c2307f..39ff0018026 100644 --- a/tools/tuning/CMakeLists.txt +++ b/tools/tuning/CMakeLists.txt @@ -3,6 +3,7 @@ set(TARGET ggml-metal-tuning) add_executable(${TARGET} main.cpp bench.cpp fa-vec.cpp) target_link_libraries(${TARGET} PRIVATE ggml ${CMAKE_THREAD_LIBS_INIT}) target_compile_features(${TARGET} PRIVATE cxx_std_17) +target_include_directories(${TARGET} PRIVATE ${CMAKE_SOURCE_DIR}/ggml/src/ggml-metal) if(LLAMA_TOOLS_INSTALL) install(TARGETS ${TARGET} RUNTIME) diff --git a/tools/tuning/bench.cpp b/tools/tuning/bench.cpp index fa51d29e89e..38527ffca14 100644 --- a/tools/tuning/bench.cpp +++ b/tools/tuning/bench.cpp @@ -153,7 +153,7 @@ cell_result measure_cell(ggml_backend_t backend, const perf_cell & cell, int rep for (size_t i = 0; i < order.size(); ++i) { res.t[order[i]] = measure_one(backend, cell, reps, set_cand, clear_cand, order[i]); - if (i % 4 != 0) { + if (i % 4 != 0) { // re-check anchor every 4 candidates: balances drift detection latency against overhead continue; } diff --git a/tools/tuning/fa-vec.cpp b/tools/tuning/fa-vec.cpp index ac235920b1f..828920acc9f 100644 --- a/tools/tuning/fa-vec.cpp +++ b/tools/tuning/fa-vec.cpp @@ -3,6 +3,7 @@ #include "ggml.h" #include "ggml-backend.h" +#include "ggml-metal-tuning.h" #include #include @@ -155,7 +156,7 @@ static void fa_init_kq_mask(ggml_tensor * t, std::mt19937 & rng, float min, floa static unsigned fa_cell_seed(const fa_shape & s, unsigned base) { unsigned h = base; for (int v : { s.dk, s.dv, s.ne01, s.ne11, (int) s.type_kv }) { - h = h*1000003u + (unsigned) v; + h = h*1000003u + (unsigned) v; // small prime, standard multiplicative hash mixing } return h; } @@ -175,18 +176,6 @@ static void fa_init_tensors(ggml_context * ctx, const fa_shape & s, unsigned bas } } -// legal NE for a (dk,dv): NL = 32/NE, require (dk/4)%NL==0 && (dv/4)%NL==0 -static std::vector fa_legal_ne(int dk, int dv) { - std::vector r; - for (int ne : { 1, 2, 4 }) { - const int nl = 32 / ne; - if ((dk/4) % nl == 0 && (dv/4) % nl == 0) { - r.push_back(ne); - } - } - return r; -} - using set_override_t = void (*)(int, int); using clear_override_t = void (*)(void); using bucket_t = int (*)(int64_t); @@ -227,7 +216,7 @@ static const char * fa_type_token(ggml_type t) { case GGML_TYPE_Q5_0: return "GGML_TYPE_Q5_0"; case GGML_TYPE_Q5_1: return "GGML_TYPE_Q5_1"; case GGML_TYPE_Q8_0: return "GGML_TYPE_Q8_0"; - default: return "GGML_TYPE_F16"; + default: GGML_ABORT("unhandled KV type in fa_type_token: %d", (int) t); } } @@ -256,7 +245,7 @@ static std::vector fa_build_cands(const fa_procs & procs, int dk, int d std::vector cands; base_i = -1; - for (int ne : fa_legal_ne(dk, dv)) { + for (int ne : ggml_metal_tuning::fa_vec_legal_ne(dk, dv)) { for (int Q : { 1, 2, 4 }) { if (Q == 1 && ne == base_ne) { base_i = (int) cands.size(); From 26bf53d829d4e77c3cdb7a8f70e37e3ded45e9c4 Mon Sep 17 00:00:00 2001 From: forforever73 <690105611@qq.com> Date: Mon, 3 Aug 2026 11:44:22 +0800 Subject: [PATCH 11/14] cleanup --- ggml/src/ggml-metal/ggml-metal-tuning.h | 7 +- tests/test-backend-ops.cpp | 14 +- tools/tuning/bench.cpp | 91 ++++---- tools/tuning/bench.h | 54 ++--- tools/tuning/fa-vec.cpp | 293 ++++++++++++------------ tools/tuning/fa-vec.h | 20 +- tools/tuning/main.cpp | 7 +- 7 files changed, 234 insertions(+), 252 deletions(-) diff --git a/ggml/src/ggml-metal/ggml-metal-tuning.h b/ggml/src/ggml-metal/ggml-metal-tuning.h index fd9a7ffd948..640ce53efba 100644 --- a/ggml/src/ggml-metal/ggml-metal-tuning.h +++ b/ggml/src/ggml-metal/ggml-metal-tuning.h @@ -19,9 +19,8 @@ int fa_vec_ne11_bucket(int64_t ne11); int fa_vec_ne01_bucket(int64_t ne01); // NE baked into each (dk,dv) baseline instantiation in kernels/fa.metal. -// Hand-maintained mirror; keep in sync with those instantiations. test-backend-ops forces -// every legal (Q,NE) on dk=128 and dk=576 under Metal, so a missing instantiation for -// those two surfaces there; the other head sizes are only covered by the offline tuner. +// Hand-maintained mirror; keep in sync with those instantiations. +// The Metal test slice covers every legal config for dk=128 and dk=576. int fa_vec_baseline_ne(int dk, int dv); // Tuned table has two row kinds. Exact rows key a (ne11_b, ne01_b) bucket. Default rows @@ -59,7 +58,7 @@ inline std::vector fa_vec_legal_ne(int dk, int dv) { std::vector r; for (int ne : { 1, 2, 4 }) { const int nl = 32 / ne; - if ((dk/4) % nl == 0 && (dv/4) % nl == 0) { + if ((dk / 4) % nl == 0 && (dv / 4) % nl == 0) { r.push_back(ne); } } diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 0a546535318..598b75ad93a 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -10574,12 +10574,10 @@ static std::vector> make_test_cases_from_file(const c } // ---- FA vec (Q,NE): forced-config numerical slice (Metal only) ---- -// metal proc_address bridges (resolved by string, not symbol linkage) using set_fa_vec_override_t = void (*)(int, int); using clear_fa_vec_override_t = void (*)(void); -// legal NE for a (dk,dv): NL = 32/NE, require (dk/4)%NL==0 && (dv/4)%NL==0. -// keep in sync with ggml_metal_tuning::fa_vec_legal_ne in ggml-metal-tuning.h (used by the tool) +// NL = 32/NE must divide both dk/4 and dv/4. static std::vector fa_vec_legal_ne(int dk, int dv) { std::vector r; for (int ne : {1, 2, 4}) { @@ -10591,12 +10589,8 @@ static std::vector fa_vec_legal_ne(int dk, int dv) { return r; } -// Forces every legal (Q,NE) on two representative shapes and compares against the CPU -// reference. Metal-only: the override is a backend-global switch, so it cannot be expressed -// per test case in the backend-agnostic case list. Covers padded rows (ne01 % Q != 0), -// per-qq sinks, kvpad, the nsg-dependent shmem offsets / parallel-reduce stride -// (ne11 -> nsg 1/2/4) and the quantized dequant-once path. -// single-threaded; g_override_set is backend-global. called only after all parallel workers have joined. +// Covers padded rows, sinks, kvpad, multi-SIMDgroup reduction, quantized K/V, and MLA views. +// The override is backend-global, so this runs after all parallel workers have joined. static bool run_fa_vec_slice(ggml_backend_t backend, ggml_backend_t backend_cpu) { auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend)); @@ -10783,8 +10777,6 @@ static bool test_backend(ggml_backend_t backend, ggml_backend_dev_t dev, test_mo output_printer->print_summary(test_summary_info(n_ok, tests_run, false)); output_printer->print_failed_tests(failed_tests); - // Metal-only: force every legal (Q,NE) on a bounded slice of shapes. Reuses the - // reference CPU backend above; a no-op on backends without the override proc. const bool slice_ok = run_fa_vec_slice(backend, backend_cpu.get()); return n_ok == tests_run && slice_ok; diff --git a/tools/tuning/bench.cpp b/tools/tuning/bench.cpp index 38527ffca14..59945506c3f 100644 --- a/tools/tuning/bench.cpp +++ b/tools/tuning/bench.cpp @@ -7,14 +7,16 @@ #include #include -perf_cell build_perf_cell(ggml_backend_t backend, const build_graph_fn & build, - const init_tensors_fn & init, const op_flops_fn & flops) { +perf_cell build_perf_cell(ggml_backend_t backend, + const build_graph_fn & build, + const init_tensors_fn & init, + const op_flops_fn & flops) { perf_cell cell; const size_t graph_nodes = 1024; ggml_init_params params = { - /* .mem_size = */ ggml_tensor_overhead()*128 + ggml_graph_overhead_custom(graph_nodes, false), + /* .mem_size = */ ggml_tensor_overhead() * 128 + ggml_graph_overhead_custom(graph_nodes, false), /* .mem_base = */ NULL, /* .no_alloc = */ true, }; @@ -28,7 +30,7 @@ perf_cell build_perf_cell(ggml_backend_t backend, const build_graph_fn & build, } cell.buf.reset(ggml_backend_alloc_ctx_tensors(cell.ctx.get(), backend)); - if (cell.buf == NULL) { + if (!cell.buf) { return cell; } @@ -38,24 +40,24 @@ perf_cell build_perf_cell(ggml_backend_t backend, const build_graph_fn & build, ggml_build_forward_expand(cell.gf, out); // replicate the op to amortize overhead (target ~50 GFLOP/compute, capped to bound graph size) - cell.n_runs = 1; - if (flops(out) > 0) { + cell.n_runs = 1; + const uint64_t n_flops = flops(out); + if (n_flops > 0) { const uint64_t target_flops = 50ULL * 1000 * 1000 * 1000; const int cap = 512; - const int by_flops = (int) std::min(cap, (int64_t) (target_flops / flops(out))); - cell.n_runs = std::max(1, std::min(by_flops, (int) (ggml_graph_size(cell.gf) - ggml_graph_n_nodes(cell.gf)))); + const int by_flops = (int) std::min(cap, (int64_t) (target_flops / n_flops)); + cell.n_runs = + std::max(1, std::min(by_flops, (int) (ggml_graph_size(cell.gf) - ggml_graph_n_nodes(cell.gf)))); } for (int i = 1; i < cell.n_runs; ++i) { ggml_graph_add_node(cell.gf, out); } - cell.ok = true; - return cell; } double time_cell_median(ggml_backend_t backend, const perf_cell & cell, int reps) { - if (!cell.ok) { + if (cell.gf == nullptr) { return -1.0; } @@ -70,15 +72,17 @@ double time_cell_median(ggml_backend_t backend, const perf_cell & cell, int reps ggml_backend_synchronize(backend); samples.push_back((double) (ggml_time_us() - t0)); } - std::nth_element(samples.begin(), samples.begin() + samples.size()/2, samples.end()); + std::nth_element(samples.begin(), samples.begin() + samples.size() / 2, samples.end()); - return samples[samples.size()/2] / cell.n_runs; + return samples[samples.size() / 2] / cell.n_runs; } -// times one candidate and returns its time; -1 on failure -static double measure_one(ggml_backend_t backend, const perf_cell & cell, int reps, - const set_candidate_fn & set_cand, const clear_candidate_fn & clear_cand, - int cand) { +static double measure_one(ggml_backend_t backend, + const perf_cell & cell, + int reps, + const set_candidate_fn & set_cand, + const clear_candidate_fn & clear_cand, + int cand) { set_cand(cand); const double t = time_cell_median(backend, cell, reps); clear_cand(); @@ -88,17 +92,22 @@ static double measure_one(ggml_backend_t backend, const perf_cell & cell, int re // waits for the anchor to come back within eps of anchor_ref, with exponential backoff. // returns the converged anchor, or -1 if it never converged within max_wait. -static double cool_until_steady(ggml_backend_t backend, const perf_cell & cell, int reps, - const set_candidate_fn & set_cand, const clear_candidate_fn & clear_cand, - int baseline_cand, double & anchor_ref, const cooldown_opts & cool, - const char * cell_label) { +static double cool_until_steady(ggml_backend_t backend, + const perf_cell & cell, + int reps, + const set_candidate_fn & set_cand, + const clear_candidate_fn & clear_cand, + int baseline_cand, + double & anchor_ref, + const cooldown_opts & cool, + const char * cell_label) { int total_wait = 0; - for (int sleep_s = 2; total_wait < cool.max_wait; sleep_s = std::min(sleep_s*2, 32)) { + for (int sleep_s = 2; total_wait < cool.max_wait; sleep_s = std::min(sleep_s * 2, 32)) { const int this_wait = std::min(sleep_s, cool.max_wait - total_wait); - fprintf(stderr, "# COOL sleeping %ds (%ds/%ds) %s\n", - this_wait, total_wait + this_wait, cool.max_wait, cell_label); + fprintf(stderr, "# COOL sleeping %ds (%ds/%ds) %s\n", this_wait, total_wait + this_wait, cool.max_wait, + cell_label); std::this_thread::sleep_for(std::chrono::seconds(this_wait)); total_wait += this_wait; @@ -112,7 +121,7 @@ static double cool_until_steady(ggml_backend_t backend, const perf_cell & cell, anchor_ref = a; } - if (a <= anchor_ref*(1.0 + cool.eps)) { + if (a <= anchor_ref * (1.0 + cool.eps)) { fprintf(stderr, "# COOL steady after %ds %s\n", total_wait, cell_label); return a; } @@ -123,14 +132,17 @@ static double cool_until_steady(ggml_backend_t backend, const perf_cell & cell, return -1.0; } -cell_result measure_cell(ggml_backend_t backend, const perf_cell & cell, int reps, - int n_cands, const std::vector & order, - const set_candidate_fn & set_cand, +cell_result measure_cell(ggml_backend_t backend, + const perf_cell & cell, + int reps, + const std::vector & order, + const set_candidate_fn & set_cand, const clear_candidate_fn & clear_cand, - int baseline_cand, const cooldown_opts & cool, - const char * cell_label) { + int baseline_cand, + const cooldown_opts & cool, + const char * cell_label) { cell_result res; - res.t.assign(n_cands, 0.0); + res.t.assign(order.size(), 0.0); double anchor_ref = 0.0; @@ -140,8 +152,8 @@ cell_result measure_cell(ggml_backend_t backend, const perf_cell & cell, int rep std::vector> anchors; auto window_start = [&]() -> size_t { - for (size_t i = anchors.size(); i-- > 0; ) { - if (anchors[i].first <= anchor_ref*(1.0 + cool.eps)) { + for (size_t i = anchors.size(); i-- > 0;) { + if (anchors[i].first <= anchor_ref * (1.0 + cool.eps)) { return anchors[i].second; } } @@ -153,7 +165,7 @@ cell_result measure_cell(ggml_backend_t backend, const perf_cell & cell, int rep for (size_t i = 0; i < order.size(); ++i) { res.t[order[i]] = measure_one(backend, cell, reps, set_cand, clear_cand, order[i]); - if (i % 4 != 0) { // re-check anchor every 4 candidates: balances drift detection latency against overhead + if (i % 4 != 0) { continue; } @@ -171,7 +183,7 @@ cell_result measure_cell(ggml_backend_t backend, const perf_cell & cell, int rep continue; } - const double drift = std::fabs(a - anchor_ref)/anchor_ref; + const double drift = std::fabs(a - anchor_ref) / anchor_ref; // a cooler anchor than any so far becomes the reference: whatever was measured // before it was measured on a hotter machine @@ -184,7 +196,7 @@ cell_result measure_cell(ggml_backend_t backend, const perf_cell & cell, int rep continue; } - fprintf(stderr, "# WARN throttling? anchor drift %.1f%% %s\n", 100.0*drift, cell_label); + fprintf(stderr, "# WARN throttling? anchor drift %.1f%% %s\n", 100.0 * drift, cell_label); if (!cool.enabled) { anchors.push_back({ a, i }); @@ -199,10 +211,8 @@ cell_result measure_cell(ggml_backend_t backend, const perf_cell & cell, int rep const size_t dirty_from = window_start(); - res.n_cooldowns++; - - const double a_cool = cool_until_steady(backend, cell, reps, set_cand, clear_cand, - baseline_cand, anchor_ref, cool, cell_label); + const double a_cool = + cool_until_steady(backend, cell, reps, set_cand, clear_cand, baseline_cand, anchor_ref, cool, cell_label); if (a_cool <= 0.0) { res.trusted = false; return res; @@ -217,7 +227,6 @@ cell_result measure_cell(ggml_backend_t backend, const perf_cell & cell, int rep fprintf(stderr, "# REDO candidates %zu..%zu %s\n", dirty_from, i, cell_label); for (size_t j = dirty_from; j <= i; ++j) { res.t[order[j]] = measure_one(backend, cell, reps, set_cand, clear_cand, order[j]); - res.n_remeasures++; } } diff --git a/tools/tuning/bench.h b/tools/tuning/bench.h index 6349d970bd5..10167ce39f3 100644 --- a/tools/tuning/bench.h +++ b/tools/tuning/bench.h @@ -1,67 +1,57 @@ #pragma once -#include "ggml.h" #include "ggml-backend.h" #include "ggml-cpp.h" +#include "ggml.h" #include #include #include -// one prebuilt op graph, replicated n_runs times so a single graph_compute amortizes -// dispatch/sync overhead. reused across candidates: an override only changes which -// pipeline is picked at encode time, so the (large) input tensors stay allocated. +// A prebuilt graph replicated to amortize dispatch and synchronization overhead. struct perf_cell { ggml_context_ptr ctx; ggml_backend_buffer_ptr buf; ggml_cgraph * gf = nullptr; int n_runs = 0; - bool ok = false; }; -// builds the op graph for one shape. returns the output tensor, or null if unsupported. -using build_graph_fn = std::function; -// fills the allocated tensors of ctx with input data +using build_graph_fn = std::function; using init_tensors_fn = std::function; -// flops of one op instance, used to size n_runs -using op_flops_fn = std::function; +using op_flops_fn = std::function; -perf_cell build_perf_cell(ggml_backend_t backend, const build_graph_fn & build, - const init_tensors_fn & init, const op_flops_fn & flops); +perf_cell build_perf_cell(ggml_backend_t backend, + const build_graph_fn & build, + const init_tensors_fn & init, + const op_flops_fn & flops); -// median per-op time (us) over the prebuilt cell for whatever config is currently set double time_cell_median(ggml_backend_t backend, const perf_cell & cell, int reps); struct cooldown_opts { bool enabled = true; double drift = 0.10; // anchor drift that triggers a cooldown double eps = 0.03; // anchor tolerance to call the GPU cool again - int max_wait = 120; // seconds of cooling per cell before giving up - int max_retry = 2; // re-measure rounds per cell before giving up + int max_wait = 120; // seconds of cooling per cell before giving up + int max_retry = 2; // re-measure rounds per cell before giving up }; -// applies candidate i (an index into the tuner's own candidate list) -using set_candidate_fn = std::function; -// undoes the last set_candidate +using set_candidate_fn = std::function; using clear_candidate_fn = std::function; -// giving up on a cell returns early with trusted == false, so a trusted cell is one every -// candidate of was measured; callers may still see a non-positive t[] from a failed measure. struct cell_result { - std::vector t; // time (us) per candidate index, <= 0 if not measured - bool trusted = true; // false -> caller must drop this cell + std::vector t; + bool trusted = true; double anchor_min = 0.0; double anchor_max = 0.0; - int n_cooldowns = 0; - int n_remeasures = 0; }; -// times every candidate over the prebuilt cell, re-measuring a periodic baseline anchor -// to watch for thermal drift. order[] gives the (shuffled) visiting order; baseline_cand is -// the candidate the anchor forces, so drift is measured against a config the tuner controls. -cell_result measure_cell(ggml_backend_t backend, const perf_cell & cell, int reps, - int n_cands, const std::vector & order, - const set_candidate_fn & set_cand, +// Times candidates in order while using baseline_cand as a thermal-drift anchor. +cell_result measure_cell(ggml_backend_t backend, + const perf_cell & cell, + int reps, + const std::vector & order, + const set_candidate_fn & set_cand, const clear_candidate_fn & clear_cand, - int baseline_cand, const cooldown_opts & cool, - const char * cell_label); + int baseline_cand, + const cooldown_opts & cool, + const char * cell_label); diff --git a/tools/tuning/fa-vec.cpp b/tools/tuning/fa-vec.cpp index 828920acc9f..2dd1c95be51 100644 --- a/tools/tuning/fa-vec.cpp +++ b/tools/tuning/fa-vec.cpp @@ -1,9 +1,9 @@ #include "fa-vec.h" -#include "bench.h" -#include "ggml.h" +#include "bench.h" #include "ggml-backend.h" #include "ggml-metal-tuning.h" +#include "ggml.h" #include #include @@ -16,9 +16,9 @@ // GQA spec-decode shape: enough query heads to keep the GPU busy so the Q>1 K/V-reuse // benefit is visible. nh KV heads, nr2 query heads each, nr3 batches. -static const int FA_NH = 4; -static const int FA_NR2 = 8; -static const int FA_NR3 = 1; +static const int FA_NH = 4; +static const int FA_NR2 = 8; +static const int FA_NR3 = 1; struct fa_shape { int dk; @@ -34,13 +34,12 @@ static ggml_tensor * fa_build_graph(ggml_context * ctx, const fa_shape & s) { const int64_t dk_padded = GGML_PAD(s.dk, ggml_blck_size(s.type_kv)); const int64_t dv_padded = GGML_PAD(s.dv, ggml_blck_size(s.type_kv)); - ggml_tensor * q = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, dk_padded, s.ne01, FA_NH*FA_NR2, FA_NR3); + ggml_tensor * q = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, dk_padded, s.ne01, FA_NH * FA_NR2, FA_NR3); ggml_set_name(q, "q"); // K/V are views of a 2x-tall parent, as they are of the KV cache in production - ggml_tensor * k0 = ggml_new_tensor_4d(ctx, s.type_kv, dk_padded, 2*s.ne11, FA_NH, FA_NR3); - ggml_tensor * k = ggml_view_4d(ctx, k0, dk_padded, s.ne11, FA_NH, FA_NR3, - k0->nb[1], k0->nb[2], k0->nb[3], 0); + ggml_tensor * k0 = ggml_new_tensor_4d(ctx, s.type_kv, dk_padded, 2 * s.ne11, FA_NH, FA_NR3); + ggml_tensor * k = ggml_view_4d(ctx, k0, dk_padded, s.ne11, FA_NH, FA_NR3, k0->nb[1], k0->nb[2], k0->nb[3], 0); ggml_set_name(k, "k"); ggml_tensor * v = nullptr; @@ -48,16 +47,15 @@ static ggml_tensor * fa_build_graph(ggml_context * ctx, const fa_shape & s) { // MLA: the V cache is a sub-view of the K cache v = ggml_view_4d(ctx, k, dv_padded, s.ne11, FA_NH, FA_NR3, k->nb[1], k->nb[2], k->nb[3], 0); } else { - ggml_tensor * v0 = ggml_new_tensor_4d(ctx, s.type_kv, dv_padded, 2*s.ne11, FA_NH, FA_NR3); - v = ggml_view_4d(ctx, v0, dv_padded, s.ne11, FA_NH, FA_NR3, - v0->nb[1], v0->nb[2], v0->nb[3], 0); + ggml_tensor * v0 = ggml_new_tensor_4d(ctx, s.type_kv, dv_padded, 2 * s.ne11, FA_NH, FA_NR3); + v = ggml_view_4d(ctx, v0, dv_padded, s.ne11, FA_NH, FA_NR3, v0->nb[1], v0->nb[2], v0->nb[3], 0); } ggml_set_name(v, "v"); ggml_tensor * m = ggml_new_tensor_4d(ctx, GGML_TYPE_F16, s.ne11, s.ne01, 1, FA_NR3); ggml_set_name(m, "m"); - ggml_tensor * out = ggml_flash_attn_ext(ctx, q, k, v, m, 1.0f/sqrtf((float) s.dk), 0.0f, 0.0f); + ggml_tensor * out = ggml_flash_attn_ext(ctx, q, k, v, m, 1.0f / sqrtf((float) s.dk), 0.0f, 0.0f); ggml_flash_attn_ext_set_prec(out, GGML_PREC_F32); ggml_set_name(out, "out"); @@ -66,21 +64,20 @@ static ggml_tensor * fa_build_graph(ggml_context * ctx, const fa_shape & s) { static uint64_t fa_op_flops(const fa_shape & s) { // Q*K^T is ne01 x dk x ne11, P*V is ne01 x ne11 x dv, per head - return (uint64_t) 2*FA_NH*FA_NR2*s.ne01*(s.dk + s.dv)*s.ne11*FA_NR3; + return (uint64_t) 2 * FA_NH * FA_NR2 * s.ne01 * (s.dk + s.dv) * s.ne11 * FA_NR3; } -// mirrors init_tensor_uniform: uniform f32 data, quantized in place for quantized types static void fa_init_uniform(ggml_tensor * t, std::mt19937 & rng, float min, float max) { const size_t nels = ggml_nelements(t); - std::vector data(nels); + std::vector data(nels); std::uniform_real_distribution dist(min, max); for (size_t i = 0; i < nels; i++) { data[i] = dist(rng); } if (t->type == GGML_TYPE_F32) { - ggml_backend_tensor_set(t, data.data(), 0, nels*sizeof(float)); + ggml_backend_tensor_set(t, data.data(), 0, nels * sizeof(float)); return; } @@ -88,10 +85,10 @@ static void fa_init_uniform(ggml_tensor * t, std::mt19937 & rng, float min, floa GGML_ASSERT(nels % ggml_blck_size(t->type) == 0); std::vector imatrix(t->ne[0], 1.0f); - const float * im = imatrix.data(); + const float * im = imatrix.data(); if (!ggml_quantize_requires_imatrix(t->type)) { // when the imatrix is optional, exercise both paths; pick via one of the random numbers - if (data[0] > 0.5f*(min + max)) { + if (data[0] > 0.5f * (min + max)) { im = nullptr; } } @@ -116,8 +113,8 @@ static void fa_init_kq_mask(ggml_tensor * t, std::mt19937 & rng, float min, floa const int32_t ne2 = (int32_t) t->ne[2]; const int32_t ne3 = (int32_t) t->ne[3]; - std::vector data_f32(size_t(ne0)*ne1*ne2*ne3); - std::vector data_f16(size_t(ne0)*ne1*ne2*ne3); + std::vector data_f32(size_t(ne0) * ne1 * ne2 * ne3); + std::vector data_f16(size_t(ne0) * ne1 * ne2 * ne3); std::uniform_real_distribution dis(min, max); for (size_t i = 0; i < data_f32.size(); i++) { @@ -127,7 +124,7 @@ static void fa_init_kq_mask(ggml_tensor * t, std::mt19937 & rng, float min, floa const int blck0 = 128; const int blck1 = 64; - const int n_inf_zero_blocks = 0.2*(ne0*ne1*ne2*ne3)/(blck0*blck1); + const int n_inf_zero_blocks = 0.2 * (ne0 * ne1 * ne2 * ne3) / (blck0 * blck1); for (int b = 0; b < n_inf_zero_blocks; b++) { const int p3 = (int) (rng() % ne3); @@ -138,7 +135,7 @@ static void fa_init_kq_mask(ggml_tensor * t, std::mt19937 & rng, float min, floa const bool inf = rng() & 1; for (int i1 = 0; i1 < blck1 && p1 + i1 < ne1; i1++) { - const int idx = p3*ne2*ne1*ne0 + p2*ne1*ne0 + (p1 + i1)*ne0 + p0; + const int idx = p3 * ne2 * ne1 * ne0 + p2 * ne1 * ne0 + (p1 + i1) * ne0 + p0; for (int i0 = 0; i0 < blck0 && p0 + i0 < ne0; i0++) { data_f32[idx + i0] = inf ? -INFINITY : 0.0f; @@ -146,17 +143,15 @@ static void fa_init_kq_mask(ggml_tensor * t, std::mt19937 & rng, float min, floa } } - ggml_fp32_to_fp16_row(data_f32.data(), data_f16.data(), ne0*ne1*ne2*ne3); + ggml_fp32_to_fp16_row(data_f32.data(), data_f16.data(), ne0 * ne1 * ne2 * ne3); - ggml_backend_tensor_set(t, data_f16.data(), 0, data_f16.size()*sizeof(ggml_fp16_t)); + ggml_backend_tensor_set(t, data_f16.data(), 0, data_f16.size() * sizeof(ggml_fp16_t)); } -// per-cell deterministic seed: the shape decides it, so a cell is reproducible -// regardless of what else the sweep visited before it static unsigned fa_cell_seed(const fa_shape & s, unsigned base) { unsigned h = base; for (int v : { s.dk, s.dv, s.ne01, s.ne11, (int) s.type_kv }) { - h = h*1000003u + (unsigned) v; // small prime, standard multiplicative hash mixing + h = h * 1000003u + (unsigned) v; } return h; } @@ -178,9 +173,9 @@ static void fa_init_tensors(ggml_context * ctx, const fa_shape & s, unsigned bas using set_override_t = void (*)(int, int); using clear_override_t = void (*)(void); -using bucket_t = int (*)(int64_t); -using baseline_ne_t = int (*)(int, int); -using device_token_t = const char * (*)(ggml_backend_dev_t); +using bucket_t = int (*)(int64_t); +using baseline_ne_t = int (*)(int, int); +using device_token_t = const char * (*) (ggml_backend_dev_t); struct fa_procs { set_override_t set_ov = nullptr; @@ -190,37 +185,25 @@ struct fa_procs { baseline_ne_t baseline_ne = nullptr; device_token_t dev_token = nullptr; - bool ok() const { - return set_ov && clr_ov && ne11_bucket && ne01_bucket && baseline_ne && dev_token; - } + bool ok() const { return set_ov && clr_ov && ne11_bucket && ne01_bucket && baseline_ne && dev_token; } }; static fa_procs fa_resolve_procs(ggml_backend_dev_t dev) { ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev); fa_procs p; - p.set_ov = (set_override_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_set_fa_vec_override"); - p.clr_ov = (clear_override_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_clear_fa_vec_override"); - p.ne11_bucket = (bucket_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_fa_vec_ne11_bucket"); - p.ne01_bucket = (bucket_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_fa_vec_ne01_bucket"); - p.baseline_ne = (baseline_ne_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_fa_vec_baseline_ne"); - p.dev_token = (device_token_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_device_token"); + p.set_ov = (set_override_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_set_fa_vec_override"); + p.clr_ov = + (clear_override_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_clear_fa_vec_override"); + p.ne11_bucket = (bucket_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_fa_vec_ne11_bucket"); + p.ne01_bucket = (bucket_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_fa_vec_ne01_bucket"); + p.baseline_ne = + (baseline_ne_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_fa_vec_baseline_ne"); + p.dev_token = (device_token_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_device_token"); return p; } -static const char * fa_type_token(ggml_type t) { - switch (t) { - case GGML_TYPE_Q4_0: return "GGML_TYPE_Q4_0"; - case GGML_TYPE_Q4_1: return "GGML_TYPE_Q4_1"; - case GGML_TYPE_Q5_0: return "GGML_TYPE_Q5_0"; - case GGML_TYPE_Q5_1: return "GGML_TYPE_Q5_1"; - case GGML_TYPE_Q8_0: return "GGML_TYPE_Q8_0"; - default: GGML_ABORT("unhandled KV type in fa_type_token: %d", (int) t); - } -} - -// "f16,q4_0" -> does it contain ggml_type_name(t)? null filter accepts everything static bool fa_filter_has(const char * filter, const char * name) { if (!filter) { return true; @@ -231,15 +214,16 @@ static bool fa_filter_has(const char * filter, const char * name) { return f.find(std::string(",") + name + ",") != std::string::npos; } -struct fa_cand { int Q, NE; }; +struct fa_cand { + int Q, NE; +}; -struct fa_point { // one swept grid point with its candidate times +struct fa_point { int dk, dv, ne11, ne01; - std::vector t; // indexed like the shape's candidate list + std::vector t; }; -// candidate list for one shape, identical for every grid point of it. base_i is the index of -// the (Q=1, baseline NE) candidate: the anchor config, and what the tuning gates compare to. +// base_i identifies the (Q=1, baseline NE) anchor configuration. static std::vector fa_build_cands(const fa_procs & procs, int dk, int dv, int & base_i) { const int base_ne = procs.baseline_ne(dk, dv); @@ -267,34 +251,56 @@ bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tune const char * dev_token = procs.dev_token(dev); - struct shape_t { int dk, dv; }; - const shape_t shapes[] = { { 32, 32 }, { 64, 64 }, { 96, 96 }, { 128, 128 }, { 192, 192 }, - { 192, 128 }, { 256, 256 }, { 320, 256 }, { 512, 512 }, { 576, 512 } }; + struct shape_t { + int dk, dv; + }; + + const shape_t shapes[] = { + { 32, 32 }, + { 64, 64 }, + { 96, 96 }, + { 128, 128 }, + { 192, 192 }, + { 192, 128 }, + { 256, 256 }, + { 320, 256 }, + { 512, 512 }, + { 576, 512 } + }; const int ne11_rep[] = { 512, 2048, 8192, 32768 }; // ne11 bucket representatives const int ne01_rep[] = { 1, 2, 3, 4, 5, 6, 7, 8, 16 }; // point buckets (1-4) + tail mod-4 cycle + anchor - const ggml_type types[] = { GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, - GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, GGML_TYPE_Q8_0 }; + + struct dtype_t { + ggml_type type; + const char * token; + }; + + const dtype_t dtypes[] = { + { GGML_TYPE_F16, "GGML_TYPE_F16" }, + { GGML_TYPE_Q4_0, "GGML_TYPE_Q4_0" }, + { GGML_TYPE_Q4_1, "GGML_TYPE_Q4_1" }, + { GGML_TYPE_Q5_0, "GGML_TYPE_Q5_0" }, + { GGML_TYPE_Q5_1, "GGML_TYPE_Q5_1" }, + { GGML_TYPE_Q8_0, "GGML_TYPE_Q8_0" }, + }; const double TUNE_TAU = 0.05; // max POINTWISE regret to ride a domain default const double TUNE_THETA = 1.05; // min AGGREGATE bucket speedup vs baseline to tune at all - cooldown_opts cool; - cool.enabled = opts.cooldown; - cool.drift = opts.cool_drift; - cool.eps = opts.cool_eps; - cool.max_wait = opts.cool_max_wait; - cool.max_retry = opts.cool_max_retry; + const cooldown_opts cool = { + opts.cooldown, opts.cool_drift, opts.cool_eps, opts.cool_max_wait, opts.cool_max_retry, + }; - fprintf(stderr, "seed=%u reps=%d cooldown=%s (drift=%.2f eps=%.2f max_wait=%ds max_retry=%d)\n", - opts.seed, opts.reps, cool.enabled ? "on" : "off", - cool.drift, cool.eps, cool.max_wait, cool.max_retry); + fprintf(stderr, "seed=%u reps=%d cooldown=%s (drift=%.2f eps=%.2f max_wait=%ds max_retry=%d)\n", opts.seed, + opts.reps, cool.enabled ? "on" : "off", cool.drift, cool.eps, cool.max_wait, cool.max_retry); fprintf(stderr, "device token: %s\n", dev_token); int n_untrusted = 0; printf("// ==== BEGIN fa_vec_tuned_table rows (%s) ====\n", dev_token); - for (ggml_type type_kv : types) { + for (const auto & dtype : dtypes) { + const ggml_type type_kv = dtype.type; if (!fa_filter_has(opts.dtype_filter, ggml_type_name(type_kv))) { continue; } @@ -308,19 +314,19 @@ bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tune continue; } - int base_i = 0; - std::vector cands = fa_build_cands(procs, s.dk, s.dv, base_i); + int base_i = 0; + std::vector cands = fa_build_cands(procs, s.dk, s.dv, base_i); for (int ne11 : ne11_rep) { for (int ne01 : ne01_rep) { const fa_shape sh = { s.dk, s.dv, ne01, ne11, type_kv }; - perf_cell cell = build_perf_cell(backend, - [&](ggml_context * ctx) { return fa_build_graph(ctx, sh); }, + perf_cell cell = build_perf_cell( + backend, [&](ggml_context * ctx) { return fa_build_graph(ctx, sh); }, [&](ggml_context * ctx) { fa_init_tensors(ctx, sh, opts.seed); }, [&](ggml_tensor *) { return fa_op_flops(sh); }); - if (!cell.ok) { + if (cell.gf == nullptr) { continue; } @@ -334,22 +340,18 @@ bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tune char label[128]; snprintf(label, sizeof(label), "dk=%d ne11=%d", s.dk, ne11); - cell_result r = measure_cell(backend, cell, opts.reps, - (int) cands.size(), order, - [&](int i) { procs.set_ov(cands[i].Q, cands[i].NE); }, - [&]() { procs.clr_ov(); }, - base_i, cool, label); + cell_result r = measure_cell( + backend, cell, opts.reps, order, [&](int i) { procs.set_ov(cands[i].Q, cands[i].NE); }, + [&]() { procs.clr_ov(); }, base_i, cool, label); - // per-cell noise floor: spread of the repeated same-config anchor if (r.anchor_min > 0.0) { - fprintf(stderr, "# noise dk=%d dv=%d ne11=%d ne01=%d spread=%.1f%%\n", - s.dk, s.dv, ne11, ne01, 100.0*(r.anchor_max - r.anchor_min)/r.anchor_min); + fprintf(stderr, "# noise dk=%d dv=%d ne11=%d ne01=%d spread=%.1f%%\n", s.dk, s.dv, ne11, ne01, + 100.0 * (r.anchor_max - r.anchor_min) / r.anchor_min); } if (!r.trusted) { n_untrusted++; - fprintf(stderr, "# DROP untrusted cell dk=%d dv=%d ne11=%d ne01=%d\n", - s.dk, s.dv, ne11, ne01); + fprintf(stderr, "# DROP untrusted cell dk=%d dv=%d ne11=%d ne01=%d\n", s.dk, s.dv, ne11, ne01); continue; } @@ -360,17 +362,17 @@ bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tune } } const double base_t = r.t[base_i]; - const bool keep = best_i >= 0 && base_t > 0.0 && r.t[best_i] < base_t*0.98; + const bool keep = best_i >= 0 && base_t > 0.0 && r.t[best_i] < base_t * 0.98; - fprintf(stderr, "# dtype=%s dk=%d dv=%d ne11=%d ne01=%d:", - ggml_type_name(type_kv), s.dk, s.dv, ne11, ne01); + fprintf(stderr, "# dtype=%s dk=%d dv=%d ne11=%d ne01=%d:", ggml_type_name(type_kv), s.dk, s.dv, + ne11, ne01); for (size_t i = 0; i < cands.size(); ++i) { fprintf(stderr, " Q%dNE%d=%.1f%s", cands[i].Q, cands[i].NE, r.t[i], (int) i == best_i ? "*" : ""); } if (keep) { - fprintf(stderr, " => Q%d,NE%d %.2fx\n", - cands[best_i].Q, cands[best_i].NE, base_t/r.t[best_i]); + fprintf(stderr, " => Q%d,NE%d %.2fx\n", cands[best_i].Q, cands[best_i].NE, + base_t / r.t[best_i]); } else { fprintf(stderr, " => baseline\n"); } @@ -385,49 +387,52 @@ bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tune // the default's pointwise regret vs the bucket target, or its aggregate slowdown vs // baseline, exceeds TUNE_TAU. std::vector rows_out; - char rbuf[192]; + char rbuf[192]; for (auto s : shapes) { if (!fa_filter_has(opts.dk_filter, std::to_string(s.dk).c_str())) { continue; } - int base_i = 0; - std::vector cands = fa_build_cands(procs, s.dk, s.dv, base_i); + int base_i = 0; + std::vector cands = fa_build_cands(procs, s.dk, s.dv, base_i); struct bkt_t { - int b11, b01, Ti; - std::vector agg; - double base_agg; + int b11, b01, Ti; + std::vector agg; std::vector bp; }; - // bucket the grid points with the runtime's bucketers, so keys match fa_vec_pick. - // short-KV points (ne11 bucket 0) are dropped: the runtime serves those from baseline. - std::set> seen; - for (const auto & p : pts) { - if (p.dk != s.dk || p.dv != s.dv) { - continue; - } - const int b11 = procs.ne11_bucket(p.ne11); + std::set> buckets; + for (int ne11 : ne11_rep) { + const int b11 = procs.ne11_bucket(ne11); if (b11 == 0) { continue; } - seen.insert({ b11, procs.ne01_bucket(p.ne01) }); + for (int ne01 : ne01_rep) { + buckets.insert({ b11, procs.ne01_bucket(ne01) }); + } } std::vector bks; - for (const auto & bb : seen) { + for (const auto & bb : buckets) { const int b11 = bb.first, b01 = bb.second; std::vector bp; for (const auto & p : pts) { - if (p.dk == s.dk && p.dv == s.dv && - procs.ne11_bucket(p.ne11) == b11 && procs.ne01_bucket(p.ne01) == b01) { + if (p.dk == s.dk && p.dv == s.dv && procs.ne11_bucket(p.ne11) == b11 && + procs.ne01_bucket(p.ne01) == b01) { bp.push_back(&p); } } + fprintf(stderr, "# bucket dk=%d dv=%d ne11_b=%d ne01_b=%d samples=%zu\n", s.dk, s.dv, b11, b01, + bp.size()); + if (bp.empty()) { + fprintf(stderr, "# WARN empty bucket dk=%d dv=%d ne11_b=%d ne01_b=%d\n", s.dk, s.dv, b11, b01); + continue; + } + std::vector agg(cands.size(), 0.0), worst(cands.size(), 0.0); for (const auto * p : bp) { double bestt = 0.0; @@ -439,35 +444,24 @@ bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tune for (size_t i = 0; i < cands.size(); ++i) { agg[i] += p->t[i]; if (p->t[i] > 0.0 && bestt > 0.0) { - worst[i] = std::max(worst[i], p->t[i]/bestt); + worst[i] = std::max(worst[i], p->t[i] / bestt); } } } int robust = 0; for (size_t i = 1; i < cands.size(); ++i) { - if (worst[i] < worst[robust] || - (worst[i] == worst[robust] && (cands[i].Q < cands[robust].Q || - (cands[i].Q == cands[robust].Q && cands[i].NE < cands[robust].NE)))) { + if (worst[i] < worst[robust] || (worst[i] == worst[robust] && (cands[i].Q < cands[robust].Q || + (cands[i].Q == cands[robust].Q && + cands[i].NE < cands[robust].NE)))) { robust = (int) i; } } const bool tune = robust != base_i && agg[base_i] > 0.0 && agg[robust] > 0.0 && - agg[base_i]/agg[robust] >= TUNE_THETA; + agg[base_i] / agg[robust] >= TUNE_THETA; - bks.push_back({ b11, b01, tune ? robust : base_i, agg, agg[base_i], bp }); - } - - // bucket coverage: a hardcoded sampling grid can't produce a wrong key, only miss - // a bucket, so report what each bucket actually got - for (const auto & b : bks) { - fprintf(stderr, "# bucket dk=%d dv=%d ne11_b=%d ne01_b=%d samples=%zu\n", - s.dk, s.dv, b.b11, b.b01, b.bp.size()); - if (b.bp.empty()) { - fprintf(stderr, "# WARN empty bucket dk=%d dv=%d ne11_b=%d ne01_b=%d\n", - s.dk, s.dv, b.b11, b.b01); - } + bks.push_back({ b11, b01, tune ? robust : base_i, agg, bp }); } // pointwise regret of default cfg d vs the bucket target: a ratio-of-sums lets a @@ -477,7 +471,7 @@ bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tune for (const auto * p : b->bp) { const double td = p->t[d], tT = p->t[b->Ti]; if (td > 0.0 && tT > 0.0) { - r = std::max(r, td/tT - 1.0); + r = std::max(r, td / tT - 1.0); } } return r; @@ -495,14 +489,15 @@ bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tune } // default cfg = the one minimizing (#rows, total achieved time, Q, NE) - int bestD = -1, bestRows = 1 << 30; + int bestD = -1, bestRows = 1 << 30; double bestTot = 0.0; for (size_t d = 0; d < cands.size(); ++d) { - int rows = ((int) d != base_i) ? 1 : 0; - double tot = 0.0; + int rows = ((int) d != base_i) ? 1 : 0; + double tot = 0.0; for (const auto * b : db) { - const double reg = reg_pointwise(b, (int) d); - const double slow = b->base_agg > 0.0 ? b->agg[d]/b->base_agg - 1.0 : 0.0; + const double base_agg = b->agg[base_i]; + const double reg = reg_pointwise(b, (int) d); + const double slow = base_agg > 0.0 ? b->agg[d] / base_agg - 1.0 : 0.0; if (reg > TUNE_TAU || slow > TUNE_TAU) { rows++; tot += b->agg[b->Ti]; @@ -510,33 +505,33 @@ bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tune tot += b->agg[d]; } } - const bool better = bestD < 0 || rows < bestRows || - (rows == bestRows && (tot < bestTot || - (tot == bestTot && (cands[d].Q < cands[bestD].Q || - (cands[d].Q == cands[bestD].Q && cands[d].NE < cands[bestD].NE))))); + const bool better = + bestD < 0 || rows < bestRows || + (rows == bestRows && + (tot < bestTot || + (tot == bestTot && (cands[d].Q < cands[bestD].Q || + (cands[d].Q == cands[bestD].Q && cands[d].NE < cands[bestD].NE))))); if (better) { - bestD = (int) d; + bestD = (int) d; bestRows = rows; - bestTot = tot; + bestTot = tot; } } - const int dom_id = (dom == 0) ? 0 : 1; // FA_VEC_DOMAIN_DECODE / FA_VEC_DOMAIN_BATCH if (bestD != base_i) { - snprintf(rbuf, sizeof(rbuf), " { { %s, %s, %d, %d, -1, %d }, { %d, %d } },", - dev_token, fa_type_token(type_kv), s.dk, s.dv, dom_id, - cands[bestD].Q, cands[bestD].NE); + snprintf(rbuf, sizeof(rbuf), " { { %s, %s, %d, %d, -1, %d }, { %d, %d } },", dev_token, + dtype.token, s.dk, s.dv, dom, cands[bestD].Q, cands[bestD].NE); rows_out.emplace_back(rbuf); } for (const auto * b : db) { - const double reg = reg_pointwise(b, bestD); - const double slow = b->base_agg > 0.0 ? b->agg[bestD]/b->base_agg - 1.0 : 0.0; + const double base_agg = b->agg[base_i]; + const double reg = reg_pointwise(b, bestD); + const double slow = base_agg > 0.0 ? b->agg[bestD] / base_agg - 1.0 : 0.0; if (reg <= TUNE_TAU && slow <= TUNE_TAU) { - continue; // rides the default / baseline + continue; } - snprintf(rbuf, sizeof(rbuf), " { { %s, %s, %d, %d, %d, %d }, { %d, %d } },", - dev_token, fa_type_token(type_kv), s.dk, s.dv, b->b11, b->b01, - cands[b->Ti].Q, cands[b->Ti].NE); + snprintf(rbuf, sizeof(rbuf), " { { %s, %s, %d, %d, %d, %d }, { %d, %d } },", dev_token, + dtype.token, s.dk, s.dv, b->b11, b->b01, cands[b->Ti].Q, cands[b->Ti].NE); rows_out.emplace_back(rbuf); } } diff --git a/tools/tuning/fa-vec.h b/tools/tuning/fa-vec.h index b015a8051f4..b815f186734 100644 --- a/tools/tuning/fa-vec.h +++ b/tools/tuning/fa-vec.h @@ -2,19 +2,17 @@ #include "ggml-backend.h" -// options shared by all tuners; parsed in main.cpp struct tuner_opts { - const char * dtype_filter = nullptr; // comma-separated, e.g. "f16,q4_0"; null = all - const char * dk_filter = nullptr; // comma-separated dk values, e.g. "128,192"; null = all - int reps = 7; - unsigned seed = 1234; - bool cooldown = true; - double cool_drift = 0.10; - double cool_eps = 0.03; - int cool_max_wait = 120; + const char * dtype_filter = nullptr; // comma-separated, e.g. "f16,q4_0"; null = all + const char * dk_filter = nullptr; // comma-separated dk values, e.g. "128,192"; null = all + int reps = 7; + unsigned seed = 1234; + bool cooldown = true; + double cool_drift = 0.10; + double cool_eps = 0.03; + int cool_max_wait = 120; int cool_max_retry = 2; }; -// runs the FA-vec (Q,NE) sweep and prints a pasteable table block on stdout. -// returns false only on environment failure (missing procs), never on perf results. +// Returns false only when the required Metal proc bridges are unavailable. bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tuner_opts & opts); diff --git a/tools/tuning/main.cpp b/tools/tuning/main.cpp index ce9eb0a9939..fbe0505936c 100644 --- a/tools/tuning/main.cpp +++ b/tools/tuning/main.cpp @@ -1,7 +1,6 @@ #include "fa-vec.h" - -#include "ggml.h" #include "ggml-backend.h" +#include "ggml.h" #include #include @@ -42,8 +41,8 @@ static void usage(const char * argv0) { } int main(int argc, char ** argv) { - const char * tuner = nullptr; - const char * bname = nullptr; + const char * tuner = nullptr; + const char * bname = nullptr; tuner_opts opts; for (int i = 1; i < argc; i++) { From 545c66e12b151f67d4d60e006c2b33b2babfc5bc Mon Sep 17 00:00:00 2001 From: forforever73 <690105611@qq.com> Date: Tue, 11 Aug 2026 15:11:45 +0800 Subject: [PATCH 12/14] honor -o in the FA vec (Q,NE) slice --- tests/test-backend-ops.cpp | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 598b75ad93a..99014df09cb 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -10589,9 +10589,37 @@ static std::vector fa_vec_legal_ne(int dk, int dv) { return r; } +static bool op_names_filter_selects(const char * op_names_filter, const char * op_name) { + if (!op_names_filter) { + return true; + } + std::string_view filter(op_names_filter); + while (!filter.empty()) { + auto comma_pos = filter.find_first_of(','); + const auto lparen_pos = filter.find_first_of('('); + std::string_view entry; + if (lparen_pos < comma_pos) { + const auto rparen_pos = filter.find_first_of(')'); + comma_pos = filter.find_first_of(',', rparen_pos); + entry = filter.substr(0, lparen_pos); + } else { + entry = filter.substr(0, comma_pos); + } + if (entry == op_name) { + return true; + } + filter = comma_pos != std::string_view::npos ? filter.substr(comma_pos + 1) : ""; + } + return false; +} + // Covers padded rows, sinks, kvpad, multi-SIMDgroup reduction, quantized K/V, and MLA views. // The override is backend-global, so this runs after all parallel workers have joined. -static bool run_fa_vec_slice(ggml_backend_t backend, ggml_backend_t backend_cpu) { +static bool run_fa_vec_slice(ggml_backend_t backend, ggml_backend_t backend_cpu, const char * op_names_filter) { + if (!op_names_filter_selects(op_names_filter, "FLASH_ATTN_EXT")) { + return true; + } + auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend)); auto set_ov = (set_fa_vec_override_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_set_fa_vec_override"); @@ -10777,7 +10805,7 @@ static bool test_backend(ggml_backend_t backend, ggml_backend_dev_t dev, test_mo output_printer->print_summary(test_summary_info(n_ok, tests_run, false)); output_printer->print_failed_tests(failed_tests); - const bool slice_ok = run_fa_vec_slice(backend, backend_cpu.get()); + const bool slice_ok = run_fa_vec_slice(backend, backend_cpu.get(), op_names_filter); return n_ok == tests_run && slice_ok; } From 7658a24107ceb02bdd07f099c558b9843fe95dc5 Mon Sep 17 00:00:00 2001 From: forforever73 <690105611@qq.com> Date: Sat, 22 Aug 2026 11:52:06 +0800 Subject: [PATCH 13/14] retune FA-vec (Q, NE) under a pointwise no-harm gate --- ggml/src/ggml-metal/ggml-metal-tuning.cpp | 83 ++++++------- tools/tuning/README.md | 6 +- tools/tuning/fa-vec.cpp | 135 ++++++++++++++++++---- 3 files changed, 152 insertions(+), 72 deletions(-) diff --git a/ggml/src/ggml-metal/ggml-metal-tuning.cpp b/ggml/src/ggml-metal/ggml-metal-tuning.cpp index 6eb56a7ef2d..112bb299743 100644 --- a/ggml/src/ggml-metal/ggml-metal-tuning.cpp +++ b/ggml/src/ggml-metal/ggml-metal-tuning.cpp @@ -65,23 +65,25 @@ fa_vec_cfg_t fa_vec_baseline_cfg(int dk, int dv) { // Generated by `ggml-metal-tuning fa-vec`; do not hand-edit. // One row per kept bucket, plus per-(dtype,dk,dv) ne11-collapsed domain defaults // (ne11_b = FA_VEC_NE11_DEFAULT, ne01_b = domain). To retune or add a device, re-run the -// sweep and paste its block. See ggml-metal-tuning.h for the row/lookup semantics. +// sweep and paste its output. See ggml-metal-tuning.h for the row/lookup semantics. constexpr fa_vec_entry_t fa_vec_tuned_table[] = { - // ---- f16: 13 rows ---- - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 32, 32, 1, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 32, 32, 2, 1 }, { 2, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 32, 32, 2, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 32, 32, 2, 4 }, { 2, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 32, 32, 3, 1 }, { 2, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 32, 32, 3, 3 }, { 2, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 64, 64, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 64, 64, 1, 2 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 64, 64, 2, 2 }, { 1, 2 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 96, 96, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 96, 96, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 96, 96, 1, 4 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 128, 128, -1, 1 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 128, 128, 1, 2 }, { 1, 1 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 192, 192, -1, 1 }, { 2, 2 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 192, 128, -1, 1 }, { 2, 2 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 256, 256, -1, 0 }, { 1, 2 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 256, 256, -1, 1 }, { 2, 2 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 320, 256, -1, 1 }, { 2, 2 } }, - // ---- q4_0: 29 rows ---- { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 32, 32, -1, 1 }, { 2, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 32, 32, 1, 2 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 32, 32, 3, 2 }, { 4, 4 } }, @@ -92,6 +94,7 @@ constexpr fa_vec_entry_t fa_vec_tuned_table[] = { { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 96, 96, -1, 1 }, { 2, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 96, 96, 1, 2 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 96, 96, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 96, 96, 3, 2 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } }, @@ -106,12 +109,13 @@ constexpr fa_vec_entry_t fa_vec_tuned_table[] = { { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 576, 512, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 576, 512, 2, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 576, 512, 3, 0 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 576, 512, -1, 1 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 576, 512, 1, 1 }, { 1, 2 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 576, 512, 1, 2 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 576, 512, 1, 3 }, { 1, 2 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_0, 576, 512, 1, 4 }, { 1, 2 } }, - // ---- q4_1: 28 rows ---- { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 32, 32, -1, 1 }, { 2, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 32, 32, 3, 2 }, { 4, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 32, 32, 3, 3 }, { 4, 4 } }, @@ -120,7 +124,10 @@ constexpr fa_vec_entry_t fa_vec_tuned_table[] = { { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 2, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 96, 96, -1, 1 }, { 2, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 96, 96, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 96, 96, 1, 4 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 96, 96, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 96, 96, 2, 4 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 96, 96, 3, 2 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 128, 128, 2, 3 }, { 2, 4 } }, @@ -138,47 +145,42 @@ constexpr fa_vec_entry_t fa_vec_tuned_table[] = { { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 256, 256, -1, 0 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 256, 256, -1, 1 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 320, 256, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 320, 256, 1, 1 }, { 1, 2 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 576, 512, -1, 0 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q4_1, 576, 512, -1, 1 }, { 1, 4 } }, - // ---- q5_0: 45 rows ---- { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 4, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 32, 32, 1, 1 }, { 2, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 32, 32, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 32, 32, 2, 4 }, { 2, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 32, 32, 3, 1 }, { 2, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 4, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 96, 96, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 96, 96, 2, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 96, 96, 2, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 96, 96, 3, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 96, 96, 3, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 96, 96, 1, 2 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 96, 96, 1, 3 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 96, 96, 2, 2 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 96, 96, 2, 3 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 96, 96, 3, 2 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 96, 96, 3, 3 }, { 4, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 128, 128, -1, 0 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 2, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 192, 192, -1, 0 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 192, 192, 1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 192, 192, 1, 2 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 192, 192, 1, 3 }, { 2, 2 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 192, 128, -1, 0 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 256, 256, 1, 1 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 256, 256, 1, 2 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 256, 256, 1, 4 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 256, 256, 2, 2 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 256, 256, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 320, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 320, 256, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 320, 256, 1, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 320, 256, 1, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 320, 256, 2, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 320, 256, 3, 2 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 320, 256, 1, 3 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 320, 256, 1, 4 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 320, 256, 2, 1 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 320, 256, 2, 3 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 320, 256, 3, 3 }, { 2, 2 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 512, 512, -1, 0 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 512, 512, -1, 1 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 576, 512, 2, 0 }, { 1, 4 } }, @@ -186,9 +188,9 @@ constexpr fa_vec_entry_t fa_vec_tuned_table[] = { { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 576, 512, 2, 2 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 576, 512, 2, 3 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_0, 576, 512, 2, 4 }, { 1, 4 } }, - // ---- q5_1: 49 rows ---- { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 4, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 32, 32, 1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 32, 32, 1, 4 }, { 2, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 32, 32, 2, 1 }, { 2, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 32, 32, 3, 1 }, { 2, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 64, 64, -1, 0 }, { 1, 4 } }, @@ -205,48 +207,38 @@ constexpr fa_vec_entry_t fa_vec_tuned_table[] = { { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 2, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 192, 192, -1, 0 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 192, 192, 1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 192, 192, 1, 2 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 192, 192, 1, 3 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 192, 192, 1, 4 }, { 2, 2 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 192, 128, -1, 0 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 2, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 256, 256, 1, 1 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 256, 256, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 256, 256, 1, 4 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 256, 256, 2, 2 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 256, 256, 3, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 320, 256, -1, 1 }, { 2, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 320, 256, 1, 1 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 320, 256, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 320, 256, 1, 3 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 320, 256, 1, 4 }, { 2, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 320, 256, 2, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 320, 256, 3, 2 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 320, 256, 1, 3 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 320, 256, 1, 4 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 320, 256, 2, 3 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 320, 256, 3, 3 }, { 2, 2 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 512, 512, -1, 0 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 512, 512, 1, 1 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 512, 512, 1, 2 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 512, 512, 1, 3 }, { 1, 2 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 512, 512, 1, 4 }, { 1, 2 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 576, 512, 2, 0 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 576, 512, 2, 1 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 576, 512, 2, 2 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 576, 512, 2, 3 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q5_1, 576, 512, 2, 4 }, { 1, 4 } }, - // ---- q8_0: 29 rows ---- { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 32, 32, 1, 2 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 32, 32, 3, 2 }, { 4, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 32, 32, 3, 3 }, { 4, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 32, 32, 3, 4 }, { 4, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 96, 96, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 96, 96, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 96, 96, 2, 2 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 2, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 128, 128, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 128, 128, 1, 4 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 128, 128, 2, 2 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 128, 128, 3, 2 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } }, @@ -255,7 +247,6 @@ constexpr fa_vec_entry_t fa_vec_tuned_table[] = { { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 2, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 192, 128, 1, 1 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 192, 128, 1, 2 }, { 1, 4 } }, - { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 192, 128, 1, 3 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 192, 128, 2, 2 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 192, 128, 3, 2 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 4 } }, diff --git a/tools/tuning/README.md b/tools/tuning/README.md index e22f2842bf4..79e0c3646b2 100644 --- a/tools/tuning/README.md +++ b/tools/tuning/README.md @@ -26,12 +26,14 @@ Sweep the grid (6 dtypes x 10 head sizes x 4 KV depths x 9 batch widths; a few h ./build/bin/ggml-metal-tuning fa-vec > fa_vec_rows.txt 2> fa_vec_sweep.log ``` -`fa_vec_rows.txt` is the finished table block, ready to paste: the min-max-regret target, the aggregate benefit gate, the short-KV drop and the pointwise compression are already applied. -`fa_vec_sweep.log` holds the per-cell timings, bucket coverage, noise floor and any cooldown activity. +`fa_vec_rows.txt` holds nothing but table rows, ready to paste into `fa_vec_tuned_table`: the min-max-regret target, the aggregate benefit gate, the short-KV drop and the pointwise compression are already applied. +A config represents a bucket only if it is no slower than the baseline config at every point that bucket covers, so a config that wins on average but loses at one batch width leaves its bucket at baseline. +`fa_vec_sweep.log` holds the per-cell timings, bucket coverage, noise floor, any cooldown activity, and every config the no-harm rule refused together with the point that refused it. Post both: the log is what makes the rows reviewable. Long sweeps can be split. `--dtype f16,q4_0` and `--dk 128,192` restrict the grid, and the emitted rows for one `(dtype, head size)` do not depend on the others. +Concatenating the shard outputs in the order the full grid would visit them gives the same rows a single run prints. Then validate the numerics, where Metal is compared against the CPU reference: diff --git a/tools/tuning/fa-vec.cpp b/tools/tuning/fa-vec.cpp index 2dd1c95be51..f904379695e 100644 --- a/tools/tuning/fa-vec.cpp +++ b/tools/tuning/fa-vec.cpp @@ -267,7 +267,10 @@ bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tune { 512, 512 }, { 576, 512 } }; - const int ne11_rep[] = { 512, 2048, 8192, 32768 }; // ne11 bucket representatives + // nsg is a pipeline specialization constant (1 up to ne11=2048, 2 up to 4096, 4 above), so ne11 + // bucket 1 takes two samples to cover both of its regimes. Bucket 0 is not sampled at all: the + // runtime leaves short KV at baseline, so no measurement there can reach the table. + const int ne11_rep[] = { 2048, 3072, 8192, 32768 }; const int ne01_rep[] = { 1, 2, 3, 4, 5, 6, 7, 8, 16 }; // point buckets (1-4) + tail mod-4 cycle + anchor struct dtype_t { @@ -297,8 +300,7 @@ bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tune int n_untrusted = 0; - printf("// ==== BEGIN fa_vec_tuned_table rows (%s) ====\n", dev_token); - + // stdout carries nothing but table rows, so the whole stream pastes into fa_vec_tuned_table for (const auto & dtype : dtypes) { const ggml_type type_kv = dtype.type; if (!fa_filter_has(opts.dtype_filter, ggml_type_name(type_kv))) { @@ -335,7 +337,7 @@ bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tune for (size_t i = 0; i < order.size(); ++i) { order[i] = (int) i; } - std::shuffle(order.begin(), order.end(), std::mt19937(opts.seed)); + std::shuffle(order.begin(), order.end(), std::mt19937(fa_cell_seed(sh, opts.seed))); char label[128]; snprintf(label, sizeof(label), "dk=%d ne11=%d", s.dk, ne11); @@ -383,9 +385,9 @@ bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tune } // compress into pasteable rows. per (dk,dv) and ne01 domain {decode==1, batch>=2}, - // emit one ne11-collapsed default cfg (ne11_b=-1) plus a per-bucket exception wherever - // the default's pointwise regret vs the bucket target, or its aggregate slowdown vs - // baseline, exceeds TUNE_TAU. + // emit one ne11-collapsed default cfg (ne11_b=-1) plus a per-bucket exception wherever the + // default's pointwise regret vs the bucket target exceeds TUNE_TAU, or the default is not + // admissible for that bucket (see never_slower / admissible below). std::vector rows_out; char rbuf[192]; @@ -403,6 +405,63 @@ bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tune std::vector bp; }; + // A config may represent a bucket only if it is no slower than baseline at every point that + // bucket covers. The aggregate gate below sums absolute times, so it can pass on the aligned + // and deep points while a misaligned ne01 pays the mod-Q padding. Nothing measured, nothing + // proven: a bucket with no surviving sample admits baseline only. + auto never_slower = [&](const std::vector & bp, int i) { + if (i == base_i) { + return true; + } + if (bp.empty()) { + return false; + } + for (const auto * p : bp) { + if (p->t[i] <= 0.0 || p->t[base_i] <= 0.0 || p->t[i] > p->t[base_i]) { + return false; + } + } + return true; + }; + + // The padded-row waste ceil(n/Q)*Q/n is largest at the smallest ne01 of each residue class + // mod Q, so one of a bucket's first Q values carries the worst padding it can ever see, and + // that value has to be sampled. Otherwise the bucket bounds nothing: a config picked on the + // aligned ne01=8,16 says nothing about ne01=9. This covers the padding term only - the + // per-row cost varies with ne01 too - so it is a floor on the evidence, not a proof. + auto admissible = [&](const std::vector & bp, int b01, int i) { + if (!never_slower(bp, i)) { + return false; + } + const int Q = cands[i].Q; + if (Q == 1) { + return true; // one row per threadgroup, no padding to witness + } + int lo = bp[0]->ne01; + for (const auto * p : bp) { + lo = std::min(lo, p->ne01); + } + while (lo > 1 && procs.ne01_bucket(lo - 1) == b01) { + lo--; // walk down to where this bucket's runtime domain starts + } + int wit = lo; + double wmax = 0.0; + for (int n = lo; n < lo + Q && procs.ne01_bucket(n) == b01; ++n) { + const int padded = ((n + Q - 1) / Q) * Q; + const double w = (double) padded / n; + if (w > wmax) { + wmax = w; + wit = n; + } + } + for (const auto * p : bp) { + if (p->ne01 == wit) { + return true; + } + } + return false; + }; + std::set> buckets; for (int ne11 : ne11_rep) { const int b11 = procs.ne11_bucket(ne11); @@ -429,7 +488,11 @@ bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tune fprintf(stderr, "# bucket dk=%d dv=%d ne11_b=%d ne01_b=%d samples=%zu\n", s.dk, s.dv, b11, b01, bp.size()); if (bp.empty()) { - fprintf(stderr, "# WARN empty bucket dk=%d dv=%d ne11_b=%d ne01_b=%d\n", s.dk, s.dv, b11, b01); + // nothing to check a config against, so pin the bucket to baseline instead of + // letting the ne11-collapsed domain default ride in unmeasured + fprintf(stderr, "# WARN empty bucket dk=%d dv=%d ne11_b=%d ne01_b=%d -> baseline\n", s.dk, s.dv, + b11, b01); + bks.push_back({ b11, b01, base_i, std::vector(cands.size(), 0.0), {} }); continue; } @@ -449,11 +512,17 @@ bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tune } } - int robust = 0; - for (size_t i = 1; i < cands.size(); ++i) { - if (worst[i] < worst[robust] || (worst[i] == worst[robust] && (cands[i].Q < cands[robust].Q || - (cands[i].Q == cands[robust].Q && - cands[i].NE < cands[robust].NE)))) { + int robust = -1, oracle_pick = -1; + for (size_t i = 0; i < cands.size(); ++i) { + auto tighter = [&](int j) { + return j < 0 || worst[i] < worst[j] || + (worst[i] == worst[j] && (cands[i].Q < cands[j].Q || + (cands[i].Q == cands[j].Q && cands[i].NE < cands[j].NE))); + }; + if (tighter(oracle_pick)) { + oracle_pick = (int) i; + } + if (admissible(bp, b01, (int) i) && tighter(robust)) { robust = (int) i; } } @@ -461,6 +530,33 @@ bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tune const bool tune = robust != base_i && agg[base_i] > 0.0 && agg[robust] > 0.0 && agg[base_i] / agg[robust] >= TUNE_THETA; + // report what the no-harm rule cost this bucket, but only when it changed the outcome: + // a sweep on another machine then shows where the winner loses, instead of just + // emitting a smaller table + const bool refused = oracle_pick != robust && oracle_pick != base_i && agg[base_i] > 0.0 && + agg[oracle_pick] > 0.0 && agg[base_i] / agg[oracle_pick] >= TUNE_THETA; + if (refused) { + double over = 0.0; + int at11 = 0, at01 = 0; + for (const auto * p : bp) { + if (p->t[base_i] > 0.0 && p->t[oracle_pick] / p->t[base_i] - 1.0 > over) { + over = p->t[oracle_pick] / p->t[base_i] - 1.0; + at11 = p->ne11; + at01 = p->ne01; + } + } + if (over > 0.0) { + fprintf(stderr, + "# reject dk=%d dv=%d ne11_b=%d ne01_b=%d Q%dNE%d: +%.2f%% vs baseline at " + "ne11=%d ne01=%d\n", + s.dk, s.dv, b11, b01, cands[oracle_pick].Q, cands[oracle_pick].NE, 100.0 * over, at11, + at01); + } else { + fprintf(stderr, "# reject dk=%d dv=%d ne11_b=%d ne01_b=%d Q%dNE%d: no padding witness\n", s.dk, + s.dv, b11, b01, cands[oracle_pick].Q, cands[oracle_pick].NE); + } + } + bks.push_back({ b11, b01, tune ? robust : base_i, agg, bp }); } @@ -495,10 +591,7 @@ bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tune int rows = ((int) d != base_i) ? 1 : 0; double tot = 0.0; for (const auto * b : db) { - const double base_agg = b->agg[base_i]; - const double reg = reg_pointwise(b, (int) d); - const double slow = base_agg > 0.0 ? b->agg[d] / base_agg - 1.0 : 0.0; - if (reg > TUNE_TAU || slow > TUNE_TAU) { + if (reg_pointwise(b, (int) d) > TUNE_TAU || !admissible(b->bp, b->b01, (int) d)) { rows++; tot += b->agg[b->Ti]; } else { @@ -524,10 +617,7 @@ bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tune rows_out.emplace_back(rbuf); } for (const auto * b : db) { - const double base_agg = b->agg[base_i]; - const double reg = reg_pointwise(b, bestD); - const double slow = base_agg > 0.0 ? b->agg[bestD] / base_agg - 1.0 : 0.0; - if (reg <= TUNE_TAU && slow <= TUNE_TAU) { + if (reg_pointwise(b, bestD) <= TUNE_TAU && admissible(b->bp, b->b01, bestD)) { continue; } snprintf(rbuf, sizeof(rbuf), " { { %s, %s, %d, %d, %d, %d }, { %d, %d } },", dev_token, @@ -537,15 +627,12 @@ bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tune } } - printf("\n // ---- %s: %zu rows ----\n", ggml_type_name(type_kv), rows_out.size()); for (const auto & r : rows_out) { printf("%s\n", r.c_str()); } fflush(stdout); } - printf("// ==== END fa_vec_tuned_table rows (%s) ====\n", dev_token); - if (n_untrusted > 0) { fprintf(stderr, "\n%d cells excluded as untrusted (see DROP lines above)\n", n_untrusted); } From efeda76b948f59ee52ea20db640bc4cf3dfe8ac1 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 24 Aug 2026 14:38:58 +0300 Subject: [PATCH 14/14] cont : add fa-vec tunings for M1 Pro, M2 Ultra, M5 Max --- ggml/src/ggml-metal/ggml-metal-tuning.cpp | 749 ++++++++++++++++++++++ 1 file changed, 749 insertions(+) diff --git a/ggml/src/ggml-metal/ggml-metal-tuning.cpp b/ggml/src/ggml-metal/ggml-metal-tuning.cpp index 112bb299743..6d8c18e6a6a 100644 --- a/ggml/src/ggml-metal/ggml-metal-tuning.cpp +++ b/ggml/src/ggml-metal/ggml-metal-tuning.cpp @@ -67,6 +67,388 @@ fa_vec_cfg_t fa_vec_baseline_cfg(int dk, int dv) { // (ne11_b = FA_VEC_NE11_DEFAULT, ne01_b = domain). To retune or add a device, re-run the // sweep and paste its output. See ggml-metal-tuning.h for the row/lookup semantics. constexpr fa_vec_entry_t fa_vec_tuned_table[] = { + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 32, 32, 3, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 64, 64, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 64, 64, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 64, 64, 1, 4 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 64, 64, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 64, 64, 3, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 64, 64, 3, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 96, 96, 1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 96, 96, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 96, 96, 3, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 128, 128, 2, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 128, 128, 2, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 128, 128, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 128, 128, 3, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 128, 128, 3, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 192, 128, 1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 192, 128, 1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 192, 128, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 192, 128, 1, 3 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 320, 256, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_F16, 320, 256, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 32, 32, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 32, 32, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 32, 32, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 32, 32, 3, 2 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 32, 32, 3, 3 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 64, 64, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 64, 64, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 64, 64, 3, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 96, 96, 1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 96, 96, 1, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 96, 96, 2, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 96, 96, 2, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 96, 96, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 96, 96, 3, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 320, 256, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 576, 512, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_0, 576, 512, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 32, 32, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 32, 32, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 32, 32, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 32, 32, 3, 2 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 32, 32, 3, 3 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 64, 64, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 96, 96, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 96, 96, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 96, 96, 1, 4 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 96, 96, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 96, 96, 3, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 128, 128, 2, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 128, 128, 2, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 128, 128, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 128, 128, 3, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 128, 128, 3, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 192, 192, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 192, 128, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 256, 256, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 256, 256, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 320, 256, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 320, 256, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 512, 512, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 512, 512, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 576, 512, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q4_1, 576, 512, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 32, 32, 1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 32, 32, 2, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 32, 32, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 96, 96, 1, 3 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 128, 128, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 128, 128, 1, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 128, 128, 1, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 128, 128, 2, 3 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 128, 128, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 128, 128, 3, 3 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 192, 192, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 192, 192, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 192, 192, 3, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 192, 128, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 192, 128, 1, 3 }, { 4, 2 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 192, 128, 2, 3 }, { 4, 2 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 192, 128, 3, 3 }, { 4, 2 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 256, 256, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 256, 256, 1, 4 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 256, 256, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 256, 256, 3, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 256, 256, 3, 4 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 320, 256, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 320, 256, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 320, 256, 1, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 320, 256, 2, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 320, 256, 2, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 512, 512, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 512, 512, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 576, 512, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_0, 576, 512, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 32, 32, 1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 32, 32, 2, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 32, 32, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 64, 64, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 96, 96, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 96, 96, 1, 3 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 96, 96, 1, 4 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 128, 128, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 128, 128, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 128, 128, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 128, 128, 2, 3 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 128, 128, 2, 4 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 128, 128, 3, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 128, 128, 3, 4 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 192, 192, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 192, 192, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 192, 192, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 192, 192, 3, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 192, 192, 3, 4 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 192, 128, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 192, 128, 1, 3 }, { 4, 2 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 192, 128, 2, 3 }, { 4, 2 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 256, 256, 1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 256, 256, 1, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 256, 256, 2, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 256, 256, 2, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 256, 256, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 256, 256, 3, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 320, 256, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 320, 256, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 320, 256, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 320, 256, 1, 4 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 320, 256, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 320, 256, 2, 4 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 320, 256, 3, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 512, 512, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 576, 512, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 576, 512, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q5_1, 576, 512, 2, 4 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 32, 32, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 32, 32, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 32, 32, 2, 4 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 32, 32, 3, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 32, 32, 3, 3 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 64, 64, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 64, 64, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 64, 64, 3, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 96, 96, 2, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 96, 96, 2, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 96, 96, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 96, 96, 3, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 96, 96, 3, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 320, 256, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 320, 256, 1, 2 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 320, 256, 1, 3 }, { 4, 2 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 320, 256, 2, 2 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 320, 256, 3, 2 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M1_PRO, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } }, + + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 64, 64, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 64, 64, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 128, 128, 2, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 128, 128, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 128, 128, 2, 3 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 128, 128, 2, 4 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 128, 128, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 128, 128, 3, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 128, 128, 3, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 192, 128, 1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 192, 128, 1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 192, 128, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 192, 128, 1, 3 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 192, 128, 1, 4 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 320, 256, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 320, 256, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 320, 256, 1, 1 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 320, 256, 1, 4 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 32, 32, 2, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 32, 32, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 32, 32, 3, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 32, 32, 3, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 64, 64, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 64, 64, 1, 4 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 64, 64, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 64, 64, 3, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 96, 96, 3, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 320, 256, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 576, 512, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 576, 512, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_0, 576, 512, 1, 1 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 32, 32, 2, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 32, 32, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 32, 32, 3, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 32, 32, 3, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 64, 64, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 64, 64, 1, 4 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 128, 128, 2, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 128, 128, 2, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 128, 128, 3, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 128, 128, 3, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 192, 192, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 192, 192, 2, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 192, 192, 2, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 192, 128, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 256, 256, 1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 256, 256, 1, 4 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 320, 256, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 320, 256, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 512, 512, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 512, 512, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 576, 512, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q4_1, 576, 512, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 32, 32, 1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 32, 32, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 32, 32, 1, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 32, 32, 2, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 32, 32, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 64, 64, 1, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 96, 96, 1, 2 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 96, 96, 1, 3 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 128, 128, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 128, 128, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 128, 128, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 128, 128, 3, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 192, 192, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 192, 192, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 192, 128, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 192, 128, 2, 3 }, { 4, 2 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 192, 128, 2, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 192, 128, 3, 3 }, { 4, 2 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 192, 128, 3, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 320, 256, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 320, 256, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 512, 512, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 512, 512, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 576, 512, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_0, 576, 512, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 32, 32, 1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 32, 32, 1, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 32, 32, 2, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 32, 32, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 64, 64, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 96, 96, 1, 2 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 96, 96, 1, 3 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 96, 96, 3, 4 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 128, 128, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 128, 128, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 128, 128, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 128, 128, 3, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 192, 192, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 192, 192, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 192, 192, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 192, 192, 3, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 192, 128, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 192, 128, 2, 3 }, { 4, 2 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 192, 128, 2, 4 }, { 4, 2 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 192, 128, 3, 3 }, { 4, 2 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 192, 128, 3, 4 }, { 4, 2 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 320, 256, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 320, 256, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 512, 512, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 576, 512, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q5_1, 576, 512, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 32, 32, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 32, 32, 3, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 32, 32, 3, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 64, 64, 1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 64, 64, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 64, 64, 3, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 64, 64, 3, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 320, 256, 2, 3 }, { 4, 2 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 320, 256, 3, 3 }, { 4, 2 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 32, 32, 2, 1 }, { 2, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 32, 32, 2, 3 }, { 2, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 32, 32, 3, 1 }, { 2, 4 } }, @@ -257,6 +639,373 @@ constexpr fa_vec_entry_t fa_vec_tuned_table[] = { { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } }, + + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 32, 32, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 32, 32, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 32, 32, 1, 4 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 32, 32, 2, 2 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 32, 32, 2, 3 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 32, 32, 3, 3 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 64, 64, 2, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 64, 64, 3, 0 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 64, 64, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 64, 64, 1, 1 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 64, 64, 3, 2 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 64, 64, 3, 3 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 64, 64, 3, 4 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 96, 96, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 96, 96, 3, 3 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 128, 128, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 128, 128, 3, 0 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 128, 128, -1, 1 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 128, 128, 1, 2 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 128, 128, 1, 4 }, { 1, 1 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 128, 128, 2, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 128, 128, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 128, 128, 3, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 128, 128, 3, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 192, 192, -1, 1 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 192, 192, 1, 2 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 192, 192, 1, 4 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 192, 128, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 192, 128, 1, 4 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 192, 128, 3, 2 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 256, 256, -1, 0 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 256, 256, -1, 1 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 256, 256, 1, 2 }, { 1, 1 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 256, 256, 1, 4 }, { 1, 1 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 320, 256, 3, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 320, 256, -1, 1 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 320, 256, 1, 2 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 320, 256, 1, 4 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 512, 512, 2, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 512, 512, 1, 1 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 512, 512, 2, 3 }, { 4, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 512, 512, 3, 3 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 512, 512, 3, 4 }, { 4, 1 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 576, 512, 2, 0 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 576, 512, -1, 1 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 576, 512, 1, 1 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 576, 512, 1, 2 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 576, 512, 1, 4 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 576, 512, 2, 2 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_F16, 576, 512, 2, 3 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 32, 32, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 32, 32, 1, 2 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 32, 32, 2, 2 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 32, 32, 2, 3 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 32, 32, 3, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 32, 32, 3, 4 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 64, 64, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 64, 64, 2, 2 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 64, 64, 2, 3 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 64, 64, 3, 2 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 64, 64, 3, 4 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 96, 96, 1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 96, 96, 1, 3 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 96, 96, 2, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 96, 96, 2, 2 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 96, 96, 2, 3 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 96, 96, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 96, 96, 3, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 96, 96, 3, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 128, 128, 1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 128, 128, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 128, 128, 2, 4 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 128, 128, 3, 1 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 192, 192, 3, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 192, 192, 1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 192, 192, 2, 1 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 192, 192, 2, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 192, 192, 2, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 192, 128, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 192, 128, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 256, 256, 1, 1 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 256, 256, 1, 4 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 256, 256, 2, 3 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 256, 256, 2, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 256, 256, 3, 3 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 256, 256, 3, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 320, 256, 3, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 320, 256, 1, 4 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 320, 256, 2, 4 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 320, 256, 3, 4 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 512, 512, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 512, 512, 2, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 512, 512, 2, 2 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 512, 512, 2, 3 }, { 4, 1 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 512, 512, 3, 1 }, { 4, 1 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 512, 512, 3, 2 }, { 4, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 512, 512, 3, 3 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 576, 512, 2, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 576, 512, 2, 1 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 576, 512, 2, 2 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 576, 512, 2, 3 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 576, 512, 2, 4 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 576, 512, 3, 1 }, { 4, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_0, 576, 512, 3, 2 }, { 4, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 32, 32, -1, 1 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 32, 32, 1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 32, 32, 1, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 32, 32, 2, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 32, 32, 2, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 32, 32, 2, 4 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 32, 32, 3, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 32, 32, 3, 4 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 64, 64, 3, 0 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 64, 64, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 64, 64, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 64, 64, 2, 3 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 96, 96, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 96, 96, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 96, 96, 1, 4 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 96, 96, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 96, 96, 2, 3 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 96, 96, 3, 4 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 128, 128, 1, 1 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 128, 128, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 128, 128, 1, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 128, 128, 2, 1 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 128, 128, 2, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 128, 128, 3, 1 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 128, 128, 3, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 128, 128, 3, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 192, 192, 1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 192, 192, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 192, 192, 1, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 192, 192, 1, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 192, 192, 2, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 192, 192, 2, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 192, 192, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 192, 192, 3, 3 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 192, 128, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 192, 128, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 192, 128, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 192, 128, 2, 3 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 192, 128, 3, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 256, 256, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 256, 256, 3, 0 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 256, 256, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 256, 256, 2, 2 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 256, 256, 2, 3 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 256, 256, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 256, 256, 3, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 320, 256, 1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 320, 256, 2, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 320, 256, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 320, 256, 1, 4 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 320, 256, 2, 2 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 320, 256, 2, 4 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 320, 256, 3, 2 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 512, 512, -1, 0 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 512, 512, 1, 2 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 512, 512, 1, 3 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 512, 512, 2, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 512, 512, 2, 3 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 512, 512, 3, 1 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 512, 512, 3, 2 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 576, 512, 2, 0 }, { 4, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 576, 512, 1, 3 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 576, 512, 2, 1 }, { 4, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q4_1, 576, 512, 3, 2 }, { 4, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 32, 32, 1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 32, 32, 2, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 32, 32, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 32, 32, 3, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 64, 64, 3, 2 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 96, 96, 1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 96, 96, 2, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 96, 96, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 128, 128, 1, 0 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 128, 128, 3, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 128, 128, 1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 128, 128, 2, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 128, 128, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 192, 192, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 192, 192, 1, 2 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 192, 192, 1, 3 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 192, 192, 2, 4 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 192, 192, 3, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 192, 192, 3, 4 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 192, 128, 1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 192, 128, 2, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 192, 128, 1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 192, 128, 2, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 192, 128, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 256, 256, 1, 0 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 256, 256, 1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 256, 256, 1, 2 }, { 1, 1 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 256, 256, 2, 2 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 256, 256, 2, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 256, 256, 3, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 320, 256, 2, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 320, 256, 3, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 320, 256, -1, 1 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 320, 256, 2, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 320, 256, 2, 2 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 320, 256, 2, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 320, 256, 3, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 512, 512, 2, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 512, 512, 3, 0 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 512, 512, 1, 2 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 512, 512, 1, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 512, 512, 2, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 512, 512, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 512, 512, 2, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 512, 512, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 512, 512, 3, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 512, 512, 3, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 576, 512, 2, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 576, 512, 3, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 576, 512, 1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 576, 512, 2, 1 }, { 4, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 576, 512, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 576, 512, 2, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 576, 512, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 576, 512, 3, 2 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 576, 512, 3, 3 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_0, 576, 512, 3, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 32, 32, 1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 32, 32, 2, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 32, 32, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 64, 64, 2, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 64, 64, 3, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 64, 64, 3, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 96, 96, 1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 96, 96, 2, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 96, 96, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 128, 128, 2, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 128, 128, 3, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 128, 128, 1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 128, 128, 2, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 128, 128, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 192, 192, 1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 192, 192, 3, 0 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 192, 192, 1, 1 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 192, 192, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 192, 192, 3, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 192, 192, 3, 3 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 192, 128, 1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 192, 128, 2, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 192, 128, 1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 192, 128, 2, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 192, 128, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 256, 256, 2, 0 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 256, 256, 1, 2 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 256, 256, 2, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 256, 256, 3, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 256, 256, 3, 4 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 320, 256, 1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 320, 256, 3, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 320, 256, -1, 1 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 320, 256, 2, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 320, 256, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 512, 512, 2, 0 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 512, 512, 3, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 512, 512, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 512, 512, 1, 3 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 512, 512, 1, 4 }, { 1, 1 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 512, 512, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 512, 512, 2, 4 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 512, 512, 3, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 576, 512, 1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 576, 512, 1, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 576, 512, 2, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 576, 512, 2, 2 }, { 4, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 576, 512, 2, 3 }, { 4, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 576, 512, 3, 2 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q5_1, 576, 512, 3, 3 }, { 4, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 32, 32, 1, 3 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 32, 32, 2, 2 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 32, 32, 2, 3 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 32, 32, 3, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 32, 32, 3, 3 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 64, 64, 1, 3 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 64, 64, 3, 2 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 96, 96, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 96, 96, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 96, 96, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 96, 96, 3, 4 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 128, 128, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 128, 128, 1, 3 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 128, 128, 2, 2 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 128, 128, 2, 4 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 128, 128, 3, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 128, 128, 3, 2 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 192, 192, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 192, 192, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 192, 192, 3, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 192, 192, 3, 2 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 192, 128, 1, 2 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 192, 128, 3, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 256, 256, 3, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 320, 256, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 320, 256, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 512, 512, 2, 0 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 512, 512, 1, 3 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 512, 512, 2, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 512, 512, 2, 2 }, { 4, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 512, 512, 2, 3 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 512, 512, 2, 4 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 512, 512, 3, 1 }, { 4, 1 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 512, 512, 3, 2 }, { 4, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 576, 512, 1, 4 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 576, 512, 2, 1 }, { 4, 4 } }, }; static enum ggml_metal_device_id fa_vec_family_representative(int gpu_family) {