From c25de61569dd241f7f44375b222c9b49089937a4 Mon Sep 17 00:00:00 2001 From: Ruben Ortlam Date: Tue, 7 Jul 2026 14:48:02 +0200 Subject: [PATCH 1/9] vulkan: use spec constant for mul mat type_a vulkan: use map for mul_mm shapes cleanup fix indentation fix cm2 and shmem init fix cm2 spec constants fix cm2 bindings consolidate shmem tables and reduce size by type spec constant fix compiler warning fix missing Q2_0 type fix unused warning when integer dot glslc support is missing use minimal shmem size 8 instead of 1 to workaround cm2 compiler bug fix missing Q2_0 type in cm2 matmul fix types --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 1510 ++++++----------- .../vulkan-shaders/dequant_funcs.glsl | 20 +- .../vulkan-shaders/dequant_funcs_cm2.glsl | 50 +- .../vulkan-shaders/dequant_iq2_s.comp | 2 +- .../vulkan-shaders/dequant_iq2_xs.comp | 2 +- .../vulkan-shaders/dequant_iq2_xxs.comp | 2 +- .../vulkan-shaders/dequant_iq3_s.comp | 2 +- .../vulkan-shaders/dequant_iq3_xxs.comp | 4 +- .../ggml-vulkan/vulkan-shaders/fa_types.glsl | 42 +- .../vulkan-shaders/flash_attn.comp | 6 +- .../vulkan-shaders/flash_attn_base.glsl | 4 +- .../vulkan-shaders/flash_attn_cm2.comp | 58 +- .../vulkan-shaders/flash_attn_dequant.glsl | 32 +- .../vulkan-shaders/flash_attn_mmq_funcs.glsl | 72 +- .../vulkan-shaders/ggml_type_ids.glsl | 33 + .../vulkan-shaders/iq_shmem_init.glsl | 66 + .../vulkan-shaders/lightning_indexer.comp | 10 +- .../vulkan-shaders/mul_mat_vec_iq2_s.comp | 2 +- .../vulkan-shaders/mul_mat_vec_iq2_xs.comp | 2 +- .../vulkan-shaders/mul_mat_vec_iq2_xxs.comp | 4 +- .../vulkan-shaders/mul_mat_vec_iq3_s.comp | 4 +- .../vulkan-shaders/mul_mat_vec_iq3_xxs.comp | 4 +- .../ggml-vulkan/vulkan-shaders/mul_mm.comp | 96 +- .../vulkan-shaders/mul_mm_cm2.comp | 208 ++- .../vulkan-shaders/mul_mm_funcs.glsl | 1259 +++++++------- .../src/ggml-vulkan/vulkan-shaders/types.glsl | 126 +- .../vulkan-shaders/vulkan-shaders-gen.cpp | 67 +- 27 files changed, 1870 insertions(+), 1817 deletions(-) create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/ggml_type_ids.glsl create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/iq_shmem_init.glsl diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index a04a6b27a8c..dac862f014c 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -250,27 +250,35 @@ typedef std::weak_ptr vk_pipeline_ref; static void ggml_vk_destroy_pipeline(vk::Device& device, vk_pipeline& pipeline); -struct vk_matmul_pipeline_struct { - vk_pipeline l, m, s; - vk_pipeline a_l, a_m, a_s; - // Returns true when all unaligned pipelines are null. - // We only check for unaligned variants since one of the unaligned pipelines must exist - // while aligned pipelines are optional - bool is_empty() const { - return l == nullptr && m == nullptr && s == nullptr; +struct vk_matmul_pipeline_key { + ggml_type type_a; + ggml_type type_b; + bool mul_mat_id; + bool f16acc; + + bool operator<(const vk_matmul_pipeline_key & o) const { + return std::tie(type_a, type_b, mul_mat_id, f16acc) + < std::tie(o.type_a, o.type_b, o.mul_mat_id, o.f16acc); } }; -typedef std::shared_ptr vk_matmul_pipeline; -struct vk_matmul_pipeline2 { - vk_matmul_pipeline2() { - f16acc = std::make_shared(); - f32acc = std::make_shared(); - } - vk_matmul_pipeline f32acc; - vk_matmul_pipeline f16acc; +struct vk_matmul_pipeline_pair { + vk_pipeline unaligned; + vk_pipeline aligned; + uint32_t align; +}; + +struct vk_tile_config { + std::vector warptile; + std::array wg_denoms; + uint32_t align; }; +using matmul_tile_selector_t = std::function& configs)>; + + struct vk_device_struct; typedef std::shared_ptr vk_device; typedef std::weak_ptr vk_device_ref; @@ -921,23 +929,9 @@ struct vk_device_struct { vk::DescriptorSetLayout dsl; - vk_matmul_pipeline pipeline_matmul_f32 {}; - vk_matmul_pipeline pipeline_matmul_f32_f16 {}; - vk_matmul_pipeline pipeline_matmul_bf16 {}; - vk_matmul_pipeline2 pipeline_matmul_f16; - vk_matmul_pipeline2 pipeline_matmul_f16_f32; - - vk_matmul_pipeline2 pipeline_dequant_mul_mat_mat[GGML_TYPE_COUNT]; - vk_matmul_pipeline2 pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_COUNT]; - vk_matmul_pipeline2 pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_COUNT]; - - vk_matmul_pipeline pipeline_matmul_id_f32 {}; - vk_matmul_pipeline pipeline_matmul_id_bf16 {}; - vk_matmul_pipeline2 pipeline_matmul_id_f16; - vk_matmul_pipeline2 pipeline_matmul_id_f16_f32; - - vk_matmul_pipeline2 pipeline_dequant_mul_mat_mat_id[GGML_TYPE_COUNT]; - vk_matmul_pipeline2 pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_COUNT]; + std::map> pipeline_matmul; + matmul_tile_selector_t matmul_tile_selector; + matmul_tile_selector_t matmul_id_tile_selector; vk_pipeline pipeline_matmul_split_k_reduce; vk_pipeline pipeline_quantize_q8_1_x4; @@ -4315,9 +4309,9 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { // spec constants and tile sizes for quant matmul_id const uint32_t mmqid_bk = device->coopmat2_decode_vector ? 64u : 32u; - l_warptile_mmqid = { 256, 128, 128, mmqid_bk, 1, device->subgroup_size }; - m_warptile_mmqid = { 256, 128, 64, mmqid_bk, 0, device->subgroup_size }; - s_warptile_mmqid = { 256, 128, 64, mmqid_bk, 0, device->subgroup_size }; + l_warptile_mmqid = { 256, 128, 128, mmqid_bk, 1 }; + m_warptile_mmqid = { 256, 128, 64, mmqid_bk, 0 }; + s_warptile_mmqid = { 256, 128, 64, mmqid_bk, 0 }; l_mmqid_wg_denoms = { 128, 128, 1 }; m_mmqid_wg_denoms = { 128, 64, 1 }; s_mmqid_wg_denoms = { 128, 64, 1 }; @@ -4458,22 +4452,6 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { } } - if (!device->pipeline_matmul_f32) { - device->pipeline_matmul_f32 = std::make_shared(); - } - if (!device->pipeline_matmul_f32_f16) { - device->pipeline_matmul_f32_f16 = std::make_shared(); - } - if (!device->pipeline_matmul_id_f32) { - device->pipeline_matmul_id_f32 = std::make_shared(); - } - if (!device->pipeline_matmul_bf16) { - device->pipeline_matmul_bf16 = std::make_shared(); - } - if (!device->pipeline_matmul_id_bf16) { - device->pipeline_matmul_id_bf16 = std::make_shared(); - } - auto const &ggml_vk_create_pipeline = [&](vk_device& device, vk_pipeline& base_pipeline, const char *name, size_t spv_size, const void* spv_data, const char *entrypoint, uint32_t parameter_count, uint32_t push_constant_size, std::array wg_denoms, const std::vector& specialization_constants, uint32_t align, bool disable_robustness = false, bool require_full_subgroups = false, uint32_t required_subgroup_size = 0) { @@ -4682,594 +4660,500 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { return spec; }; + auto const &ggml_vk_mul_mm_spec_quant = [&device](std::vector spec, bool aligned, uint32_t type) { + spec.push_back(aligned ? 1u : 0u); // constantID=11: ALIGNED + spec.push_back(type); // constantID=12: MmTypeA + if (device->vendor_id == VK_VENDOR_ID_INTEL && device->coopmat_support && + device->driver_id == vk::DriverId::eIntelProprietaryWindows) { + spec.push_back(0u); // constantID=13: SHMEM_STRIDE_PAD = 0 + spec.push_back(1u); // constantID=14: APPLY_SLM_A_RESHAPE = true + } + return spec; + }; + + static const ggml_type quant_types[] = { + GGML_TYPE_Q1_0, GGML_TYPE_Q2_0, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, GGML_TYPE_Q8_0, + GGML_TYPE_Q2_K, GGML_TYPE_TQ2_0, GGML_TYPE_Q3_K, GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K, + GGML_TYPE_IQ1_S, GGML_TYPE_IQ1_M, GGML_TYPE_IQ2_XXS, GGML_TYPE_IQ2_XS, GGML_TYPE_IQ2_S, + GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ3_S, GGML_TYPE_IQ4_XS, GGML_TYPE_IQ4_NL, + GGML_TYPE_MXFP4, GGML_TYPE_NVFP4, + }; + const int mul_mat_id_param_count = 5; + using spec_fn_t = std::function(const std::vector&, bool)>; + auto const &create_mm_pipelines = [&]( + const vk_matmul_pipeline_key& key, + const std::vector& tile_configs, + const std::string& shader_name, size_t spv_len, const void* spv_data, + uint32_t push_constant_size, uint32_t param_count, + const spec_fn_t& spec_fn, + bool disable_robustness = false, bool require_full_subgroups = false, uint32_t required_subgroup_size = 0, + bool create_aligned = true + ) { + auto& vec = device->pipeline_matmul[key]; + const bool first_call = vec.empty(); + for (size_t i = 0; i < tile_configs.size(); i++) { + const auto& tc = tile_configs[i]; + + if (first_call) { + vk_matmul_pipeline_pair pair{}; + pair.align = tc.align; + std::string suffix = "_" + std::to_string(i); + pair.unaligned = std::make_shared(); + if (create_aligned) { + pair.aligned = std::make_shared(); + } + vec.push_back(pair); + } + + ggml_vk_create_pipeline(device, vec[i].unaligned, + vec[i].unaligned->name.empty() ? (shader_name + "_" + std::to_string(i)).c_str() : vec[i].unaligned->name.c_str(), + spv_len, spv_data, "main", param_count, push_constant_size, + tc.wg_denoms, spec_fn(tc.warptile, false), 1, + disable_robustness, require_full_subgroups, required_subgroup_size); + + if (vec[i].aligned) { + ggml_vk_create_pipeline(device, vec[i].aligned, + vec[i].aligned->name.empty() ? (shader_name + "_aligned_" + std::to_string(i)).c_str() : vec[i].aligned->name.c_str(), + spv_len, spv_data, "main", param_count, push_constant_size, + tc.wg_denoms, spec_fn(tc.warptile, true), tc.align, + disable_robustness, require_full_subgroups, required_subgroup_size); + } + } + }; + + auto filter_tc = [&](const std::vector& configs, ggml_type type, bool is_id, bool is_int = false) -> std::vector { + std::vector result; + bool enabled[3]; + if (is_int) { + enabled[0] = is_id ? device->mul_mat_id_s_int[type] : device->mul_mat_s_int[type]; + enabled[1] = is_id ? device->mul_mat_id_m_int[type] : device->mul_mat_m_int[type]; + enabled[2] = is_id ? device->mul_mat_id_l_int[type] : device->mul_mat_l_int[type]; + } else { + enabled[0] = is_id ? device->mul_mat_id_s[type] : device->mul_mat_s[type]; + enabled[1] = is_id ? device->mul_mat_id_m[type] : device->mul_mat_m[type]; + enabled[2] = is_id ? device->mul_mat_id_l[type] : device->mul_mat_l[type]; + } + for (size_t i = 0; i < configs.size() && i < 3; i++) { + if (enabled[i]) result.push_back(configs[i]); + } + return result; + }; + + // Build tile config vectors + std::vector tc_mm = {{s_warptile, s_wg_denoms, s_align}, {m_warptile, m_wg_denoms, m_align}, {l_warptile, l_wg_denoms, l_align}}; + std::vector tc_mmq = {{s_warptile_mmq, s_mmq_wg_denoms, s_align}, {m_warptile_mmq, m_mmq_wg_denoms, m_align}, {l_warptile_mmq, l_mmq_wg_denoms, l_align}}; + #if defined(VK_NV_cooperative_matrix2) && defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT) if (device->coopmat2) { - auto const &ggml_vk_mul_mm_cm2_spec = [](std::vector spec, bool aligned, bool mul_mat_id) { - if (mul_mat_id && spec.size() > 5) { - spec.insert(spec.begin() + 5, aligned ? 1u : 0u); - } else { - spec.push_back(aligned ? 1u : 0u); - } - if (mul_mat_id && spec.size() == 6) { - spec.push_back(32); + auto const &ggml_vk_mul_mm_cm2_spec = [&](std::vector spec, bool aligned, uint32_t type = UINT32_MAX) { + spec.push_back(aligned ? 1u : 0u); // ALIGNED + spec.push_back(device->subgroup_size); // subgroup_size + if (type != UINT32_MAX) { + spec.push_back(type); // MmTypeA } return spec; }; - // Create 6 variants, {s,m,l}x{unaligned,aligned} -#define CREATE_MM(PIPELINE_NAME, NAMELC, F16ACC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->l, #NAMELC #F16ACC "_l", NAMELC ## F16ACC ## _cm2_len, NAMELC ## F16ACC ## _cm2_data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_cm2_spec(l_ ## WARPTILE, false, PARAMCOUNT == mul_mat_id_param_count), 1, true); \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->m, #NAMELC #F16ACC "_m", NAMELC ## F16ACC ## _cm2_len, NAMELC ## F16ACC ## _cm2_data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_cm2_spec(m_ ## WARPTILE, false, PARAMCOUNT == mul_mat_id_param_count), 1, true); \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->s, #NAMELC #F16ACC "_s", NAMELC ## F16ACC ## _cm2_len, NAMELC ## F16ACC ## _cm2_data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_cm2_spec(s_ ## WARPTILE, false, PARAMCOUNT == mul_mat_id_param_count), 1, true); \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_l, #NAMELC #F16ACC "_aligned_l", NAMELC ## F16ACC ## _cm2_len, NAMELC ## F16ACC ## _cm2_data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_cm2_spec(l_ ## WARPTILE, true, PARAMCOUNT == mul_mat_id_param_count), l_align, true); \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_m, #NAMELC #F16ACC "_aligned_m", NAMELC ## F16ACC ## _cm2_len, NAMELC ## F16ACC ## _cm2_data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_cm2_spec(m_ ## WARPTILE, true, PARAMCOUNT == mul_mat_id_param_count), m_align, true); \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_s, #NAMELC #F16ACC "_aligned_s", NAMELC ## F16ACC ## _cm2_len, NAMELC ## F16ACC ## _cm2_data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_cm2_spec(s_ ## WARPTILE, true, PARAMCOUNT == mul_mat_id_param_count), s_align, true); \ - - // Create 2 variants, {f16,f32} accumulator -#define CREATE_MM2(PIPELINE_NAME, NAMELC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT) \ - CREATE_MM(PIPELINE_NAME . f16acc, NAMELC, _f16acc, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT) \ - CREATE_MM(PIPELINE_NAME . f32acc, NAMELC, , WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT) \ - - CREATE_MM2(pipeline_matmul_f16, matmul_f16, wg_denoms, warptile, vk_mat_mat_push_constants, 3) + std::vector tc_mmq_k = {{s_warptile_mmq_k, s_mmq_wg_denoms_k, s_align}, {m_warptile_mmq_k, m_mmq_wg_denoms_k, m_align}, {l_warptile_mmq_k, l_mmq_wg_denoms_k, l_align}}; + std::vector tc_mmqid = {{s_warptile_mmqid, s_mmqid_wg_denoms, s_align}, {m_warptile_mmqid, m_mmqid_wg_denoms, m_align}, {l_warptile_mmqid, l_mmqid_wg_denoms, l_align}}; + + spec_fn_t cm2_spec = [&](const std::vector& wt, bool a) { return ggml_vk_mul_mm_cm2_spec(wt, a); }; + + // F16 x F16 + create_mm_pipelines({GGML_TYPE_F16, GGML_TYPE_F16, false, true}, tc_mm, "matmul_f16_f16acc", matmul_f16_f16acc_cm2_len, matmul_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); + create_mm_pipelines({GGML_TYPE_F16, GGML_TYPE_F16, false, false}, tc_mm, "matmul_f16", matmul_f16_cm2_len, matmul_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); #if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) if (device->coopmat_bf16_support) { - CREATE_MM(pipeline_matmul_bf16, matmul_bf16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3) + create_mm_pipelines({GGML_TYPE_BF16, GGML_TYPE_BF16, false, false}, tc_mm, "matmul_bf16", matmul_bf16_cm2_len, matmul_bf16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); } #endif - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q1_0], matmul_q1_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q2_0], matmul_q2_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q4_0], matmul_q4_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q4_1], matmul_q4_1_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q5_0], matmul_q5_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q5_1], matmul_q5_1_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q8_0], matmul_q8_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q2_K], matmul_q2_k_f16, mmq_wg_denoms_k, warptile_mmq_k, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_TQ2_0], matmul_tq2_0_f16, mmq_wg_denoms_k, warptile_mmq_k, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q3_K], matmul_q3_k_f16, mmq_wg_denoms_k, warptile_mmq_k, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q4_K], matmul_q4_k_f16, mmq_wg_denoms_k, warptile_mmq_k, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q5_K], matmul_q5_k_f16, mmq_wg_denoms_k, warptile_mmq_k, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q6_K], matmul_q6_k_f16, mmq_wg_denoms_k, warptile_mmq_k, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ1_S], matmul_iq1_s_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ1_M], matmul_iq1_m_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ2_XXS], matmul_iq2_xxs_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ2_XS], matmul_iq2_xs_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ2_S], matmul_iq2_s_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ3_XXS], matmul_iq3_xxs_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ3_S], matmul_iq3_s_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ4_XS], matmul_iq4_xs_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ4_NL], matmul_iq4_nl_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) + for (const auto type : quant_types) { + auto& tc = ((type >= GGML_TYPE_Q2_K && type <= GGML_TYPE_Q6_K) || type == GGML_TYPE_TQ2_0) ? tc_mmq_k : tc_mmq; + spec_fn_t qs = [&, type](const std::vector& wt, bool a) { return ggml_vk_mul_mm_cm2_spec(wt, a, (uint32_t)type); }; #if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) - if (device->ocp_fp4) { - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_MXFP4], matmul_mxfp4_f16_ocp, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_NVFP4], matmul_nvfp4_f16_ocp, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - } else + if (device->ocp_fp4 && (type == GGML_TYPE_MXFP4 || type == GGML_TYPE_NVFP4)) { + create_mm_pipelines({type, GGML_TYPE_F16, false, true}, tc, "matmul_quant_f16_ocp_f16acc", matmul_quant_f16_ocp_f16acc_cm2_len, matmul_quant_f16_ocp_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, qs, true); + create_mm_pipelines({type, GGML_TYPE_F16, false, false}, tc, "matmul_quant_f16_ocp", matmul_quant_f16_ocp_cm2_len, matmul_quant_f16_ocp_cm2_data, sizeof(vk_mat_mat_push_constants), 3, qs, true); + } else #endif - { - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_MXFP4], matmul_mxfp4_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_NVFP4], matmul_nvfp4_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) + { + create_mm_pipelines({type, GGML_TYPE_F16, false, true}, tc, "matmul_quant_f16_f16acc", matmul_quant_f16_f16acc_cm2_len, matmul_quant_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, qs, true); + create_mm_pipelines({type, GGML_TYPE_F16, false, false}, tc, "matmul_quant_f16", matmul_quant_f16_cm2_len, matmul_quant_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, qs, true); + } } GGML_ASSERT(device->subgroup_ballot); - CREATE_MM2(pipeline_matmul_id_f16, matmul_id_subgroup_f16, wg_denoms, warptile, vk_mat_mat_id_push_constants, 5) + // ID: F16 + create_mm_pipelines({GGML_TYPE_F16, GGML_TYPE_F16, true, true}, tc_mm, "matmul_id_subgroup_f16_f16acc", matmul_id_subgroup_f16_f16acc_cm2_len, matmul_id_subgroup_f16_f16acc_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); + create_mm_pipelines({GGML_TYPE_F16, GGML_TYPE_F16, true, false}, tc_mm, "matmul_id_subgroup_f16", matmul_id_subgroup_f16_cm2_len, matmul_id_subgroup_f16_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); #if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) if (device->coopmat_bf16_support) { - CREATE_MM(pipeline_matmul_id_bf16, matmul_id_subgroup_bf16, , wg_denoms, warptile, vk_mat_mat_id_push_constants, 5) + create_mm_pipelines({GGML_TYPE_BF16, GGML_TYPE_BF16, true, false}, tc_mm, "matmul_id_subgroup_bf16", matmul_id_subgroup_bf16_cm2_len, matmul_id_subgroup_bf16_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); } #endif - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q1_0], matmul_id_subgroup_q1_0_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_0], matmul_id_subgroup_q2_0_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_0], matmul_id_subgroup_q4_0_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_1], matmul_id_subgroup_q4_1_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_0], matmul_id_subgroup_q5_0_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_1], matmul_id_subgroup_q5_1_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q8_0], matmul_id_subgroup_q8_0_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_K], matmul_id_subgroup_q2_k_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_TQ2_0], matmul_id_subgroup_tq2_0_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q3_K], matmul_id_subgroup_q3_k_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_K], matmul_id_subgroup_q4_k_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_K], matmul_id_subgroup_q5_k_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q6_K], matmul_id_subgroup_q6_k_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_S], matmul_id_subgroup_iq1_s_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_M], matmul_id_subgroup_iq1_m_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XXS], matmul_id_subgroup_iq2_xxs_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XS], matmul_id_subgroup_iq2_xs_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_S], matmul_id_subgroup_iq2_s_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_XXS], matmul_id_subgroup_iq3_xxs_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_S], matmul_id_subgroup_iq3_s_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_XS], matmul_id_subgroup_iq4_xs_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_NL], matmul_id_subgroup_iq4_nl_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) + for (const auto type : quant_types) { + spec_fn_t qs_id = [&, type](const std::vector& wt, bool a) { return ggml_vk_mul_mm_cm2_spec(wt, a, (uint32_t)type); }; #if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) - if (device->ocp_fp4) { - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_MXFP4], matmul_id_subgroup_mxfp4_f16_ocp, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_NVFP4], matmul_id_subgroup_nvfp4_f16_ocp, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - } else + if (device->ocp_fp4 && (type == GGML_TYPE_MXFP4 || type == GGML_TYPE_NVFP4)) { + create_mm_pipelines({type, GGML_TYPE_F16, true, true}, tc_mmqid, "matmul_id_subgroup_quant_f16_ocp_f16acc", matmul_id_subgroup_quant_f16_ocp_f16acc_cm2_len, matmul_id_subgroup_quant_f16_ocp_f16acc_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, qs_id, true); + create_mm_pipelines({type, GGML_TYPE_F16, true, false}, tc_mmqid, "matmul_id_subgroup_quant_f16_ocp", matmul_id_subgroup_quant_f16_ocp_cm2_len, matmul_id_subgroup_quant_f16_ocp_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, qs_id, true); + } else #endif - { - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_MXFP4], matmul_id_subgroup_mxfp4_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_NVFP4], matmul_id_subgroup_nvfp4_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) + { + create_mm_pipelines({type, GGML_TYPE_F16, true, true}, tc_mmqid, "matmul_id_subgroup_quant_f16_f16acc", matmul_id_subgroup_quant_f16_f16acc_cm2_len, matmul_id_subgroup_quant_f16_f16acc_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, qs_id, true); + create_mm_pipelines({type, GGML_TYPE_F16, true, false}, tc_mmqid, "matmul_id_subgroup_quant_f16", matmul_id_subgroup_quant_f16_cm2_len, matmul_id_subgroup_quant_f16_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, qs_id, true); + } } -#undef CREATE_MM -#undef CREATE_MM2 } else #endif // defined(VK_NV_cooperative_matrix2) && defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT) #if defined(VK_KHR_cooperative_matrix) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) if (device->coopmat_support) { - // Create 6 variants, {s,m,l}x{unaligned,aligned} -#define CREATE_MM(TYPE, PIPELINE_NAME, NAMELC, F16ACC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID) \ - if (device->mul_mat ## ID ## _l[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->l, #NAMELC #F16ACC "_l", NAMELC ## F16ACC ## _cm1_len, NAMELC ## F16ACC ## _cm1_data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_spec(l_ ## WARPTILE, false), 1, false, true); \ - if (device->mul_mat ## ID ## _m[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->m, #NAMELC #F16ACC "_m", NAMELC ## F16ACC ## _cm1_len, NAMELC ## F16ACC ## _cm1_data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_spec(m_ ## WARPTILE, false), 1, false, true); \ - if (device->mul_mat ## ID ## _s[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->s, #NAMELC #F16ACC "_s", NAMELC ## F16ACC ## _cm1_len, NAMELC ## F16ACC ## _cm1_data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_spec(s_ ## WARPTILE, false), 1, false, true); \ - if (device->mul_mat ## ID ## _l[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_l, #NAMELC #F16ACC "_aligned_l", NAMELC ## F16ACC ## _cm1_len, NAMELC ## F16ACC ## _cm1_data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_spec(l_ ## WARPTILE, true), l_align, false, true); \ - if (device->mul_mat ## ID ## _m[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_m, #NAMELC #F16ACC "_aligned_m", NAMELC ## F16ACC ## _cm1_len, NAMELC ## F16ACC ## _cm1_data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_spec(m_ ## WARPTILE, true), m_align, false, true); \ - if (device->mul_mat ## ID ## _s[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_s, #NAMELC #F16ACC "_aligned_s", NAMELC ## F16ACC ## _cm1_len, NAMELC ## F16ACC ## _cm1_data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_spec(s_ ## WARPTILE, true), s_align, false, true); \ - - // Create 2 variants, {f16,f32} accumulator -#define CREATE_MM2(TYPE, PIPELINE_NAME, NAMELC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID) \ - if (device->coopmat_acc_f16_support) { \ - CREATE_MM(TYPE, PIPELINE_NAME . f16acc, NAMELC, _f16acc, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID) \ - } \ - if (device->coopmat_acc_f32_support) { \ - CREATE_MM(TYPE, PIPELINE_NAME . f32acc, NAMELC, , WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID) \ - } \ - - CREATE_MM(GGML_TYPE_F32, pipeline_matmul_f32, matmul_f32_f32, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, ); - CREATE_MM(GGML_TYPE_F32, pipeline_matmul_f32_f16, matmul_f32_f16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_f16, matmul_f16, wg_denoms, warptile, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_f16_f32, matmul_f16_f32, wg_denoms, warptile, vk_mat_mat_push_constants, 3, ); + spec_fn_t cm1_spec = [&](const std::vector& wt, bool a) { return ggml_vk_mul_mm_spec(wt, a); }; + + auto cm1_create = [&](vk_matmul_pipeline_key key, const std::vector& tc_base, + const std::string& name, size_t len, const void* data, uint32_t pc_size, uint32_t pc) { + auto tc = filter_tc(tc_base, key.type_a, key.mul_mat_id); + if (!tc.empty()) create_mm_pipelines(key, tc, name, len, data, pc_size, pc, cm1_spec, false, true, 0); + }; + auto cm1_create_quant = [&](vk_matmul_pipeline_key key, const std::vector& tc_base, + const std::string& name, size_t len, const void* data, uint32_t pc_size, uint32_t pc) { + spec_fn_t qs = [&, type_a=key.type_a](const std::vector& wt, bool a) { return ggml_vk_mul_mm_spec_quant(wt, a, (uint32_t)type_a); }; + auto tc = filter_tc(tc_base, key.type_a, key.mul_mat_id); + if (!tc.empty()) create_mm_pipelines(key, tc, name, len, data, pc_size, pc, qs, false, true, 0); + }; + + cm1_create({GGML_TYPE_F32, GGML_TYPE_F32, false, false}, tc_mm, "matmul_f32_f32", matmul_f32_f32_cm1_len, matmul_f32_f32_cm1_data, sizeof(vk_mat_mat_push_constants), 3); + cm1_create({GGML_TYPE_F32, GGML_TYPE_F16, false, false}, tc_mm, "matmul_f32_f16", matmul_f32_f16_cm1_len, matmul_f32_f16_cm1_data, sizeof(vk_mat_mat_push_constants), 3); + if (device->coopmat_acc_f16_support) { + cm1_create({GGML_TYPE_F16, GGML_TYPE_F16, false, true}, tc_mm, "matmul_f16_f16acc", matmul_f16_f16acc_cm1_len, matmul_f16_f16acc_cm1_data, sizeof(vk_mat_mat_push_constants), 3); + cm1_create({GGML_TYPE_F16, GGML_TYPE_F32, false, true}, tc_mm, "matmul_f16_f32_f16acc", matmul_f16_f32_f16acc_cm1_len, matmul_f16_f32_f16acc_cm1_data, sizeof(vk_mat_mat_push_constants), 3); + } + if (device->coopmat_acc_f32_support) { + cm1_create({GGML_TYPE_F16, GGML_TYPE_F16, false, false}, tc_mm, "matmul_f16", matmul_f16_cm1_len, matmul_f16_cm1_data, sizeof(vk_mat_mat_push_constants), 3); + cm1_create({GGML_TYPE_F16, GGML_TYPE_F32, false, false}, tc_mm, "matmul_f16_f32", matmul_f16_f32_cm1_len, matmul_f16_f32_cm1_data, sizeof(vk_mat_mat_push_constants), 3); + } #if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) if (device->coopmat_bf16_support) { - CREATE_MM(GGML_TYPE_BF16, pipeline_matmul_bf16, matmul_bf16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, ) + cm1_create({GGML_TYPE_BF16, GGML_TYPE_BF16, false, false}, tc_mm, "matmul_bf16", matmul_bf16_cm1_len, matmul_bf16_cm1_data, sizeof(vk_mat_mat_push_constants), 3); } #endif - - CREATE_MM2(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q1_0], matmul_q1_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q2_0], matmul_q2_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_0], matmul_q4_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_1], matmul_q4_1_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_0], matmul_q5_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_1], matmul_q5_1_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q8_0], matmul_q8_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - - CREATE_MM2(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q2_K], matmul_q2_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_TQ2_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_TQ2_0], matmul_tq2_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q3_K], matmul_q3_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_K], matmul_q4_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_K], matmul_q5_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q6_K], matmul_q6_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ1_S], matmul_iq1_s_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ1_M], matmul_iq1_m_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ2_XXS], matmul_iq2_xxs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ2_XS], matmul_iq2_xs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ2_S], matmul_iq2_s_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ3_XXS], matmul_iq3_xxs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ3_S], matmul_iq3_s_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ4_XS], matmul_iq4_xs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ4_NL], matmul_iq4_nl_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - + for (const auto type : quant_types) { #if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) - if (device->ocp_fp4) { - CREATE_MM2(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat[GGML_TYPE_MXFP4], matmul_mxfp4_f32_ocp, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat[GGML_TYPE_NVFP4], matmul_nvfp4_f32_ocp, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - } else + if (device->ocp_fp4 && (type == GGML_TYPE_MXFP4 || type == GGML_TYPE_NVFP4)) { + if (device->coopmat_acc_f16_support) { + cm1_create_quant({type, GGML_TYPE_F32, false, true}, tc_mmq, "matmul_quant_f32_ocp_f16acc", matmul_quant_f32_ocp_f16acc_cm1_len, matmul_quant_f32_ocp_f16acc_cm1_data, sizeof(vk_mat_mat_push_constants), 3); + } + if (device->coopmat_acc_f32_support) { + cm1_create_quant({type, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_quant_f32_ocp", matmul_quant_f32_ocp_cm1_len, matmul_quant_f32_ocp_cm1_data, sizeof(vk_mat_mat_push_constants), 3); + } + } else #endif - { - CREATE_MM2(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat[GGML_TYPE_MXFP4], matmul_mxfp4_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat[GGML_TYPE_NVFP4], matmul_nvfp4_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); + { + if (device->coopmat_acc_f16_support) { + cm1_create_quant({type, GGML_TYPE_F32, false, true}, tc_mmq, "matmul_quant_f32_f16acc", matmul_quant_f32_f16acc_cm1_len, matmul_quant_f32_f16acc_cm1_data, sizeof(vk_mat_mat_push_constants), 3); + } + if (device->coopmat_acc_f32_support) { + cm1_create_quant({type, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_quant_f32", matmul_quant_f32_cm1_len, matmul_quant_f32_cm1_data, sizeof(vk_mat_mat_push_constants), 3); + } + } } GGML_ASSERT(device->subgroup_ballot); - CREATE_MM(GGML_TYPE_F32, pipeline_matmul_id_f32, matmul_id_subgroup_f32_f32, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_id_f16, matmul_id_subgroup_f16, wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_id_f16_f32, matmul_id_subgroup_f16_f32, wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); + cm1_create({GGML_TYPE_F32, GGML_TYPE_F32, true, false}, tc_mm, "matmul_id_subgroup_f32_f32", matmul_id_subgroup_f32_f32_cm1_len, matmul_id_subgroup_f32_f32_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + if (device->coopmat_acc_f16_support) { + cm1_create({GGML_TYPE_F16, GGML_TYPE_F16, true, true}, tc_mm, "matmul_id_subgroup_f16_f16acc", matmul_id_subgroup_f16_f16acc_cm1_len, matmul_id_subgroup_f16_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + cm1_create({GGML_TYPE_F16, GGML_TYPE_F32, true, true}, tc_mm, "matmul_id_subgroup_f16_f32_f16acc", matmul_id_subgroup_f16_f32_f16acc_cm1_len, matmul_id_subgroup_f16_f32_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + } + if (device->coopmat_acc_f32_support) { + cm1_create({GGML_TYPE_F16, GGML_TYPE_F16, true, false}, tc_mm, "matmul_id_subgroup_f16", matmul_id_subgroup_f16_cm1_len, matmul_id_subgroup_f16_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + cm1_create({GGML_TYPE_F16, GGML_TYPE_F32, true, false}, tc_mm, "matmul_id_subgroup_f16_f32", matmul_id_subgroup_f16_f32_cm1_len, matmul_id_subgroup_f16_f32_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + } #if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) if (device->coopmat_bf16_support) { - CREATE_MM(GGML_TYPE_BF16, pipeline_matmul_id_bf16, matmul_id_subgroup_bf16, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); + cm1_create({GGML_TYPE_BF16, GGML_TYPE_BF16, true, false}, tc_mm, "matmul_id_subgroup_bf16", matmul_id_subgroup_bf16_cm1_len, matmul_id_subgroup_bf16_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); } #endif - - CREATE_MM2(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q1_0], matmul_id_subgroup_q1_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_0], matmul_id_subgroup_q2_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_0], matmul_id_subgroup_q4_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_1], matmul_id_subgroup_q4_1_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_0], matmul_id_subgroup_q5_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_1], matmul_id_subgroup_q5_1_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q8_0], matmul_id_subgroup_q8_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_K], matmul_id_subgroup_q2_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_TQ2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_TQ2_0], matmul_id_subgroup_tq2_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q3_K], matmul_id_subgroup_q3_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_K], matmul_id_subgroup_q4_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_K], matmul_id_subgroup_q5_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q6_K], matmul_id_subgroup_q6_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_S], matmul_id_subgroup_iq1_s_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_M], matmul_id_subgroup_iq1_m_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XXS], matmul_id_subgroup_iq2_xxs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XS], matmul_id_subgroup_iq2_xs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_S], matmul_id_subgroup_iq2_s_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_XXS], matmul_id_subgroup_iq3_xxs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_S], matmul_id_subgroup_iq3_s_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_XS], matmul_id_subgroup_iq4_xs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_NL], matmul_id_subgroup_iq4_nl_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); + for (const auto type : quant_types) { #if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) - if (device->ocp_fp4) { - CREATE_MM2(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_MXFP4], matmul_id_subgroup_mxfp4_f32_ocp, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_NVFP4], matmul_id_subgroup_nvfp4_f32_ocp, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - } else + if (device->ocp_fp4 && (type == GGML_TYPE_MXFP4 || type == GGML_TYPE_NVFP4)) { + if (device->coopmat_acc_f16_support) { + cm1_create_quant({type, GGML_TYPE_F32, true, true}, tc_mmq, "matmul_id_subgroup_quant_f32_ocp_f16acc", matmul_id_subgroup_quant_f32_ocp_f16acc_cm1_len, matmul_id_subgroup_quant_f32_ocp_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + } + if (device->coopmat_acc_f32_support) { + cm1_create_quant({type, GGML_TYPE_F32, true, false}, tc_mmq, "matmul_id_subgroup_quant_f32_ocp", matmul_id_subgroup_quant_f32_ocp_cm1_len, matmul_id_subgroup_quant_f32_ocp_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + } + } else #endif - { - CREATE_MM2(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_MXFP4], matmul_id_subgroup_mxfp4_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_NVFP4], matmul_id_subgroup_nvfp4_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); + { + if (device->coopmat_acc_f16_support) { + cm1_create_quant({type, GGML_TYPE_F32, true, true}, tc_mmq, "matmul_id_subgroup_quant_f32_f16acc", matmul_id_subgroup_quant_f32_f16acc_cm1_len, matmul_id_subgroup_quant_f32_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + } + if (device->coopmat_acc_f32_support) { + cm1_create_quant({type, GGML_TYPE_F32, true, false}, tc_mmq, "matmul_id_subgroup_quant_f32", matmul_id_subgroup_quant_f32_cm1_len, matmul_id_subgroup_quant_f32_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + } + } } -#undef CREATE_MM2 -#undef CREATE_MM } else #endif // defined(VK_KHR_cooperative_matrix) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) - if (device->fp16) { - // Create 6 variants, {s,m,l}x{unaligned,aligned} - // Selects dot2 SPIR-V variant at runtime when device->dot2_f16 is true -#define CREATE_MM(TYPE, PIPELINE_NAME, NAMELC, F16ACC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE) \ - if (device->mul_mat ## ID ## _l[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->l, #NAMELC #F16ACC "_l", (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _len : NAMELC ## F16ACC ## _len), (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _data : NAMELC ## F16ACC ## _data), "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_spec(l_ ## WARPTILE, false), 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _m[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->m, #NAMELC #F16ACC "_m", (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _len : NAMELC ## F16ACC ## _len), (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _data : NAMELC ## F16ACC ## _data), "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_spec(m_ ## WARPTILE, false), 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _s[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->s, #NAMELC #F16ACC "_s", (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _len : NAMELC ## F16ACC ## _len), (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _data : NAMELC ## F16ACC ## _data), "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_spec(s_ ## WARPTILE, false), 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _l[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_l, #NAMELC #F16ACC "_aligned_l", (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _len : NAMELC ## F16ACC ## _len), (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _data : NAMELC ## F16ACC ## _data), "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_spec(l_ ## WARPTILE, true), l_align, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _m[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_m, #NAMELC #F16ACC "_aligned_m", (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _len : NAMELC ## F16ACC ## _len), (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _data : NAMELC ## F16ACC ## _data), "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_spec(m_ ## WARPTILE, true), m_align, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _s[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_s, #NAMELC #F16ACC "_aligned_s", (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _len : NAMELC ## F16ACC ## _len), (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _data : NAMELC ## F16ACC ## _data), "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_spec(s_ ## WARPTILE, true), s_align, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - - // bf16 scalar path promotes to f32, no dot2 variant -#define CREATE_MM_NODOT2(TYPE, PIPELINE_NAME, NAMELC, F16ACC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE) \ - if (device->mul_mat ## ID ## _l[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->l, #NAMELC #F16ACC "_l", NAMELC ## F16ACC ## _len, NAMELC ## F16ACC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_spec(l_ ## WARPTILE, false), 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _m[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->m, #NAMELC #F16ACC "_m", NAMELC ## F16ACC ## _len, NAMELC ## F16ACC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_spec(m_ ## WARPTILE, false), 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _s[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->s, #NAMELC #F16ACC "_s", NAMELC ## F16ACC ## _len, NAMELC ## F16ACC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_spec(s_ ## WARPTILE, false), 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _l[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_l, #NAMELC #F16ACC "_aligned_l", NAMELC ## F16ACC ## _len, NAMELC ## F16ACC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_spec(l_ ## WARPTILE, true), l_align, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _m[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_m, #NAMELC #F16ACC "_aligned_m", NAMELC ## F16ACC ## _len, NAMELC ## F16ACC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_spec(m_ ## WARPTILE, true), m_align, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _s[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_s, #NAMELC #F16ACC "_aligned_s", NAMELC ## F16ACC ## _len, NAMELC ## F16ACC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_spec(s_ ## WARPTILE, true), s_align, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - -#define CREATE_MMQ(TYPE, PIPELINE_NAME, NAMELC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE) \ - if (device->mul_mat ## ID ## _l_int[TYPE]) { \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME .f32acc->l, #NAMELC "_l", NAMELC ## _len, NAMELC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, l_ ## WARPTILE, 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - } \ - if (device->mul_mat ## ID ## _m_int[TYPE]) { \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME .f32acc->m, #NAMELC "_m", NAMELC ## _len, NAMELC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, m_ ## WARPTILE, 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - } \ - if (device->mul_mat ## ID ## _s_int[TYPE]) { \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME .f32acc->s, #NAMELC "_s", NAMELC ## _len, NAMELC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, s_ ## WARPTILE, 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - } \ - - // Create 2 variants, {f16,f32} accumulator -#define CREATE_MM2(TYPE, PIPELINE_NAME, NAMELC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE) \ - CREATE_MM(TYPE, PIPELINE_NAME . f16acc, NAMELC, _f16acc, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE) \ - CREATE_MM(TYPE, PIPELINE_NAME . f32acc, NAMELC, , WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE) \ - - CREATE_MM(GGML_TYPE_F32, pipeline_matmul_f32, matmul_f32_f32, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_F32, pipeline_matmul_f32_f16, matmul_f32_f16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_f16, matmul_f16, wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_f16_f32, matmul_f16_f32, wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0); - - CREATE_MM_NODOT2(GGML_TYPE_BF16, pipeline_matmul_bf16, matmul_bf16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0); - - CREATE_MM2(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q1_0], matmul_q1_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q2_0], matmul_q2_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_0], matmul_q4_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_1], matmul_q4_1_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_0], matmul_q5_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_1], matmul_q5_1_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q8_0], matmul_q8_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q2_K], matmul_q2_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_TQ2_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_TQ2_0], matmul_tq2_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q3_K], matmul_q3_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_K], matmul_q4_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_K], matmul_q5_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q6_K], matmul_q6_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ1_S], matmul_iq1_s_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ1_M], matmul_iq1_m_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ2_XXS], matmul_iq2_xxs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ2_XS], matmul_iq2_xs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ2_S], matmul_iq2_s_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ3_XXS], matmul_iq3_xxs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ3_S], matmul_iq3_s_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ4_XS], matmul_iq4_xs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ4_NL], matmul_iq4_nl_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat[GGML_TYPE_MXFP4], matmul_mxfp4_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat[GGML_TYPE_NVFP4], matmul_nvfp4_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - + { + // Helper for subgroup path with dot2 selection and filtering + auto sg_create = [&](vk_matmul_pipeline_key key, const std::vector& tc_base, + const std::string& name, size_t len, const void* data, uint32_t pc_size, uint32_t pc, + uint32_t rsgs = 0) { + auto tc = filter_tc(tc_base, key.type_a, key.mul_mat_id); + if (!tc.empty()) create_mm_pipelines(key, tc, name, len, data, pc_size, pc, + [&](const std::vector& wt, bool a) { return ggml_vk_mul_mm_spec(wt, a); }, + false, rsgs > 0, rsgs); + }; + auto sg_create_quant = [&](vk_matmul_pipeline_key key, const std::vector& tc_base, + const std::string& name, size_t len, const void* data, uint32_t pc_size, uint32_t pc, + uint32_t rsgs = 0) { + auto tc = filter_tc(tc_base, key.type_a, key.mul_mat_id); + if (!tc.empty()) { + spec_fn_t qs = [&, type_a=key.type_a](const std::vector& wt, bool a) { return ggml_vk_mul_mm_spec_quant(wt, a, (uint32_t)type_a); }; + create_mm_pipelines(key, tc, name, len, data, pc_size, pc, qs, false, rsgs > 0, rsgs); + } + }; #if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) - if (device->integer_dot_product) { - CREATE_MMQ(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q2_0], matmul_q2_0_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, , 0); - CREATE_MMQ(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q4_0], matmul_q4_0_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, , 0); - CREATE_MMQ(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q4_1], matmul_q4_1_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, , 0); - CREATE_MMQ(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q5_0], matmul_q5_0_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, , 0); - CREATE_MMQ(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q5_1], matmul_q5_1_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, , 0); - CREATE_MMQ(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q8_0], matmul_q8_0_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, , 0); - - CREATE_MMQ(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_MXFP4], matmul_mxfp4_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, , 0); - - CREATE_MMQ(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q2_K], matmul_q2_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, , 0); - CREATE_MMQ(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q3_K], matmul_q3_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, , 0); - CREATE_MMQ(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q4_K], matmul_q4_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, , 0); - CREATE_MMQ(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q5_K], matmul_q5_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, , 0); - CREATE_MMQ(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q6_K], matmul_q6_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, , 0); - } + auto sg_create_mmq = [&](vk_matmul_pipeline_key key, const std::vector& tc_base, + const std::string& name, size_t len, const void* data, uint32_t pc_size, uint32_t pc, + uint32_t rsgs = 0) { + auto tc = filter_tc(tc_base, key.type_a, key.mul_mat_id, true); + if (!tc.empty()) { + spec_fn_t identity = [](const std::vector& wt, bool) { return wt; }; + create_mm_pipelines(key, tc, name, len, data, pc_size, pc, identity, false, rsgs > 0, rsgs, false); + } + }; #endif - if (device->subgroup_ballot && device->subgroup_require_full_support && subgroup_min_size_16) { - CREATE_MM(GGML_TYPE_F32, pipeline_matmul_id_f32, matmul_id_subgroup_f32_f32, , wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_id_f16, matmul_id_subgroup_f16, wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_id_f16_f32, matmul_id_subgroup_f16_f32, wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MM_NODOT2(GGML_TYPE_BF16, pipeline_matmul_id_bf16, matmul_id_subgroup_bf16, , wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MM2(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q1_0], matmul_id_subgroup_q1_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_0], matmul_id_subgroup_q2_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_0], matmul_id_subgroup_q4_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_1], matmul_id_subgroup_q4_1_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_0], matmul_id_subgroup_q5_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_1], matmul_id_subgroup_q5_1_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q8_0], matmul_id_subgroup_q8_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_K], matmul_id_subgroup_q2_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_TQ2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_TQ2_0], matmul_id_subgroup_tq2_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q3_K], matmul_id_subgroup_q3_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_K], matmul_id_subgroup_q4_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_K], matmul_id_subgroup_q5_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q6_K], matmul_id_subgroup_q6_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_S], matmul_id_subgroup_iq1_s_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_M], matmul_id_subgroup_iq1_m_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XXS], matmul_id_subgroup_iq2_xxs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XS], matmul_id_subgroup_iq2_xs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_S], matmul_id_subgroup_iq2_s_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_XXS], matmul_id_subgroup_iq3_xxs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_S], matmul_id_subgroup_iq3_s_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_XS], matmul_id_subgroup_iq4_xs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_NL], matmul_id_subgroup_iq4_nl_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_MXFP4], matmul_id_subgroup_mxfp4_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_NVFP4], matmul_id_subgroup_nvfp4_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); + std::vector tc_id = {{s_warptile_id, s_wg_denoms, s_align}, {m_warptile_id, m_wg_denoms, m_align}, {l_warptile_id, l_wg_denoms, l_align}}; + std::vector tc_mmqid = {{s_warptile_mmqid, s_mmq_wg_denoms, s_align}, {m_warptile_mmqid, m_mmq_wg_denoms, m_align}, {l_warptile_mmqid, l_mmq_wg_denoms, l_align}}; + + if (device->fp16) { + // FP16 subgroup path - with dot2 runtime selection + #define SPV_DOT2(NAME) (device->dot2_f16 ? NAME ## _dot2_len : NAME ## _len), (device->dot2_f16 ? NAME ## _dot2_data : NAME ## _data) + #define SPV_DOT2_F16ACC(NAME) (device->dot2_f16 ? NAME ## _dot2_f16acc_len : NAME ## _f16acc_len), (device->dot2_f16 ? NAME ## _dot2_f16acc_data : NAME ## _f16acc_data) + + sg_create({GGML_TYPE_F32, GGML_TYPE_F32, false, false}, tc_mm, "matmul_f32_f32", SPV_DOT2(matmul_f32_f32), sizeof(vk_mat_mat_push_constants), 3); + sg_create({GGML_TYPE_F32, GGML_TYPE_F16, false, false}, tc_mm, "matmul_f32_f16", SPV_DOT2(matmul_f32_f16), sizeof(vk_mat_mat_push_constants), 3); + sg_create({GGML_TYPE_F16, GGML_TYPE_F16, false, true}, tc_mm, "matmul_f16_f16acc", SPV_DOT2_F16ACC(matmul_f16), sizeof(vk_mat_mat_push_constants), 3); + sg_create({GGML_TYPE_F16, GGML_TYPE_F16, false, false}, tc_mm, "matmul_f16", SPV_DOT2(matmul_f16), sizeof(vk_mat_mat_push_constants), 3); + sg_create({GGML_TYPE_F16, GGML_TYPE_F32, false, true}, tc_mm, "matmul_f16_f32_f16acc", SPV_DOT2_F16ACC(matmul_f16_f32), sizeof(vk_mat_mat_push_constants), 3); + sg_create({GGML_TYPE_F16, GGML_TYPE_F32, false, false}, tc_mm, "matmul_f16_f32", SPV_DOT2(matmul_f16_f32), sizeof(vk_mat_mat_push_constants), 3); + // BF16 - no dot2 + sg_create({GGML_TYPE_BF16, GGML_TYPE_BF16, false, false}, tc_mm, "matmul_bf16", matmul_bf16_len, matmul_bf16_data, sizeof(vk_mat_mat_push_constants), 3); + + for (const auto type : quant_types) { + sg_create_quant({type, GGML_TYPE_F32, false, true}, tc_mmq, "matmul_quant_f32_f16acc", SPV_DOT2_F16ACC(matmul_quant_f32), sizeof(vk_mat_mat_push_constants), 3); + sg_create_quant({type, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_quant_f32", SPV_DOT2(matmul_quant_f32), sizeof(vk_mat_mat_push_constants), 3); + } #if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) if (device->integer_dot_product) { - CREATE_MMQ(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q2_0], matmul_id_subgroup_q2_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MMQ(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q4_0], matmul_id_subgroup_q4_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MMQ(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q4_1], matmul_id_subgroup_q4_1_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MMQ(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q5_0], matmul_id_subgroup_q5_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MMQ(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q5_1], matmul_id_subgroup_q5_1_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MMQ(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q8_0], matmul_id_subgroup_q8_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - - CREATE_MMQ(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_MXFP4], matmul_id_subgroup_mxfp4_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - - CREATE_MMQ(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q2_K], matmul_id_subgroup_q2_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MMQ(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q3_K], matmul_id_subgroup_q3_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MMQ(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q4_K], matmul_id_subgroup_q4_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MMQ(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q5_K], matmul_id_subgroup_q5_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MMQ(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q6_K], matmul_id_subgroup_q6_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); + std::vector tc_mmq_int = {{s_warptile_mmq_int, s_mmq_wg_denoms, s_align}, {m_warptile_mmq_int, m_mmq_wg_denoms, m_align}, {l_warptile_mmq_int, l_mmq_wg_denoms, l_align}}; + std::vector tc_mmq_int_k = {{s_warptile_mmq_int_k, s_mmq_wg_denoms, s_align}, {m_warptile_mmq_int_k, m_mmq_wg_denoms, m_align}, {l_warptile_mmq_int_k, l_mmq_wg_denoms, l_align}}; + sg_create_mmq({GGML_TYPE_Q2_0, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q2_0_q8_1", matmul_q2_0_q8_1_len, matmul_q2_0_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q4_0, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q4_0_q8_1", matmul_q4_0_q8_1_len, matmul_q4_0_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q4_1, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q4_1_q8_1", matmul_q4_1_q8_1_len, matmul_q4_1_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q5_0, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q5_0_q8_1", matmul_q5_0_q8_1_len, matmul_q5_0_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q5_1, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q5_1_q8_1", matmul_q5_1_q8_1_len, matmul_q5_1_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q8_0, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q8_0_q8_1", matmul_q8_0_q8_1_len, matmul_q8_0_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_MXFP4, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_mxfp4_q8_1", matmul_mxfp4_q8_1_len, matmul_mxfp4_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q2_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q2_k_q8_1", matmul_q2_k_q8_1_len, matmul_q2_k_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q3_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q3_k_q8_1", matmul_q3_k_q8_1_len, matmul_q3_k_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q4_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q4_k_q8_1", matmul_q4_k_q8_1_len, matmul_q4_k_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q5_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q5_k_q8_1", matmul_q5_k_q8_1_len, matmul_q5_k_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q6_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q6_k_q8_1", matmul_q6_k_q8_1_len, matmul_q6_k_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); } #endif + + if (device->subgroup_ballot && device->subgroup_require_full_support && subgroup_min_size_16) { + sg_create({GGML_TYPE_F32, GGML_TYPE_F32, true, false}, tc_id, "matmul_id_subgroup_f32_f32", SPV_DOT2(matmul_id_subgroup_f32_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + sg_create({GGML_TYPE_F16, GGML_TYPE_F16, true, true}, tc_id, "matmul_id_subgroup_f16_f16acc", SPV_DOT2_F16ACC(matmul_id_subgroup_f16), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + sg_create({GGML_TYPE_F16, GGML_TYPE_F16, true, false}, tc_id, "matmul_id_subgroup_f16", SPV_DOT2(matmul_id_subgroup_f16), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + sg_create({GGML_TYPE_F16, GGML_TYPE_F32, true, true}, tc_id, "matmul_id_subgroup_f16_f32_f16acc", SPV_DOT2_F16ACC(matmul_id_subgroup_f16_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + sg_create({GGML_TYPE_F16, GGML_TYPE_F32, true, false}, tc_id, "matmul_id_subgroup_f16_f32", SPV_DOT2(matmul_id_subgroup_f16_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + // BF16 id - no dot2 + sg_create({GGML_TYPE_BF16, GGML_TYPE_BF16, true, false}, tc_id, "matmul_id_subgroup_bf16", matmul_id_subgroup_bf16_len, matmul_id_subgroup_bf16_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + for (const auto type : quant_types) { + sg_create_quant({type, GGML_TYPE_F32, true, true}, tc_mmqid, "matmul_id_subgroup_quant_f32_f16acc", SPV_DOT2_F16ACC(matmul_id_subgroup_quant_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); + sg_create_quant({type, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_subgroup_quant_f32", SPV_DOT2(matmul_id_subgroup_quant_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); + } +#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) + if (device->integer_dot_product) { + std::vector tc_mmqid_int = {{s_warptile_mmqid_int, s_mmq_wg_denoms, s_align}, {m_warptile_mmqid_int, m_mmq_wg_denoms, m_align}, {l_warptile_mmqid_int, l_mmq_wg_denoms, l_align}}; + std::vector tc_mmqid_int_k = {{s_warptile_mmqid_int_k, s_mmq_wg_denoms, s_align}, {m_warptile_mmqid_int_k, m_mmq_wg_denoms, m_align}, {l_warptile_mmqid_int_k, l_mmq_wg_denoms, l_align}}; + sg_create_mmq({GGML_TYPE_Q2_0, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_subgroup_q2_0_q8_1", matmul_id_subgroup_q2_0_q8_1_len, matmul_id_subgroup_q2_0_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); + sg_create_mmq({GGML_TYPE_Q4_0, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_subgroup_q4_0_q8_1", matmul_id_subgroup_q4_0_q8_1_len, matmul_id_subgroup_q4_0_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); + sg_create_mmq({GGML_TYPE_Q4_1, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_subgroup_q4_1_q8_1", matmul_id_subgroup_q4_1_q8_1_len, matmul_id_subgroup_q4_1_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); + sg_create_mmq({GGML_TYPE_Q5_0, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_subgroup_q5_0_q8_1", matmul_id_subgroup_q5_0_q8_1_len, matmul_id_subgroup_q5_0_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); + sg_create_mmq({GGML_TYPE_Q5_1, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_subgroup_q5_1_q8_1", matmul_id_subgroup_q5_1_q8_1_len, matmul_id_subgroup_q5_1_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); + sg_create_mmq({GGML_TYPE_Q8_0, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_subgroup_q8_0_q8_1", matmul_id_subgroup_q8_0_q8_1_len, matmul_id_subgroup_q8_0_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); + sg_create_mmq({GGML_TYPE_MXFP4, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_subgroup_mxfp4_q8_1", matmul_id_subgroup_mxfp4_q8_1_len, matmul_id_subgroup_mxfp4_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); + sg_create_mmq({GGML_TYPE_Q2_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_subgroup_q2_k_q8_1", matmul_id_subgroup_q2_k_q8_1_len, matmul_id_subgroup_q2_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + sg_create_mmq({GGML_TYPE_Q3_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_subgroup_q3_k_q8_1", matmul_id_subgroup_q3_k_q8_1_len, matmul_id_subgroup_q3_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + sg_create_mmq({GGML_TYPE_Q4_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_subgroup_q4_k_q8_1", matmul_id_subgroup_q4_k_q8_1_len, matmul_id_subgroup_q4_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + sg_create_mmq({GGML_TYPE_Q5_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_subgroup_q5_k_q8_1", matmul_id_subgroup_q5_k_q8_1_len, matmul_id_subgroup_q5_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + sg_create_mmq({GGML_TYPE_Q6_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_subgroup_q6_k_q8_1", matmul_id_subgroup_q6_k_q8_1_len, matmul_id_subgroup_q6_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + } +#endif + } else { + sg_create({GGML_TYPE_F32, GGML_TYPE_F32, true, false}, tc_mm, "matmul_id_f32_f32", SPV_DOT2(matmul_id_f32_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create({GGML_TYPE_F16, GGML_TYPE_F16, true, true}, tc_mm, "matmul_id_f16_f16acc", SPV_DOT2_F16ACC(matmul_id_f16), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create({GGML_TYPE_F16, GGML_TYPE_F16, true, false}, tc_mm, "matmul_id_f16", SPV_DOT2(matmul_id_f16), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create({GGML_TYPE_F16, GGML_TYPE_F32, true, true}, tc_mm, "matmul_id_f16_f32_f16acc", SPV_DOT2_F16ACC(matmul_id_f16_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create({GGML_TYPE_F16, GGML_TYPE_F32, true, false}, tc_mm, "matmul_id_f16_f32", SPV_DOT2(matmul_id_f16_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + // BF16 id - no dot2 + sg_create({GGML_TYPE_BF16, GGML_TYPE_BF16, true, false}, tc_mm, "matmul_id_bf16", matmul_id_bf16_len, matmul_id_bf16_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + for (const auto type : quant_types) { + sg_create_quant({type, GGML_TYPE_F32, true, true}, tc_mmqid, "matmul_id_quant_f32_f16acc", SPV_DOT2_F16ACC(matmul_id_quant_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create_quant({type, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_quant_f32", SPV_DOT2(matmul_id_quant_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + } +#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) + if (device->integer_dot_product) { + std::vector tc_mmqid_int = {{s_warptile_mmqid_int, s_mmq_wg_denoms, s_align}, {m_warptile_mmqid_int, m_mmq_wg_denoms, m_align}, {l_warptile_mmqid_int, l_mmq_wg_denoms, l_align}}; + std::vector tc_mmqid_int_k = {{s_warptile_mmqid_int_k, s_mmq_wg_denoms, s_align}, {m_warptile_mmqid_int_k, m_mmq_wg_denoms, m_align}, {l_warptile_mmqid_int_k, l_mmq_wg_denoms, l_align}}; + sg_create_mmq({GGML_TYPE_Q2_0, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_q2_0_q8_1", matmul_id_q2_0_q8_1_len, matmul_id_q2_0_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create_mmq({GGML_TYPE_Q4_0, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_q4_0_q8_1", matmul_id_q4_0_q8_1_len, matmul_id_q4_0_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create_mmq({GGML_TYPE_Q4_1, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_q4_1_q8_1", matmul_id_q4_1_q8_1_len, matmul_id_q4_1_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create_mmq({GGML_TYPE_Q5_0, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_q5_0_q8_1", matmul_id_q5_0_q8_1_len, matmul_id_q5_0_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create_mmq({GGML_TYPE_Q5_1, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_q5_1_q8_1", matmul_id_q5_1_q8_1_len, matmul_id_q5_1_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create_mmq({GGML_TYPE_Q8_0, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_q8_0_q8_1", matmul_id_q8_0_q8_1_len, matmul_id_q8_0_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create_mmq({GGML_TYPE_MXFP4, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_mxfp4_q8_1", matmul_id_mxfp4_q8_1_len, matmul_id_mxfp4_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create_mmq({GGML_TYPE_Q2_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_q2_k_q8_1", matmul_id_q2_k_q8_1_len, matmul_id_q2_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create_mmq({GGML_TYPE_Q3_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_q3_k_q8_1", matmul_id_q3_k_q8_1_len, matmul_id_q3_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create_mmq({GGML_TYPE_Q4_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_q4_k_q8_1", matmul_id_q4_k_q8_1_len, matmul_id_q4_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create_mmq({GGML_TYPE_Q5_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_q5_k_q8_1", matmul_id_q5_k_q8_1_len, matmul_id_q5_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create_mmq({GGML_TYPE_Q6_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_q6_k_q8_1", matmul_id_q6_k_q8_1_len, matmul_id_q6_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + } +#endif + } + #undef SPV_DOT2 + #undef SPV_DOT2_F16ACC } else { - CREATE_MM(GGML_TYPE_F32, pipeline_matmul_id_f32, matmul_id_f32_f32, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_id_f16, matmul_id_f16, wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_id_f16_f32, matmul_id_f16_f32, wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM_NODOT2(GGML_TYPE_BF16, pipeline_matmul_id_bf16, matmul_id_bf16, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q1_0], matmul_id_q1_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_0], matmul_id_q2_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_0], matmul_id_q4_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_1], matmul_id_q4_1_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_0], matmul_id_q5_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_1], matmul_id_q5_1_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q8_0], matmul_id_q8_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_K], matmul_id_q2_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_TQ2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_TQ2_0], matmul_id_tq2_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q3_K], matmul_id_q3_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_K], matmul_id_q4_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_K], matmul_id_q5_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q6_K], matmul_id_q6_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_S], matmul_id_iq1_s_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_M], matmul_id_iq1_m_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XXS], matmul_id_iq2_xxs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XS], matmul_id_iq2_xs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_S], matmul_id_iq2_s_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_XXS], matmul_id_iq3_xxs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_S], matmul_id_iq3_s_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_XS], matmul_id_iq4_xs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_NL], matmul_id_iq4_nl_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_MXFP4], matmul_id_mxfp4_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_NVFP4], matmul_id_nvfp4_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); + // FP32-only fallback path + sg_create({GGML_TYPE_F32, GGML_TYPE_F32, false, false}, tc_mm, "matmul_f32_f32", matmul_f32_f32_fp32_len, matmul_f32_f32_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create({GGML_TYPE_F32, GGML_TYPE_F16, false, false}, tc_mm, "matmul_f32_f16", matmul_f32_f16_fp32_len, matmul_f32_f16_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create({GGML_TYPE_F16, GGML_TYPE_F16, false, false}, tc_mm, "matmul_f16", matmul_f16_fp32_len, matmul_f16_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create({GGML_TYPE_F16, GGML_TYPE_F32, false, false}, tc_mm, "matmul_f16_f32", matmul_f16_f32_fp32_len, matmul_f16_f32_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create({GGML_TYPE_BF16, GGML_TYPE_BF16, false, false}, tc_mm, "matmul_bf16", matmul_bf16_fp32_len, matmul_bf16_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + + for (const auto type : quant_types) { + sg_create_quant({type, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_quant_f32", matmul_quant_f32_fp32_len, matmul_quant_f32_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + } #if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) if (device->integer_dot_product) { - CREATE_MMQ(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q2_0], matmul_id_q2_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MMQ(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q4_0], matmul_id_q4_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MMQ(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q4_1], matmul_id_q4_1_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MMQ(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q5_0], matmul_id_q5_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MMQ(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q5_1], matmul_id_q5_1_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MMQ(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q8_0], matmul_id_q8_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - - CREATE_MMQ(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_MXFP4], matmul_id_mxfp4_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - - CREATE_MMQ(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q2_K], matmul_id_q2_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MMQ(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q3_K], matmul_id_q3_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MMQ(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q4_K], matmul_id_q4_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MMQ(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q5_K], matmul_id_q5_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MMQ(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q6_K], matmul_id_q6_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); + std::vector tc_mmq_int = {{s_warptile_mmq_int, s_mmq_wg_denoms, s_align}, {m_warptile_mmq_int, m_mmq_wg_denoms, m_align}, {l_warptile_mmq_int, l_mmq_wg_denoms, l_align}}; + std::vector tc_mmq_int_k = {{s_warptile_mmq_int_k, s_mmq_wg_denoms, s_align}, {m_warptile_mmq_int_k, m_mmq_wg_denoms, m_align}, {l_warptile_mmq_int_k, l_mmq_wg_denoms, l_align}}; + sg_create_mmq({GGML_TYPE_Q2_0, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q2_0_q8_1", matmul_q2_0_q8_1_fp32_len, matmul_q2_0_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q4_0, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q4_0_q8_1", matmul_q4_0_q8_1_fp32_len, matmul_q4_0_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q4_1, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q4_1_q8_1", matmul_q4_1_q8_1_fp32_len, matmul_q4_1_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q5_0, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q5_0_q8_1", matmul_q5_0_q8_1_fp32_len, matmul_q5_0_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q5_1, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q5_1_q8_1", matmul_q5_1_q8_1_fp32_len, matmul_q5_1_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q8_0, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q8_0_q8_1", matmul_q8_0_q8_1_fp32_len, matmul_q8_0_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q2_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q2_k_q8_1", matmul_q2_k_q8_1_fp32_len, matmul_q2_k_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q3_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q3_k_q8_1", matmul_q3_k_q8_1_fp32_len, matmul_q3_k_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q4_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q4_k_q8_1", matmul_q4_k_q8_1_fp32_len, matmul_q4_k_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q5_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q5_k_q8_1", matmul_q5_k_q8_1_fp32_len, matmul_q5_k_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q6_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q6_k_q8_1", matmul_q6_k_q8_1_fp32_len, matmul_q6_k_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3); } #endif - } -#undef CREATE_MM2 -#undef CREATE_MMQ -#undef CREATE_MM -#undef CREATE_MM_NODOT2 - } else { - // Create 6 variants, {s,m,l}x{unaligned,aligned} -#define CREATE_MM(TYPE, PIPELINE_NAME, NAMELC, F16ACC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE) \ - if (device->mul_mat ## ID ## _l[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->l, #NAMELC #F16ACC "_l", NAMELC ## F16ACC ## _fp32_len, NAMELC ## F16ACC ## _fp32_data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_spec(l_ ## WARPTILE, false), 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _m[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->m, #NAMELC #F16ACC "_m", NAMELC ## F16ACC ## _fp32_len, NAMELC ## F16ACC ## _fp32_data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_spec(m_ ## WARPTILE, false), 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _s[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->s, #NAMELC #F16ACC "_s", NAMELC ## F16ACC ## _fp32_len, NAMELC ## F16ACC ## _fp32_data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_spec(s_ ## WARPTILE, false), 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _l[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_l, #NAMELC #F16ACC "_aligned_l", NAMELC ## F16ACC ## _fp32_len, NAMELC ## F16ACC ## _fp32_data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_spec(l_ ## WARPTILE, true), l_align, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _m[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_m, #NAMELC #F16ACC "_aligned_m", NAMELC ## F16ACC ## _fp32_len, NAMELC ## F16ACC ## _fp32_data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_spec(m_ ## WARPTILE, true), m_align, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _s[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_s, #NAMELC #F16ACC "_aligned_s", NAMELC ## F16ACC ## _fp32_len, NAMELC ## F16ACC ## _fp32_data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_spec(s_ ## WARPTILE, true), s_align, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - -#define CREATE_MMQ(TYPE, PIPELINE_NAME, NAMELC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID) \ - if (device->mul_mat ## ID ## _l_int[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->l, #NAMELC "_l", NAMELC ## _fp32_len, NAMELC ## _fp32_data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, l_ ## WARPTILE, 1); \ - if (device->mul_mat ## ID ## _m_int[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->m, #NAMELC "_m", NAMELC ## _fp32_len, NAMELC ## _fp32_data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, m_ ## WARPTILE, 1); \ - if (device->mul_mat ## ID ## _s_int[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->s, #NAMELC "_s", NAMELC ## _fp32_len, NAMELC ## _fp32_data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, s_ ## WARPTILE, 1); \ - - CREATE_MM(GGML_TYPE_F32, pipeline_matmul_f32, matmul_f32_f32, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_F32, pipeline_matmul_f32_f16, matmul_f32_f16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_F16, pipeline_matmul_f16.f32acc, matmul_f16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_F16, pipeline_matmul_f16_f32.f32acc, matmul_f16_f32, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0); - - CREATE_MM(GGML_TYPE_BF16, pipeline_matmul_bf16, matmul_bf16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0); - - CREATE_MM(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q1_0].f32acc, matmul_q1_0_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q2_0].f32acc, matmul_q2_0_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_0].f32acc, matmul_q4_0_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_1].f32acc, matmul_q4_1_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_0].f32acc, matmul_q5_0_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_1].f32acc, matmul_q5_1_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q8_0].f32acc, matmul_q8_0_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - - CREATE_MM(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q2_K].f32acc, matmul_q2_k_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_TQ2_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_TQ2_0].f32acc, matmul_tq2_0_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q3_K].f32acc, matmul_q3_k_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_K].f32acc, matmul_q4_k_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_K].f32acc, matmul_q5_k_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q6_K].f32acc, matmul_q6_k_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ1_S].f32acc, matmul_iq1_s_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ1_M].f32acc, matmul_iq1_m_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ2_XXS].f32acc, matmul_iq2_xxs_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ2_XS].f32acc, matmul_iq2_xs_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ2_S].f32acc, matmul_iq2_s_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ3_XXS].f32acc, matmul_iq3_xxs_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ3_S].f32acc, matmul_iq3_s_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ4_XS].f32acc, matmul_iq4_xs_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ4_NL].f32acc, matmul_iq4_nl_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat[GGML_TYPE_MXFP4].f32acc, matmul_mxfp4_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat[GGML_TYPE_NVFP4].f32acc, matmul_nvfp4_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); -#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) - if (device->integer_dot_product) { - CREATE_MMQ(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q2_0].f32acc, matmul_q2_0_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, ); - CREATE_MMQ(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q4_0].f32acc, matmul_q4_0_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, ); - CREATE_MMQ(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q4_1].f32acc, matmul_q4_1_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, ); - CREATE_MMQ(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q5_0].f32acc, matmul_q5_0_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, ); - CREATE_MMQ(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q5_1].f32acc, matmul_q5_1_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, ); - CREATE_MMQ(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q8_0].f32acc, matmul_q8_0_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, ); - - CREATE_MMQ(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q2_K].f32acc, matmul_q2_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, ); - CREATE_MMQ(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q3_K].f32acc, matmul_q3_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, ); - CREATE_MMQ(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q4_K].f32acc, matmul_q4_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, ); - CREATE_MMQ(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q5_K].f32acc, matmul_q5_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, ); - CREATE_MMQ(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q6_K].f32acc, matmul_q6_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, ); + if (device->subgroup_ballot && device->subgroup_require_full_support && subgroup_min_size_16) { + std::vector tc_id = {{s_warptile_id, s_wg_denoms, s_align}, {m_warptile_id, m_wg_denoms, m_align}, {l_warptile_id, l_wg_denoms, l_align}}; + std::vector tc_mmqid = {{s_warptile_mmqid, s_mmq_wg_denoms, s_align}, {m_warptile_mmqid, m_mmq_wg_denoms, m_align}, {l_warptile_mmqid, l_mmq_wg_denoms, l_align}}; + sg_create({GGML_TYPE_F32, GGML_TYPE_F32, true, false}, tc_id, "matmul_id_subgroup_f32_f32", matmul_id_subgroup_f32_f32_fp32_len, matmul_id_subgroup_f32_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + sg_create({GGML_TYPE_F16, GGML_TYPE_F16, true, false}, tc_id, "matmul_id_subgroup_f16", matmul_id_subgroup_f16_fp32_len, matmul_id_subgroup_f16_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + sg_create({GGML_TYPE_F16, GGML_TYPE_F32, true, false}, tc_id, "matmul_id_subgroup_f16_f32", matmul_id_subgroup_f16_f32_fp32_len, matmul_id_subgroup_f16_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + sg_create({GGML_TYPE_BF16, GGML_TYPE_BF16, true, false}, tc_id, "matmul_id_subgroup_bf16", matmul_id_subgroup_bf16_fp32_len, matmul_id_subgroup_bf16_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + for (const auto type : quant_types) { + sg_create_quant({type, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_subgroup_quant_f32", matmul_id_subgroup_quant_f32_fp32_len, matmul_id_subgroup_quant_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); + } + } else { + sg_create({GGML_TYPE_F32, GGML_TYPE_F32, true, false}, tc_mm, "matmul_id_f32_f32", matmul_id_f32_f32_fp32_len, matmul_id_f32_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create({GGML_TYPE_F16, GGML_TYPE_F16, true, false}, tc_mm, "matmul_id_f16", matmul_id_f16_fp32_len, matmul_id_f16_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create({GGML_TYPE_F16, GGML_TYPE_F32, true, false}, tc_mm, "matmul_id_f16_f32", matmul_id_f16_f32_fp32_len, matmul_id_f16_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create({GGML_TYPE_BF16, GGML_TYPE_BF16, true, false}, tc_mm, "matmul_id_bf16", matmul_id_bf16_fp32_len, matmul_id_bf16_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + for (const auto type : quant_types) { + sg_create_quant({type, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_quant_f32", matmul_id_quant_f32_fp32_len, matmul_id_quant_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + } + } } -#endif - - if (device->subgroup_ballot && device->subgroup_require_full_support && subgroup_min_size_16) { - CREATE_MM(GGML_TYPE_F32, pipeline_matmul_id_f32, matmul_id_subgroup_f32_f32, , wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MM(GGML_TYPE_F16, pipeline_matmul_id_f16.f32acc, matmul_id_subgroup_f16, , wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MM(GGML_TYPE_F16, pipeline_matmul_id_f16_f32.f32acc, matmul_id_subgroup_f16_f32, , wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MM(GGML_TYPE_BF16, pipeline_matmul_id_bf16, matmul_id_subgroup_bf16, , wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - - CREATE_MM(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q1_0].f32acc, matmul_id_subgroup_q1_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_0].f32acc, matmul_id_subgroup_q2_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_0].f32acc, matmul_id_subgroup_q4_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_1].f32acc, matmul_id_subgroup_q4_1_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_0].f32acc, matmul_id_subgroup_q5_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_1].f32acc, matmul_id_subgroup_q5_1_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q8_0].f32acc, matmul_id_subgroup_q8_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_K].f32acc, matmul_id_subgroup_q2_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_TQ2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_TQ2_0].f32acc, matmul_id_subgroup_tq2_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q3_K].f32acc, matmul_id_subgroup_q3_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_K].f32acc, matmul_id_subgroup_q4_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_K].f32acc, matmul_id_subgroup_q5_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q6_K].f32acc, matmul_id_subgroup_q6_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_S].f32acc, matmul_id_subgroup_iq1_s_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_M].f32acc, matmul_id_subgroup_iq1_m_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XXS].f32acc, matmul_id_subgroup_iq2_xxs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XS].f32acc, matmul_id_subgroup_iq2_xs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_S].f32acc, matmul_id_subgroup_iq2_s_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_XXS].f32acc, matmul_id_subgroup_iq3_xxs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_S].f32acc, matmul_id_subgroup_iq3_s_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_XS].f32acc, matmul_id_subgroup_iq4_xs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_NL].f32acc, matmul_id_subgroup_iq4_nl_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_MXFP4].f32acc, matmul_id_subgroup_mxfp4_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_NVFP4].f32acc, matmul_id_subgroup_nvfp4_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - } else { - CREATE_MM(GGML_TYPE_F32, pipeline_matmul_id_f32, matmul_id_f32_f32, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_F16, pipeline_matmul_id_f16.f32acc, matmul_id_f16, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_F16, pipeline_matmul_id_f16_f32.f32acc, matmul_id_f16_f32, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_BF16, pipeline_matmul_id_bf16, matmul_id_bf16, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - - CREATE_MM(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q1_0].f32acc, matmul_id_q1_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_0].f32acc, matmul_id_q2_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_0].f32acc, matmul_id_q4_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_1].f32acc, matmul_id_q4_1_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_0].f32acc, matmul_id_q5_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_1].f32acc, matmul_id_q5_1_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q8_0].f32acc, matmul_id_q8_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_K].f32acc, matmul_id_q2_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_TQ2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_TQ2_0].f32acc, matmul_id_tq2_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q3_K].f32acc, matmul_id_q3_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_K].f32acc, matmul_id_q4_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_K].f32acc, matmul_id_q5_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q6_K].f32acc, matmul_id_q6_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_S].f32acc, matmul_id_iq1_s_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_M].f32acc, matmul_id_iq1_m_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XXS].f32acc, matmul_id_iq2_xxs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XS].f32acc, matmul_id_iq2_xs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_S].f32acc, matmul_id_iq2_s_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_XXS].f32acc, matmul_id_iq3_xxs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_S].f32acc, matmul_id_iq3_s_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_XS].f32acc, matmul_id_iq4_xs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_NL].f32acc, matmul_id_iq4_nl_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_MXFP4].f32acc, matmul_id_mxfp4_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_NVFP4].f32acc, matmul_id_nvfp4_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - } - } - // reusing CREATE_MM from the fp32 path + } + // BF16 fallback for coopmat devices without bf16 coopmat support if ((device->coopmat2 || device->coopmat_support) #if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) && !device->coopmat_bf16_support #endif ) { - const uint32_t s_warptile_wm = device->subgroup_size == 8 ? 8 : 32; - - // use scalar tile sizes - l_warptile = { 128, 128, 128, 16, mm_warp_8 * 2, 64, 2, 4, 4, 1, mm_warp_8 }; - m_warptile = { 128, 64, 64, 16, mm_warp_8, 32, 2, 4, 2, 1, mm_warp_8 }; - s_warptile = { subgroup_size_32, 32, 32, 16, s_warptile_wm, 32, 2, 2, 2, 1, subgroup_size_8 }; - - l_wg_denoms = {128, 128, 1 }; - m_wg_denoms = { 64, 64, 1 }; - s_wg_denoms = { 32, 32, 1 }; + const uint32_t s_warptile_wm_bf16 = device->subgroup_size == 8 ? 8 : 32; + std::vector tc_bf16_fb = { + {{ subgroup_size_32, 32, 32, 16, s_warptile_wm_bf16, 32, 2, 2, 2, 1, subgroup_size_8 }, {32, 32, 1}, s_align}, + {{ 128, 64, 64, 16, mm_warp_8, 32, 2, 4, 2, 1, mm_warp_8 }, {64, 64, 1}, m_align}, + {{ 128, 128, 128, 16, mm_warp_8 * 2, 64, 2, 4, 4, 1, mm_warp_8 }, {128, 128, 1}, l_align}, + }; + auto tc_bf16_filtered = filter_tc(tc_bf16_fb, GGML_TYPE_BF16, false); + auto tc_bf16_id_filtered = filter_tc(tc_bf16_fb, GGML_TYPE_BF16, true); + spec_fn_t bf16_spec = [&](const std::vector& wt, bool a) { return ggml_vk_mul_mm_spec(wt, a); }; + if (!tc_bf16_filtered.empty()) { + create_mm_pipelines({GGML_TYPE_BF16, GGML_TYPE_BF16, false, false}, tc_bf16_filtered, "matmul_bf16", matmul_bf16_fp32_len, matmul_bf16_fp32_data, sizeof(vk_mat_mat_push_constants), 3, bf16_spec); + } + if (!tc_bf16_id_filtered.empty()) { + create_mm_pipelines({GGML_TYPE_BF16, GGML_TYPE_BF16, true, false}, tc_bf16_id_filtered, "matmul_id_bf16", matmul_id_bf16_fp32_len, matmul_id_bf16_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, bf16_spec); + } + } - CREATE_MM(GGML_TYPE_BF16, pipeline_matmul_bf16, matmul_bf16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_BF16, pipeline_matmul_id_bf16, matmul_id_bf16, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); + // Set up tile selector functions + if (device->coopmat2) { + device->matmul_tile_selector = [](uint32_t m, uint32_t n, uint32_t /*k*/, uint32_t shader_core_count, + const std::vector& configs) -> uint32_t { + if (configs.size() <= 1) return 0; + uint32_t last = (uint32_t)configs.size() - 1; + if (configs.size() == 2) { + uint32_t crossover = configs[0].unaligned->wg_denoms[1]; + return (n > crossover) ? 1 : 0; + } + // 3+ configs: s=0, m=1, l=2 + const uint32_t tiles_l = CEIL_DIV(m, configs[last].unaligned->wg_denoms[0]) * CEIL_DIV(n, configs[last].unaligned->wg_denoms[1]); + const uint32_t tiles_m = CEIL_DIV(m, configs[1].unaligned->wg_denoms[0]) * CEIL_DIV(n, configs[1].unaligned->wg_denoms[1]); + uint32_t crossover_large = configs[1].unaligned->wg_denoms[1]; + bool prefer_large = tiles_m > shader_core_count || tiles_l > shader_core_count || + (tiles_l <= shader_core_count / 3 && tiles_m > shader_core_count / 2); + if (n > crossover_large && prefer_large) return last; + uint32_t crossover_medium = configs[0].unaligned->wg_denoms[1]; + if (n > crossover_medium) return 1; + return 0; + }; + device->matmul_id_tile_selector = [](uint32_t /*m*/, uint32_t n, uint32_t /*k*/, uint32_t /*shader_core_count*/, + const std::vector& configs) -> uint32_t { + if (configs.size() <= 1) return 0; + uint32_t last = (uint32_t)configs.size() - 1; + if (configs.size() == 2) { + uint32_t crossover = configs[0].unaligned->wg_denoms[1]; + return (n > crossover) ? 1 : 0; + } + uint32_t crossover_large = configs[1].unaligned->wg_denoms[1]; + if (n > crossover_large) return last; + uint32_t crossover_medium = configs[0].unaligned->wg_denoms[1]; + if (n > crossover_medium) return 1; + return 0; + }; + } else { + device->matmul_tile_selector = [](uint32_t m, uint32_t n, uint32_t /*k*/, uint32_t /*shader_core_count*/, + const std::vector& configs) -> uint32_t { + if (configs.size() <= 1) return 0; + if (m <= 32 || n <= 32) return 0; + if (configs.size() == 2) return 1; + if (m <= 64 || n <= 64) return 1; + return (uint32_t)configs.size() - 1; + }; + device->matmul_id_tile_selector = device->matmul_tile_selector; } -#undef CREATE_MM // mul mat vec @@ -7795,88 +7679,6 @@ static vk_pipeline ggml_vk_get_to_fp16(ggml_backend_vk_context * ctx, ggml_type return ctx->device->pipeline_dequant[type]; } -static vk_matmul_pipeline ggml_vk_get_mul_mat_mat_pipeline(ggml_backend_vk_context * ctx, ggml_type src0_type, ggml_type src1_type, ggml_prec prec) { - VK_LOG_DEBUG("ggml_vk_get_mul_mat_mat_pipeline(" << ggml_type_name(src0_type) << ", " << ggml_type_name(src1_type) << ", " << prec << ")"); - if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_F32) { - return ctx->device->pipeline_matmul_f32; - } - if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_F16) { - return ctx->device->pipeline_matmul_f32_f16; - } - if (src0_type == GGML_TYPE_BF16 && src1_type == GGML_TYPE_BF16) { - return ctx->device->pipeline_matmul_bf16; - } - if (prec == GGML_PREC_DEFAULT && ctx->device->fp16 && !(ctx->device->coopmat_support && !ctx->device->coopmat_acc_f16_support)) { - if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F32) { - return ctx->device->pipeline_matmul_f16_f32.f16acc; - } - if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F16) { - return ctx->device->pipeline_matmul_f16.f16acc; - } - } else { - if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F32) { - return ctx->device->pipeline_matmul_f16_f32.f32acc; - } - if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F16) { - return ctx->device->pipeline_matmul_f16.f32acc; - } - } - - // MMQ - if (src1_type == GGML_TYPE_Q8_1) { - vk_matmul_pipeline pipelines = ctx->device->pipeline_dequant_mul_mat_mat_q8_1[src0_type].f32acc; - - if (pipelines->is_empty()) { - return nullptr; - } - - return pipelines; - } - - if (src1_type != GGML_TYPE_F32 && !ctx->device->coopmat2) { - return nullptr; - } - - switch (src0_type) { - case GGML_TYPE_Q1_0: - case GGML_TYPE_Q2_0: - case GGML_TYPE_Q4_0: - case GGML_TYPE_Q4_1: - case GGML_TYPE_Q5_0: - case GGML_TYPE_Q5_1: - case GGML_TYPE_Q8_0: - case GGML_TYPE_Q2_K: - case GGML_TYPE_Q3_K: - case GGML_TYPE_Q4_K: - case GGML_TYPE_Q5_K: - case GGML_TYPE_Q6_K: - case GGML_TYPE_IQ1_S: - case GGML_TYPE_IQ1_M: - case GGML_TYPE_IQ2_XXS: - case GGML_TYPE_IQ2_XS: - case GGML_TYPE_IQ2_S: - case GGML_TYPE_IQ3_XXS: - case GGML_TYPE_IQ3_S: - case GGML_TYPE_IQ4_XS: - case GGML_TYPE_IQ4_NL: - case GGML_TYPE_MXFP4: - case GGML_TYPE_NVFP4: - case GGML_TYPE_TQ2_0: - break; - default: - return nullptr; - } - - if (ctx->device->coopmat2) { - assert(src1_type == GGML_TYPE_F16); - return prec == GGML_PREC_DEFAULT ? ctx->device->pipeline_dequant_mul_mat_mat_f16[src0_type].f16acc : ctx->device->pipeline_dequant_mul_mat_mat_f16[src0_type].f32acc; - } - if (ctx->device->coopmat_support) { - return (ctx->device->fp16 && ctx->device->coopmat_acc_f16_support && prec == GGML_PREC_DEFAULT) ? ctx->device->pipeline_dequant_mul_mat_mat[src0_type].f16acc : ctx->device->pipeline_dequant_mul_mat_mat[src0_type].f32acc; - } - return (ctx->device->fp16 && prec == GGML_PREC_DEFAULT) ? ctx->device->pipeline_dequant_mul_mat_mat[src0_type].f16acc : ctx->device->pipeline_dequant_mul_mat_mat[src0_type].f32acc; -} - static vk_pipeline ggml_vk_get_dequantize_mul_mat_vec(ggml_backend_vk_context * ctx, ggml_type a_type, ggml_type b_type, uint32_t num_cols, uint32_t m, uint32_t k) { VK_LOG_DEBUG("ggml_vk_get_dequantize_mul_mat_vec()"); GGML_ASSERT(b_type == GGML_TYPE_F32 || b_type == GGML_TYPE_F16 || b_type == GGML_TYPE_Q8_1); @@ -7964,87 +7766,6 @@ static vk_pipeline ggml_vk_get_dequantize_mul_mat_vec(ggml_backend_vk_context * return b_type == GGML_TYPE_F32 ? ctx->device->pipeline_dequant_mul_mat_vec_f32_f32[dmmv_wg][a_type][num_cols-1] : ctx->device->pipeline_dequant_mul_mat_vec_f16_f32[dmmv_wg][a_type][num_cols-1]; } -static vk_matmul_pipeline ggml_vk_get_mul_mat_mat_id_pipeline(ggml_backend_vk_context * ctx, ggml_type src0_type, ggml_type src1_type, ggml_prec prec) { - VK_LOG_DEBUG("ggml_vk_get_mul_mat_mat_id_pipeline()"); - if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_F32) { - return ctx->device->pipeline_matmul_id_f32; - } - if (src0_type == GGML_TYPE_BF16 && src1_type == GGML_TYPE_BF16) { - return ctx->device->pipeline_matmul_id_bf16; - } - if (prec == GGML_PREC_DEFAULT && ctx->device->fp16 && !(ctx->device->coopmat_support && !ctx->device->coopmat_acc_f16_support)) { - if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F32) { - return ctx->device->pipeline_matmul_id_f16_f32.f16acc; - } - if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F16) { - return ctx->device->pipeline_matmul_id_f16.f16acc; - } - } else { - if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F32) { - return ctx->device->pipeline_matmul_id_f16_f32.f32acc; - } - if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F16) { - return ctx->device->pipeline_matmul_id_f16.f32acc; - } - } - - // MMQ - if (src1_type == GGML_TYPE_Q8_1) { - vk_matmul_pipeline pipelines = ctx->device->pipeline_dequant_mul_mat_mat_id_q8_1[src0_type].f32acc; - - if (pipelines->is_empty()) { - return nullptr; - } - - return pipelines; - } - - GGML_ASSERT(src1_type == GGML_TYPE_F32 || (ctx->device->coopmat2 && src1_type == GGML_TYPE_F16)); - - switch (src0_type) { - case GGML_TYPE_Q1_0: - case GGML_TYPE_Q2_0: - case GGML_TYPE_Q4_0: - case GGML_TYPE_Q4_1: - case GGML_TYPE_Q5_0: - case GGML_TYPE_Q5_1: - case GGML_TYPE_Q8_0: - case GGML_TYPE_Q2_K: - case GGML_TYPE_Q3_K: - case GGML_TYPE_Q4_K: - case GGML_TYPE_Q5_K: - case GGML_TYPE_Q6_K: - case GGML_TYPE_IQ1_S: - case GGML_TYPE_IQ1_M: - case GGML_TYPE_IQ2_XXS: - case GGML_TYPE_IQ2_XS: - case GGML_TYPE_IQ2_S: - case GGML_TYPE_IQ3_XXS: - case GGML_TYPE_IQ3_S: - case GGML_TYPE_IQ4_XS: - case GGML_TYPE_IQ4_NL: - case GGML_TYPE_MXFP4: - case GGML_TYPE_NVFP4: - case GGML_TYPE_TQ2_0: - break; - default: - return nullptr; - } - - vk_matmul_pipeline2& mmp = ctx->device->pipeline_dequant_mul_mat_mat_id[src0_type]; - // XXX TODO 'prec' is not actually allowed in mul_mat_id. - bool prefer_fp16acc = ctx->device->fp16 /*&& prec == GGML_PREC_DEFAULT*/; - bool support_fp16acc = !mmp.f16acc->is_empty(); - bool support_fp32acc = !mmp.f32acc->is_empty(); - - if (support_fp16acc && (prefer_fp16acc || !support_fp32acc)) { - return mmp.f16acc; - } else { - GGML_ASSERT(support_fp32acc); - return mmp.f32acc; - } -} - static vk_pipeline ggml_vk_get_dequantize_mul_mat_vec_id(ggml_backend_vk_context * ctx, ggml_type a_type, ggml_type b_type, uint32_t m, uint32_t k) { VK_LOG_DEBUG("ggml_vk_get_dequantize_mul_mat_vec_id()"); GGML_ASSERT(b_type == GGML_TYPE_F32 || b_type == GGML_TYPE_Q8_1); @@ -8878,56 +8599,7 @@ static uint32_t ggml_vk_guess_split_k(ggml_backend_vk_context * ctx, uint32_t m, return split_k; } -static vk_pipeline ggml_vk_guess_matmul_pipeline(ggml_backend_vk_context * ctx, vk_matmul_pipeline& mmp, uint32_t m, uint32_t n, bool aligned, ggml_type src0_type, ggml_type src1_type) { - VK_LOG_DEBUG("ggml_vk_guess_matmul_pipeline(" << m << ", " << n << ", " << aligned << ", " << ggml_type_name(src0_type) << ", " << ggml_type_name(src1_type) << ")"); - - // The q8_1 (integer dot) mmq path uses a different shader with its own - // shared-memory layout, so use the int-specific availability flags. - const bool is_q8_1 = (src1_type == GGML_TYPE_Q8_1); - const bool mm_l = is_q8_1 ? ctx->device->mul_mat_l_int[src0_type] : ctx->device->mul_mat_l[src0_type]; - const bool mm_m = is_q8_1 ? ctx->device->mul_mat_m_int[src0_type] : ctx->device->mul_mat_m[src0_type]; - const bool mm_s = is_q8_1 ? ctx->device->mul_mat_s_int[src0_type] : ctx->device->mul_mat_s[src0_type]; - - if (ctx->device->coopmat2) { - const uint32_t shader_core_count = ctx->device->shader_core_count; - const uint32_t tiles_l = CEIL_DIV(m, mmp->a_l->wg_denoms[0]) * CEIL_DIV(n, mmp->a_l->wg_denoms[1]); - const uint32_t tiles_m = CEIL_DIV(m, mmp->a_m->wg_denoms[0]) * CEIL_DIV(n, mmp->a_m->wg_denoms[1]); - // Use large shader when the N dimension is greater than the medium shader's tile size - uint32_t crossover_large = mmp->m->wg_denoms[1]; - - // Prefer large over medium if either: - // - medium or large tiles would overfill the GPU - // - large tiles with a split_k==3 fits in the GPU and medium tiles with split_k==2 does not - // (medium with split_k==2 is probably better if it fits - more workgroups running and less split_k overhead) - bool prefer_large = tiles_m > shader_core_count || tiles_l > shader_core_count || - // split_k==3 with large tiles likely better than medium tiles with no split_k. - (tiles_l <= shader_core_count / 3 && tiles_m > shader_core_count / 2); - - if ((mm_l && (n > crossover_large && prefer_large)) || (!mm_m && !mm_s)) { - return aligned ? mmp->a_l : mmp->l; - } - // Use medium shader when the N dimension is greater than the small shader's tile size - uint32_t crossover_medium = mmp->s->wg_denoms[1]; - if ((mm_m && (n > crossover_medium)) || !mm_s) { - return aligned ? mmp->a_m : mmp->m; - } - return aligned ? mmp->a_s : mmp->s; - } - - if ((mm_s && (m <= 32 || n <= 32)) || (!mm_m && !mm_l)) { - return aligned ? mmp->a_s : mmp->s; - } - if ((mm_m && (m <= 64 || n <= 64)) || !mm_l) { - return aligned ? mmp->a_m : mmp->m; - } - return aligned ? mmp->a_l : mmp->l; -} - -static uint32_t ggml_vk_guess_matmul_pipeline_align(ggml_backend_vk_context * ctx, vk_matmul_pipeline& mmp, int m, int n, ggml_type src0_type, ggml_type src1_type) { - VK_LOG_DEBUG("ggml_vk_guess_matmul_pipeline_align(" << m << ", " << n << ", " << ggml_type_name(src0_type) << ", " << ggml_type_name(src1_type) << ")"); - return ggml_vk_guess_matmul_pipeline(ctx, mmp, m, n, true, src0_type, src1_type)->align; -} static void ggml_vk_matmul( ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline& pipeline, @@ -8978,42 +8650,58 @@ static void ggml_vk_matmul( ctx->prealloc_split_k_need_sync = true; } -static vk_pipeline ggml_vk_guess_matmul_id_pipeline(ggml_backend_vk_context * ctx, vk_matmul_pipeline& mmp, uint32_t m, uint32_t n, bool aligned, ggml_type src0_type, ggml_type src1_type) { - VK_LOG_DEBUG("ggml_vk_guess_matmul_id_pipeline(" << m << ", " << n << ", " << aligned << ", " << ggml_type_name(src0_type) << ", " << ggml_type_name(src1_type) << ")"); - // The q8_1 (integer dot) mmq path uses a different shader with its own - // shared-memory layout, so use the int-specific availability flags. - const bool is_q8_1 = (src1_type == GGML_TYPE_Q8_1); - const bool mm_l = is_q8_1 ? ctx->device->mul_mat_id_l_int[src0_type] : ctx->device->mul_mat_id_l[src0_type]; - const bool mm_m = is_q8_1 ? ctx->device->mul_mat_id_m_int[src0_type] : ctx->device->mul_mat_id_m[src0_type]; - const bool mm_s = is_q8_1 ? ctx->device->mul_mat_id_s_int[src0_type] : ctx->device->mul_mat_id_s[src0_type]; +// New map-based pipeline lookup helpers +static bool ggml_vk_get_mul_mat_mat_f16acc(ggml_backend_vk_context * ctx, ggml_type src0_type, ggml_type src1_type, ggml_prec prec) { + if (src0_type == GGML_TYPE_F32 || src0_type == GGML_TYPE_BF16) return false; + if (src1_type == GGML_TYPE_Q8_1) return false; + if (src0_type == GGML_TYPE_F16) { + return prec == GGML_PREC_DEFAULT && ctx->device->fp16 && !(ctx->device->coopmat_support && !ctx->device->coopmat_acc_f16_support); + } + // quant types if (ctx->device->coopmat2) { - // Use large shader when the N dimension is greater than the medium shader's tile size - uint32_t crossover_large = mmp->m->wg_denoms[1]; - if ((mm_l && (n > crossover_large)) || (!mm_m && !mm_s)) { - return aligned ? mmp->a_l : mmp->l; - } - // Use medium shader when the N dimension is greater than the small shader's tile size - uint32_t crossover_medium = mmp->s->wg_denoms[1]; - if ((mm_m && (n > crossover_medium)) || !mm_s) { - return aligned ? mmp->a_m : mmp->m; - } - return aligned ? mmp->a_s : mmp->s; + return prec == GGML_PREC_DEFAULT; } - - if ((mm_s && (m <= 32 || n <= 32)) || (!mm_m && !mm_l)) { - return aligned ? mmp->a_s : mmp->s; + if (ctx->device->coopmat_support) { + return ctx->device->fp16 && ctx->device->coopmat_acc_f16_support && prec == GGML_PREC_DEFAULT; } - if ((mm_m && (m <= 64 || n <= 64)) || !mm_l) { - return aligned ? mmp->a_m : mmp->m; + return ctx->device->fp16 && prec == GGML_PREC_DEFAULT; +} + +static const std::vector* ggml_vk_get_mul_mat_mat_pipeline_map( + ggml_backend_vk_context * ctx, ggml_type src0_type, ggml_type src1_type, ggml_prec prec, bool mul_mat_id = false) { + bool f16acc = ggml_vk_get_mul_mat_mat_f16acc(ctx, src0_type, src1_type, prec); + vk_matmul_pipeline_key key{src0_type, src1_type, mul_mat_id, f16acc}; + auto it = ctx->device->pipeline_matmul.find(key); + if (it == ctx->device->pipeline_matmul.end() || it->second.empty()) { + // Try without f16acc + if (f16acc) { + key.f16acc = false; + it = ctx->device->pipeline_matmul.find(key); + if (it != ctx->device->pipeline_matmul.end() && !it->second.empty()) return &it->second; + } + return nullptr; } - return aligned ? mmp->a_l : mmp->l; + return &it->second; } -static uint32_t ggml_vk_guess_matmul_id_pipeline_align(ggml_backend_vk_context * ctx, vk_matmul_pipeline& mmp, int m, int n, ggml_type src0_type, ggml_type src1_type) { - VK_LOG_DEBUG("ggml_vk_guess_matmul_pipeline_align(" << m << ", " << n << ", " << ggml_type_name(src0_type) << ", " << ggml_type_name(src1_type) << ")"); - return ggml_vk_guess_matmul_id_pipeline(ctx, mmp, m, n, true, src0_type, src1_type)->align; +static vk_pipeline ggml_vk_guess_matmul_pipeline_map(ggml_backend_vk_context * ctx, + const std::vector& configs, + uint32_t m, uint32_t n, bool aligned, bool mul_mat_id) { + auto& selector = mul_mat_id ? ctx->device->matmul_id_tile_selector : ctx->device->matmul_tile_selector; + uint32_t idx = selector(m, n, 0, ctx->device->shader_core_count, configs); + if (idx >= configs.size()) idx = (uint32_t)configs.size() - 1; + return (aligned && configs[idx].aligned) ? configs[idx].aligned : configs[idx].unaligned; +} + +static uint32_t ggml_vk_guess_matmul_pipeline_align_map(ggml_backend_vk_context * ctx, + const std::vector& configs, + uint32_t m, uint32_t n, bool mul_mat_id) { + auto& selector = mul_mat_id ? ctx->device->matmul_id_tile_selector : ctx->device->matmul_tile_selector; + uint32_t idx = selector(m, n, 0, ctx->device->shader_core_count, configs); + if (idx >= configs.size()) idx = (uint32_t)configs.size() - 1; + return configs[idx].align; } static void ggml_vk_matmul_id( @@ -9337,31 +9025,31 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub bool quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0; // Check for mmq first - vk_matmul_pipeline mmp = quantize_y ? ggml_vk_get_mul_mat_mat_pipeline(ctx, src0->type, GGML_TYPE_Q8_1, (ggml_prec)dst->op_params[0]) : nullptr; + const std::vector* mmp_map = quantize_y ? ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0->type, GGML_TYPE_Q8_1, (ggml_prec)dst->op_params[0]) : nullptr; - if (mmp == nullptr) { + if (mmp_map == nullptr) { // Fall back to f16 dequant mul mat - mmp = ggml_vk_get_mul_mat_mat_pipeline(ctx, src0->type, y_non_contig ? f16_type : src1->type, (ggml_prec)dst->op_params[0]); + mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0->type, y_non_contig ? f16_type : src1->type, (ggml_prec)dst->op_params[0]); quantize_y = false; } - const bool qx_needs_dequant = mmp == nullptr || x_non_contig; + const bool qx_needs_dequant = mmp_map == nullptr || x_non_contig; const bool qy_needs_dequant = !quantize_y && ((src1->type != f16_type && !y_f32_kernel) || y_non_contig); if (qx_needs_dequant) { // Fall back to dequant + f16 mulmat - mmp = ggml_vk_get_mul_mat_mat_pipeline(ctx, f16_type, y_f32_kernel ? GGML_TYPE_F32 : f16_type, (ggml_prec)dst->op_params[0]); + mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, f16_type, y_f32_kernel ? GGML_TYPE_F32 : f16_type, (ggml_prec)dst->op_params[0]); } // Not implemented GGML_ASSERT(y_non_contig || !qy_needs_dequant); // NOLINT - const ggml_type effective_src1_type = quantize_y ? GGML_TYPE_Q8_1 : (y_f32_kernel ? GGML_TYPE_F32 : src1->type); + GGML_ASSERT(mmp_map != nullptr); - const uint32_t kpad = quantize_y ? 0 : ggml_vk_align_size(ne10, ggml_vk_guess_matmul_pipeline_align(ctx, mmp, ne01, ne11, qx_needs_dequant ? f16_type : src0->type, effective_src1_type)); + const uint32_t kpad = quantize_y ? 0 : ggml_vk_align_size(ne10, ggml_vk_guess_matmul_pipeline_align_map(ctx, *mmp_map, ne01, ne11, false)); const bool aligned = !quantize_y && ne10 == kpad && ne01 > 8 && ne11 > 8; - vk_pipeline pipeline = ggml_vk_guess_matmul_pipeline(ctx, mmp, ne01, ne11, aligned, qx_needs_dequant ? f16_type : src0->type, effective_src1_type); + vk_pipeline pipeline = ggml_vk_guess_matmul_pipeline_map(ctx, *mmp_map, ne01, ne11, aligned, false); if (ggml_nbytes(src0) > ctx->device->properties.limits.maxStorageBufferRange) { pipeline = ggml_vk_get_64b_indexing_pipeline(ctx, pipeline); @@ -10277,25 +9965,22 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context& bool quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0; // Check for mmq first - vk_matmul_pipeline mmp = quantize_y ? ggml_vk_get_mul_mat_mat_id_pipeline(ctx, src0->type, GGML_TYPE_Q8_1, (ggml_prec)dst->op_params[0]) : nullptr; + const std::vector* mmp_map = quantize_y ? ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0->type, GGML_TYPE_Q8_1, (ggml_prec)dst->op_params[0], true) : nullptr; - if (mmp == nullptr) { + if (mmp_map == nullptr) { // Fall back to f16 dequant mul mat - mmp = ggml_vk_get_mul_mat_mat_id_pipeline(ctx, src0->type, y_non_contig ? f16_type : src1->type, (ggml_prec)dst->op_params[0]); + mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0->type, y_non_contig ? f16_type : src1->type, (ggml_prec)dst->op_params[0], true); quantize_y = false; } - const bool qx_needs_dequant = mmp == nullptr || x_non_contig; + const bool qx_needs_dequant = mmp_map == nullptr || x_non_contig; bool qy_needs_dequant = !quantize_y && ((src1->type != f16_type && !y_f32_kernel) || y_non_contig); if (qx_needs_dequant) { // Fall back to dequant + f16 mulmat - mmp = ggml_vk_get_mul_mat_mat_id_pipeline(ctx, f16_type, y_f32_kernel ? GGML_TYPE_F32 : f16_type, (ggml_prec)dst->op_params[0]); + mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, f16_type, y_f32_kernel ? GGML_TYPE_F32 : f16_type, (ggml_prec)dst->op_params[0], true); } - const ggml_type effective_src1_type = quantize_y ? GGML_TYPE_Q8_1 : (y_f32_kernel ? GGML_TYPE_F32 : src1->type); - - const uint32_t kpad = quantize_y ? 0 : ggml_vk_align_size(ne10, ggml_vk_guess_matmul_id_pipeline_align(ctx, mmp, ne01, nei1, qx_needs_dequant ? f16_type : src0->type, effective_src1_type)); // Coopmat2 MUL_MAT_ID BK specialization constants in ggml_vk_load_shaders are at most 64. const uint32_t y_staged_row_stride = ctx->device->coopmat2 && !quantize_y ? ggml_vk_align_size(ne10, 64) : ne10; const bool y_needs_k_padding = ne10 != y_staged_row_stride; @@ -10305,9 +9990,12 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context& // Not implemented GGML_ASSERT(y_needs_reformat || !qy_needs_dequant); // NOLINT + GGML_ASSERT(mmp_map != nullptr); + + const uint32_t kpad = quantize_y ? 0 : ggml_vk_align_size(ne10, ggml_vk_guess_matmul_pipeline_align_map(ctx, *mmp_map, ne01, nei1, true)); const bool aligned = !quantize_y && ne10 == kpad && ne01 > 8 && nei1 > 8; - vk_pipeline pipeline = ggml_vk_guess_matmul_id_pipeline(ctx, mmp, ne01, nei1, aligned, qx_needs_dequant ? f16_type : src0->type, effective_src1_type); + vk_pipeline pipeline = ggml_vk_guess_matmul_pipeline_map(ctx, *mmp_map, ne01, nei1, aligned, true); if (ggml_nbytes(src0) > ctx->device->properties.limits.maxStorageBufferRange) { pipeline = ggml_vk_get_64b_indexing_pipeline(ctx, pipeline); @@ -14759,106 +14447,22 @@ static void ggml_vk_test_matmul(ggml_backend_vk_context * ctx, size_t m, size_t const size_t y_ne = k * n * batch; const size_t d_ne = m * n * batch; - vk_pipeline p; - std::string shname; - if (shader_size == 0) { - if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f32->a_s; - shname = "F32_ALIGNED_S"; - } else if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f32_f16->a_s; - shname = "F32_F16_ALIGNED_S"; - } else if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f16_f32.f32acc->a_s; - shname = "F16_F32_ALIGNED_S"; - } else if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f16.f32acc->a_s; - shname = "F16_ALIGNED_S"; - } else { - GGML_ABORT("fatal error"); - } - } else if (shader_size == 1) { - if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f32->a_m; - shname = "F32_ALIGNED_M"; - } else if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f32_f16->a_m; - shname = "F32_F16_ALIGNED_M"; - } else if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f16_f32.f32acc->a_m; - shname = "F16_F32_ALIGNED_M"; - } else if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f16.f32acc->a_m; - shname = "F16_ALIGNED_M"; - } else { - GGML_ABORT("fatal error"); - } - } else if (shader_size == 2) { - if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f32->a_l; - shname = "F32_ALIGNED_L"; - } else if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f32_f16->a_l; - shname = "F32_F16_ALIGNED_L"; - } else if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f16_f32.f32acc->a_l; - shname = "F16_F32_ALIGNED_L"; - } else if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f16.f32acc->a_l; - shname = "F16_ALIGNED_L"; - } else { - GGML_ABORT("fatal error"); - } - } else { - GGML_ASSERT(0); - } + ggml_type x_type = std::is_same() ? GGML_TYPE_F32 : GGML_TYPE_F16; + ggml_type y_type = std::is_same() ? GGML_TYPE_F32 : GGML_TYPE_F16; + vk_matmul_pipeline_key mm_test_key{x_type, y_type, false, false}; + auto mm_test_it = ctx->device->pipeline_matmul.find(mm_test_key); + GGML_ASSERT(mm_test_it != ctx->device->pipeline_matmul.end() && !mm_test_it->second.empty()); + auto& mm_test_configs = mm_test_it->second; + GGML_ASSERT(shader_size >= 0 && shader_size < (int)mm_test_configs.size()); + + std::string shname = std::string(ggml_type_name(x_type)) + "_" + std::string(ggml_type_name(y_type)) + "_ALIGNED_" + std::to_string(shader_size); + vk_pipeline p = mm_test_configs[shader_size].aligned ? mm_test_configs[shader_size].aligned : mm_test_configs[shader_size].unaligned; - const size_t kpad = ggml_vk_align_size(k, p->align); + const size_t kpad = ggml_vk_align_size(k, mm_test_configs[shader_size].align); if (k != kpad) { - if (shader_size == 0) { - if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f32->s; - shname = "F32_S"; - } else if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f32_f16->s; - shname = "F32_F16_S"; - } else if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f16_f32.f32acc->s; - shname = "F16_F32_S"; - } else if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f16.f32acc->s; - shname = "F16_S"; - } - } else if (shader_size == 1) { - if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f32->m; - shname = "F32_M"; - } else if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f32_f16->m; - shname = "F32_F16_M"; - } else if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f16_f32.f32acc->m; - shname = "F16_F32_M"; - } else if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f16.f32acc->m; - shname = "F16_M"; - } - } else if (shader_size == 2) { - if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f32->l; - shname = "F32_L"; - } else if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f32_f16->l; - shname = "F32_F16_L"; - } else if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f16_f32.f32acc->l; - shname = "F16_F32_L"; - } else if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f16.f32acc->l; - shname = "F16_L"; - } - } + p = mm_test_configs[shader_size].unaligned; + shname = std::string(ggml_type_name(x_type)) + "_" + std::string(ggml_type_name(y_type)) + "_" + std::to_string(shader_size); } if (split_k > 1) { @@ -15293,46 +14897,34 @@ static void ggml_vk_test_dequant_matmul(ggml_backend_vk_context * ctx, size_t m, const size_t y_ne = k * n * batch; const size_t d_ne = m * n * batch; - vk_matmul_pipeline2 * pipelines; - - if (mmq) { - pipelines = ctx->device->pipeline_dequant_mul_mat_mat_q8_1; - } else { - pipelines = ctx->device->pipeline_dequant_mul_mat_mat; - } - - const bool fp16acc = ctx->device->fp16; - - vk_pipeline p; - std::string shname; - if (shader_size == 0) { - p = fp16acc ? pipelines[quant].f16acc->a_s : pipelines[quant].f32acc->a_s; - shname = std::string(ggml_type_name(quant)) + "_ALIGNED_S"; - } else if (shader_size == 1) { - p = fp16acc ? pipelines[quant].f16acc->a_m : pipelines[quant].f32acc->a_m; - shname = std::string(ggml_type_name(quant)) + "_ALIGNED_M"; - } else if (shader_size == 2) { - p = fp16acc ? pipelines[quant].f16acc->a_l : pipelines[quant].f32acc->a_l; - shname = std::string(ggml_type_name(quant)) + "_ALIGNED_L"; - } else { - GGML_ASSERT(0); + ggml_type b_type = mmq ? GGML_TYPE_Q8_1 : GGML_TYPE_F32; + bool f16acc = ctx->device->fp16 && !mmq; + vk_matmul_pipeline_key dq_key{quant, b_type, false, f16acc}; + auto dq_it = ctx->device->pipeline_matmul.find(dq_key); + if (dq_it == ctx->device->pipeline_matmul.end() || dq_it->second.empty()) { + if (f16acc) { + dq_key.f16acc = false; + dq_it = ctx->device->pipeline_matmul.find(dq_key); + } } + if (dq_it == ctx->device->pipeline_matmul.end() || dq_it->second.empty()) { + std::cerr << "error: no pipeline for ggml_vk_test_dequant_matmul " << ggml_type_name(quant) << std::endl; + return; + } + auto& dq_configs = dq_it->second; + if (shader_size >= (int)dq_configs.size()) { + std::cerr << "error: shader_size " << shader_size << " >= configs.size() " << dq_configs.size() << " for " << ggml_type_name(quant) << std::endl; + return; + } + + std::string shname = std::string(ggml_type_name(quant)) + "_ALIGNED_" + std::to_string(shader_size); + vk_pipeline p = dq_configs[shader_size].aligned ? dq_configs[shader_size].aligned : dq_configs[shader_size].unaligned; - const size_t kpad = mmq ? 0 : ggml_vk_align_size(k, p->align); + const size_t kpad = mmq ? 0 : ggml_vk_align_size(k, dq_configs[shader_size].align); if (mmq || k != kpad) { - if (shader_size == 0) { - p = fp16acc ? pipelines[quant].f16acc->s : pipelines[quant].f32acc->s; - shname = std::string(ggml_type_name(quant)) + "_S"; - } else if (shader_size == 1) { - p = fp16acc ? pipelines[quant].f16acc->m : pipelines[quant].f32acc->m; - shname = std::string(ggml_type_name(quant)) + "_M"; - } else if (shader_size == 2) { - p = fp16acc ? pipelines[quant].f16acc->l : pipelines[quant].f32acc->l; - shname = std::string(ggml_type_name(quant)) + "_L"; - } else { - GGML_ASSERT(0); - } + p = dq_configs[shader_size].unaligned; + shname = std::string(ggml_type_name(quant)) + "_" + std::to_string(shader_size); } if (p == nullptr) { diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl index 627932bd354..6c852fb4266 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl @@ -244,7 +244,7 @@ vec2 dequantize(uint ib, uint iqs, uint a_offset) { // Add parity bit const uint sign8 = sign7 | (bitCount(sign7) << 7); const uint sign = sign8 >> (iqs % 8); - const u8vec4 grid = unpack8(iq2xxs_grid[qs][(iqs % 8) / 4] >> (8 * (iqs % 4))); + const u8vec4 grid = unpack8(iq2_grid[qs][(iqs % 8) / 4] >> (8 * (iqs % 4))); bool sign0 = (sign & 1) != 0; bool sign1 = (sign & 2) != 0; return db * vec2( @@ -264,7 +264,7 @@ vec4 dequantize4(uint ib, uint iqs, uint a_offset) { // Add parity bit const uint sign8 = sign7 | (bitCount(sign7) << 7); const uint sign = sign8 >> (iqs % 8); - const u8vec4 grid = unpack8(iq2xxs_grid[qs][(iqs % 8) / 4] >> (8 * (iqs % 4))); + const u8vec4 grid = unpack8(iq2_grid[qs][(iqs % 8) / 4] >> (8 * (iqs % 4))); bool sign0 = (sign & 1) != 0; bool sign1 = (sign & 2) != 0; bool sign2 = (sign & 4) != 0; @@ -287,7 +287,7 @@ vec2 dequantize(uint ib, uint iqs, uint a_offset) { // Add parity bit const uint sign8 = sign7 | (bitCount(sign7) << 7); const uint sign = sign8 >> (iqs % 8); - const u8vec4 grid = unpack8(iq2xs_grid[qs & 511][(iqs % 8) / 4] >> (8 * (iqs % 4))); + const u8vec4 grid = unpack8(iq2_grid[qs & 511][(iqs % 8) / 4] >> (8 * (iqs % 4))); bool sign0 = (sign & 1) != 0; bool sign1 = (sign & 2) != 0; return db * vec2( @@ -303,7 +303,7 @@ vec4 dequantize4(uint ib, uint iqs, uint a_offset) { // Add parity bit const uint sign8 = sign7 | (bitCount(sign7) << 7); const uint sign = sign8 >> (iqs % 8); - const u8vec4 grid = unpack8(iq2xs_grid[qs & 511][(iqs % 8) / 4] >> (8 * (iqs % 4))); + const u8vec4 grid = unpack8(iq2_grid[qs & 511][(iqs % 8) / 4] >> (8 * (iqs % 4))); bool sign0 = (sign & 1) != 0; bool sign1 = (sign & 2) != 0; bool sign2 = (sign & 4) != 0; @@ -329,7 +329,7 @@ vec2 dequantize(uint ib, uint iqs, uint a_offset) { const uint sign = data_a[a_offset + ib].qs[QUANT_K / 8 + ib8] >> (iqs % 8); const float db = 0.25 * (0.5 + scale); - const u8vec4 grid = unpack8(iq2s_grid[qs | ((qh << (8 - qhshift)) & 0x300)][(iqs % 8) / 4]); + const u8vec4 grid = unpack8(iq2_grid[qs | ((qh << (8 - qhshift)) & 0x300)][(iqs % 8) / 4]); bool sign0 = (sign & 1) != 0; bool sign1 = (sign & 2) != 0; return db * vec2( @@ -348,7 +348,7 @@ vec4 dequantize4(uint ib, uint iqs, uint a_offset) { const uint sign = data_a[a_offset + ib].qs[QUANT_K / 8 + ib8] >> (iqs % 8); const float db = 0.25 * (0.5 + scale); - const u8vec4 grid = unpack8(iq2s_grid[qs | ((qh << (8 - qhshift)) & 0x300)][(iqs % 8) / 4]); + const u8vec4 grid = unpack8(iq2_grid[qs | ((qh << (8 - qhshift)) & 0x300)][(iqs % 8) / 4]); bool sign0 = (sign & 1) != 0; bool sign1 = (sign & 2) != 0; bool sign2 = (sign & 4) != 0; @@ -376,7 +376,7 @@ vec2 dequantize(uint ib, uint iqs, uint a_offset) { // Add parity bit const uint sign8 = sign7 | (bitCount(sign7) << 7); const uint sign = sign8 >> (iqs % 8); - const u8vec4 grid = unpack8(iq3xxs_grid[qs] >> (8 * (iqs % 4))); + const u8vec4 grid = unpack8(iq3_grid[qs] >> (8 * (iqs % 4))); bool sign0 = (sign & 1) != 0; bool sign1 = (sign & 2) != 0; return db * vec2( @@ -396,7 +396,7 @@ vec4 dequantize4(uint ib, uint iqs, uint a_offset) { // Add parity bit const uint sign8 = sign7 | (bitCount(sign7) << 7); const uint sign = sign8 >> (iqs % 8); - const u8vec4 grid = unpack8(iq3xxs_grid[qs]); + const u8vec4 grid = unpack8(iq3_grid[qs]); bool sign0 = (sign & 1) != 0; bool sign1 = (sign & 2) != 0; bool sign2 = (sign & 4) != 0; @@ -419,7 +419,7 @@ vec2 dequantize(uint ib, uint iqs, uint a_offset) { bool sign0 = (sign & 1) != 0; bool sign1 = (sign & 2) != 0; const float db = 1 + 2 * ((scale >> (4 * ((iqs / 32) & 1))) & 0xf); - const uint32_t grid = iq3s_grid[qs | ((qh << (8 - ((iqs / 4) % 8))) & 256)] >> (8 * (iqs % 4)); + const uint32_t grid = iq3_grid[qs | ((qh << (8 - ((iqs / 4) % 8))) & 256)] >> (8 * (iqs % 4)); return db * vec2( int(grid & 0xFF) * (sign0 ? -1.0 : 1.0), int((grid >> 8) & 0xFF) * (sign1 ? -1.0 : 1.0) @@ -437,7 +437,7 @@ vec4 dequantize4(uint ib, uint iqs, uint a_offset) { bool sign2 = (sign & 4) != 0; bool sign3 = (sign & 8) != 0; const float db = 1 + 2 * ((scale >> (4 * (ib32 & 1))) & 0xf); - const uint32_t grid = iq3s_grid[qs | ((qh << (8 - ib4 % 8)) & 256)] >> (8 * (iqs % 4)); + const uint32_t grid = iq3_grid[qs | ((qh << (8 - ib4 % 8)) & 256)] >> (8 * (iqs % 4)); return db * vec4( int(grid & 0xFF) * (sign0 ? -1.0 : 1.0), int((grid >> 8) & 0xFF) * (sign1 ? -1.0 : 1.0), diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl index 46cc69cb26e..55d2c3912f6 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl @@ -804,7 +804,7 @@ f16vec4 dequantFuncQ6_K_v(const in decodeBufQ6_K bl, const in uint blockCoords[2 return f16vec4((vec4(qi) - vec4(32.0f)) * vec4(float(dscale))); } -#if defined(DATA_A_IQ1_S) +#if defined(DATA_A_IQ1_S) || defined(MULMAT_QUANT) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ1_S { block_iq1_s block; }; @@ -851,7 +851,7 @@ f16vec4 dequantFuncIQ1_S_v(const in decodeBufIQ1_S bl, const in uint blockCoords } #endif -#if defined(DATA_A_IQ1_M) +#if defined(DATA_A_IQ1_M) || defined(MULMAT_QUANT) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ1_M { block_iq1_m block; }; @@ -910,7 +910,7 @@ f16vec4 dequantFuncIQ1_M_v(const in decodeBufIQ1_M bl, const in uint blockCoords } #endif -#if defined(DATA_A_IQ2_XXS) +#if defined(DATA_A_IQ2_XXS) || defined(MULMAT_QUANT) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ2_XXS { block_iq2_xxs block; }; @@ -936,7 +936,7 @@ float16_t dequantFuncIQ2_XXS(const in decodeBufIQ2_XXS bl, const in uint blockCo uint sign = bitfieldExtract(signscale, 7 * int(ib8), 7); sign |= bitCount(sign) << 7; - uint g2 = iq2xxs_grid[qs][(idx & 4) >> 2]; + uint g2 = iq2_grid[qs][(idx & 4) >> 2]; g2 >>= (idx & 2) * 8; const vec2 g = vec2(unpack8(g2)); @@ -961,7 +961,7 @@ f16vec4 dequantFuncIQ2_XXS_v(const in decodeBufIQ2_XXS bl, const in uint blockCo sign |= bitCount(sign) << 7; const uint sb = sign >> (idx & 7u); - const uint g2 = iq2xxs_grid[qs][(idx & 4) >> 2]; + const uint g2 = iq2_grid[qs][(idx & 4) >> 2]; const u8vec4 g = unpack8(g2); return f16vec4( @@ -972,7 +972,7 @@ f16vec4 dequantFuncIQ2_XXS_v(const in decodeBufIQ2_XXS bl, const in uint blockCo } #endif -#if defined(DATA_A_IQ2_XS) +#if defined(DATA_A_IQ2_XS) || defined(MULMAT_QUANT) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ2_XS { block_iq2_xs block; }; @@ -991,7 +991,7 @@ float16_t dequantFuncIQ2_XS(const in decodeBufIQ2_XS bl, const in uint blockCoor uint sign = uint(qs >> 9); sign |= bitCount(sign) << 7; - uint g2 = iq2xs_grid[qs & 0x1FF][(idx & 4) >> 2]; + uint g2 = iq2_grid[qs & 0x1FF][(idx & 4) >> 2]; g2 >>= (idx & 2) * 8; const vec2 g = vec2(unpack8(g2)); @@ -1014,7 +1014,7 @@ f16vec4 dequantFuncIQ2_XS_v(const in decodeBufIQ2_XS bl, const in uint blockCoor sign |= bitCount(sign) << 7; const uint sb = sign >> (idx & 7u); - const uint g2 = iq2xs_grid[qs & 0x1FF][(idx & 4) >> 2]; + const uint g2 = iq2_grid[qs & 0x1FF][(idx & 4) >> 2]; const u8vec4 g = unpack8(g2); return f16vec4( @@ -1025,7 +1025,7 @@ f16vec4 dequantFuncIQ2_XS_v(const in decodeBufIQ2_XS bl, const in uint blockCoor } #endif -#if defined(DATA_A_IQ2_S) +#if defined(DATA_A_IQ2_S) || defined(MULMAT_QUANT) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ2_S { block_iq2_s block; }; @@ -1041,12 +1041,12 @@ float16_t dequantFuncIQ2_S(const in decodeBufIQ2_S bl, const in uint blockCoords const uint scale = (bl.block.scales[ib32] >> ((idx & 0x10) >> 2)) & 0xf; const uint qs = bl.block.qs[ib8]; const uint qh = bl.block.qh[ib32]; - const uint sign = bl.block.qs[QUANT_K / 8 + ib8] >> (idx & 0x6); + const uint sign = bl.block.qs[QUANT_K_IQ2_S / 8 + ib8] >> (idx & 0x6); const float d = float(bl.block.d); const float db = d * 0.25 * (0.5 + scale); const ivec2 sign01 = 1 - (2 & ivec2(sign << 1, sign)); - uint g2 = iq2s_grid[qs | ((qh << (8 - qhshift)) & 0x300)][(idx & 4) >> 2]; + uint g2 = iq2_grid[qs | ((qh << (8 - qhshift)) & 0x300)][(idx & 4) >> 2]; g2 >>= (idx & 2) * 8; const vec2 v = db * vec2(sign01) * vec2(unpack8(g2)); return float16_t(v[idx & 1]); @@ -1063,12 +1063,12 @@ f16vec4 dequantFuncIQ2_S_v(const in decodeBufIQ2_S bl, const in uint blockCoords const uint scale = (bl.block.scales[ib32] >> ((idx & 0x10) >> 2)) & 0xf; const uint qs = bl.block.qs[ib8]; const uint qh = bl.block.qh[ib32]; - const uint sb = uint(bl.block.qs[QUANT_K / 8 + ib8]) >> (idx & 0x6u); + const uint sb = uint(bl.block.qs[QUANT_K_IQ2_S / 8 + ib8]) >> (idx & 0x6u); const float d = float(bl.block.d); const float db = d * 0.25 * (0.5 + scale); - const uint g2 = iq2s_grid[qs | ((qh << (8 - qhshift)) & 0x300)][(idx & 4) >> 2]; + const uint g2 = iq2_grid[qs | ((qh << (8 - qhshift)) & 0x300)][(idx & 4) >> 2]; const u8vec4 g = unpack8(g2); return f16vec4( @@ -1079,7 +1079,7 @@ f16vec4 dequantFuncIQ2_S_v(const in decodeBufIQ2_S bl, const in uint blockCoords } #endif -#if defined(DATA_A_IQ3_XXS) +#if defined(DATA_A_IQ3_XXS) || defined(MULMAT_QUANT) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ3_XXS { block_iq3_xxs block; }; @@ -1094,7 +1094,7 @@ float16_t dequantFuncIQ3_XXS(const in decodeBufIQ3_XXS bl, const in uint blockCo uint idx = coordInBlock[1]; const uint iqs = (idx & 0xFC) >> 2; // 0..63 - const uint is = QUANT_K / 4 + ((idx & 0xE0) >> 3);// 8 values + const uint is = QUANT_K_IQ3_XXS / 4 + ((idx & 0xE0) >> 3);// 8 values const float d = float(bl.block.d); const uint qs = bl.block.qs[iqs]; @@ -1106,7 +1106,7 @@ float16_t dequantFuncIQ3_XXS(const in decodeBufIQ3_XXS bl, const in uint blockCo const uint32_t sign7 = bitfieldExtract(signs, 7 * (int(iqs / 2) % 4), 7); const uint sign = (sign7 | (bitCount(sign7) << 7)) >> (idx & 0x6); const ivec2 sign01 = ivec2(1 - (2 & ivec2(sign << 1, sign))); - const uint grid = iq3xxs_grid[qs] >> (16 * ((idx & 2) >> 1)); + const uint grid = iq3_grid[qs] >> (16 * ((idx & 2) >> 1)); const vec2 v = db * vec2(sign01) * vec2(unpack8(grid).xy); return float16_t(v[idx & 1]); } @@ -1117,7 +1117,7 @@ f16vec4 dequantFuncIQ3_XXS_v(const in decodeBufIQ3_XXS bl, const in uint blockCo const uint idx = coordInBlock[1]; const uint iqs = idx >> 2; - const uint is = QUANT_K / 4 + ((idx & 0xE0) >> 3); + const uint is = QUANT_K_IQ3_XXS / 4 + ((idx & 0xE0) >> 3); const float d = float(bl.block.d); const uint qs = bl.block.qs[iqs]; @@ -1127,7 +1127,7 @@ f16vec4 dequantFuncIQ3_XXS_v(const in decodeBufIQ3_XXS bl, const in uint blockCo const uint sign7 = bitfieldExtract(signs, 7 * (int(iqs / 2) % 4), 7); const uint sb = (sign7 | (bitCount(sign7) << 7)) >> (idx & 0x6u); - const uint grid = iq3xxs_grid[qs]; + const uint grid = iq3_grid[qs]; const u8vec4 g = unpack8(grid); return f16vec4( @@ -1138,7 +1138,7 @@ f16vec4 dequantFuncIQ3_XXS_v(const in decodeBufIQ3_XXS bl, const in uint blockCo } #endif -#if defined(DATA_A_IQ3_S) +#if defined(DATA_A_IQ3_S) || defined(MULMAT_QUANT) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ3_S { block_iq3_s block; }; @@ -1157,7 +1157,7 @@ float16_t dequantFuncIQ3_S(const in decodeBufIQ3_S bl, const in uint blockCoords const uint scale = bl.block.scales[iqs / 16]; const ivec2 sign01 = ivec2(1 - (2 & ivec2(sign << 1, sign))); const float db = d * (1 + 2 * ((scale >> (4 * (iqh & 1))) & 0xf)); - const uint32_t grid = iq3s_grid[qs | ((qh << (8 - (iqs % 8))) & 256)] >> ((idx & 2) << 3); + const uint32_t grid = iq3_grid[qs | ((qh << (8 - (iqs % 8))) & 256)] >> ((idx & 2) << 3); const vec2 v = db * vec2(sign01) * vec2(unpack8(grid).xy); return float16_t(v[idx & 1]); @@ -1177,7 +1177,7 @@ f16vec4 dequantFuncIQ3_S_v(const in decodeBufIQ3_S bl, const in uint blockCoords const uint scale = bl.block.scales[iqs / 16]; const float db = d * (1 + 2 * ((scale >> (4 * (iqh & 1))) & 0xf)); - const uint grid = iq3s_grid[qs | ((qh << (8 - (iqs % 8))) & 256)]; + const uint grid = iq3_grid[qs | ((qh << (8 - (iqs % 8))) & 256)]; const u8vec4 g = unpack8(grid); return f16vec4( @@ -1188,7 +1188,7 @@ f16vec4 dequantFuncIQ3_S_v(const in decodeBufIQ3_S bl, const in uint blockCoords } #endif -#if defined(DATA_A_IQ4_XS) +#if defined(DATA_A_IQ4_XS) || defined(MULMAT_QUANT) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ4_XS { block_iq4_xs block; }; @@ -1238,7 +1238,7 @@ f16vec4 dequantFuncIQ4_XS_v(const in decodeBufIQ4_XS bl, const in uint blockCoor } #endif -#if defined(DATA_A_IQ4_NL) +#if defined(DATA_A_IQ4_NL) || defined(MULMAT_QUANT) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ4_NL { block_iq4_nl block; }; @@ -1279,7 +1279,7 @@ f16vec4 dequantFuncIQ4_NL_v(const in decodeBufIQ4_NL bl, const in uint blockCoor } #endif -#if defined(DATA_A_MXFP4) +#if defined(DATA_A_MXFP4) || defined(MULMAT_QUANT) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufMXFP4 { block_mxfp4 block; }; @@ -1333,7 +1333,7 @@ f16vec4 dequantFuncMXFP4_v(const in decodeBufMXFP4 bl, const in uint blockCoords } #endif -#if defined(DATA_A_NVFP4) +#if defined(DATA_A_NVFP4) || defined(MULMAT_QUANT) layout(buffer_reference, std430, buffer_reference_align = 4) buffer decodeBufNVFP4 { block_nvfp4 block; }; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp index 78490162cd1..f834b785629 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp @@ -29,7 +29,7 @@ void main() { uint qs = data_a[ib].qs[4 * ib32 + l]; const uint8_t sign = data_a[ib].qs[QUANT_K / 8 + 4 * ib32 + l]; qs |= (qh << (8 - 2 * l)) & 0x300; - const uvec2 grid = iq2s_grid[qs]; + const uvec2 grid = iq2_grid[qs]; const u8vec4 grid0 = unpack8(grid.x); const u8vec4 grid1 = unpack8(grid.y); data_b[b_idx + 8 * l + 0] = D_TYPE(db[l/2] * grid0.x * ((sign & 1) != 0 ? -1.0 : 1.0)); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp index 9b8ce0a7f81..f5bba538f33 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp @@ -28,7 +28,7 @@ void main() { uint16_t qs = data_a[ib].qs[4 * ib32 + l]; const uint sign7 = qs >> 9; const uint sign8 = sign7 | (bitCount(sign7) << 7); // parity bit - const uvec2 grid = iq2xs_grid[qs & 511]; + const uvec2 grid = iq2_grid[qs & 511]; const u8vec4 grid0 = unpack8(grid.x); const u8vec4 grid1 = unpack8(grid.y); data_b[b_idx + 8 * l + 0] = D_TYPE(db[l/2] * grid0.x * ((sign8 & 1) != 0 ? -1.0 : 1.0)); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp index aacf07d0f8e..dca5b84c6ac 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp @@ -34,7 +34,7 @@ void main() { const uint sign7 = bitfieldExtract(signscale, 7 * int(l), 7); const uint sign8 = sign7 | (bitCount(sign7) << 7); // parity bit const uint qs = data_a[ib].qs[8 * is + l]; - const uvec2 grid = iq2xxs_grid[qs]; + const uvec2 grid = iq2_grid[qs]; const u8vec4 grid0 = unpack8(grid.x); const u8vec4 grid1 = unpack8(grid.y); data_b[b_idx + 8 * l + 0] = D_TYPE(db * grid0.x * ((sign8 & 1) != 0 ? -1.0 : 1.0)); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp index f2c20b1d2c0..b18a9d0cd11 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp @@ -31,7 +31,7 @@ void main() { const uint qs = data_a[ib].qs[iqs]; const uint gidx = qs | ((qh << (8 - l)) & 256); const uint8_t signs = data_a[ib].signs[iqs / 2] >> (4 * (l & 1)); - const u8vec4 grid = unpack8(iq3s_grid[gidx]); + const u8vec4 grid = unpack8(iq3_grid[gidx]); data_b[b_idx + 4 * l + 0] = D_TYPE(db * grid.x * ((signs & 1) != 0 ? -1.0 : 1.0)); data_b[b_idx + 4 * l + 1] = D_TYPE(db * grid.y * ((signs & 2) != 0 ? -1.0 : 1.0)); data_b[b_idx + 4 * l + 2] = D_TYPE(db * grid.z * ((signs & 4) != 0 ? -1.0 : 1.0)); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp index 671c1f4a0d3..60c5318e3be 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp @@ -37,8 +37,8 @@ void main() { const uint sign8 = sign7 | (bitCount(sign7) << 7); const uint qs0 = data_a[ib].qs[8 * is + 2 * l]; const uint qs1 = data_a[ib].qs[8 * is + 2 * l + 1]; - const u8vec4 grid0 = unpack8(iq3xxs_grid[qs0]); - const u8vec4 grid1 = unpack8(iq3xxs_grid[qs1]); + const u8vec4 grid0 = unpack8(iq3_grid[qs0]); + const u8vec4 grid1 = unpack8(iq3_grid[qs1]); data_b[b_idx + 8 * l + 0] = D_TYPE(db * grid0.x * ((sign8 & 1) != 0 ? -1.0 : 1.0)); data_b[b_idx + 8 * l + 1] = D_TYPE(db * grid0.y * ((sign8 & 2) != 0 ? -1.0 : 1.0)); data_b[b_idx + 8 * l + 2] = D_TYPE(db * grid0.z * ((sign8 & 4) != 0 ? -1.0 : 1.0)); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glsl index 6f414ded123..1e732a9a29c 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glsl @@ -1,32 +1,22 @@ #if !defined(GGML_FA_TYPES_COMP) #define GGML_FA_TYPES_COMP -// FaTypeK / FaTypeV spec constant values. These mirror enum ggml_type so the -// host can pass the type directly. Keep in sync with ggml.h. -#define FA_TYPE_F32 0u -#define FA_TYPE_F16 1u -#define FA_TYPE_Q4_0 2u -#define FA_TYPE_Q4_1 3u -#define FA_TYPE_Q5_0 6u -#define FA_TYPE_Q5_1 7u -#define FA_TYPE_Q8_0 8u -#define FA_TYPE_IQ4_NL 20u -#define FA_TYPE_BF16 30u +#include "ggml_type_ids.glsl" // Number of matrix elements per buffer block, derived from the K/V type spec // constant. F32 is treated as a vec4 "block" of 4 floats. F16 uses block size 1 // and bypasses the dequant path entirely. Quants follow their ggml block sizes. uint fa_block_elems(uint ty) { switch (ty) { - case FA_TYPE_F32: return 4u; - case FA_TYPE_F16: return 1u; - case FA_TYPE_Q4_0: return uint(QUANT_K_Q4_0); - case FA_TYPE_Q4_1: return uint(QUANT_K_Q4_1); - case FA_TYPE_Q5_0: return uint(QUANT_K_Q5_0); - case FA_TYPE_Q5_1: return uint(QUANT_K_Q5_1); - case FA_TYPE_Q8_0: return uint(QUANT_K_Q8_0); - case FA_TYPE_IQ4_NL: return uint(QUANT_K_IQ4_NL); - case FA_TYPE_BF16: return 1u; + case GGML_TYPE_F32: return 4u; + case GGML_TYPE_F16: return 1u; + case GGML_TYPE_Q4_0: return uint(QUANT_K_Q4_0); + case GGML_TYPE_Q4_1: return uint(QUANT_K_Q4_1); + case GGML_TYPE_Q5_0: return uint(QUANT_K_Q5_0); + case GGML_TYPE_Q5_1: return uint(QUANT_K_Q5_1); + case GGML_TYPE_Q8_0: return uint(QUANT_K_Q8_0); + case GGML_TYPE_IQ4_NL: return uint(QUANT_K_IQ4_NL); + case GGML_TYPE_BF16: return 1u; default: return 1u; } } @@ -36,18 +26,18 @@ uint fa_block_elems(uint ty) { // of int32s per 32-element block on the MMQ K path: ints_per_block == 8 / R. uint fa_quant_r_mmq(uint ty) { switch (ty) { - case FA_TYPE_Q4_0: return uint(QUANT_R_Q4_0); - case FA_TYPE_Q4_1: return uint(QUANT_R_Q4_1); - case FA_TYPE_Q5_0: return uint(QUANT_R_Q5_0); - case FA_TYPE_Q5_1: return uint(QUANT_R_Q5_1); - case FA_TYPE_Q8_0: return uint(QUANT_R_Q8_0); + case GGML_TYPE_Q4_0: return uint(QUANT_R_Q4_0); + case GGML_TYPE_Q4_1: return uint(QUANT_R_Q4_1); + case GGML_TYPE_Q5_0: return uint(QUANT_R_Q5_0); + case GGML_TYPE_Q5_1: return uint(QUANT_R_Q5_1); + case GGML_TYPE_Q8_0: return uint(QUANT_R_Q8_0); default: return 1u; } } bool fa_type_needs_shmem(uint ty) { switch (ty) { - case FA_TYPE_IQ4_NL: return true; + case GGML_TYPE_IQ4_NL: return true; default: return false; } } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp index 0c1b6d0673e..9a12cdfb881 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp @@ -134,7 +134,7 @@ void main() { // Q8_0 K only needs (qd, _); the asymmetric Q4_*/Q5_* family also stores // the row-sum scaled by qd, used in k_dot_correction. - if (FaTypeK == FA_TYPE_Q8_0) { + if (FaTypeK == GGML_TYPE_Q8_0) { if (buf_iqs == 0) { Qf[buf_ib].ds = FLOAT_TYPEV2(qd, 0.0f); } @@ -367,7 +367,7 @@ void main() { // Q4_*/Q5_* take the block-8 fast path when one step covers a full // block; Q8_0 always goes through the per-int get_k_qs* helpers // (its qs is byte-packed, not nibble-packed). - const bool block8_fast = (d_per_step == 8) && (FaTypeK != FA_TYPE_Q8_0); + const bool block8_fast = (d_per_step == 8) && (FaTypeK != GGML_TYPE_Q8_0); if (SHMEM_STAGING != 0) { const uint k_block_idx = (d_tid * (HSK_per_thread / 4) + d_block) / 8; @@ -375,7 +375,7 @@ void main() { k_dm = ACC_TYPEV2(kblocksh[buf_ib].dm); if (block8_fast) { - const bool has_qh = (FaTypeK == FA_TYPE_Q5_0) || (FaTypeK == FA_TYPE_Q5_1); + const bool has_qh = (FaTypeK == GGML_TYPE_Q5_0) || (FaTypeK == GGML_TYPE_Q5_1); [[unroll]] for (uint32_t d = 0; d < 4; d++) { uint vui = kblocksh[buf_ib].qs[d]; k_quants[d ] = int32_t( vui & 0x0F0F0F0F); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl index 0ce4503a884..a4be1ebf98e 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl @@ -105,8 +105,8 @@ layout (binding = 6) readonly buffer MO {uint32_t data_mask_opt[];}; #define BLOCK_SIZE_V fa_block_elems(FaTypeV) // F16 reads f16 elements directly from the binding; everything else routes // through dequantize4 / the MMQ helpers to unpack from the packed block layout. -#define USE_DECODE_K (FaTypeK != FA_TYPE_F16) -#define USE_DECODE_V (FaTypeV != FA_TYPE_F16) +#define USE_DECODE_K (FaTypeK != GGML_TYPE_F16) +#define USE_DECODE_V (FaTypeV != GGML_TYPE_F16) #define CEIL_DIV(a, b) (((a) + (b) - 1) / (b)) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp index 31741115308..5a9abe2265f 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp @@ -40,26 +40,28 @@ layout(buffer_reference, std430, buffer_reference_align = 1) buffer decodeBufFA_ #if !defined(BFLOAT16) float16_t faDecodeK(const decodeBufFA_K bl_in, const uint blockCoords[2], const uint coordInBlock[2]) { switch (FaTypeK) { - case FA_TYPE_F32: return dequantFuncF32 (decodeBufF32 (bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q4_0: return dequantFuncQ4_0(decodeBufQ4_0(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q4_1: return dequantFuncQ4_1(decodeBufQ4_1(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q5_0: return dequantFuncQ5_0(decodeBufQ5_0(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q5_1: return dequantFuncQ5_1(decodeBufQ5_1(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q8_0: return dequantFuncQ8_0(decodeBufQ8_0(bl_in), blockCoords, coordInBlock); - case FA_TYPE_IQ4_NL: return dequantFuncIQ4_NL(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_F32: return dequantFuncF32 (decodeBufF32 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q4_0: return dequantFuncQ4_0(decodeBufQ4_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q4_1: return dequantFuncQ4_1(decodeBufQ4_1(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q5_0: return dequantFuncQ5_0(decodeBufQ5_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q5_1: return dequantFuncQ5_1(decodeBufQ5_1(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q8_0: return dequantFuncQ8_0(decodeBufQ8_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_IQ4_NL: return dequantFuncIQ4_NL(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q1_0: return dequantFuncQ1_0(decodeBufQ1_0(bl_in), blockCoords, coordInBlock); default: return float16_t(0); } } float16_t faDecodeV(const decodeBufFA_V bl_in, const uint blockCoords[2], const uint coordInBlock[2]) { switch (FaTypeV) { - case FA_TYPE_F32: return dequantFuncF32 (decodeBufF32 (bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q4_0: return dequantFuncQ4_0(decodeBufQ4_0(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q4_1: return dequantFuncQ4_1(decodeBufQ4_1(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q5_0: return dequantFuncQ5_0(decodeBufQ5_0(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q5_1: return dequantFuncQ5_1(decodeBufQ5_1(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q8_0: return dequantFuncQ8_0(decodeBufQ8_0(bl_in), blockCoords, coordInBlock); - case FA_TYPE_IQ4_NL: return dequantFuncIQ4_NL(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_F32: return dequantFuncF32 (decodeBufF32 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q4_0: return dequantFuncQ4_0(decodeBufQ4_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q4_1: return dequantFuncQ4_1(decodeBufQ4_1(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q5_0: return dequantFuncQ5_0(decodeBufQ5_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q5_1: return dequantFuncQ5_1(decodeBufQ5_1(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q8_0: return dequantFuncQ8_0(decodeBufQ8_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_IQ4_NL: return dequantFuncIQ4_NL(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q1_0: return dequantFuncQ1_0(decodeBufQ1_0(bl_in), blockCoords, coordInBlock); default: return float16_t(0); } } @@ -67,26 +69,26 @@ float16_t faDecodeV(const decodeBufFA_V bl_in, const uint blockCoords[2], const // V=4 vector decode for K/V; dispatches to per-format _v decoders. f16vec4 faDecodeKVector(const decodeBufFA_K bl_in, const uint blockCoords[2], const uint coordInBlock[2]) { switch (FaTypeK) { - case FA_TYPE_F32: return f16vec4(decodeBufF32(bl_in).block); - case FA_TYPE_Q4_0: return dequantFuncQ4_0_v(decodeBufQ4_0(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q4_1: return dequantFuncQ4_1_v(decodeBufQ4_1(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q5_0: return dequantFuncQ5_0_v(decodeBufQ5_0(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q5_1: return dequantFuncQ5_1_v(decodeBufQ5_1(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q8_0: return dequantFuncQ8_0_v(decodeBufQ8_0(bl_in), blockCoords, coordInBlock); - case FA_TYPE_IQ4_NL: return dequantFuncIQ4_NL_v(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_F32: return f16vec4(decodeBufF32(bl_in).block); + case GGML_TYPE_Q4_0: return dequantFuncQ4_0_v(decodeBufQ4_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q4_1: return dequantFuncQ4_1_v(decodeBufQ4_1(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q5_0: return dequantFuncQ5_0_v(decodeBufQ5_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q5_1: return dequantFuncQ5_1_v(decodeBufQ5_1(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q8_0: return dequantFuncQ8_0_v(decodeBufQ8_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_IQ4_NL: return dequantFuncIQ4_NL_v(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock); default: return f16vec4(0); } } f16vec4 faDecodeVVector(const decodeBufFA_V bl_in, const uint blockCoords[2], const uint coordInBlock[2]) { switch (FaTypeV) { - case FA_TYPE_F32: return f16vec4(decodeBufF32(bl_in).block); - case FA_TYPE_Q4_0: return dequantFuncQ4_0_v(decodeBufQ4_0(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q4_1: return dequantFuncQ4_1_v(decodeBufQ4_1(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q5_0: return dequantFuncQ5_0_v(decodeBufQ5_0(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q5_1: return dequantFuncQ5_1_v(decodeBufQ5_1(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q8_0: return dequantFuncQ8_0_v(decodeBufQ8_0(bl_in), blockCoords, coordInBlock); - case FA_TYPE_IQ4_NL: return dequantFuncIQ4_NL_v(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_F32: return f16vec4(decodeBufF32(bl_in).block); + case GGML_TYPE_Q4_0: return dequantFuncQ4_0_v(decodeBufQ4_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q4_1: return dequantFuncQ4_1_v(decodeBufQ4_1(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q5_0: return dequantFuncQ5_0_v(decodeBufQ5_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q5_1: return dequantFuncQ5_1_v(decodeBufQ5_1(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q8_0: return dequantFuncQ8_0_v(decodeBufQ8_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_IQ4_NL: return dequantFuncIQ4_NL_v(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock); default: return f16vec4(0); } } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl index 8ba4725f334..4fcf7c1f4f6 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl @@ -121,25 +121,25 @@ layout (binding = 1) readonly buffer K_PACKED_Q5_1_P32 { block_q5_1_packed32 dat FLOAT_TYPEV4 dequantize4(uint ib, uint iqs, uint a_offset, uint binding_idx) { if (binding_idx == BINDING_IDX_K) { switch (FaTypeK) { - case FA_TYPE_F32: FA_DEQUANT4_F32 (k_packed_f32) - case FA_TYPE_Q4_0: FA_DEQUANT4_Q4_0(k_packed_q4_0) - case FA_TYPE_Q4_1: FA_DEQUANT4_Q4_1(k_packed_q4_1) - case FA_TYPE_Q5_0: FA_DEQUANT4_Q5_0(k_packed_q5_0) - case FA_TYPE_Q5_1: FA_DEQUANT4_Q5_1(k_packed_q5_1) - case FA_TYPE_Q8_0: FA_DEQUANT4_Q8_0(k_packed_q8_0) - case FA_TYPE_IQ4_NL: FA_DEQUANT4_IQ4_NL(k_packed_iq4_nl) - case FA_TYPE_BF16: FA_DEQUANT4_BF16(k_packed_bf16) + case GGML_TYPE_F32: FA_DEQUANT4_F32 (k_packed_f32) + case GGML_TYPE_Q4_0: FA_DEQUANT4_Q4_0(k_packed_q4_0) + case GGML_TYPE_Q4_1: FA_DEQUANT4_Q4_1(k_packed_q4_1) + case GGML_TYPE_Q5_0: FA_DEQUANT4_Q5_0(k_packed_q5_0) + case GGML_TYPE_Q5_1: FA_DEQUANT4_Q5_1(k_packed_q5_1) + case GGML_TYPE_Q8_0: FA_DEQUANT4_Q8_0(k_packed_q8_0) + case GGML_TYPE_IQ4_NL: FA_DEQUANT4_IQ4_NL(k_packed_iq4_nl) + case GGML_TYPE_BF16: FA_DEQUANT4_BF16(k_packed_bf16) } } else { switch (FaTypeV) { - case FA_TYPE_F32: FA_DEQUANT4_F32 (v_packed_f32) - case FA_TYPE_Q4_0: FA_DEQUANT4_Q4_0(v_packed_q4_0) - case FA_TYPE_Q4_1: FA_DEQUANT4_Q4_1(v_packed_q4_1) - case FA_TYPE_Q5_0: FA_DEQUANT4_Q5_0(v_packed_q5_0) - case FA_TYPE_Q5_1: FA_DEQUANT4_Q5_1(v_packed_q5_1) - case FA_TYPE_Q8_0: FA_DEQUANT4_Q8_0(v_packed_q8_0) - case FA_TYPE_IQ4_NL: FA_DEQUANT4_IQ4_NL(v_packed_iq4_nl) - case FA_TYPE_BF16: FA_DEQUANT4_BF16(v_packed_bf16) + case GGML_TYPE_F32: FA_DEQUANT4_F32 (v_packed_f32) + case GGML_TYPE_Q4_0: FA_DEQUANT4_Q4_0(v_packed_q4_0) + case GGML_TYPE_Q4_1: FA_DEQUANT4_Q4_1(v_packed_q4_1) + case GGML_TYPE_Q5_0: FA_DEQUANT4_Q5_0(v_packed_q5_0) + case GGML_TYPE_Q5_1: FA_DEQUANT4_Q5_1(v_packed_q5_1) + case GGML_TYPE_Q8_0: FA_DEQUANT4_Q8_0(v_packed_q8_0) + case GGML_TYPE_IQ4_NL: FA_DEQUANT4_IQ4_NL(v_packed_iq4_nl) + case GGML_TYPE_BF16: FA_DEQUANT4_BF16(v_packed_bf16) } } return FLOAT_TYPEV4(0); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glsl index 6bf10a7cffd..49900aa5aeb 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glsl @@ -4,20 +4,20 @@ int32_t get_k_qs(uint ib, uint iqs, uint a_offset) { switch (FaTypeK) { - case FA_TYPE_Q4_0: { + case GGML_TYPE_Q4_0: { uint vui = pack32(u16vec2(k_packed_q4_0.data[a_offset + ib].qs[(iqs & 0xF) / 2 + 0], k_packed_q4_0.data[a_offset + ib].qs[(iqs & 0xF) / 2 + 1])); uint shift = (iqs & 0x10) >> 2; vui >>= shift; return int32_t(vui & 0x0F0F0F0F); } - case FA_TYPE_Q4_1: { // uses packed32 alias + case GGML_TYPE_Q4_1: { // uses packed32 alias uint vui = k_packed_q4_1_p32.data[a_offset + ib].qs[(iqs & 0xF) / 4]; uint shift = (iqs & 0x10) >> 2; vui >>= shift; return int32_t(vui & 0x0F0F0F0F); } - case FA_TYPE_Q5_0: { + case GGML_TYPE_Q5_0: { uint vui = pack32(u16vec2(k_packed_q5_0.data[a_offset + ib].qs[(iqs & 0xF) / 2 + 0], k_packed_q5_0.data[a_offset + ib].qs[(iqs & 0xF) / 2 + 1])); uint qh = pack32(u16vec2(k_packed_q5_0.data[a_offset + ib].qh[0], @@ -27,7 +27,7 @@ int32_t get_k_qs(uint ib, uint iqs, uint a_offset) { uint qh_bits = (qh >> iqs) & 0xF; return int32_t(vui & 0x0F0F0F0F) | int32_t((qh_bits * 0x02040810u) & 0x10101010u); } - case FA_TYPE_Q5_1: { // qs via packed32, qh via packed16 + case GGML_TYPE_Q5_1: { // qs via packed32, qh via packed16 uint vui = k_packed_q5_1_p32.data[a_offset + ib].qs[(iqs & 0xF) / 4]; uint qh = k_packed_q5_1.data[a_offset + ib].qh; uint shift = (iqs & 0x10) >> 2; @@ -35,7 +35,7 @@ int32_t get_k_qs(uint ib, uint iqs, uint a_offset) { uint qh_bits = (qh >> iqs) & 0xF; return int32_t(vui & 0x0F0F0F0F) | int32_t((qh_bits * 0x02040810u) & 0x10101010u); } - case FA_TYPE_Q8_0: { + case GGML_TYPE_Q8_0: { return pack32(i16vec2(k_packed_q8_0.data[a_offset + ib].qs[iqs / 2], k_packed_q8_0.data[a_offset + ib].qs[iqs / 2 + 1])); } @@ -47,11 +47,11 @@ int32_t get_k_qs(uint ib, uint iqs, uint a_offset) { // return (d, 0) so call sites always see the same shape. FLOAT_TYPEV2 get_k_scale(uint ib, uint a_offset) { switch (FaTypeK) { - case FA_TYPE_Q4_0: return FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q4_0.data[a_offset + ib].d), 0.0); - case FA_TYPE_Q4_1: return FLOAT_TYPEV2(k_packed_q4_1_p32.data[a_offset + ib].dm); - case FA_TYPE_Q5_0: return FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q5_0.data[a_offset + ib].d), 0.0); - case FA_TYPE_Q5_1: return FLOAT_TYPEV2(k_packed_q5_1_p32.data[a_offset + ib].dm); - case FA_TYPE_Q8_0: return FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q8_0.data[a_offset + ib].d), 0.0); + case GGML_TYPE_Q4_0: return FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q4_0.data[a_offset + ib].d), 0.0); + case GGML_TYPE_Q4_1: return FLOAT_TYPEV2(k_packed_q4_1_p32.data[a_offset + ib].dm); + case GGML_TYPE_Q5_0: return FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q5_0.data[a_offset + ib].d), 0.0); + case GGML_TYPE_Q5_1: return FLOAT_TYPEV2(k_packed_q5_1_p32.data[a_offset + ib].dm); + case GGML_TYPE_Q8_0: return FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q8_0.data[a_offset + ib].d), 0.0); default: return FLOAT_TYPEV2(0); } } @@ -61,16 +61,16 @@ void k_block_to_shmem(const uint buf_ib, const uint global_ib, const uint iqs, c // explicit casts. The bit pattern is what we care about here -- the actual // signed/unsigned interpretation happens downstream in the dot product. switch (FaTypeK) { - case FA_TYPE_Q4_0: { + case GGML_TYPE_Q4_0: { kblocksh[buf_ib].qs[iqs] = int32_t(pack32(u16vec2(k_packed_q4_0.data[a_offset + global_ib].qs[iqs * 2], k_packed_q4_0.data[a_offset + global_ib].qs[iqs * 2 + 1]))); break; } - case FA_TYPE_Q4_1: { + case GGML_TYPE_Q4_1: { kblocksh[buf_ib].qs[iqs] = int32_t(k_packed_q4_1_p32.data[a_offset + global_ib].qs[iqs]); break; } - case FA_TYPE_Q5_0: { + case GGML_TYPE_Q5_0: { kblocksh[buf_ib].qs[iqs] = int32_t(pack32(u16vec2(k_packed_q5_0.data[a_offset + global_ib].qs[iqs * 2], k_packed_q5_0.data[a_offset + global_ib].qs[iqs * 2 + 1]))); if (iqs == 0) { @@ -79,14 +79,14 @@ void k_block_to_shmem(const uint buf_ib, const uint global_ib, const uint iqs, c } break; } - case FA_TYPE_Q5_1: { + case GGML_TYPE_Q5_1: { kblocksh[buf_ib].qs[iqs] = int32_t(k_packed_q5_1_p32.data[a_offset + global_ib].qs[iqs]); if (iqs == 0) { kblocksh[buf_ib].qh = k_packed_q5_1.data[a_offset + global_ib].qh; } break; } - case FA_TYPE_Q8_0: { + case GGML_TYPE_Q8_0: { kblocksh[buf_ib].qs[iqs] = pack32(i16vec2(k_packed_q8_0.data[a_offset + global_ib].qs[iqs * 2], k_packed_q8_0.data[a_offset + global_ib].qs[iqs * 2 + 1])); break; @@ -96,11 +96,11 @@ void k_block_to_shmem(const uint buf_ib, const uint global_ib, const uint iqs, c if (iqs == 0) { // Q4_0/Q5_0/Q8_0 store dm.x = d; Q4_1/Q5_1 store dm = (d, m) pair. switch (FaTypeK) { - case FA_TYPE_Q4_0: kblocksh[buf_ib].dm = FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q4_0.data[a_offset + global_ib].d), 0.0); break; - case FA_TYPE_Q4_1: kblocksh[buf_ib].dm = FLOAT_TYPEV2(k_packed_q4_1_p32.data[a_offset + global_ib].dm); break; - case FA_TYPE_Q5_0: kblocksh[buf_ib].dm = FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q5_0.data[a_offset + global_ib].d), 0.0); break; - case FA_TYPE_Q5_1: kblocksh[buf_ib].dm = FLOAT_TYPEV2(k_packed_q5_1_p32.data[a_offset + global_ib].dm); break; - case FA_TYPE_Q8_0: kblocksh[buf_ib].dm = FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q8_0.data[a_offset + global_ib].d), 0.0); break; + case GGML_TYPE_Q4_0: kblocksh[buf_ib].dm = FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q4_0.data[a_offset + global_ib].d), 0.0); break; + case GGML_TYPE_Q4_1: kblocksh[buf_ib].dm = FLOAT_TYPEV2(k_packed_q4_1_p32.data[a_offset + global_ib].dm); break; + case GGML_TYPE_Q5_0: kblocksh[buf_ib].dm = FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q5_0.data[a_offset + global_ib].d), 0.0); break; + case GGML_TYPE_Q5_1: kblocksh[buf_ib].dm = FLOAT_TYPEV2(k_packed_q5_1_p32.data[a_offset + global_ib].dm); break; + case GGML_TYPE_Q8_0: kblocksh[buf_ib].dm = FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q8_0.data[a_offset + global_ib].d), 0.0); break; } } } @@ -121,31 +121,31 @@ struct fa_k_qs_block8 { fa_k_qs_block8 get_k_qs_block8(uint ib, uint a_offset) { fa_k_qs_block8 r; uint qh = 0; - if (FaTypeK == FA_TYPE_Q5_0) { + if (FaTypeK == GGML_TYPE_Q5_0) { qh = pack32(u16vec2(k_packed_q5_0.data[a_offset + ib].qh[0], k_packed_q5_0.data[a_offset + ib].qh[1])); - } else if (FaTypeK == FA_TYPE_Q5_1) { + } else if (FaTypeK == GGML_TYPE_Q5_1) { qh = k_packed_q5_1.data[a_offset + ib].qh; } - const bool has_qh = (FaTypeK == FA_TYPE_Q5_0) || (FaTypeK == FA_TYPE_Q5_1); + const bool has_qh = (FaTypeK == GGML_TYPE_Q5_0) || (FaTypeK == GGML_TYPE_Q5_1); [[unroll]] for (uint32_t d = 0; d < 4; d++) { uint vui = 0; switch (FaTypeK) { - case FA_TYPE_Q4_0: { // packed16 + case GGML_TYPE_Q4_0: { // packed16 vui = pack32(u16vec2(k_packed_q4_0.data[a_offset + ib].qs[d * 2 + 0], k_packed_q4_0.data[a_offset + ib].qs[d * 2 + 1])); break; } - case FA_TYPE_Q4_1: { // packed32 alias + case GGML_TYPE_Q4_1: { // packed32 alias vui = k_packed_q4_1_p32.data[a_offset + ib].qs[d]; break; } - case FA_TYPE_Q5_0: { // packed16 + case GGML_TYPE_Q5_0: { // packed16 vui = pack32(u16vec2(k_packed_q5_0.data[a_offset + ib].qs[d * 2 + 0], k_packed_q5_0.data[a_offset + ib].qs[d * 2 + 1])); break; } - case FA_TYPE_Q5_1: { // packed32 alias + case GGML_TYPE_Q5_1: { // packed32 alias vui = k_packed_q5_1_p32.data[a_offset + ib].qs[d]; break; } @@ -164,21 +164,21 @@ fa_k_qs_block8 get_k_qs_block8(uint ib, uint a_offset) { int32_t get_k_qs_shmem(const uint buf_ib, const uint pos) { switch (FaTypeK) { - case FA_TYPE_Q4_0: - case FA_TYPE_Q4_1: { + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: { uint sub = pos % 4; uint shift = ((pos % 8) >= 4) ? 4u : 0u; return int32_t((uint(kblocksh[buf_ib].qs[sub]) >> shift) & 0x0F0F0F0Fu); } - case FA_TYPE_Q5_0: - case FA_TYPE_Q5_1: { + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: { uint sub = pos % 4; uint shift = ((pos % 8) >= 4) ? 4u : 0u; int32_t result = int32_t((uint(kblocksh[buf_ib].qs[sub]) >> shift) & 0x0F0F0F0Fu); uint qh_bits = (kblocksh[buf_ib].qh >> (pos * 4u)) & 0xFu; return result | int32_t((qh_bits * 0x02040810u) & 0x10101010u); } - case FA_TYPE_Q8_0: { + case GGML_TYPE_Q8_0: { return kblocksh[buf_ib].qs[pos]; } default: return 0; @@ -187,10 +187,10 @@ int32_t get_k_qs_shmem(const uint buf_ib, const uint pos) { ACC_TYPE k_dot_correction(const uint qib, const ACC_TYPEV2 k_dm) { switch (FaTypeK) { - case FA_TYPE_Q4_0: return -ACC_TYPE(8.0) * ACC_TYPE(Qf[qib].ds.y) * k_dm.x; - case FA_TYPE_Q5_0: return -ACC_TYPE(16.0) * ACC_TYPE(Qf[qib].ds.y) * k_dm.x; - case FA_TYPE_Q4_1: - case FA_TYPE_Q5_1: return ACC_TYPE(Qf[qib].ds.y) * k_dm.y; + case GGML_TYPE_Q4_0: return -ACC_TYPE(8.0) * ACC_TYPE(Qf[qib].ds.y) * k_dm.x; + case GGML_TYPE_Q5_0: return -ACC_TYPE(16.0) * ACC_TYPE(Qf[qib].ds.y) * k_dm.x; + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_1: return ACC_TYPE(Qf[qib].ds.y) * k_dm.y; default: return ACC_TYPE(0.0); } } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/ggml_type_ids.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/ggml_type_ids.glsl new file mode 100644 index 00000000000..5f7da985a25 --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/ggml_type_ids.glsl @@ -0,0 +1,33 @@ +#if !defined(GGML_TYPE_IDS_COMP) +#define GGML_TYPE_IDS_COMP + +// ggml_type enum values — must match ggml.h +#define GGML_TYPE_F32 0u +#define GGML_TYPE_F16 1u +#define GGML_TYPE_Q4_0 2u +#define GGML_TYPE_Q4_1 3u +#define GGML_TYPE_Q5_0 6u +#define GGML_TYPE_Q5_1 7u +#define GGML_TYPE_Q8_0 8u +#define GGML_TYPE_Q2_K 10u +#define GGML_TYPE_Q3_K 11u +#define GGML_TYPE_Q4_K 12u +#define GGML_TYPE_Q5_K 13u +#define GGML_TYPE_Q6_K 14u +#define GGML_TYPE_IQ2_XXS 16u +#define GGML_TYPE_IQ2_XS 17u +#define GGML_TYPE_IQ3_XXS 18u +#define GGML_TYPE_IQ1_S 19u +#define GGML_TYPE_IQ4_NL 20u +#define GGML_TYPE_IQ3_S 21u +#define GGML_TYPE_IQ2_S 22u +#define GGML_TYPE_IQ4_XS 23u +#define GGML_TYPE_IQ1_M 29u +#define GGML_TYPE_BF16 30u +#define GGML_TYPE_TQ2_0 35u +#define GGML_TYPE_Q1_0 41u +#define GGML_TYPE_Q2_0 42u +#define GGML_TYPE_MXFP4 39u +#define GGML_TYPE_NVFP4 40u + +#endif // !defined(GGML_TYPE_IDS_COMP) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/iq_shmem_init.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/iq_shmem_init.glsl new file mode 100644 index 00000000000..5fb2ba9ab8d --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/iq_shmem_init.glsl @@ -0,0 +1,66 @@ +void init_iq_shmem(uvec3 wgsize) { + if (MmTypeA == GGML_TYPE_IQ1_S || MmTypeA == GGML_TYPE_IQ1_M) { + [[unroll]] for (uint i = 0; i < iq1s_grid_const.length(); i += wgsize.x) { + uint idx = i + gl_LocalInvocationIndex.x; + if (iq1s_grid_const.length() % wgsize.x == 0 || idx < iq1s_grid_const.length()) { + u16vec2 g = unpack16(iq1s_grid_const[idx]); + iq1s_grid[2*idx+0] = g.x; + iq1s_grid[2*idx+1] = g.y; + } + } + barrier(); + } else if (MmTypeA == GGML_TYPE_IQ2_XXS) { + [[unroll]] for (uint i = 0; i < iq2_grid.length(); i += wgsize.x) { + if (iq2xxs_grid_const.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq2xxs_grid_const.length()) { + iq2_grid[i + gl_LocalInvocationIndex.x] = iq2xxs_grid_const[i + gl_LocalInvocationIndex.x]; + } + } + barrier(); + } else if (MmTypeA == GGML_TYPE_IQ2_XS) { + [[unroll]] for (uint i = 0; i < iq2_grid.length(); i += wgsize.x) { + if (iq2xs_grid_const.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq2xs_grid_const.length()) { + iq2_grid[i + gl_LocalInvocationIndex.x] = iq2xs_grid_const[i + gl_LocalInvocationIndex.x]; + } + } + barrier(); + } else if (MmTypeA == GGML_TYPE_IQ2_S) { + [[unroll]] for (uint i = 0; i < iq2_grid.length(); i += wgsize.x) { + if (iq2s_grid_const.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq2s_grid_const.length()) { + iq2_grid[i + gl_LocalInvocationIndex.x] = iq2s_grid_const[i + gl_LocalInvocationIndex.x]; + } + } + barrier(); + } else if (MmTypeA == GGML_TYPE_IQ3_XXS) { + [[unroll]] for (uint i = 0; i < iq3_grid.length(); i += wgsize.x) { + if (iq3xxs_grid_const.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq3xxs_grid_const.length()) { + iq3_grid[i + gl_LocalInvocationIndex.x] = iq3xxs_grid_const[i + gl_LocalInvocationIndex.x]; + } + } + barrier(); + } else if (MmTypeA == GGML_TYPE_IQ3_S) { + [[unroll]] for (uint i = 0; i < iq3_grid.length(); i += wgsize.x) { + if (iq3s_grid_const.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq3s_grid_const.length()) { + iq3_grid[i + gl_LocalInvocationIndex.x] = iq3s_grid_const[i + gl_LocalInvocationIndex.x]; + } + } + barrier(); + } else if (MmTypeA == GGML_TYPE_IQ4_XS || MmTypeA == GGML_TYPE_IQ4_NL) { + for (uint i = gl_LocalInvocationIndex.x; i < kvalues_iq4nl.length(); i += wgsize.x) { + kvalues_iq4nl[i] = FLOAT_TYPE(kvalues_iq4nl_const[i]); + } + barrier(); + } +#if !defined(USE_OCP_FP4) + else if (MmTypeA == GGML_TYPE_MXFP4 || MmTypeA == GGML_TYPE_NVFP4) { + for (uint i = gl_LocalInvocationIndex.x; i < kvalues_mxfp4.length(); i += wgsize.x) { + kvalues_mxfp4[i] = kvalues_mxfp4_const[i]; + } + if (MmTypeA == GGML_TYPE_NVFP4) { + for (uint i = gl_LocalInvocationIndex.x; i < 128u; i += wgsize.x) { + ue4m3_fp32_lut[i] = ue4m3_to_fp32_build(i); + } + } + barrier(); + } +#endif +} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.comp b/ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.comp index ba76ec72ca6..9b34d836622 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.comp @@ -12,9 +12,9 @@ #include "types.glsl" #include "fa_types.glsl" -#define FaTypeV FA_TYPE_F32 +#define FaTypeV GGML_TYPE_F32 -layout(constant_id = 0) const uint FaTypeK = FA_TYPE_F32; +layout(constant_id = 0) const uint FaTypeK = GGML_TYPE_F32; layout(constant_id = 1) const uint FaBlockBytesK = 4; layout(constant_id = 2) const uint SUBGROUP_SIZE = 32; @@ -84,11 +84,11 @@ void main() { const uint k_block_elems = fa_block_elems(FaTypeK); const uint k_elem_bytes = FaBlockBytesK / k_block_elems; - if (FaTypeK == FA_TYPE_F16) { + if (FaTypeK == GGML_TYPE_F16) { k_row[tid] = float(k_f16[k_offset / k_elem_bytes + tid]); - } else if (FaTypeK == FA_TYPE_F32) { + } else if (FaTypeK == GGML_TYPE_F32) { k_row[tid] = k_f32[k_offset / k_elem_bytes + tid]; - } else if (FaTypeK == FA_TYPE_BF16) { + } else if (FaTypeK == GGML_TYPE_BF16) { k_row[tid] = bf16_to_fp32(uint(k_bf16[k_offset / k_elem_bytes + tid])); } else if (4 * tid < HEAD_SIZE) { const uint coord = 4 * tid; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp index b4f6d1d6b64..27c461d60dc 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp @@ -24,7 +24,7 @@ void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, [[unroll]] for (uint l = 0; l < 2; ++l) { const uint8_t sign = sign16[l]; const uint qs = qs16[l] | ((qh << (8 - nibble_shift - 2 * l)) & 0x300); - const uvec2 grid = iq2s_grid[qs]; + const uvec2 grid = iq2_grid[qs]; const vec4 grid0 = vec4(unpack8(grid.x)); const vec4 grid1 = vec4(unpack8(grid.y)); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp index d8dafe5f709..1a202e1297e 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp @@ -31,7 +31,7 @@ void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, const uint qs = data_a[ibi].qs[2 * itid + l]; sign_vals[l] = qs >> 9; sign7_vals[l] = bitCount(sign_vals[l]); - const uvec2 grid_data = iq2xs_grid[qs & 511]; + const uvec2 grid_data = iq2_grid[qs & 511]; grid0_vals[l] = vec4(unpack8(grid_data.x)); grid1_vals[l] = vec4(unpack8(grid_data.y)); } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp index f75dcf8331d..f55ee10e8a7 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp @@ -22,8 +22,8 @@ void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, const uint qs = data_a[ibi].qs[8 * ib32 + 2 * (itid & 1) + l]; const uint sign = bitfieldExtract(signscale, 7 * int(2 * (itid & 1) + l), 7); const uint sign7 = bitCount(sign); - const vec4 grid0 = vec4(unpack8(iq2xxs_grid[qs].x)); - const vec4 grid1 = vec4(unpack8(iq2xxs_grid[qs].y)); + const vec4 grid0 = vec4(unpack8(iq2_grid[qs].x)); + const vec4 grid1 = vec4(unpack8(iq2_grid[qs].y)); [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { const vec4 b0 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 0]); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp index 42f52b4a127..e41538997f4 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp @@ -27,8 +27,8 @@ void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, const uint l = l0 + ll; const u8vec2 qs = unpack8(uint32_t(data_a_packed16[ibi].qs[4 * ib32 + l])).xy; // vec4 used due to #12147 const uint sign = data_a[ibi].signs[4 * ib32 + l]; - const vec4 grid0 = vec4(unpack8(iq3s_grid[qs.x | ((qh << (8 - 2*l)) & 0x100)])); - const vec4 grid1 = vec4(unpack8(iq3s_grid[qs.y | ((qh << (7 - 2*l)) & 0x100)])); + const vec4 grid0 = vec4(unpack8(iq3_grid[qs.x | ((qh << (8 - 2*l)) & 0x100)])); + const vec4 grid1 = vec4(unpack8(iq3_grid[qs.y | ((qh << (7 - 2*l)) & 0x100)])); [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { const vec4 b0 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 0]); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp index a88898109ab..5f7a89bc1c3 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp @@ -23,8 +23,8 @@ void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, const uint qs1 = data_a[ibi].qs[8 * ib32 + 4 * (itid & 1) + 2 * l + 1]; const uint sign = bitfieldExtract(signscale, 7 * int(2 * (itid & 1) + l), 7); const uint sign7 = bitCount(sign); - const vec4 grid0 = vec4(unpack8(iq3xxs_grid[qs0])); - const vec4 grid1 = vec4(unpack8(iq3xxs_grid[qs1])); + const vec4 grid0 = vec4(unpack8(iq3_grid[qs0])); + const vec4 grid1 = vec4(unpack8(iq3_grid[qs1])); [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { const vec4 b0 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 0]); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp index 63c4aaebcb1..7540fd8711a 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp @@ -9,6 +9,9 @@ #if defined(DATA_A_IQ1_M) #extension GL_EXT_shader_explicit_arithmetic_types_int16 : require #endif +#if !defined(DATA_A_F32) && !defined(DATA_A_F16) && !defined(DATA_A_BF16) +#extension GL_EXT_shader_explicit_arithmetic_types_int16 : require +#endif #if defined(DATA_A_BF16) && defined(COOPMAT) #extension GL_EXT_bfloat16 : enable @@ -28,24 +31,65 @@ #extension GL_EXT_shader_explicit_arithmetic_types_int16 : require #endif +#ifdef MULMAT_QUANT +#include "ggml_type_ids.glsl" +layout (constant_id = 12) const uint MmTypeA = 0; +#endif + #include "types.glsl" #include "dot_product_funcs.glsl" +#ifndef MULMAT_QUANT #ifndef LOAD_VEC_A #define LOAD_VEC_A 1 #endif +#endif #ifndef LOAD_VEC_B #define LOAD_VEC_B 1 #endif layout (constant_id = 11) const uint ALIGNED = 0; +#ifdef MULMAT_QUANT + +uint mm_load_vec_a() { + switch (MmTypeA) { + case GGML_TYPE_Q1_0: + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_1: + case GGML_TYPE_IQ1_S: + case GGML_TYPE_IQ1_M: + case GGML_TYPE_IQ2_XXS: + case GGML_TYPE_IQ2_XS: + case GGML_TYPE_IQ2_S: + return 8u; + case GGML_TYPE_Q2_0: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q8_0: + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_IQ3_XXS: + case GGML_TYPE_IQ3_S: + case GGML_TYPE_IQ4_XS: + case GGML_TYPE_IQ4_NL: + case GGML_TYPE_MXFP4: + case GGML_TYPE_NVFP4: + return 4u; + default: + return 2u; + } +} +#endif + #if !defined(TO_FLOAT_TYPE) #define TO_FLOAT_TYPE FLOAT_TYPE #endif layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; +#ifndef MULMAT_QUANT layout (binding = 0) readonly buffer A {A_TYPE data_a[];}; #if defined(DATA_A_F32) layout (binding = 0) readonly buffer A_SCALAR {float data_a_scalar[];}; @@ -60,6 +104,42 @@ layout (binding = 0) readonly buffer A_PACKED16 {A_TYPE_PACKED16 data_a_packed16 #if defined(A_TYPE_PACKED32) layout (binding = 0) readonly buffer A_PACKED32 {A_TYPE_PACKED32 data_a_packed32[];}; #endif +#else +// Unpacked struct aliases +layout (binding = 0) readonly buffer BUF_Q1_0 { block_q1_0 data[]; } a_q1_0; +layout (binding = 0) readonly buffer BUF_Q2_0 { block_q2_0 data[]; } a_q2_0; +layout (binding = 0) readonly buffer BUF_Q2_K { block_q2_K data[]; } a_q2_k; +layout (binding = 0) readonly buffer BUF_TQ2_0 { block_tq2_0 data[]; } a_tq2_0; +layout (binding = 0) readonly buffer BUF_Q3_K { block_q3_K data[]; } a_q3_k; +layout (binding = 0) readonly buffer BUF_Q4_K { block_q4_K data[]; } a_q4_k; +layout (binding = 0) readonly buffer BUF_Q5_K { block_q5_K data[]; } a_q5_k; +layout (binding = 0) readonly buffer BUF_Q6_K { block_q6_K data[]; } a_q6_k; +layout (binding = 0) readonly buffer BUF_IQ1_S { block_iq1_s data[]; } a_iq1_s; +layout (binding = 0) readonly buffer BUF_IQ1_M { block_iq1_m data[]; } a_iq1_m; +layout (binding = 0) readonly buffer BUF_IQ2_XXS { block_iq2_xxs data[]; } a_iq2_xxs; +layout (binding = 0) readonly buffer BUF_IQ2_XS { block_iq2_xs data[]; } a_iq2_xs; +layout (binding = 0) readonly buffer BUF_IQ2_S { block_iq2_s data[]; } a_iq2_s; +layout (binding = 0) readonly buffer BUF_IQ3_XXS { block_iq3_xxs data[]; } a_iq3_xxs; +layout (binding = 0) readonly buffer BUF_IQ3_S { block_iq3_s data[]; } a_iq3_s; +layout (binding = 0) readonly buffer BUF_IQ4_XS { block_iq4_xs data[]; } a_iq4_xs; +layout (binding = 0) readonly buffer BUF_MXFP4 { block_mxfp4 data[]; } a_mxfp4; +layout (binding = 0) readonly buffer BUF_NVFP4 { block_nvfp4 data[]; } a_nvfp4; +// Packed16 aliases +layout (binding = 0) readonly buffer BUF_Q4_0_P16 { block_q4_0_packed16 data[]; } a_q4_0_p16; +layout (binding = 0) readonly buffer BUF_Q5_0_P16 { block_q5_0_packed16 data[]; } a_q5_0_p16; +layout (binding = 0) readonly buffer BUF_Q8_0_P16 { block_q8_0_packed16 data[]; } a_q8_0_p16; +layout (binding = 0) readonly buffer BUF_Q3_K_P16 { block_q3_K_packed16 data[]; } a_q3_k_p16; +layout (binding = 0) readonly buffer BUF_Q6_K_P16 { block_q6_K_packed16 data[]; } a_q6_k_p16; +layout (binding = 0) readonly buffer BUF_IQ3_XXS_P16{ block_iq3_xxs_packed16 data[];} a_iq3_xxs_p16; +layout (binding = 0) readonly buffer BUF_IQ4_NL_P16 { block_iq4_nl_packed16 data[]; } a_iq4_nl_p16; +// Packed32 aliases +layout (binding = 0) readonly buffer BUF_Q4_1_P32 { block_q4_1_packed32 data[]; } a_q4_1_p32; +layout (binding = 0) readonly buffer BUF_Q5_1_P32 { block_q5_1_packed32 data[]; } a_q5_1_p32; +layout (binding = 0) readonly buffer BUF_Q2_K_P32 { block_q2_K_packed32 data[]; } a_q2_k_p32; +layout (binding = 0) readonly buffer BUF_Q4_K_P32 { block_q4_K_packed32 data[]; } a_q4_k_p32; +layout (binding = 0) readonly buffer BUF_Q5_K_P32 { block_q5_K_packed32 data[]; } a_q5_k_p32; +layout (binding = 0) readonly buffer BUF_IQ4_XS_P32 { block_iq4_xs_packed32 data[]; } a_iq4_xs_p32; +#endif layout (binding = 1) readonly buffer B {B_TYPE data_b[];}; layout (binding = 1) readonly buffer B_SCALAR {B_TYPE_SCALAR data_b_scalar[];}; @@ -121,8 +201,13 @@ layout (constant_id = 3) const uint BK = 16; // Assumed to be 32 if working wit #endif #ifdef COOPMAT +#ifdef MULMAT_QUANT +layout(constant_id = 13) const uint SHMEM_STRIDE_PAD = 4; +layout(constant_id = 14) const bool APPLY_SLM_A_RESHAPE = false; +#else layout(constant_id = 12) const uint SHMEM_STRIDE_PAD = 4; layout(constant_id = 13) const bool APPLY_SLM_A_RESHAPE = false; +#endif #else const uint SHMEM_STRIDE_PAD = 1; const bool APPLY_SLM_A_RESHAPE = false; @@ -141,6 +226,10 @@ shared ACC_TYPE coopmat_stage[TM * TN * NUM_WARPS]; #include "mul_mm_id_funcs.glsl" #include "mul_mm_funcs.glsl" +#ifdef MULMAT_QUANT +#include "iq_shmem_init.glsl" +#endif + void main() { const uint ic = gl_WorkGroupID.y; @@ -150,7 +239,7 @@ void main() { return; } #endif -#ifdef NEEDS_INIT_IQ_SHMEM +#if defined(NEEDS_INIT_IQ_SHMEM) || defined(MULMAT_QUANT) init_iq_shmem(gl_WorkGroupSize); #endif @@ -200,9 +289,12 @@ void main() { #if defined(DATA_A_F32) || defined(DATA_A_F16) || defined(DATA_A_BF16) const uint LOAD_VEC_A_EFF = (ALIGNED != 0) ? LOAD_VEC_A : 1; const uint LOAD_VEC_BATCH_A = (ALIGNED != 0) ? 1 : 2; -#else +#elif !defined(MULMAT_QUANT) const uint LOAD_VEC_A_EFF = LOAD_VEC_A; const uint LOAD_VEC_BATCH_A = 1; +#else + const uint LOAD_VEC_A_EFF = mm_load_vec_a(); + const uint LOAD_VEC_BATCH_A = 1; #endif const uint LOAD_VEC_B_EFF = (ALIGNED != 0) ? LOAD_VEC_B : 1; const uint LOAD_VEC_BATCH_B = (ALIGNED != 0) ? 1 : 2; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp index 27f3178e7f2..b31e929d872 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp @@ -21,6 +21,12 @@ #extension GL_EXT_bfloat16 : enable #endif +#include "ggml_type_ids.glsl" + +#ifdef MULMAT_QUANT +layout (constant_id = 7) const uint MmTypeA = 0; +#endif + #include "types.glsl" #include "utils.glsl" @@ -37,6 +43,25 @@ layout (constant_id = 4) const bool enable_smaller_matrices = false; const uint BNover2 = enable_smaller_matrices ? (BN / 2) : BN; const uint BNover4 = enable_smaller_matrices ? (BN / 4) : BN; layout (constant_id = 5) const uint ALIGNED = 0; +layout (constant_id = 6) const uint subgroup_size = 32; + +#ifdef MULMAT_QUANT + +uint mm_quant_k() { + switch (MmTypeA) { + case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: case GGML_TYPE_IQ4_NL: case GGML_TYPE_MXFP4: + return 32u; + case GGML_TYPE_Q1_0: + return 128u; + case GGML_TYPE_Q2_0: + case GGML_TYPE_NVFP4: + return 64u; + default: // K-quants and IQ types + return 256u; + } +} +#endif layout (push_constant) uniform parameter { @@ -72,21 +97,108 @@ layout (push_constant) uniform parameter } p; +#ifndef MULMAT_QUANT layout (binding = 0) readonly buffer A {A_TYPE data_a[];}; +#else +layout (binding = 0) readonly buffer BUF_Q1_0 { block_q1_0 data[]; } a_q1_0; +layout (binding = 0) readonly buffer BUF_Q2_0 { block_q2_0 data[]; } a_q2_0; +layout (binding = 0) readonly buffer BUF_Q4_0 { block_q4_0 data[]; } a_q4_0; +layout (binding = 0) readonly buffer BUF_Q4_1 { block_q4_1 data[]; } a_q4_1; +layout (binding = 0) readonly buffer BUF_Q5_0 { block_q5_0 data[]; } a_q5_0; +layout (binding = 0) readonly buffer BUF_Q5_1 { block_q5_1 data[]; } a_q5_1; +layout (binding = 0) readonly buffer BUF_Q8_0 { block_q8_0 data[]; } a_q8_0; +layout (binding = 0) readonly buffer BUF_Q2_K { block_q2_K data[]; } a_q2_k; +layout (binding = 0) readonly buffer BUF_TQ2_0 { block_tq2_0 data[]; } a_tq2_0; +layout (binding = 0) readonly buffer BUF_Q3_K { block_q3_K data[]; } a_q3_k; +layout (binding = 0) readonly buffer BUF_Q4_K { block_q4_K data[]; } a_q4_k; +layout (binding = 0) readonly buffer BUF_Q5_K { block_q5_K data[]; } a_q5_k; +layout (binding = 0) readonly buffer BUF_Q6_K { block_q6_K data[]; } a_q6_k; +layout (binding = 0) readonly buffer BUF_IQ1_S { block_iq1_s data[]; } a_iq1_s; +layout (binding = 0) readonly buffer BUF_IQ1_M { block_iq1_m data[]; } a_iq1_m; +layout (binding = 0) readonly buffer BUF_IQ2_XXS { block_iq2_xxs data[]; } a_iq2_xxs; +layout (binding = 0) readonly buffer BUF_IQ2_XS { block_iq2_xs data[]; } a_iq2_xs; +layout (binding = 0) readonly buffer BUF_IQ2_S { block_iq2_s data[]; } a_iq2_s; +layout (binding = 0) readonly buffer BUF_IQ3_XXS { block_iq3_xxs data[]; } a_iq3_xxs; +layout (binding = 0) readonly buffer BUF_IQ3_S { block_iq3_s data[]; } a_iq3_s; +layout (binding = 0) readonly buffer BUF_IQ4_XS { block_iq4_xs data[]; } a_iq4_xs; +layout (binding = 0) readonly buffer BUF_IQ4_NL { block_iq4_nl data[]; } a_iq4_nl; +layout (binding = 0) readonly buffer BUF_MXFP4 { block_mxfp4 data[]; } a_mxfp4; +layout (binding = 0) readonly buffer BUF_NVFP4 { block_nvfp4 data[]; } a_nvfp4; +#endif layout (binding = 1) readonly buffer B {B_TYPE data_b[];}; layout (binding = 2) writeonly buffer D {D_TYPE data_d[];}; #if defined(MUL_MAT_ID) && defined(GGML_VULKAN_COOPMAT2_DECODE_VECTOR) layout (binding = 1) readonly buffer B4 {B_TYPEV4 data_b_v4[];}; #endif -#if QUANT_K > 1 +#if defined(MULMAT_QUANT) || QUANT_K > 1 #include "dequant_funcs_cm2.glsl" +#ifndef MULMAT_QUANT +// Per-type path: use the alias set by dequant_funcs_cm2.glsl #if defined(dequantFuncA_v) && defined(GGML_VULKAN_COOPMAT2_DECODE_VECTOR) #define DECODEFUNCA , dequantFuncA, dequantFuncA_v #else #define DECODEFUNCA , dequantFuncA #endif #else +// Dispatch coopMatLoadTensorNV to the right decode function +// based on the MmTypeA spec constant. The compiler will dead-code-eliminate +// all but the selected branch. +#ifdef GGML_VULKAN_COOPMAT2_DECODE_VECTOR +#define COOPMAT_LOAD_A(mat, tensor_layout) \ + if (MmTypeA == GGML_TYPE_Q1_0) { coopMatLoadTensorNV(mat, a_q1_0.data, pos_a, tensor_layout, dequantFuncQ1_0, dequantFuncQ1_0_v); } \ + else if (MmTypeA == GGML_TYPE_Q2_0) { coopMatLoadTensorNV(mat, a_q2_0.data, pos_a, tensor_layout, dequantFuncQ2_0, dequantFuncQ2_0_v); } \ + else if (MmTypeA == GGML_TYPE_Q4_0) { coopMatLoadTensorNV(mat, a_q4_0.data, pos_a, tensor_layout, dequantFuncQ4_0, dequantFuncQ4_0_v); } \ + else if (MmTypeA == GGML_TYPE_Q4_1) { coopMatLoadTensorNV(mat, a_q4_1.data, pos_a, tensor_layout, dequantFuncQ4_1, dequantFuncQ4_1_v); } \ + else if (MmTypeA == GGML_TYPE_Q5_0) { coopMatLoadTensorNV(mat, a_q5_0.data, pos_a, tensor_layout, dequantFuncQ5_0, dequantFuncQ5_0_v); } \ + else if (MmTypeA == GGML_TYPE_Q5_1) { coopMatLoadTensorNV(mat, a_q5_1.data, pos_a, tensor_layout, dequantFuncQ5_1, dequantFuncQ5_1_v); } \ + else if (MmTypeA == GGML_TYPE_Q8_0) { coopMatLoadTensorNV(mat, a_q8_0.data, pos_a, tensor_layout, dequantFuncQ8_0, dequantFuncQ8_0_v); } \ + else if (MmTypeA == GGML_TYPE_Q2_K) { coopMatLoadTensorNV(mat, a_q2_k.data, pos_a, tensor_layout, dequantFuncQ2_K, dequantFuncQ2_K_v); } \ + else if (MmTypeA == GGML_TYPE_TQ2_0) { coopMatLoadTensorNV(mat, a_tq2_0.data, pos_a, tensor_layout, dequantFuncTQ2_0, dequantFuncTQ2_0_v); } \ + else if (MmTypeA == GGML_TYPE_Q3_K) { coopMatLoadTensorNV(mat, a_q3_k.data, pos_a, tensor_layout, dequantFuncQ3_K, dequantFuncQ3_K_v); } \ + else if (MmTypeA == GGML_TYPE_Q4_K) { coopMatLoadTensorNV(mat, a_q4_k.data, pos_a, tensor_layout, dequantFuncQ4_K, dequantFuncQ4_K_v); } \ + else if (MmTypeA == GGML_TYPE_Q5_K) { coopMatLoadTensorNV(mat, a_q5_k.data, pos_a, tensor_layout, dequantFuncQ5_K, dequantFuncQ5_K_v); } \ + else if (MmTypeA == GGML_TYPE_Q6_K) { coopMatLoadTensorNV(mat, a_q6_k.data, pos_a, tensor_layout, dequantFuncQ6_K, dequantFuncQ6_K_v); } \ + else if (MmTypeA == GGML_TYPE_IQ1_S) { coopMatLoadTensorNV(mat, a_iq1_s.data, pos_a, tensor_layout, dequantFuncIQ1_S, dequantFuncIQ1_S_v); } \ + else if (MmTypeA == GGML_TYPE_IQ1_M) { coopMatLoadTensorNV(mat, a_iq1_m.data, pos_a, tensor_layout, dequantFuncIQ1_M, dequantFuncIQ1_M_v); } \ + else if (MmTypeA == GGML_TYPE_IQ2_XXS) { coopMatLoadTensorNV(mat, a_iq2_xxs.data, pos_a, tensor_layout, dequantFuncIQ2_XXS, dequantFuncIQ2_XXS_v); } \ + else if (MmTypeA == GGML_TYPE_IQ2_XS) { coopMatLoadTensorNV(mat, a_iq2_xs.data, pos_a, tensor_layout, dequantFuncIQ2_XS, dequantFuncIQ2_XS_v); } \ + else if (MmTypeA == GGML_TYPE_IQ2_S) { coopMatLoadTensorNV(mat, a_iq2_s.data, pos_a, tensor_layout, dequantFuncIQ2_S, dequantFuncIQ2_S_v); } \ + else if (MmTypeA == GGML_TYPE_IQ3_XXS) { coopMatLoadTensorNV(mat, a_iq3_xxs.data, pos_a, tensor_layout, dequantFuncIQ3_XXS, dequantFuncIQ3_XXS_v); } \ + else if (MmTypeA == GGML_TYPE_IQ3_S) { coopMatLoadTensorNV(mat, a_iq3_s.data, pos_a, tensor_layout, dequantFuncIQ3_S, dequantFuncIQ3_S_v); } \ + else if (MmTypeA == GGML_TYPE_IQ4_XS) { coopMatLoadTensorNV(mat, a_iq4_xs.data, pos_a, tensor_layout, dequantFuncIQ4_XS, dequantFuncIQ4_XS_v); } \ + else if (MmTypeA == GGML_TYPE_IQ4_NL) { coopMatLoadTensorNV(mat, a_iq4_nl.data, pos_a, tensor_layout, dequantFuncIQ4_NL, dequantFuncIQ4_NL_v); } \ + else if (MmTypeA == GGML_TYPE_MXFP4) { coopMatLoadTensorNV(mat, a_mxfp4.data, pos_a, tensor_layout, dequantFuncMXFP4, dequantFuncMXFP4_v); } \ + else if (MmTypeA == GGML_TYPE_NVFP4) { coopMatLoadTensorNV(mat, a_nvfp4.data, pos_a, tensor_layout, dequantFuncNVFP4, dequantFuncNVFP4_v); } +#else +#define COOPMAT_LOAD_A(mat, tensor_layout) \ + if (MmTypeA == GGML_TYPE_Q1_0) { coopMatLoadTensorNV(mat, a_q1_0.data, pos_a, tensor_layout, dequantFuncQ1_0); } \ + else if (MmTypeA == GGML_TYPE_Q2_0) { coopMatLoadTensorNV(mat, a_q2_0.data, pos_a, tensor_layout, dequantFuncQ2_0); } \ + else if (MmTypeA == GGML_TYPE_Q4_0) { coopMatLoadTensorNV(mat, a_q4_0.data, pos_a, tensor_layout, dequantFuncQ4_0); } \ + else if (MmTypeA == GGML_TYPE_Q4_1) { coopMatLoadTensorNV(mat, a_q4_1.data, pos_a, tensor_layout, dequantFuncQ4_1); } \ + else if (MmTypeA == GGML_TYPE_Q5_0) { coopMatLoadTensorNV(mat, a_q5_0.data, pos_a, tensor_layout, dequantFuncQ5_0); } \ + else if (MmTypeA == GGML_TYPE_Q5_1) { coopMatLoadTensorNV(mat, a_q5_1.data, pos_a, tensor_layout, dequantFuncQ5_1); } \ + else if (MmTypeA == GGML_TYPE_Q8_0) { coopMatLoadTensorNV(mat, a_q8_0.data, pos_a, tensor_layout, dequantFuncQ8_0); } \ + else if (MmTypeA == GGML_TYPE_Q2_K) { coopMatLoadTensorNV(mat, a_q2_k.data, pos_a, tensor_layout, dequantFuncQ2_K); } \ + else if (MmTypeA == GGML_TYPE_TQ2_0) { coopMatLoadTensorNV(mat, a_tq2_0.data, pos_a, tensor_layout, dequantFuncTQ2_0); } \ + else if (MmTypeA == GGML_TYPE_Q3_K) { coopMatLoadTensorNV(mat, a_q3_k.data, pos_a, tensor_layout, dequantFuncQ3_K); } \ + else if (MmTypeA == GGML_TYPE_Q4_K) { coopMatLoadTensorNV(mat, a_q4_k.data, pos_a, tensor_layout, dequantFuncQ4_K); } \ + else if (MmTypeA == GGML_TYPE_Q5_K) { coopMatLoadTensorNV(mat, a_q5_k.data, pos_a, tensor_layout, dequantFuncQ5_K); } \ + else if (MmTypeA == GGML_TYPE_Q6_K) { coopMatLoadTensorNV(mat, a_q6_k.data, pos_a, tensor_layout, dequantFuncQ6_K); } \ + else if (MmTypeA == GGML_TYPE_IQ1_S) { coopMatLoadTensorNV(mat, a_iq1_s.data, pos_a, tensor_layout, dequantFuncIQ1_S); } \ + else if (MmTypeA == GGML_TYPE_IQ1_M) { coopMatLoadTensorNV(mat, a_iq1_m.data, pos_a, tensor_layout, dequantFuncIQ1_M); } \ + else if (MmTypeA == GGML_TYPE_IQ2_XXS) { coopMatLoadTensorNV(mat, a_iq2_xxs.data, pos_a, tensor_layout, dequantFuncIQ2_XXS); } \ + else if (MmTypeA == GGML_TYPE_IQ2_XS) { coopMatLoadTensorNV(mat, a_iq2_xs.data, pos_a, tensor_layout, dequantFuncIQ2_XS); } \ + else if (MmTypeA == GGML_TYPE_IQ2_S) { coopMatLoadTensorNV(mat, a_iq2_s.data, pos_a, tensor_layout, dequantFuncIQ2_S); } \ + else if (MmTypeA == GGML_TYPE_IQ3_XXS) { coopMatLoadTensorNV(mat, a_iq3_xxs.data, pos_a, tensor_layout, dequantFuncIQ3_XXS); } \ + else if (MmTypeA == GGML_TYPE_IQ3_S) { coopMatLoadTensorNV(mat, a_iq3_s.data, pos_a, tensor_layout, dequantFuncIQ3_S); } \ + else if (MmTypeA == GGML_TYPE_IQ4_XS) { coopMatLoadTensorNV(mat, a_iq4_xs.data, pos_a, tensor_layout, dequantFuncIQ4_XS); } \ + else if (MmTypeA == GGML_TYPE_IQ4_NL) { coopMatLoadTensorNV(mat, a_iq4_nl.data, pos_a, tensor_layout, dequantFuncIQ4_NL); } \ + else if (MmTypeA == GGML_TYPE_MXFP4) { coopMatLoadTensorNV(mat, a_mxfp4.data, pos_a, tensor_layout, dequantFuncMXFP4); } \ + else if (MmTypeA == GGML_TYPE_NVFP4) { coopMatLoadTensorNV(mat, a_nvfp4.data, pos_a, tensor_layout, dequantFuncNVFP4); } +#endif +#endif +#else #define DECODEFUNCA #endif @@ -114,7 +226,6 @@ layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufB { }; uint _ne1; -layout (constant_id = 6) const uint subgroup_size = 32; shared uvec4 ballots_sh[BLOCK_SIZE / subgroup_size]; B_TYPE decodeFuncB(const in decodeBufB bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -246,6 +357,10 @@ void load_row_ids_hoisted(uint expert_idx, uint ic) { } #endif +#ifdef MULMAT_QUANT +#include "iq_shmem_init.glsl" +#endif + void main() { const uint tid = gl_LocalInvocationIndex; const uint ic = gl_WorkGroupID.y; @@ -259,12 +374,12 @@ void main() { if (tid < BN) { row_ids[tid] = u16vec4(0); } -#if !defined(NEEDS_INIT_IQ_SHMEM) +#if !defined(NEEDS_INIT_IQ_SHMEM) && !defined(MULMAT_QUANT) barrier(); #endif #endif -#ifdef NEEDS_INIT_IQ_SHMEM +#if defined(NEEDS_INIT_IQ_SHMEM) || defined(MULMAT_QUANT) init_iq_shmem(gl_WorkGroupSize); #endif @@ -305,22 +420,28 @@ void main() { const uint end_k = min(p.K, (ik + 1) * p.k_split); #endif +#ifdef MULMAT_QUANT + const uint qk = mm_quant_k(); +#else + const uint qk = QUANT_K; +#endif + #ifdef MUL_MAT_ID - uint pos_a = expert_idx * (p.batch_stride_a / QUANT_K); + uint pos_a = expert_idx * (p.batch_stride_a / qk); uint pos_b = 0; #else - uint pos_a = batch_idx_a * (p.batch_stride_a / QUANT_K); + uint pos_a = batch_idx_a * (p.batch_stride_a / qk); uint pos_b = batch_idx * p.batch_stride_b; uint pos_d = batch_idx * p.batch_stride_d + ik * p.batch_stride_d * p.num_batches; #endif - uint stride_a = p.stride_a / QUANT_K; + uint stride_a = p.stride_a / qk; uint stride_b = p.stride_b; // Hint to the compiler that values are aligned (want 16B alignment). // Quants are always block-aligned, no alignment needed. if (ALIGNED != 0) { -#if QUANT_K == 1 +#if !defined(MULMAT_QUANT) && QUANT_K == 1 stride_a &= ~7; #endif stride_b &= ~7; @@ -335,10 +456,15 @@ void main() { #endif tensorLayoutNV<2, gl_CooperativeMatrixClampModeConstantNV> tensorLayoutD = createTensorLayoutNV(2, gl_CooperativeMatrixClampModeConstantNV); +#ifndef MULMAT_QUANT #if QUANT_K > 1 tensorLayoutA = setTensorLayoutBlockSizeNV(tensorLayoutA, 1, QUANT_K); tensorLayoutAClamp = setTensorLayoutBlockSizeNV(tensorLayoutAClamp, 1, QUANT_K); #endif +#else + tensorLayoutA = setTensorLayoutBlockSizeNV(tensorLayoutA, 1, qk); + tensorLayoutAClamp = setTensorLayoutBlockSizeNV(tensorLayoutAClamp, 1, qk); +#endif #if defined(MUL_MAT_ID) && defined(GGML_VULKAN_COOPMAT2_DECODE_VECTOR) tensorLayoutB = setTensorLayoutBlockSizeNV(tensorLayoutB, 1, BK); #endif @@ -368,19 +494,19 @@ void main() { const uint START_ALIGN_K = 256; // For Qi_K (block size 256), unroll whole 256 element tiles. // For legacy quants (block size 32), unroll 8x. - const uint UNROLL_K = (QUANT_K == 256) ? 256 : (BK * 8); + const uint UNROLL_K = (qk == 256) ? 256 : (BK * 8); const uint unroll_count = UNROLL_K / BK; // Detect a fast path where all loads are entirely in bounds and no clamping is required if ((ir + 1) * BM <= p.M && (ic + 1) * BN <= p.padded_N && (start_k % START_ALIGN_K) == 0 && (end_k % BK) == 0 && -#if QUANT_K == 1 +#if !defined(MULMAT_QUANT) && QUANT_K == 1 (stride_a % 8) == 0 && #endif (stride_b % 8) == 0) { // Hint to the compiler that values are aligned (want 16B alignment) start_k &= ~(START_ALIGN_K-1); stride_b &= ~7; -#if QUANT_K == 1 +#if !defined(MULMAT_QUANT) && QUANT_K == 1 stride_a &= ~7; #endif @@ -408,7 +534,11 @@ void main() { coopmat mat_a; coopmat mat_b; +#ifndef MULMAT_QUANT coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA); +#else + COOPMAT_LOAD_A(mat_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK)); +#endif coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, ic * BN, BNover4, block_k, BK), tensorViewTranspose); sum = coopMatMulAdd(mat_a, mat_b, sum); @@ -423,7 +553,11 @@ void main() { coopmat mat_a; coopmat mat_b; +#ifndef MULMAT_QUANT coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA); +#else + COOPMAT_LOAD_A(mat_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK)); +#endif coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, ic * BN, BNover4, block_k, BK), tensorViewTranspose); sum = coopMatMulAdd(mat_a, mat_b, sum); @@ -451,7 +585,11 @@ void main() { coopmat mat_a; coopmat mat_b; +#ifndef MULMAT_QUANT coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA); +#else + COOPMAT_LOAD_A(mat_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK)); +#endif coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, ic * BN, BNover2, block_k, BK), tensorViewTranspose); sum = coopMatMulAdd(mat_a, mat_b, sum); @@ -466,7 +604,11 @@ void main() { coopmat mat_a; coopmat mat_b; +#ifndef MULMAT_QUANT coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA); +#else + COOPMAT_LOAD_A(mat_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK)); +#endif coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, ic * BN, BNover2, block_k, BK), tensorViewTranspose); sum = coopMatMulAdd(mat_a, mat_b, sum); @@ -495,7 +637,11 @@ void main() { coopmat mat_a; coopmat mat_b; +#ifndef MULMAT_QUANT coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA); +#else + COOPMAT_LOAD_A(mat_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK)); +#endif coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, ic * BN, BN, block_k, BK), tensorViewTranspose); sum = coopMatMulAdd(mat_a, mat_b, sum); @@ -510,7 +656,11 @@ void main() { coopmat mat_a; coopmat mat_b; +#ifndef MULMAT_QUANT coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA); +#else + COOPMAT_LOAD_A(mat_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK)); +#endif coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, ic * BN, BN, block_k, BK), tensorViewTranspose); sum = coopMatMulAdd(mat_a, mat_b, sum); @@ -551,10 +701,10 @@ void main() { [[dont_unroll]] for (uint block_k = start_k, i = 0; i < k_iters; block_k += BK, ++i) { - if ((block_k % QUANT_K) == 0) { + if ((block_k % qk) == 0) { store_scales(tid); } - if (block_k + BK < end_k && ((block_k + BK) % QUANT_K) == 0) { + if (block_k + BK < end_k && ((block_k + BK) % qk) == 0) { fetch_scales(ir * BM, pos_a, stride_a, block_k + BK, tid, false); } @@ -562,7 +712,11 @@ void main() { coopmat mat_a; coopmat mat_b; +#ifndef MULMAT_QUANT coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA); +#else + COOPMAT_LOAD_A(mat_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK)); +#endif coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, 0, BNover4, block_k, BK), tensorViewTranspose DECODEFUNCB); sum = coopMatMulAdd(mat_a, mat_b, sum); @@ -570,7 +724,11 @@ void main() { coopmat mat_a; coopmat mat_b; +#ifndef MULMAT_QUANT coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutAClamp, ir * BM, BM, block_k, BK) DECODEFUNCA); +#else + COOPMAT_LOAD_A(mat_a, sliceTensorLayoutNV(tensorLayoutAClamp, ir * BM, BM, block_k, BK)); +#endif coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, 0, BNover4, block_k, BK), tensorViewTranspose DECODEFUNCB); sum = coopMatMulAdd(mat_a, mat_b, sum); @@ -595,10 +753,10 @@ void main() { [[dont_unroll]] for (uint block_k = start_k, i = 0; i < k_iters; block_k += BK, ++i) { - if ((block_k % QUANT_K) == 0) { + if ((block_k % qk) == 0) { store_scales(tid); } - if (block_k + BK < end_k && ((block_k + BK) % QUANT_K) == 0) { + if (block_k + BK < end_k && ((block_k + BK) % qk) == 0) { fetch_scales(ir * BM, pos_a, stride_a, block_k + BK, tid, false); } @@ -606,7 +764,11 @@ void main() { coopmat mat_a; coopmat mat_b; +#ifndef MULMAT_QUANT coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA); +#else + COOPMAT_LOAD_A(mat_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK)); +#endif coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, 0, BNover2, block_k, BK), tensorViewTranspose DECODEFUNCB); sum = coopMatMulAdd(mat_a, mat_b, sum); @@ -614,7 +776,11 @@ void main() { coopmat mat_a; coopmat mat_b; +#ifndef MULMAT_QUANT coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutAClamp, ir * BM, BM, block_k, BK) DECODEFUNCA); +#else + COOPMAT_LOAD_A(mat_a, sliceTensorLayoutNV(tensorLayoutAClamp, ir * BM, BM, block_k, BK)); +#endif coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, 0, BNover2, block_k, BK), tensorViewTranspose DECODEFUNCB); sum = coopMatMulAdd(mat_a, mat_b, sum); @@ -639,10 +805,10 @@ void main() { [[dont_unroll]] for (uint block_k = start_k, i = 0; i < k_iters; block_k += BK, ++i) { - if ((block_k % QUANT_K) == 0) { + if ((block_k % qk) == 0) { store_scales(tid); } - if (block_k + BK < end_k && ((block_k + BK) % QUANT_K) == 0) { + if (block_k + BK < end_k && ((block_k + BK) % qk) == 0) { fetch_scales(ir * BM, pos_a, stride_a, block_k + BK, tid, false); } @@ -650,7 +816,11 @@ void main() { coopmat mat_a; coopmat mat_b; +#ifndef MULMAT_QUANT coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA); +#else + COOPMAT_LOAD_A(mat_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK)); +#endif #ifdef MUL_MAT_ID coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, 0, BN, block_k, BK), tensorViewTranspose DECODEFUNCB); #else @@ -662,7 +832,11 @@ void main() { coopmat mat_a; coopmat mat_b; +#ifndef MULMAT_QUANT coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutAClamp, ir * BM, BM, block_k, BK) DECODEFUNCA); +#else + COOPMAT_LOAD_A(mat_a, sliceTensorLayoutNV(tensorLayoutAClamp, ir * BM, BM, block_k, BK)); +#endif #ifdef MUL_MAT_ID coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, 0, BN, block_k, BK), tensorViewTranspose DECODEFUNCB); #else diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl index 7d852dced8a..c29219d3f46 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl @@ -20,651 +20,682 @@ void store_a(uint m, uint k_pair, FLOAT_TYPEV2 value) { void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uint idx_m, const uint block, const uint end_k) { #if defined(DATA_A_F32) || defined(DATA_A_F16) #if LOAD_VEC_A == 8 - if (ALIGNED != 0) { - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - const uint k_pair = row * LOAD_VEC_A / 2; - FLOAT_TYPEV8 aa = FLOAT_TYPEV8(data_a[idx]); - store_a(col, k_pair, aa[0].xy); - store_a(col, k_pair + 1, aa[0].zw); - store_a(col, k_pair + 2, aa[1].xy); - store_a(col, k_pair + 3, aa[1].zw); - return; - } + if (ALIGNED != 0) { + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; + FLOAT_TYPEV8 aa = FLOAT_TYPEV8(data_a[idx]); + store_a(col, k_pair, aa[0].xy); + store_a(col, k_pair + 1, aa[0].zw); + store_a(col, k_pair + 2, aa[1].xy); + store_a(col, k_pair + 3, aa[1].zw); + return; + } #elif LOAD_VEC_A == 4 - if (ALIGNED != 0) { - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - const uint k_pair = row * LOAD_VEC_A / 2; - FLOAT_TYPEV4 aa = FLOAT_TYPEV4(data_a[idx]); - store_a(col, k_pair, aa.xy); - store_a(col, k_pair + 1, aa.zw); - return; - } + if (ALIGNED != 0) { + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; + FLOAT_TYPEV4 aa = FLOAT_TYPEV4(data_a[idx]); + store_a(col, k_pair, aa.xy); + store_a(col, k_pair + 1, aa.zw); + return; + } #endif - const uint idx = pos_a + col * p.stride_a + row * 2; - if (idx_m < p.M && block + row * 2 + 1 < end_k) { - store_a(col, row, FLOAT_TYPEV2(data_a_scalar[idx], - data_a_scalar[idx + 1])); - } else if (idx_m < p.M && block + row * 2 < end_k) { - store_a(col, row, FLOAT_TYPEV2(data_a_scalar[idx], 0.0f)); - } else { - store_a(col, row, FLOAT_TYPEV2(0.0f)); - } + const uint idx = pos_a + col * p.stride_a + row * 2; + if (idx_m < p.M && block + row * 2 + 1 < end_k) { + store_a(col, row, FLOAT_TYPEV2(data_a_scalar[idx], + data_a_scalar[idx + 1])); + } else if (idx_m < p.M && block + row * 2 < end_k) { + store_a(col, row, FLOAT_TYPEV2(data_a_scalar[idx], 0.0f)); + } else { + store_a(col, row, FLOAT_TYPEV2(0.0f)); + } #elif defined(DATA_A_BF16) #if LOAD_VEC_A == 4 - if (ALIGNED != 0) { - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - const uint k_pair = row * LOAD_VEC_A / 2; - FLOAT_TYPEV4 aa = FLOAT_TYPEV4(TO_FLOAT_TYPE(data_a[idx])); - store_a(col, k_pair, aa.xy); - store_a(col, k_pair + 1, aa.zw); - return; - } + if (ALIGNED != 0) { + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; + FLOAT_TYPEV4 aa = FLOAT_TYPEV4(TO_FLOAT_TYPE(data_a[idx])); + store_a(col, k_pair, aa.xy); + store_a(col, k_pair + 1, aa.zw); + return; + } #endif - const uint idx = pos_a + col * p.stride_a + row * 2; - if (idx_m < p.M && block + row * 2 + 1 < end_k) { - store_a(col, row, FLOAT_TYPEV2(TO_FLOAT_TYPE(data_a_scalar[idx]), - TO_FLOAT_TYPE(data_a_scalar[idx + 1]))); - } else if (idx_m < p.M && block + row * 2 < end_k) { - store_a(col, row, FLOAT_TYPEV2(TO_FLOAT_TYPE(data_a_scalar[idx]), 0.0f)); - } else { - store_a(col, row, FLOAT_TYPEV2(0.0f)); - } -#elif defined(DATA_A_Q4_0) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 4; - const uint iqs = idx & 0x03; - - const float d = float(data_a_packed16[ib].d); - const uint vui = uint(data_a_packed16[ib].qs[2*iqs]) | (uint(data_a_packed16[ib].qs[2*iqs + 1]) << 16); - const vec4 v0 = (vec4(unpack8(vui & 0x0F0F0F0F)) - 8.0f) * d; - const vec4 v1 = (vec4(unpack8((vui >> 4) & 0x0F0F0F0F)) - 8.0f) * d; - - const uint k_pair = row * LOAD_VEC_A / 4; - store_a(col, k_pair, FLOAT_TYPEV2(v0.xy)); - store_a(col, k_pair + 1, FLOAT_TYPEV2(v0.zw)); - store_a(col, k_pair + 8, FLOAT_TYPEV2(v1.xy)); - store_a(col, k_pair + 9, FLOAT_TYPEV2(v1.zw)); -#elif defined(DATA_A_Q4_1) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 4; - const uint iqs = idx & 0x03; - - const vec2 dm = vec2(data_a_packed32[ib].dm); - const uint vui = data_a_packed32[ib].qs[iqs]; - const vec4 v0 = vec4(unpack8(vui & 0x0F0F0F0F)) * dm.x + dm.y; - const vec4 v1 = vec4(unpack8((vui >> 4) & 0x0F0F0F0F)) * dm.x + dm.y; - - const uint k_pair = row * LOAD_VEC_A / 4; - store_a(col, k_pair, FLOAT_TYPEV2(v0.xy)); - store_a(col, k_pair + 1, FLOAT_TYPEV2(v0.zw)); - store_a(col, k_pair + 8, FLOAT_TYPEV2(v1.xy)); - store_a(col, k_pair + 9, FLOAT_TYPEV2(v1.zw)); -#elif defined(DATA_A_Q5_0) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 8; - const uint iqs = idx & 0x07; - - const float d = float(data_a_packed16[ib].d); - const uint uint_qh = uint(data_a_packed16[ib].qh[1]) << 16 | uint(data_a_packed16[ib].qh[0]); - const ivec2 qh0 = ivec2(((uint_qh >> 2*iqs) << 4) & 0x10, (uint_qh >> (2*iqs + 12)) & 0x10); - const ivec2 qh1 = ivec2(((uint_qh >> (2*iqs + 1)) << 4) & 0x10, (uint_qh >> (2*iqs + 13)) & 0x10); - - const uint vui = uint(data_a_packed16[ib].qs[iqs]); - const vec4 v = (vec4((vui & 0xF) | qh0.x, ((vui >> 4) & 0xF) | qh0.y, ((vui >> 8) & 0xF) | qh1.x, (vui >> 12) | qh1.y) - 16.0f) * d; - store_a(col, row, FLOAT_TYPEV2(v.xz)); - store_a(col, row + 8, FLOAT_TYPEV2(v.yw)); -#elif defined(DATA_A_Q5_1) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 4; - const uint iqs = idx & 0x03; - - const vec2 dm = vec2(data_a_packed32[ib].dm); - const uint uint_qh = data_a_packed32[ib].qh; - const uvec2 qh0 = uvec2(((uint_qh >> 4*iqs) << 4) & 0x10, (uint_qh >> (4*iqs + 12)) & 0x10); - const uvec2 qh1 = uvec2(((uint_qh >> (4*iqs + 1)) << 4) & 0x10, (uint_qh >> (4*iqs + 13)) & 0x10); - const uvec2 qh2 = uvec2(((uint_qh >> (4*iqs + 2)) << 4) & 0x10, (uint_qh >> (4*iqs + 14)) & 0x10); - const uvec2 qh3 = uvec2(((uint_qh >> (4*iqs + 3)) << 4) & 0x10, (uint_qh >> (4*iqs + 15)) & 0x10); - - const uint vui = data_a_packed32[ib].qs[iqs]; - const vec4 v0 = vec4((vui & 0xF) | qh0.x, ((vui >> 4) & 0xF) | qh0.y, ((vui >> 8) & 0xF) | qh1.x, ((vui >> 12) & 0xF) | qh1.y) * dm.x + dm.y; - const vec4 v1 = vec4(((vui >> 16) & 0xF) | qh2.x, ((vui >> 20) & 0xF) | qh2.y, ((vui >> 24) & 0xF) | qh3.x, ((vui >> 28) & 0xF) | qh3.y) * dm.x + dm.y; - - const uint k_pair = row * LOAD_VEC_A / 4; - store_a(col, k_pair, FLOAT_TYPEV2(v0.xz)); - store_a(col, k_pair + 1, FLOAT_TYPEV2(v1.xz)); - store_a(col, k_pair + 8, FLOAT_TYPEV2(v0.yw)); - store_a(col, k_pair + 9, FLOAT_TYPEV2(v1.yw)); -#elif defined(DATA_A_Q8_0) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 8; - const uint iqs = idx & 0x07; - - const float d = float(data_a_packed16[ib].d); - const i8vec2 v0 = unpack8(int32_t(data_a_packed16[ib].qs[2*iqs])).xy; // vec4 used due to #12147 - const i8vec2 v1 = unpack8(int32_t(data_a_packed16[ib].qs[2*iqs + 1])).xy; - const vec4 v = vec4(v0.x, v0.y, v1.x, v1.y) * d; - - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, FLOAT_TYPEV2(v.xy)); - store_a(col, k_pair + 1, FLOAT_TYPEV2(v.zw)); -#elif defined(DATA_A_Q1_0) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 16; - const uint iqs = idx & 0xfu; - - const float d = float(data_a[ib].d); - const uint bits = uint(data_a[ib].qs[iqs]); - - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, FLOAT_TYPEV2((bits & 0x01u) != 0u ? d : -d, (bits & 0x02u) != 0u ? d : -d)); - store_a(col, k_pair + 1, FLOAT_TYPEV2((bits & 0x04u) != 0u ? d : -d, (bits & 0x08u) != 0u ? d : -d)); - store_a(col, k_pair + 2, FLOAT_TYPEV2((bits & 0x10u) != 0u ? d : -d, (bits & 0x20u) != 0u ? d : -d)); - store_a(col, k_pair + 3, FLOAT_TYPEV2((bits & 0x40u) != 0u ? d : -d, (bits & 0x80u) != 0u ? d : -d)); -#elif defined(DATA_A_Q2_0) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 16; - const uint iqs = idx & 0xfu; - - const FLOAT_TYPE d = FLOAT_TYPE(data_a[ib].d); - const uint bits = uint(data_a[ib].qs[iqs]); - - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, d * (FLOAT_TYPEV2(bits & 3u, (bits >> 2u) & 3u) - FLOAT_TYPEV2(1.0f))); - store_a(col, k_pair + 1, d * (FLOAT_TYPEV2((bits >> 4u) & 3u, bits >> 6u) - FLOAT_TYPEV2(1.0f))); -#elif defined(DATA_A_Q2_K) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 64; // 4 values per idx - const uint iqs = (idx % 64) * 2; // 0,2,4..126 - - const uint qsi = (iqs / 64) * 16 + (iqs % 16); // 0..15 - const uint scalesi = iqs / 8; // 0..15 - const uint qsshift = ((iqs % 64) / 16) * 2; // 0,2,4,6 - - const vec4 qs = vec4(unpack8((data_a_packed32[ib].qs[qsi / 2] >> qsshift) & 0x03030303)); - const uint scales = data_a[ib].scales[scalesi]; - const vec2 dm = vec2(data_a[ib].dm); - - const vec4 v = dm.x * float(scales & 0xF) * qs - dm.y * float(scales >> 4); - - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, FLOAT_TYPEV2(v.xy)); - store_a(col, k_pair + 1, FLOAT_TYPEV2(v.zw)); -#elif defined(DATA_A_TQ2_0) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 128; // 2 values per idx - const uint iqs = (idx % 128) * 2; // elem 0,2,4..254 - - const uint qsi = (iqs / 128) * 32 + (iqs % 32); // byte pair start - const uint shift = 2 * ((iqs % 128) / 32); // 0,2,4,6 - - const uvec2 qs = uvec2(data_a[ib].qs[qsi], data_a[ib].qs[qsi + 1]); - const float d = float(data_a[ib].d); - - const vec2 v = d * (vec2((qs >> shift) & 3) - 1.0); - - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, FLOAT_TYPEV2(v.xy)); -#elif defined(DATA_A_Q3_K) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 128; // 2 values per idx - const uint iqs = idx % 128; // 0..127 - - const uint n = iqs / 64; // 0,1 - const uint qsi = n * 32 + (iqs % 16) * 2; // 0,2,4..62 - const uint hmi = (iqs % 16) * 2; // 0,2,4..30 - const uint j = (iqs % 64) / 4; // 0..3 - const uint is = iqs / 8; // 0..15 - const uint halfsplit = ((iqs % 64) / 16); // 0,1,2,3 - const uint qsshift = halfsplit * 2; // 0,2,4,6 - - const int8_t us = int8_t(((data_a[ib].scales[is % 8] >> (4 * int(is / 8))) & 0xF) - | (((data_a[ib].scales[8 + (is % 4)] >> (2 * int(is / 4))) & 3) << 4)); - const float dl = float(data_a[ib].d) * float(us - 32); - - const vec2 qs = vec2(unpack8((uint(data_a_packed16[ib].qs[qsi / 2]) >> qsshift) & 0x0303).xy); - const vec2 hm = vec2(unpack8(((uint(data_a_packed16[ib].hmask[hmi / 2]) >> (4 * n + halfsplit)) & 0x0101 ^ 0x0101) << 2).xy); - - store_a(col, row * LOAD_VEC_A / 2, FLOAT_TYPEV2(dl * (qs.x - hm.x), - dl * (qs.y - hm.y))); -#elif defined(DATA_A_Q4_K) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 64; // 4 values per idx - const uint iqs = (idx % 64) * 2; // 0,2,4..126 - - const uint n = iqs / 32; // 0,1,2,3 - const uint b = (iqs % 32) / 16; // 0,1 - const uint is = 2 * n + b; // 0..7 - const uint qsi = n * 32 + (iqs % 16) * 2; // 0,2,4..126 - - const vec2 loadd = vec2(data_a[ib].dm); - - const uvec3 scales = uvec3(data_a_packed32[ib].scales[0], - data_a_packed32[ib].scales[1], - data_a_packed32[ib].scales[2]); - const uint scalesoffs = (is & 3) * 8; - - const uint scidx0 = (is < 4) ? 0 : 2; - const uint scidxshift0 = scalesoffs; - const uint scidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2; - const uint mbidx0 = (is < 4) ? 1 : 2; - const uint mbidxshift0 = (is < 4) ? scalesoffs : scalesoffs + 4; - const uint mbidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2; - - const uint8_t sc = uint8_t(((scales[scidx0] >> scidxshift0) & 0xF) | ((scales[0] >> scidxshift1) & 0x30)); - const uint8_t mbyte = uint8_t(((scales[mbidx0] >> mbidxshift0) & 0xF) | ((scales[1] >> mbidxshift1) & 0x30)); - - const float d = loadd.x * sc; - const float m = -loadd.y * mbyte; - - const vec4 q = vec4(unpack8((data_a_packed32[ib].qs[qsi / 4] >> (b * 4)) & 0x0F0F0F0F)); - - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, FLOAT_TYPEV2(fma(d, q.x, m), fma(d, q.y, m))); - store_a(col, k_pair + 1, FLOAT_TYPEV2(fma(d, q.z, m), fma(d, q.w, m))); -#elif defined(DATA_A_Q5_K) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 64; // 4 values per idx - const uint iqs = (idx % 64) * 2; // 0,2,4..126 - - const uint n = iqs / 32; // 0,1,2,3 - const uint b = (iqs % 32) / 16; // 0,1 - const uint is = 2 * n + b; // 0..7 - const uint qsi = n * 32 + (iqs % 16) * 2; // 0,2,4..126 - const uint qhi = (iqs % 16) * 2; // 0,2,4..30 - - const vec2 loadd = vec2(data_a[ib].dm); - - const uvec3 scales = uvec3(data_a_packed32[ib].scales[0], - data_a_packed32[ib].scales[1], - data_a_packed32[ib].scales[2]); - const uint scalesoffs = (is & 3) * 8; - - const uint scidx0 = (is < 4) ? 0 : 2; - const uint scidxshift0 = scalesoffs; - const uint scidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2; - const uint mbidx0 = (is < 4) ? 1 : 2; - const uint mbidxshift0 = (is < 4) ? scalesoffs : scalesoffs + 4; - const uint mbidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2; - - const uint8_t sc = uint8_t(((scales[scidx0] >> scidxshift0) & 0xF) | ((scales[0] >> scidxshift1) & 0x30)); - const uint8_t mbyte = uint8_t(((scales[mbidx0] >> mbidxshift0) & 0xF) | ((scales[1] >> mbidxshift1) & 0x30)); - - const float d = loadd.x * sc; - const float m = -loadd.y * mbyte; - - const uint qs = (data_a_packed32[ib].qs[qsi / 4] >> (b * 4)) & 0x0F0F0F0F; - const uint qh = ((data_a_packed32[ib].qh[qhi / 4] >> (iqs / 16)) & 0x01010101) << 4; - const vec4 q = vec4(unpack8(qs | qh)); - - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, FLOAT_TYPEV2(fma(d, q.x, m), fma(d, q.y, m))); - store_a(col, k_pair + 1, FLOAT_TYPEV2(fma(d, q.z, m), fma(d, q.w, m))); -#elif defined(DATA_A_Q6_K) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 128; // 2 values per idx - const uint iqs = idx % 128; // 0..127 - - const uint n = iqs / 64; // 0,1 - const uint b = ((iqs % 64) / 32) * 4; // 0,4 - const uint is_b = (iqs % 16) / 8; // 0,1 - const uint qhshift = ((iqs % 64) / 16) * 2; // 0,2,4,6 - const uint is = 8 * n + qhshift + is_b; // 0..15 - const uint qsi = n * 32 + (iqs % 32); // 0..63 - const uint qhi = n * 16 + (iqs % 16); // 0..31 - - const float dscale = float(data_a[ib].d) * float(data_a[ib].scales[is]); - - const uint ql = (uint(data_a_packed16[ib].ql[qsi]) >> b) & 0x0F0F; - const uint qh = (uint(data_a_packed16[ib].qh[qhi]) >> qhshift) & 0x0303; - const vec2 q = (vec2(unpack8(ql | (qh << 4)).xy) - 32) * dscale; - - store_a(col, row * LOAD_VEC_A / 2, FLOAT_TYPEV2(q.x, q.y)); -#elif defined(DATA_A_IQ1_S) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 32; // 8 values per idx - const uint ib32 = (idx % 32) / 4; // 0..7 - const uint ib8 = idx % 32; - - const float d = float(data_a[ib].d); - const uint qh = data_a[ib].qh[ib32]; - const uint qs = data_a[ib].qs[ib8]; - const float dl = d * (2 * bitfieldExtract(qh, 12, 3) + 1); - const float delta = ((qh & 0x8000) != 0) ? -IQ1S_DELTA : IQ1S_DELTA; - const int16_t grid = int16_t(iq1s_grid[qs | (bitfieldExtract(qh, 3 * int(ib8 & 3), 3) << 8)]); - - const uint k_pair = row * LOAD_VEC_A / 2; - [[unroll]] for (int k = 0; k < 4; ++k) { - store_a(col, k_pair + k, FLOAT_TYPEV2(dl * (bitfieldExtract(grid, 4 * k , 2) + delta), - dl * (bitfieldExtract(grid, 4 * k + 2, 2) + delta))); - } -#elif defined(DATA_A_IQ1_M) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 32; // 8 values per idx - const uint ib8 = idx % 32; - const uint ib16 = ib8 / 2; - - const uint16_t[4] scales = data_a[ib].scales; - const u16vec4 s = u16vec4(scales[0], scales[1], scales[2], scales[3]) >> 12; - const float d = float(unpackHalf2x16(s.x | (s.y << 4) | (s.z << 8) | (s.w << 12)).x); - const uint sc = scales[ib8 / 8]; - const uint qs = data_a[ib].qs[ib8]; - const uint qh = data_a[ib].qh[ib16] >> (4 * (ib8 & 1)); - const float dl = d * (2 * bitfieldExtract(sc, 3 * int(ib16 & 3), 3) + 1); - const float delta = ((qh & 8) != 0) ? -IQ1M_DELTA : IQ1M_DELTA; - const int16_t grid = int16_t(iq1s_grid[qs | ((qh & 7) << 8)]); - - const uint k_pair = row * LOAD_VEC_A / 2; - [[unroll]] for (int k = 0; k < 4; ++k) { - store_a(col, k_pair + k, FLOAT_TYPEV2(dl * (bitfieldExtract(grid, 4 * k , 2) + delta), - dl * (bitfieldExtract(grid, 4 * k + 2, 2) + delta))); - } -#elif defined(DATA_A_IQ2_XXS) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 32; // 8 values per idx - const uint ib32 = (idx % 32) / 4; // 0..7 - const uint ib8 = idx % 4; - - const float d = float(data_a[ib].d); - const uint qs = data_a[ib].qs[8 * ib32 + ib8]; - const uint signs = pack32(u8vec4( - data_a[ib].qs[8*ib32 + 4], - data_a[ib].qs[8*ib32 + 5], - data_a[ib].qs[8*ib32 + 6], - data_a[ib].qs[8*ib32 + 7] - )); - const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + (signs >> 28))); - const uint32_t sign7 = bitfieldExtract(signs, 7 * int(ib8), 7); - const uint sign = sign7 | (bitCount(sign7) << 7); - const uvec2 grid = iq2xxs_grid[qs]; - const vec4 grid0 = vec4(unpack8(grid.x)); - const vec4 grid1 = vec4(unpack8(grid.y)); - - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x, - (sign & 2) != 0 ? -grid0.y : grid0.y)); - store_a(col, k_pair + 1, db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z, - (sign & 8) != 0 ? -grid0.w : grid0.w)); - store_a(col, k_pair + 2, db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x, - (sign & 32) != 0 ? -grid1.y : grid1.y)); - store_a(col, k_pair + 3, db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z, - (sign & 128) != 0 ? -grid1.w : grid1.w)); -#elif defined(DATA_A_IQ2_XS) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 32; // 8 values per idx - const uint ib32 = (idx % 32) / 4; // 0..7 - const uint ib8 = idx % 4; // 0..3 - - const float d = float(data_a[ib].d); - const uint scale = (data_a[ib].scales[ib32] >> (2 * (ib8 & 2))) & 0xf; - const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + scale)); - const uint qs = data_a[ib].qs[4 * ib32 + ib8]; - const uint sign7 = qs >> 9; - const uint sign = sign7 | (bitCount(sign7) << 7); - const uvec2 grid = iq2xs_grid[qs & 511]; - const vec4 grid0 = vec4(unpack8(grid.x)); - const vec4 grid1 = vec4(unpack8(grid.y)); - - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x, - (sign & 2) != 0 ? -grid0.y : grid0.y)); - store_a(col, k_pair + 1, db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z, - (sign & 8) != 0 ? -grid0.w : grid0.w)); - store_a(col, k_pair + 2, db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x, - (sign & 32) != 0 ? -grid1.y : grid1.y)); - store_a(col, k_pair + 3, db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z, - (sign & 128) != 0 ? -grid1.w : grid1.w)); -#elif defined(DATA_A_IQ2_S) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 32; // 8 values per idx - const uint ib8 = idx % 32; // 0..31 - const uint ib32 = ib8 / 4; // 0..7 - - const uint scale = (data_a[ib].scales[ib32] >> (2 * (ib8 & 2))) & 0xf; - const uint qs = data_a[ib].qs[ib8]; - const uint qh = data_a[ib].qh[ib32]; - const uint qhshift = 2 * (ib8 % 4); - const uint sign = data_a[ib].qs[QUANT_K / 8 + ib8]; - - const float d = float(data_a[ib].d); - const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + scale)); - const uvec2 grid = iq2s_grid[qs | ((qh << (8 - qhshift)) & 0x300)]; - const vec4 grid0 = vec4(unpack8(grid.x)); - const vec4 grid1 = vec4(unpack8(grid.y)); - - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x, - (sign & 2) != 0 ? -grid0.y : grid0.y)); - store_a(col, k_pair + 1, db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z, - (sign & 8) != 0 ? -grid0.w : grid0.w)); - store_a(col, k_pair + 2, db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x, - (sign & 32) != 0 ? -grid1.y : grid1.y)); - store_a(col, k_pair + 3, db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z, - (sign & 128) != 0 ? -grid1.w : grid1.w)); -#elif defined(DATA_A_IQ3_XXS) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 64; // 4 values per idx - const uint iqs = idx % 64; // 0..63 - const uint is = QUANT_K / 4 + 4 * (iqs / 8); // 8 values - - const float d = float(data_a[ib].d); - const uint qs = data_a[ib].qs[iqs]; - const uint signs = pack32(u16vec2( - data_a_packed16[ib].qs[is/2], - data_a_packed16[ib].qs[is/2+1] - )); - const float db = d * 0.5 * (0.5 + (signs >> 28)); - const uint32_t sign7 = bitfieldExtract(signs, 7 * (int(iqs / 2) % 4), 7); - const uint sign = (sign7 | (bitCount(sign7) << 7)) >> (4 * (idx % 2)); - const uint grid = iq3xxs_grid[qs]; - const vec4 v = db * vec4(unpack8(grid)); - - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, FLOAT_TYPEV2((sign & 1) != 0 ? -v.x : v.x, - (sign & 2) != 0 ? -v.y : v.y)); - store_a(col, k_pair + 1, FLOAT_TYPEV2((sign & 4) != 0 ? -v.z : v.z, - (sign & 8) != 0 ? -v.w : v.w)); -#elif defined(DATA_A_IQ3_S) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 64; // 4 values per idx - const uint iqs = idx % 64; // 0..63 - const uint iqh = iqs / 8; - - const float d = float(data_a[ib].d); - const uint qs = data_a[ib].qs[iqs]; - const uint qh = data_a[ib].qh[iqh]; - const int8_t sign = int8_t(data_a[ib].signs[iqs / 2] >> (4 * (idx % 2))); - const uint scale = data_a[ib].scales[iqs / 16]; - const i8vec2 sign01 = i8vec2(1 - (2 & i8vec2(sign << 1, sign))); - const float db = d * (1 + 2 * ((scale >> (4 * (iqh & 1))) & 0xf)); - const uint32_t grid = iq3s_grid[qs | ((qh << (8 - (iqs % 8))) & 256)]; - const vec4 v = db * vec4(unpack8(grid)); - - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, FLOAT_TYPEV2((sign & 1) != 0 ? -v.x : v.x, - (sign & 2) != 0 ? -v.y : v.y)); - store_a(col, k_pair + 1, FLOAT_TYPEV2((sign & 4) != 0 ? -v.z : v.z, - (sign & 8) != 0 ? -v.w : v.w)); -#elif defined(DATA_A_IQ4_XS) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 64; // 4 values per idx - const uint ib32 = (idx % 64) / 8; // 0..7 - const uint iq = 4 * ib32 + (idx % 4); - - const uint sl = (data_a[ib].scales_l[ib32/2] >> (4 * (ib32 & 1))) & 0xF; - const uint sh = ((data_a[ib].scales_h) >> (2 * ib32)) & 3; - const uint qshift = idx & 4; - u8vec4 qs = unpack8((uint(data_a_packed32[ib].qs[iq]) >> qshift) & 0x0F0F0F0F); - - const float d = float(data_a[ib].d); - const vec4 v = d * float(int(sl | (sh << 4)) - 32) * vec4(kvalues_iq4nl[qs.x], kvalues_iq4nl[qs.y], kvalues_iq4nl[qs.z], kvalues_iq4nl[qs.w]); - - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, FLOAT_TYPEV2(v.xy)); - store_a(col, k_pair + 1, FLOAT_TYPEV2(v.zw)); -#elif defined(DATA_A_IQ4_NL) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 8; - const uint iqs = idx & 0x07; - - const FLOAT_TYPE d = FLOAT_TYPE(data_a_packed16[ib].d); - const uint vui = uint(data_a_packed16[ib].qs[iqs]); - - const uint k_pair = row * LOAD_VEC_A / 4; - store_a(col, k_pair, d * FLOAT_TYPEV2(kvalues_iq4nl[vui & 0xF], - kvalues_iq4nl[bitfieldExtract(vui, 8, 4)])); - store_a(col, k_pair + 8, d * FLOAT_TYPEV2(kvalues_iq4nl[bitfieldExtract(vui, 4, 4)], - kvalues_iq4nl[vui >> 12])); -#elif defined(DATA_A_MXFP4) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 8; - const uint iqs = (idx & 0x07) * 2; - - const uint vui = uint(data_a[ib].qs[iqs]); - const uint vui2 = uint(data_a[ib].qs[iqs+1]); + const uint idx = pos_a + col * p.stride_a + row * 2; + if (idx_m < p.M && block + row * 2 + 1 < end_k) { + store_a(col, row, FLOAT_TYPEV2(TO_FLOAT_TYPE(data_a_scalar[idx]), + TO_FLOAT_TYPE(data_a_scalar[idx + 1]))); + } else if (idx_m < p.M && block + row * 2 < end_k) { + store_a(col, row, FLOAT_TYPEV2(TO_FLOAT_TYPE(data_a_scalar[idx]), 0.0f)); + } else { + store_a(col, row, FLOAT_TYPEV2(0.0f)); + } +#else + if (MmTypeA == GGML_TYPE_Q4_0) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 4; + + const uint ib = idx / 4; + const uint iqs = idx & 0x03; + + const float d = float(a_q4_0_p16.data[ib].d); + const uint vui = uint(a_q4_0_p16.data[ib].qs[2*iqs]) | (uint(a_q4_0_p16.data[ib].qs[2*iqs + 1]) << 16); + const vec4 v0 = (vec4(unpack8(vui & 0x0F0F0F0F)) - 8.0f) * d; + const vec4 v1 = (vec4(unpack8((vui >> 4) & 0x0F0F0F0F)) - 8.0f) * d; + + store_a(col, k_pair, FLOAT_TYPEV2(v0.xy)); + store_a(col, k_pair + 1, FLOAT_TYPEV2(v0.zw)); + store_a(col, k_pair + 8, FLOAT_TYPEV2(v1.xy)); + store_a(col, k_pair + 9, FLOAT_TYPEV2(v1.zw)); + } else if (MmTypeA == GGML_TYPE_Q4_1) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 4; + + const uint ib = idx / 4; + const uint iqs = idx & 0x03; + + const vec2 dm = vec2(a_q4_1_p32.data[ib].dm); + const uint vui = a_q4_1_p32.data[ib].qs[iqs]; + const vec4 v0 = vec4(unpack8(vui & 0x0F0F0F0F)) * dm.x + dm.y; + const vec4 v1 = vec4(unpack8((vui >> 4) & 0x0F0F0F0F)) * dm.x + dm.y; + + store_a(col, k_pair, FLOAT_TYPEV2(v0.xy)); + store_a(col, k_pair + 1, FLOAT_TYPEV2(v0.zw)); + store_a(col, k_pair + 8, FLOAT_TYPEV2(v1.xy)); + store_a(col, k_pair + 9, FLOAT_TYPEV2(v1.zw)); + } else if (MmTypeA == GGML_TYPE_Q5_0) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 4; + + const uint ib = idx / 8; + const uint iqs = idx & 0x07; + + const float d = float(a_q5_0_p16.data[ib].d); + const uint uint_qh = uint(a_q5_0_p16.data[ib].qh[1]) << 16 | uint(a_q5_0_p16.data[ib].qh[0]); + const ivec2 qh0 = ivec2(((uint_qh >> 2*iqs) << 4) & 0x10, (uint_qh >> (2*iqs + 12)) & 0x10); + const ivec2 qh1 = ivec2(((uint_qh >> (2*iqs + 1)) << 4) & 0x10, (uint_qh >> (2*iqs + 13)) & 0x10); + + const uint vui = uint(a_q5_0_p16.data[ib].qs[iqs]); + const vec4 v = (vec4((vui & 0xF) | qh0.x, ((vui >> 4) & 0xF) | qh0.y, ((vui >> 8) & 0xF) | qh1.x, (vui >> 12) | qh1.y) - 16.0f) * d; + + store_a(col, k_pair, FLOAT_TYPEV2(v.xz)); + store_a(col, k_pair + 8, FLOAT_TYPEV2(v.yw)); + } else if (MmTypeA == GGML_TYPE_Q5_1) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 4; + + const uint ib = idx / 4; + const uint iqs = idx & 0x03; + + const vec2 dm = vec2(a_q5_1_p32.data[ib].dm); + const uint uint_qh = a_q5_1_p32.data[ib].qh; + const uvec2 qh0 = uvec2(((uint_qh >> 4*iqs) << 4) & 0x10, (uint_qh >> (4*iqs + 12)) & 0x10); + const uvec2 qh1 = uvec2(((uint_qh >> (4*iqs + 1)) << 4) & 0x10, (uint_qh >> (4*iqs + 13)) & 0x10); + const uvec2 qh2 = uvec2(((uint_qh >> (4*iqs + 2)) << 4) & 0x10, (uint_qh >> (4*iqs + 14)) & 0x10); + const uvec2 qh3 = uvec2(((uint_qh >> (4*iqs + 3)) << 4) & 0x10, (uint_qh >> (4*iqs + 15)) & 0x10); + + const uint vui = a_q5_1_p32.data[ib].qs[iqs]; + const vec4 v0 = vec4((vui & 0xF) | qh0.x, ((vui >> 4) & 0xF) | qh0.y, ((vui >> 8) & 0xF) | qh1.x, ((vui >> 12) & 0xF) | qh1.y) * dm.x + dm.y; + const vec4 v1 = vec4(((vui >> 16) & 0xF) | qh2.x, ((vui >> 20) & 0xF) | qh2.y, ((vui >> 24) & 0xF) | qh3.x, ((vui >> 28) & 0xF) | qh3.y) * dm.x + dm.y; + + store_a(col, k_pair, FLOAT_TYPEV2(v0.xz)); + store_a(col, k_pair + 1, FLOAT_TYPEV2(v1.xz)); + store_a(col, k_pair + 8, FLOAT_TYPEV2(v0.yw)); + store_a(col, k_pair + 9, FLOAT_TYPEV2(v1.yw)); + } else if (MmTypeA == GGML_TYPE_Q8_0) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 8; + const uint iqs = idx & 0x07; + + const float d = float(a_q8_0_p16.data[ib].d); + const i8vec2 v0 = unpack8(int32_t(a_q8_0_p16.data[ib].qs[2*iqs])).xy; // vec4 used due to #12147 + const i8vec2 v1 = unpack8(int32_t(a_q8_0_p16.data[ib].qs[2*iqs + 1])).xy; + const vec4 v = vec4(v0.x, v0.y, v1.x, v1.y) * d; + + store_a(col, k_pair, FLOAT_TYPEV2(v.xy)); + store_a(col, k_pair + 1, FLOAT_TYPEV2(v.zw)); + } else if (MmTypeA == GGML_TYPE_Q1_0) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 16; + const uint iqs = idx & 0xfu; + + const float d = float(a_q1_0.data[ib].d); + const uint bits = uint(a_q1_0.data[ib].qs[iqs]); + + store_a(col, k_pair, FLOAT_TYPEV2((bits & 0x01u) != 0u ? d : -d, (bits & 0x02u) != 0u ? d : -d)); + store_a(col, k_pair + 1, FLOAT_TYPEV2((bits & 0x04u) != 0u ? d : -d, (bits & 0x08u) != 0u ? d : -d)); + store_a(col, k_pair + 2, FLOAT_TYPEV2((bits & 0x10u) != 0u ? d : -d, (bits & 0x20u) != 0u ? d : -d)); + store_a(col, k_pair + 3, FLOAT_TYPEV2((bits & 0x40u) != 0u ? d : -d, (bits & 0x80u) != 0u ? d : -d)); + } else if (MmTypeA == GGML_TYPE_Q2_0) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 16; + const uint iqs = idx & 0xfu; + + const FLOAT_TYPE d = FLOAT_TYPE(a_q2_0.data[ib].d); + const uint bits = uint(a_q2_0.data[ib].qs[iqs]); + + store_a(col, k_pair, d * (FLOAT_TYPEV2(bits & 3u, (bits >> 2u) & 3u) - FLOAT_TYPEV2(1.0f))); + store_a(col, k_pair + 1, d * (FLOAT_TYPEV2((bits >> 4u) & 3u, bits >> 6u) - FLOAT_TYPEV2(1.0f))); + } else if (MmTypeA == GGML_TYPE_Q2_K) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 64; // 4 values per idx + const uint iqs = (idx % 64) * 2; // 0,2,4..126 + + const uint qsi = (iqs / 64) * 16 + (iqs % 16); // 0..15 + const uint scalesi = iqs / 8; // 0..15 + const uint qsshift = ((iqs % 64) / 16) * 2; // 0,2,4,6 + + const vec4 qs = vec4(unpack8((a_q2_k_p32.data[ib].qs[qsi / 2] >> qsshift) & 0x03030303)); + const uint scales = a_q2_k.data[ib].scales[scalesi]; + const vec2 dm = vec2(a_q2_k.data[ib].dm); + + const vec4 v = dm.x * float(scales & 0xF) * qs - dm.y * float(scales >> 4); + + store_a(col, k_pair, FLOAT_TYPEV2(v.xy)); + store_a(col, k_pair + 1, FLOAT_TYPEV2(v.zw)); + } else if (MmTypeA == GGML_TYPE_TQ2_0) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + + const uint ib = idx / 128; // 2 values per idx + const uint iqs = (idx % 128) * 2; // elem 0,2,4..254 + + const uint qsi = (iqs / 128) * 32 + (iqs % 32); // byte pair start + const uint shift = 2 * ((iqs % 128) / 32); // 0,2,4,6 + + const uvec2 qs = uvec2(a_tq2_0.data[ib].qs[qsi], a_tq2_0.data[ib].qs[qsi + 1]); + const float d = float(a_tq2_0.data[ib].d); + + const vec2 v = d * (vec2((qs >> shift) & 3) - 1.0); + + const uint k_pair = row * mm_load_vec_a() / 2; + store_a(col, k_pair, FLOAT_TYPEV2(v.xy)); + } else if (MmTypeA == GGML_TYPE_Q3_K) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 128; // 2 values per idx + const uint iqs = idx % 128; // 0..127 + + const uint n = iqs / 64; // 0,1 + const uint qsi = n * 32 + (iqs % 16) * 2; // 0,2,4..62 + const uint hmi = (iqs % 16) * 2; // 0,2,4..30 + const uint j = (iqs % 64) / 4; // 0..3 + const uint is = iqs / 8; // 0..15 + const uint halfsplit = ((iqs % 64) / 16); // 0,1,2,3 + const uint qsshift = halfsplit * 2; // 0,2,4,6 + + const int8_t us = int8_t(((a_q3_k.data[ib].scales[is % 8] >> (4 * int(is / 8))) & 0xF) + | (((a_q3_k.data[ib].scales[8 + (is % 4)] >> (2 * int(is / 4))) & 3) << 4)); + const float dl = float(a_q3_k.data[ib].d) * float(us - 32); + + const vec2 qs = vec2(unpack8((uint(a_q3_k_p16.data[ib].qs[qsi / 2]) >> qsshift) & 0x0303).xy); + const vec2 hm = vec2(unpack8(((uint(a_q3_k_p16.data[ib].hmask[hmi / 2]) >> (4 * n + halfsplit)) & 0x0101 ^ 0x0101) << 2).xy); + + store_a(col, k_pair, FLOAT_TYPEV2(dl * (qs.x - hm.x), + dl * (qs.y - hm.y))); + + } else if (MmTypeA == GGML_TYPE_Q4_K) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 64; // 4 values per idx + const uint iqs = (idx % 64) * 2; // 0,2,4..126 + + const uint n = iqs / 32; // 0,1,2,3 + const uint b = (iqs % 32) / 16; // 0,1 + const uint is = 2 * n + b; // 0..7 + const uint qsi = n * 32 + (iqs % 16) * 2; // 0,2,4..126 + + const vec2 loadd = vec2(a_q4_k.data[ib].dm); + + const uvec3 scales = uvec3(a_q4_k_p32.data[ib].scales[0], + a_q4_k_p32.data[ib].scales[1], + a_q4_k_p32.data[ib].scales[2]); + const uint scalesoffs = (is & 3) * 8; + + const uint scidx0 = (is < 4) ? 0 : 2; + const uint scidxshift0 = scalesoffs; + const uint scidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2; + const uint mbidx0 = (is < 4) ? 1 : 2; + const uint mbidxshift0 = (is < 4) ? scalesoffs : scalesoffs + 4; + const uint mbidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2; + + const uint8_t sc = uint8_t(((scales[scidx0] >> scidxshift0) & 0xF) | ((scales[0] >> scidxshift1) & 0x30)); + const uint8_t mbyte = uint8_t(((scales[mbidx0] >> mbidxshift0) & 0xF) | ((scales[1] >> mbidxshift1) & 0x30)); + + const float d = loadd.x * sc; + const float m = -loadd.y * mbyte; + + const vec4 q = vec4(unpack8((a_q4_k_p32.data[ib].qs[qsi / 4] >> (b * 4)) & 0x0F0F0F0F)); + + store_a(col, k_pair, FLOAT_TYPEV2(fma(d, q.x, m), fma(d, q.y, m))); + store_a(col, k_pair + 1, FLOAT_TYPEV2(fma(d, q.z, m), fma(d, q.w, m))); + } else if (MmTypeA == GGML_TYPE_Q5_K) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 64; // 4 values per idx + const uint iqs = (idx % 64) * 2; // 0,2,4..126 + + const uint n = iqs / 32; // 0,1,2,3 + const uint b = (iqs % 32) / 16; // 0,1 + const uint is = 2 * n + b; // 0..7 + const uint qsi = n * 32 + (iqs % 16) * 2; // 0,2,4..126 + const uint qhi = (iqs % 16) * 2; // 0,2,4..30 + + const vec2 loadd = vec2(a_q5_k.data[ib].dm); + + const uvec3 scales = uvec3(a_q5_k_p32.data[ib].scales[0], + a_q5_k_p32.data[ib].scales[1], + a_q5_k_p32.data[ib].scales[2]); + const uint scalesoffs = (is & 3) * 8; + + const uint scidx0 = (is < 4) ? 0 : 2; + const uint scidxshift0 = scalesoffs; + const uint scidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2; + const uint mbidx0 = (is < 4) ? 1 : 2; + const uint mbidxshift0 = (is < 4) ? scalesoffs : scalesoffs + 4; + const uint mbidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2; + + const uint8_t sc = uint8_t(((scales[scidx0] >> scidxshift0) & 0xF) | ((scales[0] >> scidxshift1) & 0x30)); + const uint8_t mbyte = uint8_t(((scales[mbidx0] >> mbidxshift0) & 0xF) | ((scales[1] >> mbidxshift1) & 0x30)); + + const float d = loadd.x * sc; + const float m = -loadd.y * mbyte; + + const uint qs = (a_q5_k_p32.data[ib].qs[qsi / 4] >> (b * 4)) & 0x0F0F0F0F; + const uint qh = ((a_q5_k_p32.data[ib].qh[qhi / 4] >> (iqs / 16)) & 0x01010101) << 4; + const vec4 q = vec4(unpack8(qs | qh)); + + store_a(col, k_pair, FLOAT_TYPEV2(fma(d, q.x, m), fma(d, q.y, m))); + store_a(col, k_pair + 1, FLOAT_TYPEV2(fma(d, q.z, m), fma(d, q.w, m))); + } else if (MmTypeA == GGML_TYPE_Q6_K) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 128; // 2 values per idx + const uint iqs = idx % 128; // 0..127 + + const uint n = iqs / 64; // 0,1 + const uint b = ((iqs % 64) / 32) * 4; // 0,4 + const uint is_b = (iqs % 16) / 8; // 0,1 + const uint qhshift = ((iqs % 64) / 16) * 2; // 0,2,4,6 + const uint is = 8 * n + qhshift + is_b; // 0..15 + const uint qsi = n * 32 + (iqs % 32); // 0..63 + const uint qhi = n * 16 + (iqs % 16); // 0..31 + + const float dscale = float(a_q6_k.data[ib].d) * float(a_q6_k.data[ib].scales[is]); + + const uint ql = (uint(a_q6_k_p16.data[ib].ql[qsi]) >> b) & 0x0F0F; + const uint qh = (uint(a_q6_k_p16.data[ib].qh[qhi]) >> qhshift) & 0x0303; + const vec2 q = (vec2(unpack8(ql | (qh << 4)).xy) - 32) * dscale; + + store_a(col, k_pair, FLOAT_TYPEV2(q.x, q.y)); + } else if (MmTypeA == GGML_TYPE_IQ1_S) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 32; // 8 values per idx + const uint ib32 = (idx % 32) / 4; // 0..7 + const uint ib8 = idx % 32; + + const float d = float(a_iq1_s.data[ib].d); + const uint qh = a_iq1_s.data[ib].qh[ib32]; + const uint qs = a_iq1_s.data[ib].qs[ib8]; + const float dl = d * (2 * bitfieldExtract(qh, 12, 3) + 1); + const float delta = ((qh & 0x8000) != 0) ? -IQ1S_DELTA : IQ1S_DELTA; + const int16_t grid = int16_t(iq1s_grid[qs | (bitfieldExtract(qh, 3 * int(ib8 & 3), 3) << 8)]); + + [[unroll]] for (int k = 0; k < 4; ++k) { + store_a(col, k_pair + k, FLOAT_TYPEV2(dl * (bitfieldExtract(grid, 4 * k , 2) + delta), + dl * (bitfieldExtract(grid, 4 * k + 2, 2) + delta))); + + } + } else if (MmTypeA == GGML_TYPE_IQ1_M) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 32; // 8 values per idx + const uint ib8 = idx % 32; + const uint ib16 = ib8 / 2; + + const uint16_t[4] scales = a_iq1_m.data[ib].scales; + const u16vec4 s = u16vec4(scales[0], scales[1], scales[2], scales[3]) >> 12; + const float d = float(unpackHalf2x16(s.x | (s.y << 4) | (s.z << 8) | (s.w << 12)).x); + const uint sc = scales[ib8 / 8]; + const uint qs = a_iq1_m.data[ib].qs[ib8]; + const uint qh = a_iq1_m.data[ib].qh[ib16] >> (4 * (ib8 & 1)); + const float dl = d * (2 * bitfieldExtract(sc, 3 * int(ib16 & 3), 3) + 1); + const float delta = ((qh & 8) != 0) ? -IQ1M_DELTA : IQ1M_DELTA; + const int16_t grid = int16_t(iq1s_grid[qs | ((qh & 7) << 8)]); + + [[unroll]] for (int k = 0; k < 4; ++k) { + store_a(col, k_pair + k, FLOAT_TYPEV2(dl * (bitfieldExtract(grid, 4 * k , 2) + delta), + dl * (bitfieldExtract(grid, 4 * k + 2, 2) + delta))); + + } + } else if (MmTypeA == GGML_TYPE_IQ2_XXS) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 32; // 8 values per idx + const uint ib32 = (idx % 32) / 4; // 0..7 + const uint ib8 = idx % 4; + + const float d = float(a_iq2_xxs.data[ib].d); + const uint qs = a_iq2_xxs.data[ib].qs[8 * ib32 + ib8]; + const uint signs = pack32(u8vec4( + a_iq2_xxs.data[ib].qs[8*ib32 + 4], + a_iq2_xxs.data[ib].qs[8*ib32 + 5], + a_iq2_xxs.data[ib].qs[8*ib32 + 6], + a_iq2_xxs.data[ib].qs[8*ib32 + 7] + )); + const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + (signs >> 28))); + const uint32_t sign7 = bitfieldExtract(signs, 7 * int(ib8), 7); + const uint sign = sign7 | (bitCount(sign7) << 7); + const uvec2 grid = iq2_grid[qs]; + const vec4 grid0 = vec4(unpack8(grid.x)); + const vec4 grid1 = vec4(unpack8(grid.y)); + + store_a(col, k_pair, db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x, + (sign & 2) != 0 ? -grid0.y : grid0.y)); + + store_a(col, k_pair + 1, db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z, + (sign & 8) != 0 ? -grid0.w : grid0.w)); + + store_a(col, k_pair + 2, db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x, + (sign & 32) != 0 ? -grid1.y : grid1.y)); + + store_a(col, k_pair + 3, db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z, + (sign & 128) != 0 ? -grid1.w : grid1.w)); + + } else if (MmTypeA == GGML_TYPE_IQ2_XS) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 32; // 8 values per idx + const uint ib32 = (idx % 32) / 4; // 0..7 + const uint ib8 = idx % 4; // 0..3 + + const float d = float(a_iq2_xs.data[ib].d); + const uint scale = (a_iq2_xs.data[ib].scales[ib32] >> (2 * (ib8 & 2))) & 0xf; + const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + scale)); + const uint qs = a_iq2_xs.data[ib].qs[4 * ib32 + ib8]; + const uint sign7 = qs >> 9; + const uint sign = sign7 | (bitCount(sign7) << 7); + const uvec2 grid = iq2_grid[qs & 511]; + const vec4 grid0 = vec4(unpack8(grid.x)); + const vec4 grid1 = vec4(unpack8(grid.y)); + + store_a(col, k_pair, db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x, + (sign & 2) != 0 ? -grid0.y : grid0.y)); + + store_a(col, k_pair + 1, db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z, + (sign & 8) != 0 ? -grid0.w : grid0.w)); + + store_a(col, k_pair + 2, db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x, + (sign & 32) != 0 ? -grid1.y : grid1.y)); + + store_a(col, k_pair + 3, db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z, + (sign & 128) != 0 ? -grid1.w : grid1.w)); + + } else if (MmTypeA == GGML_TYPE_IQ2_S) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 32; // 8 values per idx + const uint ib8 = idx % 32; // 0..31 + const uint ib32 = ib8 / 4; // 0..7 + + const uint scale = (a_iq2_s.data[ib].scales[ib32] >> (2 * (ib8 & 2))) & 0xf; + const uint qs = a_iq2_s.data[ib].qs[ib8]; + const uint qh = a_iq2_s.data[ib].qh[ib32]; + const uint qhshift = 2 * (ib8 % 4); + const uint sign = a_iq2_s.data[ib].qs[QUANT_K_IQ2_S / 8 + ib8]; + + const float d = float(a_iq2_s.data[ib].d); + const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + scale)); + const uvec2 grid = iq2_grid[qs | ((qh << (8 - qhshift)) & 0x300)]; + const vec4 grid0 = vec4(unpack8(grid.x)); + const vec4 grid1 = vec4(unpack8(grid.y)); + + store_a(col, k_pair, db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x, + (sign & 2) != 0 ? -grid0.y : grid0.y)); + + store_a(col, k_pair + 1, db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z, + (sign & 8) != 0 ? -grid0.w : grid0.w)); + + store_a(col, k_pair + 2, db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x, + (sign & 32) != 0 ? -grid1.y : grid1.y)); + + store_a(col, k_pair + 3, db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z, + (sign & 128) != 0 ? -grid1.w : grid1.w)); + + } else if (MmTypeA == GGML_TYPE_IQ3_XXS) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 64; // 4 values per idx + const uint iqs = idx % 64; // 0..63 + const uint is = QUANT_K_IQ3_XXS / 4 + 4 * (iqs / 8); // 8 values + + const float d = float(a_iq3_xxs.data[ib].d); + const uint qs = a_iq3_xxs.data[ib].qs[iqs]; + const uint signs = pack32(u16vec2( + a_iq3_xxs_p16.data[ib].qs[is/2], + a_iq3_xxs_p16.data[ib].qs[is/2+1] + )); + const float db = d * 0.5 * (0.5 + (signs >> 28)); + const uint32_t sign7 = bitfieldExtract(signs, 7 * (int(iqs / 2) % 4), 7); + const uint sign = (sign7 | (bitCount(sign7) << 7)) >> (4 * (idx % 2)); + const uint grid = iq3_grid[qs]; + const vec4 v = db * vec4(unpack8(grid)); + + store_a(col, k_pair, FLOAT_TYPEV2((sign & 1) != 0 ? -v.x : v.x, + (sign & 2) != 0 ? -v.y : v.y)); + + store_a(col, k_pair + 1, FLOAT_TYPEV2((sign & 4) != 0 ? -v.z : v.z, + (sign & 8) != 0 ? -v.w : v.w)); + + } else if (MmTypeA == GGML_TYPE_IQ3_S) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 64; // 4 values per idx + const uint iqs = idx % 64; // 0..63 + const uint iqh = iqs / 8; + + const float d = float(a_iq3_s.data[ib].d); + const uint qs = a_iq3_s.data[ib].qs[iqs]; + const uint qh = a_iq3_s.data[ib].qh[iqh]; + const int8_t sign = int8_t(a_iq3_s.data[ib].signs[iqs / 2] >> (4 * (idx % 2))); + const uint scale = a_iq3_s.data[ib].scales[iqs / 16]; + const i8vec2 sign01 = i8vec2(1 - (2 & i8vec2(sign << 1, sign))); + const float db = d * (1 + 2 * ((scale >> (4 * (iqh & 1))) & 0xf)); + const uint32_t grid = iq3_grid[qs | ((qh << (8 - (iqs % 8))) & 256)]; + const vec4 v = db * vec4(unpack8(grid)); + + store_a(col, k_pair, FLOAT_TYPEV2((sign & 1) != 0 ? -v.x : v.x, + (sign & 2) != 0 ? -v.y : v.y)); + + store_a(col, k_pair + 1, FLOAT_TYPEV2((sign & 4) != 0 ? -v.z : v.z, + (sign & 8) != 0 ? -v.w : v.w)); + + } else if (MmTypeA == GGML_TYPE_IQ4_XS) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 64; // 4 values per idx + const uint ib32 = (idx % 64) / 8; // 0..7 + const uint iq = 4 * ib32 + (idx % 4); + + const uint sl = (a_iq4_xs.data[ib].scales_l[ib32/2] >> (4 * (ib32 & 1))) & 0xF; + const uint sh = ((a_iq4_xs.data[ib].scales_h) >> (2 * ib32)) & 3; + const uint qshift = idx & 4; + u8vec4 qs = unpack8((uint(a_iq4_xs_p32.data[ib].qs[iq]) >> qshift) & 0x0F0F0F0F); + + const float d = float(a_iq4_xs.data[ib].d); + const vec4 v = d * float(int(sl | (sh << 4)) - 32) * vec4(kvalues_iq4nl[qs.x], kvalues_iq4nl[qs.y], kvalues_iq4nl[qs.z], kvalues_iq4nl[qs.w]); + + store_a(col, k_pair, FLOAT_TYPEV2(v.xy)); + store_a(col, k_pair + 1, FLOAT_TYPEV2(v.zw)); + } else if (MmTypeA == GGML_TYPE_IQ4_NL) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 4; + + const uint ib = idx / 8; + const uint iqs = idx & 0x07; + + const FLOAT_TYPE d = FLOAT_TYPE(a_iq4_nl_p16.data[ib].d); + const uint vui = uint(a_iq4_nl_p16.data[ib].qs[iqs]); + + store_a(col, k_pair, d * FLOAT_TYPEV2(kvalues_iq4nl[vui & 0xF], + kvalues_iq4nl[bitfieldExtract(vui, 8, 4)])); + + store_a(col, k_pair + 8, d * FLOAT_TYPEV2(kvalues_iq4nl[bitfieldExtract(vui, 4, 4)], + kvalues_iq4nl[vui >> 12])); + + } else if (MmTypeA == GGML_TYPE_MXFP4) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 4; + + const uint ib = idx / 8; + const uint iqs = (idx & 0x07) * 2; + + const uint vui = uint(a_mxfp4.data[ib].qs[iqs]); + const uint vui2 = uint(a_mxfp4.data[ib].qs[iqs+1]); #ifdef USE_OCP_FP4 - const float d = e8m0_to_fp32(data_a[ib].e); - const u8vec2 packed = u8vec2(vui, vui2); - store_a(col, row, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 0u)) * FLOAT_TYPE(d)); - store_a(col, row + 8, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 4u)) * FLOAT_TYPE(d)); + const float d = e8m0_to_fp32(a_mxfp4.data[ib].e); + const u8vec2 packed = u8vec2(vui, vui2); + store_a(col, k_pair, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 0u)) * FLOAT_TYPE(d)); + store_a(col, k_pair + 8, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 4u)) * FLOAT_TYPE(d)); #else - const float d = e8m0_to_fp32(data_a[ib].e) * 0.5; - store_a(col, row, FLOAT_TYPEV2(kvalues_mxfp4[vui & 0xF] * d, - kvalues_mxfp4[vui2 & 0xF] * d)); - store_a(col, row + 8, FLOAT_TYPEV2(kvalues_mxfp4[vui >> 4] * d, - kvalues_mxfp4[vui2 >> 4] * d)); + const float d = e8m0_to_fp32(a_mxfp4.data[ib].e) * 0.5; + store_a(col, k_pair, FLOAT_TYPEV2(kvalues_mxfp4[vui & 0xF] * d, + kvalues_mxfp4[vui2 & 0xF] * d)); + + store_a(col, k_pair + 8, FLOAT_TYPEV2(kvalues_mxfp4[vui >> 4] * d, + kvalues_mxfp4[vui2 >> 4] * d)); + #endif -#elif defined(DATA_A_NVFP4) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - const uint ib = idx / 16u; - const uint sub = (idx & 0xC) >> 2; - const uint iqs = (idx & 0xF) * 2; - const uint vui = uint(data_a[ib].qs[iqs]); - const uint vui2 = uint(data_a[ib].qs[iqs+1]); - - // lo and hi nibbles are 8 elements apart, which doesn't quite line up with - // how the thread mapping and buf_idx calculation works for other types. - const uint eff_row = (row & 3) + (row & ~3) * 2; + } else if (MmTypeA == GGML_TYPE_NVFP4) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + // lo and hi nibbles are 8 elements apart, which doesn't quite line up with + // how the thread mapping and buf_idx calculation works for other types. + const uint eff_row = (row & 3) + (row & ~3) * 2; + + const uint ib = idx / 16u; + const uint sub = (idx & 0xC) >> 2; + const uint iqs = (idx & 0xF) * 2; + const uint vui = uint(a_nvfp4.data[ib].qs[iqs]); + const uint vui2 = uint(a_nvfp4.data[ib].qs[iqs+1]); + #ifdef USE_OCP_FP4 - const FLOAT_TYPE d = FLOAT_TYPE(ue4m3_from_bits(data_a[ib].d[sub])); - const u8vec2 packed = u8vec2(vui, vui2); - store_a(col, eff_row, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 0u)) * d); - store_a(col, eff_row + 4, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 4u)) * d); + const FLOAT_TYPE d = FLOAT_TYPE(ue4m3_from_bits(a_nvfp4.data[ib].d[sub])); + const u8vec2 packed = u8vec2(vui, vui2); + store_a(col, eff_row, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 0u)) * d); + store_a(col, eff_row + 4, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 4u)) * d); #else - const float d = ue4m3_to_fp32(data_a[ib].d[sub]) * 0.5; - store_a(col, eff_row, FLOAT_TYPEV2(kvalues_mxfp4[vui & 0xF] * d, - kvalues_mxfp4[vui2 & 0xF] * d)); - store_a(col, eff_row + 4, FLOAT_TYPEV2(kvalues_mxfp4[vui >> 4] * d, - kvalues_mxfp4[vui2 >> 4] * d)); + const float d = ue4m3_to_fp32(a_nvfp4.data[ib].d[sub]) * 0.5; + store_a(col, eff_row, FLOAT_TYPEV2(kvalues_mxfp4[vui & 0xF] * d, + kvalues_mxfp4[vui2 & 0xF] * d)); + store_a(col, eff_row + 4, FLOAT_TYPEV2(kvalues_mxfp4[vui >> 4] * d, + kvalues_mxfp4[vui2 >> 4] * d)); #endif +} #endif } #if !defined(MUL_MAT_ID) void load_b_to_shmem(const uint pos_b, const uint row, const uint col, const uint idx_n, const uint block, const uint end_k) { #if LOAD_VEC_B == 8 - if (ALIGNED != 0) { - // Not supported for b_type bf16 because bf16mat2x4 does not exist - const uint idx = pos_b + col * p.stride_b / LOAD_VEC_B + row; - const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2; - FLOAT_TYPEV8 bb = FLOAT_TYPEV8(data_b[idx]); - buf_b[buf_idx + 0] = bb[0].xy; - buf_b[buf_idx + 1] = bb[0].zw; - buf_b[buf_idx + 2] = bb[1].xy; - buf_b[buf_idx + 3] = bb[1].zw; - return; - } + if (ALIGNED != 0) { + // Not supported for b_type bf16 because bf16mat2x4 does not exist + const uint idx = pos_b + col * p.stride_b / LOAD_VEC_B + row; + const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2; + FLOAT_TYPEV8 bb = FLOAT_TYPEV8(data_b[idx]); + buf_b[buf_idx + 0] = bb[0].xy; + buf_b[buf_idx + 1] = bb[0].zw; + buf_b[buf_idx + 2] = bb[1].xy; + buf_b[buf_idx + 3] = bb[1].zw; + return; + } #elif LOAD_VEC_B == 4 - if (ALIGNED != 0) { - const uint idx = pos_b + col * p.stride_b / LOAD_VEC_B + row; - const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2; + if (ALIGNED != 0) { + const uint idx = pos_b + col * p.stride_b / LOAD_VEC_B + row; + const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2; #if defined(DATA_B_BF16) - FLOAT_TYPEV4 bb = FLOAT_TYPEV4(TO_FLOAT_TYPE(data_b[idx])); + FLOAT_TYPEV4 bb = FLOAT_TYPEV4(TO_FLOAT_TYPE(data_b[idx])); #else - FLOAT_TYPEV4 bb = FLOAT_TYPEV4(data_b[idx]); + FLOAT_TYPEV4 bb = FLOAT_TYPEV4(data_b[idx]); #endif - buf_b[buf_idx + 0] = bb.xy; - buf_b[buf_idx + 1] = bb.zw; - return; - } + buf_b[buf_idx + 0] = bb.xy; + buf_b[buf_idx + 1] = bb.zw; + return; + } #endif - const uint idx = pos_b + col * p.stride_b + row * 2; - const uint buf_idx = col * SHMEM_STRIDE + row; - if (idx_n < p.N && block + row * 2 + 1 < end_k) { - buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]), - TO_FLOAT_TYPE(data_b_scalar[idx + 1])); - } else if (idx_n < p.N && block + row * 2 < end_k) { - buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]), 0.0f); - } else { - buf_b[buf_idx] = FLOAT_TYPEV2(0.0f); - } + const uint idx = pos_b + col * p.stride_b + row * 2; + const uint buf_idx = col * SHMEM_STRIDE + row; + if (idx_n < p.N && block + row * 2 + 1 < end_k) { + buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]), + TO_FLOAT_TYPE(data_b_scalar[idx + 1])); + } else if (idx_n < p.N && block + row * 2 < end_k) { + buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]), 0.0f); + } else { + buf_b[buf_idx] = FLOAT_TYPEV2(0.0f); + } } #else void load_b_to_shmem(const uint pos_b, const uint row, const uint col, const uint ic, const uint _ne1, const uint block, const uint end_k) { #if LOAD_VEC_B == 8 - if (ALIGNED != 0) { - // Not supported for b_type bf16 because bf16mat2x4 does not exist - const u16vec2 row_idx = row_ids[col]; - const uint idx = pos_b + row_idx.y * p.batch_stride_b / LOAD_VEC_B + (row_idx.x % p.ne11) * p.stride_b / LOAD_VEC_B + row; - const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2; - FLOAT_TYPEV8 bb = FLOAT_TYPEV8(data_b[idx]); - buf_b[buf_idx + 0] = bb[0].xy; - buf_b[buf_idx + 1] = bb[0].zw; - buf_b[buf_idx + 2] = bb[1].xy; - buf_b[buf_idx + 3] = bb[1].zw; - return; - } + if (ALIGNED != 0) { + // Not supported for b_type bf16 because bf16mat2x4 does not exist + const u16vec2 row_idx = row_ids[col]; + const uint idx = pos_b + row_idx.y * p.batch_stride_b / LOAD_VEC_B + (row_idx.x % p.ne11) * p.stride_b / LOAD_VEC_B + row; + const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2; + FLOAT_TYPEV8 bb = FLOAT_TYPEV8(data_b[idx]); + buf_b[buf_idx + 0] = bb[0].xy; + buf_b[buf_idx + 1] = bb[0].zw; + buf_b[buf_idx + 2] = bb[1].xy; + buf_b[buf_idx + 3] = bb[1].zw; + return; + } #elif LOAD_VEC_B == 4 - if (ALIGNED != 0) { - const u16vec2 row_idx = row_ids[col]; - const uint idx = pos_b + row_idx.y * p.batch_stride_b / LOAD_VEC_B + (row_idx.x % p.ne11) * p.stride_b / LOAD_VEC_B + row; - const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2; + if (ALIGNED != 0) { + const u16vec2 row_idx = row_ids[col]; + const uint idx = pos_b + row_idx.y * p.batch_stride_b / LOAD_VEC_B + (row_idx.x % p.ne11) * p.stride_b / LOAD_VEC_B + row; + const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2; #if defined(DATA_B_BF16) - FLOAT_TYPEV4 bb = FLOAT_TYPEV4(TO_FLOAT_TYPE(data_b[idx])); + FLOAT_TYPEV4 bb = FLOAT_TYPEV4(TO_FLOAT_TYPE(data_b[idx])); #else - FLOAT_TYPEV4 bb = FLOAT_TYPEV4(data_b[idx]); + FLOAT_TYPEV4 bb = FLOAT_TYPEV4(data_b[idx]); #endif - buf_b[buf_idx + 0] = bb.xy; - buf_b[buf_idx + 1] = bb.zw; - return; - } + buf_b[buf_idx + 0] = bb.xy; + buf_b[buf_idx + 1] = bb.zw; + return; + } #endif - const uint row_i = ic * BN + col; - const uint buf_idx = col * SHMEM_STRIDE + row; - if (row_i < _ne1 && block + row * 2 + 1 < end_k) { - const u16vec2 row_idx = row_ids[col]; - const uint idx = pos_b + row_idx.y * p.batch_stride_b + (row_idx.x % p.ne11) * p.stride_b + row * 2; - buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]), - TO_FLOAT_TYPE(data_b_scalar[idx + 1])); - } else if (row_i < _ne1 && block + row * 2 < end_k) { - const u16vec2 row_idx = row_ids[col]; - const uint idx = pos_b + row_idx.y * p.batch_stride_b + (row_idx.x % p.ne11) * p.stride_b + row * 2; - buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]), 0.0f); - } else { - buf_b[buf_idx] = FLOAT_TYPEV2(0.0f); - } + const uint row_i = ic * BN + col; + const uint buf_idx = col * SHMEM_STRIDE + row; + if (row_i < _ne1 && block + row * 2 + 1 < end_k) { + const u16vec2 row_idx = row_ids[col]; + const uint idx = pos_b + row_idx.y * p.batch_stride_b + (row_idx.x % p.ne11) * p.stride_b + row * 2; + buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]), + TO_FLOAT_TYPE(data_b_scalar[idx + 1])); + } else if (row_i < _ne1 && block + row * 2 < end_k) { + const u16vec2 row_idx = row_ids[col]; + const uint idx = pos_b + row_idx.y * p.batch_stride_b + (row_idx.x % p.ne11) * p.stride_b + row * 2; + buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]), 0.0f); + } else { + buf_b[buf_idx] = FLOAT_TYPEV2(0.0f); + } } #endif diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl index adb1bb8b32b..b5936a3493b 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl @@ -515,7 +515,7 @@ struct block_iq1_m_packed64 { #define A_TYPE_PACKED32 block_iq1_m_packed32 #endif -#if defined(DATA_A_IQ1_S) || defined(DATA_A_IQ1_M) +#if defined(DATA_A_IQ1_S) || defined(DATA_A_IQ1_M) || defined(MULMAT_QUANT) #define IQ1S_DELTA 0.125f #define IQ1M_DELTA 0.125f @@ -915,11 +915,19 @@ const uint32_t[2048] iq1s_grid_gpu_const = { }; #endif +#ifdef MULMAT_QUANT +shared uint16_t iq1s_grid[(MmTypeA == GGML_TYPE_IQ1_S || MmTypeA == GGML_TYPE_IQ1_M) ? 2048 : 8]; +#if defined(NEEDS_IQ1S_GRID_GPU) +shared uint32_t iq1s_grid_gpu[(MmTypeA == GGML_TYPE_IQ1_S || MmTypeA == GGML_TYPE_IQ1_M) ? 2048 : 8]; +#endif +#else shared uint16_t iq1s_grid[2048]; #if defined(NEEDS_IQ1S_GRID_GPU) shared uint32_t iq1s_grid_gpu[2048]; #endif +#endif +#if defined(DATA_A_IQ1_S) || defined(DATA_A_IQ1_M) #define NEEDS_INIT_IQ_SHMEM void init_iq_shmem(uvec3 wgsize) { @@ -943,6 +951,19 @@ void init_iq_shmem(uvec3 wgsize) barrier(); } #endif +#endif + +#if defined(DATA_A_IQ2_XXS) || defined(DATA_A_IQ2_XS) || defined(DATA_A_IQ2_S) || defined(MULMAT_QUANT) +#ifdef MULMAT_QUANT +shared uvec2 iq2_grid[MmTypeA == GGML_TYPE_IQ2_S ? 1024 : MmTypeA == GGML_TYPE_IQ2_XS ? 512 : MmTypeA == GGML_TYPE_IQ2_XXS ? 256 : 8]; +#elif defined(DATA_A_IQ2_S) +shared uvec2 iq2_grid[1024]; +#elif defined(DATA_A_IQ2_XS) +shared uvec2 iq2_grid[512]; +#else +shared uvec2 iq2_grid[256]; +#endif +#endif #define QUANT_K_IQ2_XXS 256 #define QUANT_R_IQ2_XXS 1 @@ -959,7 +980,7 @@ struct block_iq2_xxs_packed16 uint16_t qs[QUANT_K_IQ2_XXS/8]; }; -#if defined(DATA_A_IQ2_XXS) +#if defined(DATA_A_IQ2_XXS) || defined(MULMAT_QUANT) const uvec2[256] iq2xxs_grid_const = { uvec2(0x08080808, 0x08080808), uvec2(0x0808082b, 0x08080808), uvec2(0x08081919, 0x08080808), uvec2(0x08082b08, 0x08080808), @@ -1028,25 +1049,27 @@ const uvec2[256] iq2xxs_grid_const = { uvec2(0x08080808, 0x2b2b082b), uvec2(0x08192b08, 0x2b2b1908), uvec2(0x19190808, 0x2b2b2b08), uvec2(0x08081908, 0x2b2b2b19) }; -shared uvec2 iq2xxs_grid[256]; - +#if defined(DATA_A_IQ2_XXS) #define NEEDS_INIT_IQ_SHMEM void init_iq_shmem(uvec3 wgsize) { // copy the table into shared memory and sync - [[unroll]] for (uint i = 0; i < iq2xxs_grid.length(); i += wgsize.x) { + [[unroll]] for (uint i = 0; i < iq2_grid.length(); i += wgsize.x) { if (iq2xxs_grid_const.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq2xxs_grid_const.length()) { - iq2xxs_grid[i + gl_LocalInvocationIndex.x] = iq2xxs_grid_const[i + gl_LocalInvocationIndex.x]; + iq2_grid[i + gl_LocalInvocationIndex.x] = iq2xxs_grid_const[i + gl_LocalInvocationIndex.x]; } } barrier(); } +#endif +#if defined(DATA_A_IQ2_XXS) #define QUANT_K QUANT_K_IQ2_XXS #define QUANT_R QUANT_R_IQ2_XXS #define A_TYPE block_iq2_xxs #define A_TYPE_PACKED16 block_iq2_xxs_packed16 #endif +#endif #define QUANT_K_IQ2_XS 256 #define QUANT_R_IQ2_XS 1 @@ -1065,7 +1088,7 @@ struct block_iq2_xs_packed16 uint16_t scales[QUANT_K_IQ2_XS/64]; }; -#if defined(DATA_A_IQ2_XS) +#if defined(DATA_A_IQ2_XS) || defined(MULMAT_QUANT) const uvec2 iq2xs_grid_const[512] = { uvec2(0x08080808, 0x08080808), uvec2(0x0808082b, 0x08080808), uvec2(0x08081919, 0x08080808), uvec2(0x08082b08, 0x08080808), @@ -1198,25 +1221,27 @@ const uvec2 iq2xs_grid_const[512] = { uvec2(0x082b2b08, 0x2b2b2b2b), uvec2(0x082b2b2b, 0x2b2b2b2b), uvec2(0x2b190819, 0x2b2b2b2b), uvec2(0x2b2b2b2b, 0x2b2b2b2b), }; -shared uvec2 iq2xs_grid[512]; - +#if defined(DATA_A_IQ2_XS) #define NEEDS_INIT_IQ_SHMEM void init_iq_shmem(uvec3 wgsize) { // copy the table into shared memory and sync - [[unroll]] for (uint i = 0; i < iq2xs_grid.length(); i += wgsize.x) { - if (iq2xs_grid.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq2xs_grid_const.length()) { - iq2xs_grid[i + gl_LocalInvocationIndex.x] = iq2xs_grid_const[i + gl_LocalInvocationIndex.x]; + [[unroll]] for (uint i = 0; i < iq2_grid.length(); i += wgsize.x) { + if (iq2_grid.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq2xs_grid_const.length()) { + iq2_grid[i + gl_LocalInvocationIndex.x] = iq2xs_grid_const[i + gl_LocalInvocationIndex.x]; } } barrier(); } +#endif +#if defined(DATA_A_IQ2_XS) #define QUANT_K QUANT_K_IQ2_XS #define QUANT_R QUANT_R_IQ2_XS #define A_TYPE block_iq2_xs #define A_TYPE_PACKED16 block_iq2_xs_packed16 #endif +#endif #define QUANT_K_IQ2_S 256 #define QUANT_R_IQ2_S 1 @@ -1237,7 +1262,7 @@ struct block_iq2_s_packed16 uint16_t scales[QUANT_K_IQ2_S/64]; }; -#if defined(DATA_A_IQ2_S) +#if defined(DATA_A_IQ2_S) || defined(MULMAT_QUANT) const uvec2 iq2s_grid_const[1024] = { uvec2(0x08080808, 0x08080808), uvec2(0x0808082b, 0x08080808), uvec2(0x08081919, 0x08080808), uvec2(0x08082b08, 0x08080808), @@ -1498,25 +1523,37 @@ const uvec2 iq2s_grid_const[1024] = { uvec2(0x082b082b, 0x2b2b2b2b), uvec2(0x082b2b08, 0x2b2b2b2b), uvec2(0x2b082b08, 0x2b2b2b2b), uvec2(0x2b2b2b2b, 0x2b2b2b2b) }; -shared uvec2 iq2s_grid[1024]; - +#if defined(DATA_A_IQ2_S) #define NEEDS_INIT_IQ_SHMEM void init_iq_shmem(uvec3 wgsize) { // copy the table into shared memory and sync - [[unroll]] for (uint i = 0; i < iq2s_grid.length(); i += wgsize.x) { - if (iq2s_grid.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq2s_grid_const.length()) { - iq2s_grid[i + gl_LocalInvocationIndex.x] = iq2s_grid_const[i + gl_LocalInvocationIndex.x]; + [[unroll]] for (uint i = 0; i < iq2_grid.length(); i += wgsize.x) { + if (iq2_grid.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq2s_grid_const.length()) { + iq2_grid[i + gl_LocalInvocationIndex.x] = iq2s_grid_const[i + gl_LocalInvocationIndex.x]; } } barrier(); } +#endif +#if defined(DATA_A_IQ2_S) #define QUANT_K QUANT_K_IQ2_S #define QUANT_R QUANT_R_IQ2_S #define A_TYPE block_iq2_s #define A_TYPE_PACKED16 block_iq2_s_packed16 #endif +#endif + +#if defined(DATA_A_IQ3_XXS) || defined(DATA_A_IQ3_S) || defined(MULMAT_QUANT) +#ifdef MULMAT_QUANT +shared uint32_t iq3_grid[MmTypeA == GGML_TYPE_IQ3_S ? 512 : MmTypeA == GGML_TYPE_IQ3_XXS ? 256 : 8]; +#elif defined(DATA_A_IQ3_S) +shared uint32_t iq3_grid[512]; +#else +shared uint32_t iq3_grid[256]; +#endif +#endif #define QUANT_K_IQ3_XXS 256 #define QUANT_R_IQ3_XXS 1 @@ -1533,7 +1570,7 @@ struct block_iq3_xxs_packed16 uint16_t qs[QUANT_K_IQ3_XXS/8 + QUANT_K_IQ3_XXS/16]; }; -#if defined(DATA_A_IQ3_XXS) +#if defined(DATA_A_IQ3_XXS) || defined(MULMAT_QUANT) const uint32_t iq3xxs_grid_const[256] = { 0x04040404, 0x04040414, 0x04040424, 0x04040c0c, 0x04040c1c, 0x04040c3e, 0x04041404, 0x04041414, @@ -1570,25 +1607,27 @@ const uint32_t iq3xxs_grid_const[256] = { 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04, }; -shared uint32_t iq3xxs_grid[256]; - +#if defined(DATA_A_IQ3_XXS) #define NEEDS_INIT_IQ_SHMEM void init_iq_shmem(uvec3 wgsize) { // copy the table into shared memory and sync - [[unroll]] for (uint i = 0; i < iq3xxs_grid.length(); i += wgsize.x) { - if (iq3xxs_grid.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq3xxs_grid.length()) { - iq3xxs_grid[i + gl_LocalInvocationIndex.x] = iq3xxs_grid_const[i + gl_LocalInvocationIndex.x]; + [[unroll]] for (uint i = 0; i < iq3_grid.length(); i += wgsize.x) { + if (iq3_grid.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq3_grid.length()) { + iq3_grid[i + gl_LocalInvocationIndex.x] = iq3xxs_grid_const[i + gl_LocalInvocationIndex.x]; } } barrier(); } +#endif +#if defined(DATA_A_IQ3_XXS) #define QUANT_K QUANT_K_IQ3_XXS #define QUANT_R QUANT_R_IQ3_XXS #define A_TYPE block_iq3_xxs #define A_TYPE_PACKED16 block_iq3_xxs_packed16 #endif +#endif #define QUANT_K_IQ3_S 256 #define QUANT_R_IQ3_S 1 @@ -1611,7 +1650,7 @@ struct block_iq3_s_packed16 uint16_t scales[QUANT_K_IQ3_S/64/2]; }; -#if defined(DATA_A_IQ3_S) +#if defined(DATA_A_IQ3_S) || defined(MULMAT_QUANT) const uint32_t iq3s_grid_const[512] = { 0x01010101, 0x01010103, 0x01010105, 0x0101010b, 0x0101010f, 0x01010301, 0x01010303, 0x01010305, @@ -1680,25 +1719,27 @@ const uint32_t iq3s_grid_const[512] = { 0x0f090307, 0x0f090501, 0x0f090b01, 0x0f0b0505, 0x0f0b0905, 0x0f0d0105, 0x0f0d0703, 0x0f0f0101, }; -shared uint32_t iq3s_grid[512]; - +#if defined(DATA_A_IQ3_S) #define NEEDS_INIT_IQ_SHMEM void init_iq_shmem(uvec3 wgsize) { // copy the table into shared memory and sync - [[unroll]] for (uint i = 0; i < iq3s_grid.length(); i += wgsize.x) { - if (iq3s_grid.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq3s_grid.length()) { - iq3s_grid[i + gl_LocalInvocationIndex.x] = iq3s_grid_const[i + gl_LocalInvocationIndex.x]; + [[unroll]] for (uint i = 0; i < iq3_grid.length(); i += wgsize.x) { + if (iq3_grid.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq3_grid.length()) { + iq3_grid[i + gl_LocalInvocationIndex.x] = iq3s_grid_const[i + gl_LocalInvocationIndex.x]; } } barrier(); } +#endif +#if defined(DATA_A_IQ3_S) #define QUANT_K QUANT_K_IQ3_S #define QUANT_R QUANT_R_IQ3_S #define A_TYPE block_iq3_s #define A_TYPE_PACKED16 block_iq3_s_packed16 #endif +#endif #define QUANT_K_IQ4_XS 256 #define QUANT_R_IQ4_XS 1 @@ -1804,14 +1845,19 @@ struct block_nvfp4_packed32 #define A_TYPE_PACKED32 block_nvfp4_packed32 #endif -#if defined(DATA_A_IQ4_NL) || defined(DATA_A_IQ4_XS) +#if defined(DATA_A_IQ4_NL) || defined(DATA_A_IQ4_XS) || defined(MULMAT_QUANT) const int8_t kvalues_iq4nl_const[16] = { int8_t(-127), int8_t(-104), int8_t(-83), int8_t(-65), int8_t(-49), int8_t(-35), int8_t(-22), int8_t(-10), int8_t(1), int8_t(13), int8_t(25), int8_t(38), int8_t(53), int8_t(69), int8_t(89), int8_t(113) }; +#ifdef MULMAT_QUANT +shared FLOAT_TYPE kvalues_iq4nl[(MmTypeA == GGML_TYPE_IQ4_NL || MmTypeA == GGML_TYPE_IQ4_XS) ? 16 : 8]; +#else shared FLOAT_TYPE kvalues_iq4nl[16]; +#endif +#if defined(DATA_A_IQ4_NL) || defined(DATA_A_IQ4_XS) #define NEEDS_INIT_IQ_SHMEM void init_iq_shmem(uvec3 wgsize) { @@ -1822,20 +1868,28 @@ void init_iq_shmem(uvec3 wgsize) barrier(); } #endif +#endif -#if defined(DATA_A_MXFP4) || defined(DATA_A_NVFP4) +#if defined(DATA_A_MXFP4) || defined(DATA_A_NVFP4) || defined(MULMAT_QUANT) #if !defined(USE_OCP_FP4) const int8_t kvalues_mxfp4_const[16] = { int8_t(0), int8_t(1), int8_t(2), int8_t(3), int8_t(4), int8_t(6), int8_t(8), int8_t(12), int8_t(0), int8_t(-1), int8_t(-2), int8_t(-3), int8_t(-4), int8_t(-6), int8_t(-8), int8_t(-12), }; +#ifdef MULMAT_QUANT +shared int8_t kvalues_mxfp4[(MmTypeA == GGML_TYPE_MXFP4 || MmTypeA == GGML_TYPE_NVFP4) ? 16 : 8]; +#else shared int8_t kvalues_mxfp4[16]; #endif +#endif -#if defined(DATA_A_NVFP4) && !defined(USE_OCP_FP4) -// UE4M3 scale in NVFP4 blocks use only 7 bits; sign (bit 7) is always zero. +#if (defined(DATA_A_NVFP4) || defined(MULMAT_QUANT)) && !defined(USE_OCP_FP4) +#ifdef MULMAT_QUANT +shared float ue4m3_fp32_lut[MmTypeA == GGML_TYPE_NVFP4 ? 128 : 8]; +#else shared float ue4m3_fp32_lut[128]; +#endif float ue4m3_to_fp32_build(uint u) { if (u == 0u || u == 127u) { @@ -1851,7 +1905,7 @@ float ue4m3_to_fp32_build(uint u) { } #endif -#if !defined(USE_OCP_FP4) +#if (defined(DATA_A_MXFP4) || defined(DATA_A_NVFP4)) && !defined(USE_OCP_FP4) #define NEEDS_INIT_IQ_SHMEM void init_iq_shmem(uvec3 wgsize) { @@ -1901,7 +1955,7 @@ float e8m0_to_fp32(uint8_t x) { return uintBitsToFloat(bits); } -#if defined(DATA_A_NVFP4) +#if defined(DATA_A_NVFP4) || defined(MULMAT_QUANT) #if defined(USE_OCP_FP4) floate4m3_t ue4m3_from_bits(uint8_t x) { if (x == uint8_t(0x7F)) { diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp index 27ff68c10d5..160a1ec1e93 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -582,20 +582,31 @@ void matmul_shaders(bool fp16, MatMulIdType matmul_id_type, bool coopmat, bool c } for (const auto& tname : type_names) { - std::string load_vec_quant = "2"; - if ((tname == "q1_0") || (tname == "q4_0") || (tname == "q4_1") || (tname == "q5_1") || (tname == "iq1_s") || (tname == "iq1_m") || (tname == "iq2_xxs") || (tname == "iq2_xs") || (tname == "iq2_s")) - load_vec_quant = "8"; - else if ((tname == "q2_0") || (tname == "q5_0") || (tname == "q8_0") || (tname == "q2_k") || (tname == "q4_k") || (tname == "q5_k") || (tname == "iq3_xxs") || (tname == "iq3_s") || (tname == "iq4_xs") || (tname == "iq4_nl") || (tname == "mxfp4") || (tname == "nvfp4")) - load_vec_quant = "4"; - if (tname == "bf16") { continue; } - std::string data_a_key = "DATA_A_" + to_uppercase(tname); - // For aligned matmul loads - std::string load_vec_a = (coopmat2 || tname == "f32" || tname == "f16" || tname == "bf16") ? load_vec : load_vec_quant; + // Float types keep per-type compilation (different accumulation loop structure) + if (tname == "f32" || tname == "f16") { + std::string data_a_key = "DATA_A_" + to_uppercase(tname); + std::string load_vec_a = coopmat2 ? load_vec : load_vec; + + const std::map float_type_dict = { + {"FLOAT_TYPE", FLOAT_TYPE(1, tname)}, + {"FLOAT_TYPEV2", FLOAT_TYPE(2, tname)}, + {"FLOAT_TYPEV4", FLOAT_TYPE(4, tname)}, + {"FLOAT_TYPEV8", FLOAT_TYPE(8, tname)}, + }; + + // don't generate f32 variants for coopmat2 + if (!coopmat2) { + string_to_spv(shader_name + "_" + tname + "_f32" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"LOAD_VEC_A", load_vec_a}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f32}, {"B_TYPE_SCALAR", "float"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); + } + continue; + } + // Quant types: only MMQ (mul_mmq.comp) stays per-type + std::string data_a_key = "DATA_A_" + to_uppercase(tname); const std::map float_type_dict = { {"FLOAT_TYPE", FLOAT_TYPE(1, tname)}, {"FLOAT_TYPEV2", FLOAT_TYPE(2, tname)}, @@ -603,28 +614,36 @@ void matmul_shaders(bool fp16, MatMulIdType matmul_id_type, bool coopmat, bool c {"FLOAT_TYPEV8", FLOAT_TYPE(8, tname)}, }; - // don't generate f32 variants for coopmat2 - if (!coopmat2) { - string_to_spv(shader_name + "_" + tname + "_f32" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"LOAD_VEC_A", load_vec_a}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f32}, {"B_TYPE_SCALAR", "float"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); +#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) + if (!f16acc && !coopmat && !coopmat2 && !dot2 && (is_legacy_quant(tname) || is_k_quant(tname) || tname == "mxfp4")) { + string_to_spv(shader_name + "_" + tname + "_q8_1", "mul_mmq.comp", merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"D_TYPE", "float"},}), fp16, coopmat, coopmat2, f16acc); } +#endif + } + + // Quant shader: one SPIR-V for all quant types, selected via MmTypeA spec constant + { + const std::map quant_float_type_dict = { + {"FLOAT_TYPE", FLOAT_TYPE(1, "q4_0")}, + {"FLOAT_TYPEV2", FLOAT_TYPE(2, "q4_0")}, + {"FLOAT_TYPEV4", FLOAT_TYPE(4, "q4_0")}, + {"FLOAT_TYPEV8", FLOAT_TYPE(8, "q4_0")}, + }; + + // f16 B-type (all paths) + string_to_spv(shader_name + "_quant_f16" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, quant_float_type_dict), {{"MULMAT_QUANT", "1"}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f16}, {"B_TYPE_SCALAR", "float16_t"}, {"B_TYPEV4", "f16vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); - if (tname != "f16" && tname != "f32") { - string_to_spv(shader_name + "_" + tname + "_f16" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"LOAD_VEC_A", load_vec_a}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f16}, {"B_TYPE_SCALAR", "float16_t"}, {"B_TYPEV4", "f16vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); + // f32 B-type (non-coopmat2 only) + if (!coopmat2) { + string_to_spv(shader_name + "_quant_f32" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, quant_float_type_dict), {{"MULMAT_QUANT", "1"}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f32}, {"B_TYPE_SCALAR", "float"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); } #if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) - if ((coopmat || coopmat2) && (tname == "mxfp4" || tname == "nvfp4")) { + if (coopmat || coopmat2) { + string_to_spv(shader_name + "_quant_f16_ocp" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, quant_float_type_dict), {{"MULMAT_QUANT", "1"}, {"USE_OCP_FP4", "1"}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f16}, {"B_TYPE_SCALAR", "float16_t"}, {"B_TYPEV4", "f16vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); if (!coopmat2) { - string_to_spv(shader_name + "_" + tname + "_f32_ocp" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"USE_OCP_FP4", "1"}, {"LOAD_VEC_A", load_vec_a}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f32}, {"B_TYPE_SCALAR", "float"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); + string_to_spv(shader_name + "_quant_f32_ocp" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, quant_float_type_dict), {{"MULMAT_QUANT", "1"}, {"USE_OCP_FP4", "1"}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f32}, {"B_TYPE_SCALAR", "float"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); } - string_to_spv(shader_name + "_" + tname + "_f16_ocp" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"USE_OCP_FP4", "1"}, {"LOAD_VEC_A", load_vec_a}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f16}, {"B_TYPE_SCALAR", "float16_t"}, {"B_TYPEV4", "f16vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); - } -#endif - -#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) - // Integer dot mmq performs better with f32 accumulators (different shader, skip for dot2) - if (!f16acc && !coopmat && !coopmat2 && !dot2 && (is_legacy_quant(tname) || is_k_quant(tname) || tname == "mxfp4")) { - string_to_spv(shader_name + "_" + tname + "_q8_1", "mul_mmq.comp", merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"D_TYPE", "float"},}), fp16, coopmat, coopmat2, f16acc); } #endif } From b1e0349b7acc664e692ad4e365e9dba92792c9d4 Mon Sep 17 00:00:00 2001 From: Ruben Ortlam Date: Wed, 26 Aug 2026 04:52:28 +0200 Subject: [PATCH 2/9] remove LUT quants from unified shader --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 214 ++++--- .../vulkan-shaders/dequant_funcs_cm2.glsl | 22 +- .../vulkan-shaders/iq_shmem_init.glsl | 64 --- .../ggml-vulkan/vulkan-shaders/mul_mm.comp | 24 - .../vulkan-shaders/mul_mm_cm2.comp | 42 +- .../vulkan-shaders/mul_mm_funcs.glsl | 544 +++++++++--------- .../src/ggml-vulkan/vulkan-shaders/types.glsl | 51 +- .../vulkan-shaders/vulkan-shaders-gen.cpp | 32 +- 8 files changed, 481 insertions(+), 512 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index dac862f014c..b5a0eb957e3 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -4671,14 +4671,27 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { return spec; }; - static const ggml_type quant_types[] = { + static const ggml_type non_lut_quant_types[] = { GGML_TYPE_Q1_0, GGML_TYPE_Q2_0, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, GGML_TYPE_Q8_0, GGML_TYPE_Q2_K, GGML_TYPE_TQ2_0, GGML_TYPE_Q3_K, GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K, - GGML_TYPE_IQ1_S, GGML_TYPE_IQ1_M, GGML_TYPE_IQ2_XXS, GGML_TYPE_IQ2_XS, GGML_TYPE_IQ2_S, - GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ3_S, GGML_TYPE_IQ4_XS, GGML_TYPE_IQ4_NL, - GGML_TYPE_MXFP4, GGML_TYPE_NVFP4, }; +#define FOR_EACH_LUT_TYPE(X) \ + X(GGML_TYPE_IQ1_S, iq1_s) \ + X(GGML_TYPE_IQ1_M, iq1_m) \ + X(GGML_TYPE_IQ2_XXS, iq2_xxs) \ + X(GGML_TYPE_IQ2_XS, iq2_xs) \ + X(GGML_TYPE_IQ2_S, iq2_s) \ + X(GGML_TYPE_IQ3_XXS, iq3_xxs) \ + X(GGML_TYPE_IQ3_S, iq3_s) \ + X(GGML_TYPE_IQ4_XS, iq4_xs) \ + X(GGML_TYPE_IQ4_NL, iq4_nl) \ + X(GGML_TYPE_MXFP4, mxfp4) \ + X(GGML_TYPE_NVFP4, nvfp4) +#define FOR_EACH_LUT_FP4_TYPE(X) \ + X(GGML_TYPE_MXFP4, mxfp4) \ + X(GGML_TYPE_NVFP4, nvfp4) + const int mul_mat_id_param_count = 5; using spec_fn_t = std::function(const std::vector&, bool)>; @@ -4769,20 +4782,32 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { create_mm_pipelines({GGML_TYPE_BF16, GGML_TYPE_BF16, false, false}, tc_mm, "matmul_bf16", matmul_bf16_cm2_len, matmul_bf16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); } #endif - for (const auto type : quant_types) { + for (const auto type : non_lut_quant_types) { auto& tc = ((type >= GGML_TYPE_Q2_K && type <= GGML_TYPE_Q6_K) || type == GGML_TYPE_TQ2_0) ? tc_mmq_k : tc_mmq; spec_fn_t qs = [&, type](const std::vector& wt, bool a) { return ggml_vk_mul_mm_cm2_spec(wt, a, (uint32_t)type); }; + create_mm_pipelines({type, GGML_TYPE_F16, false, true}, tc, "matmul_quant_f16_f16acc", matmul_quant_f16_f16acc_cm2_len, matmul_quant_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, qs, true); + create_mm_pipelines({type, GGML_TYPE_F16, false, false}, tc, "matmul_quant_f16", matmul_quant_f16_cm2_len, matmul_quant_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, qs, true); + } +#define X_CM2(TYPE, tstr) \ + { auto tc = filter_tc(tc_mmq, TYPE, false); \ + if (!tc.empty()) { \ + create_mm_pipelines({TYPE, GGML_TYPE_F16, false, true}, tc, "matmul_" #tstr "_f16_f16acc", matmul_##tstr##_f16_f16acc_cm2_len, matmul_##tstr##_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); \ + create_mm_pipelines({TYPE, GGML_TYPE_F16, false, false}, tc, "matmul_" #tstr "_f16", matmul_##tstr##_f16_cm2_len, matmul_##tstr##_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); \ + } } + FOR_EACH_LUT_TYPE(X_CM2) +#undef X_CM2 #if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) - if (device->ocp_fp4 && (type == GGML_TYPE_MXFP4 || type == GGML_TYPE_NVFP4)) { - create_mm_pipelines({type, GGML_TYPE_F16, false, true}, tc, "matmul_quant_f16_ocp_f16acc", matmul_quant_f16_ocp_f16acc_cm2_len, matmul_quant_f16_ocp_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, qs, true); - create_mm_pipelines({type, GGML_TYPE_F16, false, false}, tc, "matmul_quant_f16_ocp", matmul_quant_f16_ocp_cm2_len, matmul_quant_f16_ocp_cm2_data, sizeof(vk_mat_mat_push_constants), 3, qs, true); - } else -#endif - { - create_mm_pipelines({type, GGML_TYPE_F16, false, true}, tc, "matmul_quant_f16_f16acc", matmul_quant_f16_f16acc_cm2_len, matmul_quant_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, qs, true); - create_mm_pipelines({type, GGML_TYPE_F16, false, false}, tc, "matmul_quant_f16", matmul_quant_f16_cm2_len, matmul_quant_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, qs, true); - } + if (device->ocp_fp4) { +#define X_CM2_OCP(TYPE, tstr) \ + { auto tc = filter_tc(tc_mmq, TYPE, false); \ + if (!tc.empty()) { \ + create_mm_pipelines({TYPE, GGML_TYPE_F16, false, true}, tc, "matmul_" #tstr "_f16_ocp_f16acc", matmul_##tstr##_f16_ocp_f16acc_cm2_len, matmul_##tstr##_f16_ocp_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); \ + create_mm_pipelines({TYPE, GGML_TYPE_F16, false, false}, tc, "matmul_" #tstr "_f16_ocp", matmul_##tstr##_f16_ocp_cm2_len, matmul_##tstr##_f16_ocp_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); \ + } } + FOR_EACH_LUT_FP4_TYPE(X_CM2_OCP) +#undef X_CM2_OCP } +#endif GGML_ASSERT(device->subgroup_ballot); @@ -4794,19 +4819,31 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { create_mm_pipelines({GGML_TYPE_BF16, GGML_TYPE_BF16, true, false}, tc_mm, "matmul_id_subgroup_bf16", matmul_id_subgroup_bf16_cm2_len, matmul_id_subgroup_bf16_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); } #endif - for (const auto type : quant_types) { + for (const auto type : non_lut_quant_types) { spec_fn_t qs_id = [&, type](const std::vector& wt, bool a) { return ggml_vk_mul_mm_cm2_spec(wt, a, (uint32_t)type); }; + create_mm_pipelines({type, GGML_TYPE_F16, true, true}, tc_mmqid, "matmul_id_subgroup_quant_f16_f16acc", matmul_id_subgroup_quant_f16_f16acc_cm2_len, matmul_id_subgroup_quant_f16_f16acc_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, qs_id, true); + create_mm_pipelines({type, GGML_TYPE_F16, true, false}, tc_mmqid, "matmul_id_subgroup_quant_f16", matmul_id_subgroup_quant_f16_cm2_len, matmul_id_subgroup_quant_f16_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, qs_id, true); + } +#define X_CM2_ID(TYPE, tstr) \ + { auto tc = filter_tc(tc_mmqid, TYPE, true); \ + if (!tc.empty()) { \ + create_mm_pipelines({TYPE, GGML_TYPE_F16, true, true}, tc, "matmul_id_subgroup_" #tstr "_f16_f16acc", matmul_id_subgroup_##tstr##_f16_f16acc_cm2_len, matmul_id_subgroup_##tstr##_f16_f16acc_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); \ + create_mm_pipelines({TYPE, GGML_TYPE_F16, true, false}, tc, "matmul_id_subgroup_" #tstr "_f16", matmul_id_subgroup_##tstr##_f16_cm2_len, matmul_id_subgroup_##tstr##_f16_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); \ + } } + FOR_EACH_LUT_TYPE(X_CM2_ID) +#undef X_CM2_ID #if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) - if (device->ocp_fp4 && (type == GGML_TYPE_MXFP4 || type == GGML_TYPE_NVFP4)) { - create_mm_pipelines({type, GGML_TYPE_F16, true, true}, tc_mmqid, "matmul_id_subgroup_quant_f16_ocp_f16acc", matmul_id_subgroup_quant_f16_ocp_f16acc_cm2_len, matmul_id_subgroup_quant_f16_ocp_f16acc_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, qs_id, true); - create_mm_pipelines({type, GGML_TYPE_F16, true, false}, tc_mmqid, "matmul_id_subgroup_quant_f16_ocp", matmul_id_subgroup_quant_f16_ocp_cm2_len, matmul_id_subgroup_quant_f16_ocp_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, qs_id, true); - } else -#endif - { - create_mm_pipelines({type, GGML_TYPE_F16, true, true}, tc_mmqid, "matmul_id_subgroup_quant_f16_f16acc", matmul_id_subgroup_quant_f16_f16acc_cm2_len, matmul_id_subgroup_quant_f16_f16acc_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, qs_id, true); - create_mm_pipelines({type, GGML_TYPE_F16, true, false}, tc_mmqid, "matmul_id_subgroup_quant_f16", matmul_id_subgroup_quant_f16_cm2_len, matmul_id_subgroup_quant_f16_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, qs_id, true); - } + if (device->ocp_fp4) { +#define X_CM2_ID_OCP(TYPE, tstr) \ + { auto tc = filter_tc(tc_mmqid, TYPE, true); \ + if (!tc.empty()) { \ + create_mm_pipelines({TYPE, GGML_TYPE_F16, true, true}, tc, "matmul_id_subgroup_" #tstr "_f16_ocp_f16acc", matmul_id_subgroup_##tstr##_f16_ocp_f16acc_cm2_len, matmul_id_subgroup_##tstr##_f16_ocp_f16acc_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); \ + create_mm_pipelines({TYPE, GGML_TYPE_F16, true, false}, tc, "matmul_id_subgroup_" #tstr "_f16_ocp", matmul_id_subgroup_##tstr##_f16_ocp_cm2_len, matmul_id_subgroup_##tstr##_f16_ocp_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); \ + } } + FOR_EACH_LUT_FP4_TYPE(X_CM2_ID_OCP) +#undef X_CM2_ID_OCP } +#endif } else #endif // defined(VK_NV_cooperative_matrix2) && defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT) #if defined(VK_KHR_cooperative_matrix) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) @@ -4840,26 +4877,36 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { cm1_create({GGML_TYPE_BF16, GGML_TYPE_BF16, false, false}, tc_mm, "matmul_bf16", matmul_bf16_cm1_len, matmul_bf16_cm1_data, sizeof(vk_mat_mat_push_constants), 3); } #endif - for (const auto type : quant_types) { + for (const auto type : non_lut_quant_types) { + if (device->coopmat_acc_f16_support) { + cm1_create_quant({type, GGML_TYPE_F32, false, true}, tc_mmq, "matmul_quant_f32_f16acc", matmul_quant_f32_f16acc_cm1_len, matmul_quant_f32_f16acc_cm1_data, sizeof(vk_mat_mat_push_constants), 3); + } + if (device->coopmat_acc_f32_support) { + cm1_create_quant({type, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_quant_f32", matmul_quant_f32_cm1_len, matmul_quant_f32_cm1_data, sizeof(vk_mat_mat_push_constants), 3); + } + } +#define X_CM1(TYPE, tstr) \ + if (device->coopmat_acc_f16_support) { \ + cm1_create({TYPE, GGML_TYPE_F32, false, true}, tc_mmq, "matmul_" #tstr "_f32_f16acc", matmul_##tstr##_f32_f16acc_cm1_len, matmul_##tstr##_f32_f16acc_cm1_data, sizeof(vk_mat_mat_push_constants), 3); \ + } \ + if (device->coopmat_acc_f32_support) { \ + cm1_create({TYPE, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_" #tstr "_f32", matmul_##tstr##_f32_cm1_len, matmul_##tstr##_f32_cm1_data, sizeof(vk_mat_mat_push_constants), 3); \ + } + FOR_EACH_LUT_TYPE(X_CM1) +#undef X_CM1 #if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) - if (device->ocp_fp4 && (type == GGML_TYPE_MXFP4 || type == GGML_TYPE_NVFP4)) { - if (device->coopmat_acc_f16_support) { - cm1_create_quant({type, GGML_TYPE_F32, false, true}, tc_mmq, "matmul_quant_f32_ocp_f16acc", matmul_quant_f32_ocp_f16acc_cm1_len, matmul_quant_f32_ocp_f16acc_cm1_data, sizeof(vk_mat_mat_push_constants), 3); - } - if (device->coopmat_acc_f32_support) { - cm1_create_quant({type, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_quant_f32_ocp", matmul_quant_f32_ocp_cm1_len, matmul_quant_f32_ocp_cm1_data, sizeof(vk_mat_mat_push_constants), 3); - } - } else -#endif - { - if (device->coopmat_acc_f16_support) { - cm1_create_quant({type, GGML_TYPE_F32, false, true}, tc_mmq, "matmul_quant_f32_f16acc", matmul_quant_f32_f16acc_cm1_len, matmul_quant_f32_f16acc_cm1_data, sizeof(vk_mat_mat_push_constants), 3); - } - if (device->coopmat_acc_f32_support) { - cm1_create_quant({type, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_quant_f32", matmul_quant_f32_cm1_len, matmul_quant_f32_cm1_data, sizeof(vk_mat_mat_push_constants), 3); - } + if (device->ocp_fp4) { +#define X_CM1_OCP(TYPE, tstr) \ + if (device->coopmat_acc_f16_support) { \ + cm1_create({TYPE, GGML_TYPE_F32, false, true}, tc_mmq, "matmul_" #tstr "_f32_ocp_f16acc", matmul_##tstr##_f32_ocp_f16acc_cm1_len, matmul_##tstr##_f32_ocp_f16acc_cm1_data, sizeof(vk_mat_mat_push_constants), 3); \ + } \ + if (device->coopmat_acc_f32_support) { \ + cm1_create({TYPE, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_" #tstr "_f32_ocp", matmul_##tstr##_f32_ocp_cm1_len, matmul_##tstr##_f32_ocp_cm1_data, sizeof(vk_mat_mat_push_constants), 3); \ } + FOR_EACH_LUT_FP4_TYPE(X_CM1_OCP) +#undef X_CM1_OCP } +#endif GGML_ASSERT(device->subgroup_ballot); @@ -4877,26 +4924,36 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { cm1_create({GGML_TYPE_BF16, GGML_TYPE_BF16, true, false}, tc_mm, "matmul_id_subgroup_bf16", matmul_id_subgroup_bf16_cm1_len, matmul_id_subgroup_bf16_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); } #endif - for (const auto type : quant_types) { + for (const auto type : non_lut_quant_types) { + if (device->coopmat_acc_f16_support) { + cm1_create_quant({type, GGML_TYPE_F32, true, true}, tc_mmq, "matmul_id_subgroup_quant_f32_f16acc", matmul_id_subgroup_quant_f32_f16acc_cm1_len, matmul_id_subgroup_quant_f32_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + } + if (device->coopmat_acc_f32_support) { + cm1_create_quant({type, GGML_TYPE_F32, true, false}, tc_mmq, "matmul_id_subgroup_quant_f32", matmul_id_subgroup_quant_f32_cm1_len, matmul_id_subgroup_quant_f32_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + } + } +#define X_CM1_ID(TYPE, tstr) \ + if (device->coopmat_acc_f16_support) { \ + cm1_create({TYPE, GGML_TYPE_F32, true, true}, tc_mmq, "matmul_id_subgroup_" #tstr "_f32_f16acc", matmul_id_subgroup_##tstr##_f32_f16acc_cm1_len, matmul_id_subgroup_##tstr##_f32_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); \ + } \ + if (device->coopmat_acc_f32_support) { \ + cm1_create({TYPE, GGML_TYPE_F32, true, false}, tc_mmq, "matmul_id_subgroup_" #tstr "_f32", matmul_id_subgroup_##tstr##_f32_cm1_len, matmul_id_subgroup_##tstr##_f32_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); \ + } + FOR_EACH_LUT_TYPE(X_CM1_ID) +#undef X_CM1_ID #if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) - if (device->ocp_fp4 && (type == GGML_TYPE_MXFP4 || type == GGML_TYPE_NVFP4)) { - if (device->coopmat_acc_f16_support) { - cm1_create_quant({type, GGML_TYPE_F32, true, true}, tc_mmq, "matmul_id_subgroup_quant_f32_ocp_f16acc", matmul_id_subgroup_quant_f32_ocp_f16acc_cm1_len, matmul_id_subgroup_quant_f32_ocp_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); - } - if (device->coopmat_acc_f32_support) { - cm1_create_quant({type, GGML_TYPE_F32, true, false}, tc_mmq, "matmul_id_subgroup_quant_f32_ocp", matmul_id_subgroup_quant_f32_ocp_cm1_len, matmul_id_subgroup_quant_f32_ocp_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); - } - } else -#endif - { - if (device->coopmat_acc_f16_support) { - cm1_create_quant({type, GGML_TYPE_F32, true, true}, tc_mmq, "matmul_id_subgroup_quant_f32_f16acc", matmul_id_subgroup_quant_f32_f16acc_cm1_len, matmul_id_subgroup_quant_f32_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); - } - if (device->coopmat_acc_f32_support) { - cm1_create_quant({type, GGML_TYPE_F32, true, false}, tc_mmq, "matmul_id_subgroup_quant_f32", matmul_id_subgroup_quant_f32_cm1_len, matmul_id_subgroup_quant_f32_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); - } + if (device->ocp_fp4) { +#define X_CM1_ID_OCP(TYPE, tstr) \ + if (device->coopmat_acc_f16_support) { \ + cm1_create({TYPE, GGML_TYPE_F32, true, true}, tc_mmq, "matmul_id_subgroup_" #tstr "_f32_ocp_f16acc", matmul_id_subgroup_##tstr##_f32_ocp_f16acc_cm1_len, matmul_id_subgroup_##tstr##_f32_ocp_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); \ + } \ + if (device->coopmat_acc_f32_support) { \ + cm1_create({TYPE, GGML_TYPE_F32, true, false}, tc_mmq, "matmul_id_subgroup_" #tstr "_f32_ocp", matmul_id_subgroup_##tstr##_f32_ocp_cm1_len, matmul_id_subgroup_##tstr##_f32_ocp_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); \ } + FOR_EACH_LUT_FP4_TYPE(X_CM1_ID_OCP) +#undef X_CM1_ID_OCP } +#endif } else #endif // defined(VK_KHR_cooperative_matrix) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) { @@ -4947,10 +5004,15 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { // BF16 - no dot2 sg_create({GGML_TYPE_BF16, GGML_TYPE_BF16, false, false}, tc_mm, "matmul_bf16", matmul_bf16_len, matmul_bf16_data, sizeof(vk_mat_mat_push_constants), 3); - for (const auto type : quant_types) { + for (const auto type : non_lut_quant_types) { sg_create_quant({type, GGML_TYPE_F32, false, true}, tc_mmq, "matmul_quant_f32_f16acc", SPV_DOT2_F16ACC(matmul_quant_f32), sizeof(vk_mat_mat_push_constants), 3); sg_create_quant({type, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_quant_f32", SPV_DOT2(matmul_quant_f32), sizeof(vk_mat_mat_push_constants), 3); } + #define X_SG(TYPE, tstr) \ + sg_create({TYPE, GGML_TYPE_F32, false, true}, tc_mmq, "matmul_" #tstr "_f32_f16acc", SPV_DOT2_F16ACC(matmul_##tstr##_f32), sizeof(vk_mat_mat_push_constants), 3); \ + sg_create({TYPE, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_" #tstr "_f32", SPV_DOT2(matmul_##tstr##_f32), sizeof(vk_mat_mat_push_constants), 3); + FOR_EACH_LUT_TYPE(X_SG) +#undef X_SG #if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) if (device->integer_dot_product) { @@ -4979,10 +5041,15 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { sg_create({GGML_TYPE_F16, GGML_TYPE_F32, true, false}, tc_id, "matmul_id_subgroup_f16_f32", SPV_DOT2(matmul_id_subgroup_f16_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); // BF16 id - no dot2 sg_create({GGML_TYPE_BF16, GGML_TYPE_BF16, true, false}, tc_id, "matmul_id_subgroup_bf16", matmul_id_subgroup_bf16_len, matmul_id_subgroup_bf16_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); - for (const auto type : quant_types) { + for (const auto type : non_lut_quant_types) { sg_create_quant({type, GGML_TYPE_F32, true, true}, tc_mmqid, "matmul_id_subgroup_quant_f32_f16acc", SPV_DOT2_F16ACC(matmul_id_subgroup_quant_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); sg_create_quant({type, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_subgroup_quant_f32", SPV_DOT2(matmul_id_subgroup_quant_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); } + #define X_SG_ID_SUB(TYPE, tstr) \ + sg_create({TYPE, GGML_TYPE_F32, true, true}, tc_mmqid, "matmul_id_subgroup_" #tstr "_f32_f16acc", SPV_DOT2_F16ACC(matmul_id_subgroup_##tstr##_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); \ + sg_create({TYPE, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_subgroup_" #tstr "_f32", SPV_DOT2(matmul_id_subgroup_##tstr##_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); + FOR_EACH_LUT_TYPE(X_SG_ID_SUB) +#undef X_SG_ID_SUB #if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) if (device->integer_dot_product) { std::vector tc_mmqid_int = {{s_warptile_mmqid_int, s_mmq_wg_denoms, s_align}, {m_warptile_mmqid_int, m_mmq_wg_denoms, m_align}, {l_warptile_mmqid_int, l_mmq_wg_denoms, l_align}}; @@ -5009,10 +5076,15 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { sg_create({GGML_TYPE_F16, GGML_TYPE_F32, true, false}, tc_mm, "matmul_id_f16_f32", SPV_DOT2(matmul_id_f16_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); // BF16 id - no dot2 sg_create({GGML_TYPE_BF16, GGML_TYPE_BF16, true, false}, tc_mm, "matmul_id_bf16", matmul_id_bf16_len, matmul_id_bf16_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); - for (const auto type : quant_types) { + for (const auto type : non_lut_quant_types) { sg_create_quant({type, GGML_TYPE_F32, true, true}, tc_mmqid, "matmul_id_quant_f32_f16acc", SPV_DOT2_F16ACC(matmul_id_quant_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); sg_create_quant({type, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_quant_f32", SPV_DOT2(matmul_id_quant_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); } + #define X_SG_ID(TYPE, tstr) \ + sg_create({TYPE, GGML_TYPE_F32, true, true}, tc_mmqid, "matmul_id_" #tstr "_f32_f16acc", SPV_DOT2_F16ACC(matmul_id_##tstr##_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); \ + sg_create({TYPE, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_" #tstr "_f32", SPV_DOT2(matmul_id_##tstr##_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + FOR_EACH_LUT_TYPE(X_SG_ID) +#undef X_SG_ID #if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) if (device->integer_dot_product) { std::vector tc_mmqid_int = {{s_warptile_mmqid_int, s_mmq_wg_denoms, s_align}, {m_warptile_mmqid_int, m_mmq_wg_denoms, m_align}, {l_warptile_mmqid_int, l_mmq_wg_denoms, l_align}}; @@ -5042,9 +5114,13 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { sg_create({GGML_TYPE_F16, GGML_TYPE_F32, false, false}, tc_mm, "matmul_f16_f32", matmul_f16_f32_fp32_len, matmul_f16_f32_fp32_data, sizeof(vk_mat_mat_push_constants), 3); sg_create({GGML_TYPE_BF16, GGML_TYPE_BF16, false, false}, tc_mm, "matmul_bf16", matmul_bf16_fp32_len, matmul_bf16_fp32_data, sizeof(vk_mat_mat_push_constants), 3); - for (const auto type : quant_types) { + for (const auto type : non_lut_quant_types) { sg_create_quant({type, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_quant_f32", matmul_quant_f32_fp32_len, matmul_quant_f32_fp32_data, sizeof(vk_mat_mat_push_constants), 3); } + #define X_SG_FP32(TYPE, tstr) \ + sg_create({TYPE, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_" #tstr "_f32", matmul_##tstr##_f32_fp32_len, matmul_##tstr##_f32_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + FOR_EACH_LUT_TYPE(X_SG_FP32) +#undef X_SG_FP32 #if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) if (device->integer_dot_product) { @@ -5071,20 +5147,30 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { sg_create({GGML_TYPE_F16, GGML_TYPE_F16, true, false}, tc_id, "matmul_id_subgroup_f16", matmul_id_subgroup_f16_fp32_len, matmul_id_subgroup_f16_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); sg_create({GGML_TYPE_F16, GGML_TYPE_F32, true, false}, tc_id, "matmul_id_subgroup_f16_f32", matmul_id_subgroup_f16_f32_fp32_len, matmul_id_subgroup_f16_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); sg_create({GGML_TYPE_BF16, GGML_TYPE_BF16, true, false}, tc_id, "matmul_id_subgroup_bf16", matmul_id_subgroup_bf16_fp32_len, matmul_id_subgroup_bf16_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); - for (const auto type : quant_types) { + for (const auto type : non_lut_quant_types) { sg_create_quant({type, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_subgroup_quant_f32", matmul_id_subgroup_quant_f32_fp32_len, matmul_id_subgroup_quant_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); } + #define X_SG_ID_SUB_FP32(TYPE, tstr) \ + sg_create({TYPE, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_subgroup_" #tstr "_f32", matmul_id_subgroup_##tstr##_f32_fp32_len, matmul_id_subgroup_##tstr##_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); + FOR_EACH_LUT_TYPE(X_SG_ID_SUB_FP32) +#undef X_SG_ID_SUB_FP32 } else { sg_create({GGML_TYPE_F32, GGML_TYPE_F32, true, false}, tc_mm, "matmul_id_f32_f32", matmul_id_f32_f32_fp32_len, matmul_id_f32_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); sg_create({GGML_TYPE_F16, GGML_TYPE_F16, true, false}, tc_mm, "matmul_id_f16", matmul_id_f16_fp32_len, matmul_id_f16_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); sg_create({GGML_TYPE_F16, GGML_TYPE_F32, true, false}, tc_mm, "matmul_id_f16_f32", matmul_id_f16_f32_fp32_len, matmul_id_f16_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); sg_create({GGML_TYPE_BF16, GGML_TYPE_BF16, true, false}, tc_mm, "matmul_id_bf16", matmul_id_bf16_fp32_len, matmul_id_bf16_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); - for (const auto type : quant_types) { + for (const auto type : non_lut_quant_types) { sg_create_quant({type, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_quant_f32", matmul_id_quant_f32_fp32_len, matmul_id_quant_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); } + #define X_SG_ID_FP32(TYPE, tstr) \ + sg_create({TYPE, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_" #tstr "_f32", matmul_id_##tstr##_f32_fp32_len, matmul_id_##tstr##_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + FOR_EACH_LUT_TYPE(X_SG_ID_FP32) +#undef X_SG_ID_FP32 } } } +#undef FOR_EACH_LUT_TYPE +#undef FOR_EACH_LUT_FP4_TYPE // BF16 fallback for coopmat devices without bf16 coopmat support if ((device->coopmat2 || device->coopmat_support) #if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl index 55d2c3912f6..d32a30c86f4 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl @@ -804,7 +804,7 @@ f16vec4 dequantFuncQ6_K_v(const in decodeBufQ6_K bl, const in uint blockCoords[2 return f16vec4((vec4(qi) - vec4(32.0f)) * vec4(float(dscale))); } -#if defined(DATA_A_IQ1_S) || defined(MULMAT_QUANT) +#if defined(DATA_A_IQ1_S) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ1_S { block_iq1_s block; }; @@ -851,7 +851,7 @@ f16vec4 dequantFuncIQ1_S_v(const in decodeBufIQ1_S bl, const in uint blockCoords } #endif -#if defined(DATA_A_IQ1_M) || defined(MULMAT_QUANT) +#if defined(DATA_A_IQ1_M) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ1_M { block_iq1_m block; }; @@ -910,7 +910,7 @@ f16vec4 dequantFuncIQ1_M_v(const in decodeBufIQ1_M bl, const in uint blockCoords } #endif -#if defined(DATA_A_IQ2_XXS) || defined(MULMAT_QUANT) +#if defined(DATA_A_IQ2_XXS) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ2_XXS { block_iq2_xxs block; }; @@ -972,7 +972,7 @@ f16vec4 dequantFuncIQ2_XXS_v(const in decodeBufIQ2_XXS bl, const in uint blockCo } #endif -#if defined(DATA_A_IQ2_XS) || defined(MULMAT_QUANT) +#if defined(DATA_A_IQ2_XS) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ2_XS { block_iq2_xs block; }; @@ -1025,7 +1025,7 @@ f16vec4 dequantFuncIQ2_XS_v(const in decodeBufIQ2_XS bl, const in uint blockCoor } #endif -#if defined(DATA_A_IQ2_S) || defined(MULMAT_QUANT) +#if defined(DATA_A_IQ2_S) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ2_S { block_iq2_s block; }; @@ -1079,7 +1079,7 @@ f16vec4 dequantFuncIQ2_S_v(const in decodeBufIQ2_S bl, const in uint blockCoords } #endif -#if defined(DATA_A_IQ3_XXS) || defined(MULMAT_QUANT) +#if defined(DATA_A_IQ3_XXS) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ3_XXS { block_iq3_xxs block; }; @@ -1138,7 +1138,7 @@ f16vec4 dequantFuncIQ3_XXS_v(const in decodeBufIQ3_XXS bl, const in uint blockCo } #endif -#if defined(DATA_A_IQ3_S) || defined(MULMAT_QUANT) +#if defined(DATA_A_IQ3_S) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ3_S { block_iq3_s block; }; @@ -1188,7 +1188,7 @@ f16vec4 dequantFuncIQ3_S_v(const in decodeBufIQ3_S bl, const in uint blockCoords } #endif -#if defined(DATA_A_IQ4_XS) || defined(MULMAT_QUANT) +#if defined(DATA_A_IQ4_XS) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ4_XS { block_iq4_xs block; }; @@ -1238,7 +1238,7 @@ f16vec4 dequantFuncIQ4_XS_v(const in decodeBufIQ4_XS bl, const in uint blockCoor } #endif -#if defined(DATA_A_IQ4_NL) || defined(MULMAT_QUANT) +#if defined(DATA_A_IQ4_NL) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ4_NL { block_iq4_nl block; }; @@ -1279,7 +1279,7 @@ f16vec4 dequantFuncIQ4_NL_v(const in decodeBufIQ4_NL bl, const in uint blockCoor } #endif -#if defined(DATA_A_MXFP4) || defined(MULMAT_QUANT) +#if defined(DATA_A_MXFP4) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufMXFP4 { block_mxfp4 block; }; @@ -1333,7 +1333,7 @@ f16vec4 dequantFuncMXFP4_v(const in decodeBufMXFP4 bl, const in uint blockCoords } #endif -#if defined(DATA_A_NVFP4) || defined(MULMAT_QUANT) +#if defined(DATA_A_NVFP4) layout(buffer_reference, std430, buffer_reference_align = 4) buffer decodeBufNVFP4 { block_nvfp4 block; }; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/iq_shmem_init.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/iq_shmem_init.glsl index 5fb2ba9ab8d..12e50ee9eb9 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/iq_shmem_init.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/iq_shmem_init.glsl @@ -1,66 +1,2 @@ void init_iq_shmem(uvec3 wgsize) { - if (MmTypeA == GGML_TYPE_IQ1_S || MmTypeA == GGML_TYPE_IQ1_M) { - [[unroll]] for (uint i = 0; i < iq1s_grid_const.length(); i += wgsize.x) { - uint idx = i + gl_LocalInvocationIndex.x; - if (iq1s_grid_const.length() % wgsize.x == 0 || idx < iq1s_grid_const.length()) { - u16vec2 g = unpack16(iq1s_grid_const[idx]); - iq1s_grid[2*idx+0] = g.x; - iq1s_grid[2*idx+1] = g.y; - } - } - barrier(); - } else if (MmTypeA == GGML_TYPE_IQ2_XXS) { - [[unroll]] for (uint i = 0; i < iq2_grid.length(); i += wgsize.x) { - if (iq2xxs_grid_const.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq2xxs_grid_const.length()) { - iq2_grid[i + gl_LocalInvocationIndex.x] = iq2xxs_grid_const[i + gl_LocalInvocationIndex.x]; - } - } - barrier(); - } else if (MmTypeA == GGML_TYPE_IQ2_XS) { - [[unroll]] for (uint i = 0; i < iq2_grid.length(); i += wgsize.x) { - if (iq2xs_grid_const.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq2xs_grid_const.length()) { - iq2_grid[i + gl_LocalInvocationIndex.x] = iq2xs_grid_const[i + gl_LocalInvocationIndex.x]; - } - } - barrier(); - } else if (MmTypeA == GGML_TYPE_IQ2_S) { - [[unroll]] for (uint i = 0; i < iq2_grid.length(); i += wgsize.x) { - if (iq2s_grid_const.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq2s_grid_const.length()) { - iq2_grid[i + gl_LocalInvocationIndex.x] = iq2s_grid_const[i + gl_LocalInvocationIndex.x]; - } - } - barrier(); - } else if (MmTypeA == GGML_TYPE_IQ3_XXS) { - [[unroll]] for (uint i = 0; i < iq3_grid.length(); i += wgsize.x) { - if (iq3xxs_grid_const.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq3xxs_grid_const.length()) { - iq3_grid[i + gl_LocalInvocationIndex.x] = iq3xxs_grid_const[i + gl_LocalInvocationIndex.x]; - } - } - barrier(); - } else if (MmTypeA == GGML_TYPE_IQ3_S) { - [[unroll]] for (uint i = 0; i < iq3_grid.length(); i += wgsize.x) { - if (iq3s_grid_const.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq3s_grid_const.length()) { - iq3_grid[i + gl_LocalInvocationIndex.x] = iq3s_grid_const[i + gl_LocalInvocationIndex.x]; - } - } - barrier(); - } else if (MmTypeA == GGML_TYPE_IQ4_XS || MmTypeA == GGML_TYPE_IQ4_NL) { - for (uint i = gl_LocalInvocationIndex.x; i < kvalues_iq4nl.length(); i += wgsize.x) { - kvalues_iq4nl[i] = FLOAT_TYPE(kvalues_iq4nl_const[i]); - } - barrier(); - } -#if !defined(USE_OCP_FP4) - else if (MmTypeA == GGML_TYPE_MXFP4 || MmTypeA == GGML_TYPE_NVFP4) { - for (uint i = gl_LocalInvocationIndex.x; i < kvalues_mxfp4.length(); i += wgsize.x) { - kvalues_mxfp4[i] = kvalues_mxfp4_const[i]; - } - if (MmTypeA == GGML_TYPE_NVFP4) { - for (uint i = gl_LocalInvocationIndex.x; i < 128u; i += wgsize.x) { - ue4m3_fp32_lut[i] = ue4m3_to_fp32_build(i); - } - } - barrier(); - } -#endif } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp index 7540fd8711a..40dbbcdc64b 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp @@ -58,11 +58,6 @@ uint mm_load_vec_a() { case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_1: - case GGML_TYPE_IQ1_S: - case GGML_TYPE_IQ1_M: - case GGML_TYPE_IQ2_XXS: - case GGML_TYPE_IQ2_XS: - case GGML_TYPE_IQ2_S: return 8u; case GGML_TYPE_Q2_0: case GGML_TYPE_Q5_0: @@ -70,12 +65,6 @@ uint mm_load_vec_a() { case GGML_TYPE_Q2_K: case GGML_TYPE_Q4_K: case GGML_TYPE_Q5_K: - case GGML_TYPE_IQ3_XXS: - case GGML_TYPE_IQ3_S: - case GGML_TYPE_IQ4_XS: - case GGML_TYPE_IQ4_NL: - case GGML_TYPE_MXFP4: - case GGML_TYPE_NVFP4: return 4u; default: return 2u; @@ -114,31 +103,18 @@ layout (binding = 0) readonly buffer BUF_Q3_K { block_q3_K data[]; } a_q layout (binding = 0) readonly buffer BUF_Q4_K { block_q4_K data[]; } a_q4_k; layout (binding = 0) readonly buffer BUF_Q5_K { block_q5_K data[]; } a_q5_k; layout (binding = 0) readonly buffer BUF_Q6_K { block_q6_K data[]; } a_q6_k; -layout (binding = 0) readonly buffer BUF_IQ1_S { block_iq1_s data[]; } a_iq1_s; -layout (binding = 0) readonly buffer BUF_IQ1_M { block_iq1_m data[]; } a_iq1_m; -layout (binding = 0) readonly buffer BUF_IQ2_XXS { block_iq2_xxs data[]; } a_iq2_xxs; -layout (binding = 0) readonly buffer BUF_IQ2_XS { block_iq2_xs data[]; } a_iq2_xs; -layout (binding = 0) readonly buffer BUF_IQ2_S { block_iq2_s data[]; } a_iq2_s; -layout (binding = 0) readonly buffer BUF_IQ3_XXS { block_iq3_xxs data[]; } a_iq3_xxs; -layout (binding = 0) readonly buffer BUF_IQ3_S { block_iq3_s data[]; } a_iq3_s; -layout (binding = 0) readonly buffer BUF_IQ4_XS { block_iq4_xs data[]; } a_iq4_xs; -layout (binding = 0) readonly buffer BUF_MXFP4 { block_mxfp4 data[]; } a_mxfp4; -layout (binding = 0) readonly buffer BUF_NVFP4 { block_nvfp4 data[]; } a_nvfp4; // Packed16 aliases layout (binding = 0) readonly buffer BUF_Q4_0_P16 { block_q4_0_packed16 data[]; } a_q4_0_p16; layout (binding = 0) readonly buffer BUF_Q5_0_P16 { block_q5_0_packed16 data[]; } a_q5_0_p16; layout (binding = 0) readonly buffer BUF_Q8_0_P16 { block_q8_0_packed16 data[]; } a_q8_0_p16; layout (binding = 0) readonly buffer BUF_Q3_K_P16 { block_q3_K_packed16 data[]; } a_q3_k_p16; layout (binding = 0) readonly buffer BUF_Q6_K_P16 { block_q6_K_packed16 data[]; } a_q6_k_p16; -layout (binding = 0) readonly buffer BUF_IQ3_XXS_P16{ block_iq3_xxs_packed16 data[];} a_iq3_xxs_p16; -layout (binding = 0) readonly buffer BUF_IQ4_NL_P16 { block_iq4_nl_packed16 data[]; } a_iq4_nl_p16; // Packed32 aliases layout (binding = 0) readonly buffer BUF_Q4_1_P32 { block_q4_1_packed32 data[]; } a_q4_1_p32; layout (binding = 0) readonly buffer BUF_Q5_1_P32 { block_q5_1_packed32 data[]; } a_q5_1_p32; layout (binding = 0) readonly buffer BUF_Q2_K_P32 { block_q2_K_packed32 data[]; } a_q2_k_p32; layout (binding = 0) readonly buffer BUF_Q4_K_P32 { block_q4_K_packed32 data[]; } a_q4_k_p32; layout (binding = 0) readonly buffer BUF_Q5_K_P32 { block_q5_K_packed32 data[]; } a_q5_k_p32; -layout (binding = 0) readonly buffer BUF_IQ4_XS_P32 { block_iq4_xs_packed32 data[]; } a_iq4_xs_p32; #endif layout (binding = 1) readonly buffer B {B_TYPE data_b[];}; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp index b31e929d872..35ba6d7b463 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp @@ -50,14 +50,13 @@ layout (constant_id = 6) const uint subgroup_size = 32; uint mm_quant_k() { switch (MmTypeA) { case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: case GGML_TYPE_Q5_1: - case GGML_TYPE_Q8_0: case GGML_TYPE_IQ4_NL: case GGML_TYPE_MXFP4: + case GGML_TYPE_Q8_0: return 32u; case GGML_TYPE_Q1_0: return 128u; case GGML_TYPE_Q2_0: - case GGML_TYPE_NVFP4: return 64u; - default: // K-quants and IQ types + default: return 256u; } } @@ -113,17 +112,6 @@ layout (binding = 0) readonly buffer BUF_Q3_K { block_q3_K data[]; } a_q3_ layout (binding = 0) readonly buffer BUF_Q4_K { block_q4_K data[]; } a_q4_k; layout (binding = 0) readonly buffer BUF_Q5_K { block_q5_K data[]; } a_q5_k; layout (binding = 0) readonly buffer BUF_Q6_K { block_q6_K data[]; } a_q6_k; -layout (binding = 0) readonly buffer BUF_IQ1_S { block_iq1_s data[]; } a_iq1_s; -layout (binding = 0) readonly buffer BUF_IQ1_M { block_iq1_m data[]; } a_iq1_m; -layout (binding = 0) readonly buffer BUF_IQ2_XXS { block_iq2_xxs data[]; } a_iq2_xxs; -layout (binding = 0) readonly buffer BUF_IQ2_XS { block_iq2_xs data[]; } a_iq2_xs; -layout (binding = 0) readonly buffer BUF_IQ2_S { block_iq2_s data[]; } a_iq2_s; -layout (binding = 0) readonly buffer BUF_IQ3_XXS { block_iq3_xxs data[]; } a_iq3_xxs; -layout (binding = 0) readonly buffer BUF_IQ3_S { block_iq3_s data[]; } a_iq3_s; -layout (binding = 0) readonly buffer BUF_IQ4_XS { block_iq4_xs data[]; } a_iq4_xs; -layout (binding = 0) readonly buffer BUF_IQ4_NL { block_iq4_nl data[]; } a_iq4_nl; -layout (binding = 0) readonly buffer BUF_MXFP4 { block_mxfp4 data[]; } a_mxfp4; -layout (binding = 0) readonly buffer BUF_NVFP4 { block_nvfp4 data[]; } a_nvfp4; #endif layout (binding = 1) readonly buffer B {B_TYPE data_b[];}; layout (binding = 2) writeonly buffer D {D_TYPE data_d[];}; @@ -158,18 +146,7 @@ layout (binding = 1) readonly buffer B4 {B_TYPEV4 data_b_v4[];}; else if (MmTypeA == GGML_TYPE_Q3_K) { coopMatLoadTensorNV(mat, a_q3_k.data, pos_a, tensor_layout, dequantFuncQ3_K, dequantFuncQ3_K_v); } \ else if (MmTypeA == GGML_TYPE_Q4_K) { coopMatLoadTensorNV(mat, a_q4_k.data, pos_a, tensor_layout, dequantFuncQ4_K, dequantFuncQ4_K_v); } \ else if (MmTypeA == GGML_TYPE_Q5_K) { coopMatLoadTensorNV(mat, a_q5_k.data, pos_a, tensor_layout, dequantFuncQ5_K, dequantFuncQ5_K_v); } \ - else if (MmTypeA == GGML_TYPE_Q6_K) { coopMatLoadTensorNV(mat, a_q6_k.data, pos_a, tensor_layout, dequantFuncQ6_K, dequantFuncQ6_K_v); } \ - else if (MmTypeA == GGML_TYPE_IQ1_S) { coopMatLoadTensorNV(mat, a_iq1_s.data, pos_a, tensor_layout, dequantFuncIQ1_S, dequantFuncIQ1_S_v); } \ - else if (MmTypeA == GGML_TYPE_IQ1_M) { coopMatLoadTensorNV(mat, a_iq1_m.data, pos_a, tensor_layout, dequantFuncIQ1_M, dequantFuncIQ1_M_v); } \ - else if (MmTypeA == GGML_TYPE_IQ2_XXS) { coopMatLoadTensorNV(mat, a_iq2_xxs.data, pos_a, tensor_layout, dequantFuncIQ2_XXS, dequantFuncIQ2_XXS_v); } \ - else if (MmTypeA == GGML_TYPE_IQ2_XS) { coopMatLoadTensorNV(mat, a_iq2_xs.data, pos_a, tensor_layout, dequantFuncIQ2_XS, dequantFuncIQ2_XS_v); } \ - else if (MmTypeA == GGML_TYPE_IQ2_S) { coopMatLoadTensorNV(mat, a_iq2_s.data, pos_a, tensor_layout, dequantFuncIQ2_S, dequantFuncIQ2_S_v); } \ - else if (MmTypeA == GGML_TYPE_IQ3_XXS) { coopMatLoadTensorNV(mat, a_iq3_xxs.data, pos_a, tensor_layout, dequantFuncIQ3_XXS, dequantFuncIQ3_XXS_v); } \ - else if (MmTypeA == GGML_TYPE_IQ3_S) { coopMatLoadTensorNV(mat, a_iq3_s.data, pos_a, tensor_layout, dequantFuncIQ3_S, dequantFuncIQ3_S_v); } \ - else if (MmTypeA == GGML_TYPE_IQ4_XS) { coopMatLoadTensorNV(mat, a_iq4_xs.data, pos_a, tensor_layout, dequantFuncIQ4_XS, dequantFuncIQ4_XS_v); } \ - else if (MmTypeA == GGML_TYPE_IQ4_NL) { coopMatLoadTensorNV(mat, a_iq4_nl.data, pos_a, tensor_layout, dequantFuncIQ4_NL, dequantFuncIQ4_NL_v); } \ - else if (MmTypeA == GGML_TYPE_MXFP4) { coopMatLoadTensorNV(mat, a_mxfp4.data, pos_a, tensor_layout, dequantFuncMXFP4, dequantFuncMXFP4_v); } \ - else if (MmTypeA == GGML_TYPE_NVFP4) { coopMatLoadTensorNV(mat, a_nvfp4.data, pos_a, tensor_layout, dequantFuncNVFP4, dequantFuncNVFP4_v); } + else if (MmTypeA == GGML_TYPE_Q6_K) { coopMatLoadTensorNV(mat, a_q6_k.data, pos_a, tensor_layout, dequantFuncQ6_K, dequantFuncQ6_K_v); } #else #define COOPMAT_LOAD_A(mat, tensor_layout) \ if (MmTypeA == GGML_TYPE_Q1_0) { coopMatLoadTensorNV(mat, a_q1_0.data, pos_a, tensor_layout, dequantFuncQ1_0); } \ @@ -184,18 +161,7 @@ layout (binding = 1) readonly buffer B4 {B_TYPEV4 data_b_v4[];}; else if (MmTypeA == GGML_TYPE_Q3_K) { coopMatLoadTensorNV(mat, a_q3_k.data, pos_a, tensor_layout, dequantFuncQ3_K); } \ else if (MmTypeA == GGML_TYPE_Q4_K) { coopMatLoadTensorNV(mat, a_q4_k.data, pos_a, tensor_layout, dequantFuncQ4_K); } \ else if (MmTypeA == GGML_TYPE_Q5_K) { coopMatLoadTensorNV(mat, a_q5_k.data, pos_a, tensor_layout, dequantFuncQ5_K); } \ - else if (MmTypeA == GGML_TYPE_Q6_K) { coopMatLoadTensorNV(mat, a_q6_k.data, pos_a, tensor_layout, dequantFuncQ6_K); } \ - else if (MmTypeA == GGML_TYPE_IQ1_S) { coopMatLoadTensorNV(mat, a_iq1_s.data, pos_a, tensor_layout, dequantFuncIQ1_S); } \ - else if (MmTypeA == GGML_TYPE_IQ1_M) { coopMatLoadTensorNV(mat, a_iq1_m.data, pos_a, tensor_layout, dequantFuncIQ1_M); } \ - else if (MmTypeA == GGML_TYPE_IQ2_XXS) { coopMatLoadTensorNV(mat, a_iq2_xxs.data, pos_a, tensor_layout, dequantFuncIQ2_XXS); } \ - else if (MmTypeA == GGML_TYPE_IQ2_XS) { coopMatLoadTensorNV(mat, a_iq2_xs.data, pos_a, tensor_layout, dequantFuncIQ2_XS); } \ - else if (MmTypeA == GGML_TYPE_IQ2_S) { coopMatLoadTensorNV(mat, a_iq2_s.data, pos_a, tensor_layout, dequantFuncIQ2_S); } \ - else if (MmTypeA == GGML_TYPE_IQ3_XXS) { coopMatLoadTensorNV(mat, a_iq3_xxs.data, pos_a, tensor_layout, dequantFuncIQ3_XXS); } \ - else if (MmTypeA == GGML_TYPE_IQ3_S) { coopMatLoadTensorNV(mat, a_iq3_s.data, pos_a, tensor_layout, dequantFuncIQ3_S); } \ - else if (MmTypeA == GGML_TYPE_IQ4_XS) { coopMatLoadTensorNV(mat, a_iq4_xs.data, pos_a, tensor_layout, dequantFuncIQ4_XS); } \ - else if (MmTypeA == GGML_TYPE_IQ4_NL) { coopMatLoadTensorNV(mat, a_iq4_nl.data, pos_a, tensor_layout, dequantFuncIQ4_NL); } \ - else if (MmTypeA == GGML_TYPE_MXFP4) { coopMatLoadTensorNV(mat, a_mxfp4.data, pos_a, tensor_layout, dequantFuncMXFP4); } \ - else if (MmTypeA == GGML_TYPE_NVFP4) { coopMatLoadTensorNV(mat, a_nvfp4.data, pos_a, tensor_layout, dequantFuncNVFP4); } + else if (MmTypeA == GGML_TYPE_Q6_K) { coopMatLoadTensorNV(mat, a_q6_k.data, pos_a, tensor_layout, dequantFuncQ6_K); } #endif #endif #else diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl index c29219d3f46..2c5d69c1fd3 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl @@ -69,6 +69,276 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uin } else { store_a(col, row, FLOAT_TYPEV2(0.0f)); } +#elif defined(DATA_A_IQ1_S) + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; + + const uint ib = idx / 32; + const uint ib32 = (idx % 32) / 4; + const uint ib8 = idx % 32; + + const float d = float(data_a[ib].d); + const uint qh = data_a[ib].qh[ib32]; + const uint qs = data_a[ib].qs[ib8]; + const float dl = d * (2 * bitfieldExtract(qh, 12, 3) + 1); + const float delta = ((qh & 0x8000) != 0) ? -IQ1S_DELTA : IQ1S_DELTA; + const int16_t grid = int16_t(iq1s_grid[qs | (bitfieldExtract(qh, 3 * int(ib8 & 3), 3) << 8)]); + + [[unroll]] for (int k = 0; k < 4; ++k) { + store_a(col, k_pair + k, FLOAT_TYPEV2(dl * (bitfieldExtract(grid, 4 * k , 2) + delta), + dl * (bitfieldExtract(grid, 4 * k + 2, 2) + delta))); + + } +#elif defined(DATA_A_IQ1_M) + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; + + const uint ib = idx / 32; + const uint ib8 = idx % 32; + const uint ib16 = ib8 / 2; + + const uint16_t[4] scales = data_a[ib].scales; + const u16vec4 s = u16vec4(scales[0], scales[1], scales[2], scales[3]) >> 12; + const float d = float(unpackHalf2x16(s.x | (s.y << 4) | (s.z << 8) | (s.w << 12)).x); + const uint sc = scales[ib8 / 8]; + const uint qs = data_a[ib].qs[ib8]; + const uint qh = data_a[ib].qh[ib16] >> (4 * (ib8 & 1)); + const float dl = d * (2 * bitfieldExtract(sc, 3 * int(ib16 & 3), 3) + 1); + const float delta = ((qh & 8) != 0) ? -IQ1M_DELTA : IQ1M_DELTA; + const int16_t grid = int16_t(iq1s_grid[qs | ((qh & 7) << 8)]); + + [[unroll]] for (int k = 0; k < 4; ++k) { + store_a(col, k_pair + k, FLOAT_TYPEV2(dl * (bitfieldExtract(grid, 4 * k , 2) + delta), + dl * (bitfieldExtract(grid, 4 * k + 2, 2) + delta))); + + } +#elif defined(DATA_A_IQ2_XXS) + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; + + const uint ib = idx / 32; + const uint ib32 = (idx % 32) / 4; + const uint ib8 = idx % 4; + + const float d = float(data_a[ib].d); + const uint qs = data_a[ib].qs[8 * ib32 + ib8]; + const uint signs = pack32(u8vec4( + data_a[ib].qs[8*ib32 + 4], + data_a[ib].qs[8*ib32 + 5], + data_a[ib].qs[8*ib32 + 6], + data_a[ib].qs[8*ib32 + 7] + )); + const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + (signs >> 28))); + const uint32_t sign7 = bitfieldExtract(signs, 7 * int(ib8), 7); + const uint sign = sign7 | (bitCount(sign7) << 7); + const uvec2 grid = iq2_grid[qs]; + const vec4 grid0 = vec4(unpack8(grid.x)); + const vec4 grid1 = vec4(unpack8(grid.y)); + + store_a(col, k_pair, db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x, + (sign & 2) != 0 ? -grid0.y : grid0.y)); + + store_a(col, k_pair + 1, db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z, + (sign & 8) != 0 ? -grid0.w : grid0.w)); + + store_a(col, k_pair + 2, db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x, + (sign & 32) != 0 ? -grid1.y : grid1.y)); + + store_a(col, k_pair + 3, db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z, + (sign & 128) != 0 ? -grid1.w : grid1.w)); + +#elif defined(DATA_A_IQ2_XS) + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; + + const uint ib = idx / 32; + const uint ib32 = (idx % 32) / 4; + const uint ib8 = idx % 4; + + const float d = float(data_a[ib].d); + const uint scale = (data_a[ib].scales[ib32] >> (2 * (ib8 & 2))) & 0xf; + const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + scale)); + const uint qs = data_a[ib].qs[4 * ib32 + ib8]; + const uint sign7 = qs >> 9; + const uint sign = sign7 | (bitCount(sign7) << 7); + const uvec2 grid = iq2_grid[qs & 511]; + const vec4 grid0 = vec4(unpack8(grid.x)); + const vec4 grid1 = vec4(unpack8(grid.y)); + + store_a(col, k_pair, db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x, + (sign & 2) != 0 ? -grid0.y : grid0.y)); + + store_a(col, k_pair + 1, db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z, + (sign & 8) != 0 ? -grid0.w : grid0.w)); + + store_a(col, k_pair + 2, db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x, + (sign & 32) != 0 ? -grid1.y : grid1.y)); + + store_a(col, k_pair + 3, db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z, + (sign & 128) != 0 ? -grid1.w : grid1.w)); + +#elif defined(DATA_A_IQ2_S) + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; + + const uint ib = idx / 32; + const uint ib8 = idx % 32; + const uint ib32 = ib8 / 4; + + const uint scale = (data_a[ib].scales[ib32] >> (2 * (ib8 & 2))) & 0xf; + const uint qs = data_a[ib].qs[ib8]; + const uint qh = data_a[ib].qh[ib32]; + const uint qhshift = 2 * (ib8 % 4); + const uint sign = data_a[ib].qs[QUANT_K_IQ2_S / 8 + ib8]; + + const float d = float(data_a[ib].d); + const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + scale)); + const uvec2 grid = iq2_grid[qs | ((qh << (8 - qhshift)) & 0x300)]; + const vec4 grid0 = vec4(unpack8(grid.x)); + const vec4 grid1 = vec4(unpack8(grid.y)); + + store_a(col, k_pair, db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x, + (sign & 2) != 0 ? -grid0.y : grid0.y)); + + store_a(col, k_pair + 1, db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z, + (sign & 8) != 0 ? -grid0.w : grid0.w)); + + store_a(col, k_pair + 2, db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x, + (sign & 32) != 0 ? -grid1.y : grid1.y)); + + store_a(col, k_pair + 3, db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z, + (sign & 128) != 0 ? -grid1.w : grid1.w)); + +#elif defined(DATA_A_IQ3_XXS) + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; + + const uint ib = idx / 64; + const uint iqs = idx % 64; + const uint is = QUANT_K_IQ3_XXS / 4 + 4 * (iqs / 8); + + const float d = float(data_a[ib].d); + const uint qs = data_a[ib].qs[iqs]; + const uint signs = pack32(u16vec2( + data_a_packed16[ib].qs[is/2], + data_a_packed16[ib].qs[is/2+1] + )); + const float db = d * 0.5 * (0.5 + (signs >> 28)); + const uint32_t sign7 = bitfieldExtract(signs, 7 * (int(iqs / 2) % 4), 7); + const uint sign = (sign7 | (bitCount(sign7) << 7)) >> (4 * (idx % 2)); + const uint grid = iq3_grid[qs]; + const vec4 v = db * vec4(unpack8(grid)); + + store_a(col, k_pair, FLOAT_TYPEV2((sign & 1) != 0 ? -v.x : v.x, + (sign & 2) != 0 ? -v.y : v.y)); + + store_a(col, k_pair + 1, FLOAT_TYPEV2((sign & 4) != 0 ? -v.z : v.z, + (sign & 8) != 0 ? -v.w : v.w)); + +#elif defined(DATA_A_IQ3_S) + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; + + const uint ib = idx / 64; + const uint iqs = idx % 64; + const uint iqh = iqs / 8; + + const float d = float(data_a[ib].d); + const uint qs = data_a[ib].qs[iqs]; + const uint qh = data_a[ib].qh[iqh]; + const int8_t sign = int8_t(data_a[ib].signs[iqs / 2] >> (4 * (idx % 2))); + const uint scale = data_a[ib].scales[iqs / 16]; + const i8vec2 sign01 = i8vec2(1 - (2 & i8vec2(sign << 1, sign))); + const float db = d * (1 + 2 * ((scale >> (4 * (iqh & 1))) & 0xf)); + const uint32_t grid = iq3_grid[qs | ((qh << (8 - (iqs % 8))) & 256)]; + const vec4 v = db * vec4(unpack8(grid)); + + store_a(col, k_pair, FLOAT_TYPEV2((sign & 1) != 0 ? -v.x : v.x, + (sign & 2) != 0 ? -v.y : v.y)); + + store_a(col, k_pair + 1, FLOAT_TYPEV2((sign & 4) != 0 ? -v.z : v.z, + (sign & 8) != 0 ? -v.w : v.w)); + +#elif defined(DATA_A_IQ4_XS) + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; + + const uint ib = idx / 64; + const uint ib32 = (idx % 64) / 8; + const uint iq = 4 * ib32 + (idx % 4); + + const uint sl = (data_a[ib].scales_l[ib32/2] >> (4 * (ib32 & 1))) & 0xF; + const uint sh = ((data_a[ib].scales_h) >> (2 * ib32)) & 3; + const uint qshift = idx & 4; + u8vec4 qs = unpack8((uint(data_a_packed32[ib].qs[iq]) >> qshift) & 0x0F0F0F0F); + + const float d = float(data_a[ib].d); + const vec4 v = d * float(int(sl | (sh << 4)) - 32) * vec4(kvalues_iq4nl[qs.x], kvalues_iq4nl[qs.y], kvalues_iq4nl[qs.z], kvalues_iq4nl[qs.w]); + + store_a(col, k_pair, FLOAT_TYPEV2(v.xy)); + store_a(col, k_pair + 1, FLOAT_TYPEV2(v.zw)); +#elif defined(DATA_A_IQ4_NL) + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 4; + + const uint ib = idx / 8; + const uint iqs = idx & 0x07; + + const FLOAT_TYPE d = FLOAT_TYPE(data_a_packed16[ib].d); + const uint vui = uint(data_a_packed16[ib].qs[iqs]); + + store_a(col, k_pair, d * FLOAT_TYPEV2(kvalues_iq4nl[vui & 0xF], + kvalues_iq4nl[bitfieldExtract(vui, 8, 4)])); + + store_a(col, k_pair + 8, d * FLOAT_TYPEV2(kvalues_iq4nl[bitfieldExtract(vui, 4, 4)], + kvalues_iq4nl[vui >> 12])); + +#elif defined(DATA_A_MXFP4) + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 4; + + const uint ib = idx / 8; + const uint iqs = (idx & 0x07) * 2; + + const uint vui = uint(data_a[ib].qs[iqs]); + const uint vui2 = uint(data_a[ib].qs[iqs+1]); + +#ifdef USE_OCP_FP4 + const float d = e8m0_to_fp32(data_a[ib].e); + const u8vec2 packed = u8vec2(vui, vui2); + store_a(col, k_pair, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 0u)) * FLOAT_TYPE(d)); + store_a(col, k_pair + 8, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 4u)) * FLOAT_TYPE(d)); +#else + const float d = e8m0_to_fp32(data_a[ib].e) * 0.5; + store_a(col, k_pair, FLOAT_TYPEV2(kvalues_mxfp4[vui & 0xF] * d, + kvalues_mxfp4[vui2 & 0xF] * d)); + + store_a(col, k_pair + 8, FLOAT_TYPEV2(kvalues_mxfp4[vui >> 4] * d, + kvalues_mxfp4[vui2 >> 4] * d)); + +#endif +#elif defined(DATA_A_NVFP4) + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint eff_row = (row & 3) + (row & ~3) * 2; + + const uint ib = idx / 16u; + const uint sub = (idx & 0xC) >> 2; + const uint iqs = (idx & 0xF) * 2; + const uint vui = uint(data_a[ib].qs[iqs]); + const uint vui2 = uint(data_a[ib].qs[iqs+1]); + +#ifdef USE_OCP_FP4 + const FLOAT_TYPE d = FLOAT_TYPE(ue4m3_from_bits(data_a[ib].d[sub])); + const u8vec2 packed = u8vec2(vui, vui2); + store_a(col, eff_row, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 0u)) * d); + store_a(col, eff_row + 4, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 4u)) * d); +#else + const float d = ue4m3_to_fp32(data_a[ib].d[sub]) * 0.5; + store_a(col, eff_row, FLOAT_TYPEV2(kvalues_mxfp4[vui & 0xF] * d, + kvalues_mxfp4[vui2 & 0xF] * d)); + store_a(col, eff_row + 4, FLOAT_TYPEV2(kvalues_mxfp4[vui >> 4] * d, + kvalues_mxfp4[vui2 >> 4] * d)); +#endif #else if (MmTypeA == GGML_TYPE_Q4_0) { const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; @@ -338,279 +608,7 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uin const vec2 q = (vec2(unpack8(ql | (qh << 4)).xy) - 32) * dscale; store_a(col, k_pair, FLOAT_TYPEV2(q.x, q.y)); - } else if (MmTypeA == GGML_TYPE_IQ1_S) { - const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; - const uint k_pair = row * mm_load_vec_a() / 2; - - const uint ib = idx / 32; // 8 values per idx - const uint ib32 = (idx % 32) / 4; // 0..7 - const uint ib8 = idx % 32; - - const float d = float(a_iq1_s.data[ib].d); - const uint qh = a_iq1_s.data[ib].qh[ib32]; - const uint qs = a_iq1_s.data[ib].qs[ib8]; - const float dl = d * (2 * bitfieldExtract(qh, 12, 3) + 1); - const float delta = ((qh & 0x8000) != 0) ? -IQ1S_DELTA : IQ1S_DELTA; - const int16_t grid = int16_t(iq1s_grid[qs | (bitfieldExtract(qh, 3 * int(ib8 & 3), 3) << 8)]); - - [[unroll]] for (int k = 0; k < 4; ++k) { - store_a(col, k_pair + k, FLOAT_TYPEV2(dl * (bitfieldExtract(grid, 4 * k , 2) + delta), - dl * (bitfieldExtract(grid, 4 * k + 2, 2) + delta))); - - } - } else if (MmTypeA == GGML_TYPE_IQ1_M) { - const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; - const uint k_pair = row * mm_load_vec_a() / 2; - - const uint ib = idx / 32; // 8 values per idx - const uint ib8 = idx % 32; - const uint ib16 = ib8 / 2; - - const uint16_t[4] scales = a_iq1_m.data[ib].scales; - const u16vec4 s = u16vec4(scales[0], scales[1], scales[2], scales[3]) >> 12; - const float d = float(unpackHalf2x16(s.x | (s.y << 4) | (s.z << 8) | (s.w << 12)).x); - const uint sc = scales[ib8 / 8]; - const uint qs = a_iq1_m.data[ib].qs[ib8]; - const uint qh = a_iq1_m.data[ib].qh[ib16] >> (4 * (ib8 & 1)); - const float dl = d * (2 * bitfieldExtract(sc, 3 * int(ib16 & 3), 3) + 1); - const float delta = ((qh & 8) != 0) ? -IQ1M_DELTA : IQ1M_DELTA; - const int16_t grid = int16_t(iq1s_grid[qs | ((qh & 7) << 8)]); - - [[unroll]] for (int k = 0; k < 4; ++k) { - store_a(col, k_pair + k, FLOAT_TYPEV2(dl * (bitfieldExtract(grid, 4 * k , 2) + delta), - dl * (bitfieldExtract(grid, 4 * k + 2, 2) + delta))); - - } - } else if (MmTypeA == GGML_TYPE_IQ2_XXS) { - const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; - const uint k_pair = row * mm_load_vec_a() / 2; - - const uint ib = idx / 32; // 8 values per idx - const uint ib32 = (idx % 32) / 4; // 0..7 - const uint ib8 = idx % 4; - - const float d = float(a_iq2_xxs.data[ib].d); - const uint qs = a_iq2_xxs.data[ib].qs[8 * ib32 + ib8]; - const uint signs = pack32(u8vec4( - a_iq2_xxs.data[ib].qs[8*ib32 + 4], - a_iq2_xxs.data[ib].qs[8*ib32 + 5], - a_iq2_xxs.data[ib].qs[8*ib32 + 6], - a_iq2_xxs.data[ib].qs[8*ib32 + 7] - )); - const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + (signs >> 28))); - const uint32_t sign7 = bitfieldExtract(signs, 7 * int(ib8), 7); - const uint sign = sign7 | (bitCount(sign7) << 7); - const uvec2 grid = iq2_grid[qs]; - const vec4 grid0 = vec4(unpack8(grid.x)); - const vec4 grid1 = vec4(unpack8(grid.y)); - - store_a(col, k_pair, db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x, - (sign & 2) != 0 ? -grid0.y : grid0.y)); - - store_a(col, k_pair + 1, db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z, - (sign & 8) != 0 ? -grid0.w : grid0.w)); - - store_a(col, k_pair + 2, db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x, - (sign & 32) != 0 ? -grid1.y : grid1.y)); - - store_a(col, k_pair + 3, db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z, - (sign & 128) != 0 ? -grid1.w : grid1.w)); - - } else if (MmTypeA == GGML_TYPE_IQ2_XS) { - const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; - const uint k_pair = row * mm_load_vec_a() / 2; - - const uint ib = idx / 32; // 8 values per idx - const uint ib32 = (idx % 32) / 4; // 0..7 - const uint ib8 = idx % 4; // 0..3 - - const float d = float(a_iq2_xs.data[ib].d); - const uint scale = (a_iq2_xs.data[ib].scales[ib32] >> (2 * (ib8 & 2))) & 0xf; - const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + scale)); - const uint qs = a_iq2_xs.data[ib].qs[4 * ib32 + ib8]; - const uint sign7 = qs >> 9; - const uint sign = sign7 | (bitCount(sign7) << 7); - const uvec2 grid = iq2_grid[qs & 511]; - const vec4 grid0 = vec4(unpack8(grid.x)); - const vec4 grid1 = vec4(unpack8(grid.y)); - - store_a(col, k_pair, db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x, - (sign & 2) != 0 ? -grid0.y : grid0.y)); - - store_a(col, k_pair + 1, db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z, - (sign & 8) != 0 ? -grid0.w : grid0.w)); - - store_a(col, k_pair + 2, db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x, - (sign & 32) != 0 ? -grid1.y : grid1.y)); - - store_a(col, k_pair + 3, db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z, - (sign & 128) != 0 ? -grid1.w : grid1.w)); - - } else if (MmTypeA == GGML_TYPE_IQ2_S) { - const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; - const uint k_pair = row * mm_load_vec_a() / 2; - - const uint ib = idx / 32; // 8 values per idx - const uint ib8 = idx % 32; // 0..31 - const uint ib32 = ib8 / 4; // 0..7 - - const uint scale = (a_iq2_s.data[ib].scales[ib32] >> (2 * (ib8 & 2))) & 0xf; - const uint qs = a_iq2_s.data[ib].qs[ib8]; - const uint qh = a_iq2_s.data[ib].qh[ib32]; - const uint qhshift = 2 * (ib8 % 4); - const uint sign = a_iq2_s.data[ib].qs[QUANT_K_IQ2_S / 8 + ib8]; - - const float d = float(a_iq2_s.data[ib].d); - const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + scale)); - const uvec2 grid = iq2_grid[qs | ((qh << (8 - qhshift)) & 0x300)]; - const vec4 grid0 = vec4(unpack8(grid.x)); - const vec4 grid1 = vec4(unpack8(grid.y)); - - store_a(col, k_pair, db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x, - (sign & 2) != 0 ? -grid0.y : grid0.y)); - - store_a(col, k_pair + 1, db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z, - (sign & 8) != 0 ? -grid0.w : grid0.w)); - - store_a(col, k_pair + 2, db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x, - (sign & 32) != 0 ? -grid1.y : grid1.y)); - - store_a(col, k_pair + 3, db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z, - (sign & 128) != 0 ? -grid1.w : grid1.w)); - - } else if (MmTypeA == GGML_TYPE_IQ3_XXS) { - const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; - const uint k_pair = row * mm_load_vec_a() / 2; - - const uint ib = idx / 64; // 4 values per idx - const uint iqs = idx % 64; // 0..63 - const uint is = QUANT_K_IQ3_XXS / 4 + 4 * (iqs / 8); // 8 values - - const float d = float(a_iq3_xxs.data[ib].d); - const uint qs = a_iq3_xxs.data[ib].qs[iqs]; - const uint signs = pack32(u16vec2( - a_iq3_xxs_p16.data[ib].qs[is/2], - a_iq3_xxs_p16.data[ib].qs[is/2+1] - )); - const float db = d * 0.5 * (0.5 + (signs >> 28)); - const uint32_t sign7 = bitfieldExtract(signs, 7 * (int(iqs / 2) % 4), 7); - const uint sign = (sign7 | (bitCount(sign7) << 7)) >> (4 * (idx % 2)); - const uint grid = iq3_grid[qs]; - const vec4 v = db * vec4(unpack8(grid)); - - store_a(col, k_pair, FLOAT_TYPEV2((sign & 1) != 0 ? -v.x : v.x, - (sign & 2) != 0 ? -v.y : v.y)); - - store_a(col, k_pair + 1, FLOAT_TYPEV2((sign & 4) != 0 ? -v.z : v.z, - (sign & 8) != 0 ? -v.w : v.w)); - - } else if (MmTypeA == GGML_TYPE_IQ3_S) { - const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; - const uint k_pair = row * mm_load_vec_a() / 2; - - const uint ib = idx / 64; // 4 values per idx - const uint iqs = idx % 64; // 0..63 - const uint iqh = iqs / 8; - - const float d = float(a_iq3_s.data[ib].d); - const uint qs = a_iq3_s.data[ib].qs[iqs]; - const uint qh = a_iq3_s.data[ib].qh[iqh]; - const int8_t sign = int8_t(a_iq3_s.data[ib].signs[iqs / 2] >> (4 * (idx % 2))); - const uint scale = a_iq3_s.data[ib].scales[iqs / 16]; - const i8vec2 sign01 = i8vec2(1 - (2 & i8vec2(sign << 1, sign))); - const float db = d * (1 + 2 * ((scale >> (4 * (iqh & 1))) & 0xf)); - const uint32_t grid = iq3_grid[qs | ((qh << (8 - (iqs % 8))) & 256)]; - const vec4 v = db * vec4(unpack8(grid)); - - store_a(col, k_pair, FLOAT_TYPEV2((sign & 1) != 0 ? -v.x : v.x, - (sign & 2) != 0 ? -v.y : v.y)); - - store_a(col, k_pair + 1, FLOAT_TYPEV2((sign & 4) != 0 ? -v.z : v.z, - (sign & 8) != 0 ? -v.w : v.w)); - - } else if (MmTypeA == GGML_TYPE_IQ4_XS) { - const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; - const uint k_pair = row * mm_load_vec_a() / 2; - - const uint ib = idx / 64; // 4 values per idx - const uint ib32 = (idx % 64) / 8; // 0..7 - const uint iq = 4 * ib32 + (idx % 4); - - const uint sl = (a_iq4_xs.data[ib].scales_l[ib32/2] >> (4 * (ib32 & 1))) & 0xF; - const uint sh = ((a_iq4_xs.data[ib].scales_h) >> (2 * ib32)) & 3; - const uint qshift = idx & 4; - u8vec4 qs = unpack8((uint(a_iq4_xs_p32.data[ib].qs[iq]) >> qshift) & 0x0F0F0F0F); - - const float d = float(a_iq4_xs.data[ib].d); - const vec4 v = d * float(int(sl | (sh << 4)) - 32) * vec4(kvalues_iq4nl[qs.x], kvalues_iq4nl[qs.y], kvalues_iq4nl[qs.z], kvalues_iq4nl[qs.w]); - - store_a(col, k_pair, FLOAT_TYPEV2(v.xy)); - store_a(col, k_pair + 1, FLOAT_TYPEV2(v.zw)); - } else if (MmTypeA == GGML_TYPE_IQ4_NL) { - const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; - const uint k_pair = row * mm_load_vec_a() / 4; - - const uint ib = idx / 8; - const uint iqs = idx & 0x07; - - const FLOAT_TYPE d = FLOAT_TYPE(a_iq4_nl_p16.data[ib].d); - const uint vui = uint(a_iq4_nl_p16.data[ib].qs[iqs]); - - store_a(col, k_pair, d * FLOAT_TYPEV2(kvalues_iq4nl[vui & 0xF], - kvalues_iq4nl[bitfieldExtract(vui, 8, 4)])); - - store_a(col, k_pair + 8, d * FLOAT_TYPEV2(kvalues_iq4nl[bitfieldExtract(vui, 4, 4)], - kvalues_iq4nl[vui >> 12])); - - } else if (MmTypeA == GGML_TYPE_MXFP4) { - const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; - const uint k_pair = row * mm_load_vec_a() / 4; - - const uint ib = idx / 8; - const uint iqs = (idx & 0x07) * 2; - - const uint vui = uint(a_mxfp4.data[ib].qs[iqs]); - const uint vui2 = uint(a_mxfp4.data[ib].qs[iqs+1]); - -#ifdef USE_OCP_FP4 - const float d = e8m0_to_fp32(a_mxfp4.data[ib].e); - const u8vec2 packed = u8vec2(vui, vui2); - store_a(col, k_pair, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 0u)) * FLOAT_TYPE(d)); - store_a(col, k_pair + 8, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 4u)) * FLOAT_TYPE(d)); -#else - const float d = e8m0_to_fp32(a_mxfp4.data[ib].e) * 0.5; - store_a(col, k_pair, FLOAT_TYPEV2(kvalues_mxfp4[vui & 0xF] * d, - kvalues_mxfp4[vui2 & 0xF] * d)); - - store_a(col, k_pair + 8, FLOAT_TYPEV2(kvalues_mxfp4[vui >> 4] * d, - kvalues_mxfp4[vui2 >> 4] * d)); - -#endif - } else if (MmTypeA == GGML_TYPE_NVFP4) { - const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; - // lo and hi nibbles are 8 elements apart, which doesn't quite line up with - // how the thread mapping and buf_idx calculation works for other types. - const uint eff_row = (row & 3) + (row & ~3) * 2; - - const uint ib = idx / 16u; - const uint sub = (idx & 0xC) >> 2; - const uint iqs = (idx & 0xF) * 2; - const uint vui = uint(a_nvfp4.data[ib].qs[iqs]); - const uint vui2 = uint(a_nvfp4.data[ib].qs[iqs+1]); - -#ifdef USE_OCP_FP4 - const FLOAT_TYPE d = FLOAT_TYPE(ue4m3_from_bits(a_nvfp4.data[ib].d[sub])); - const u8vec2 packed = u8vec2(vui, vui2); - store_a(col, eff_row, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 0u)) * d); - store_a(col, eff_row + 4, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 4u)) * d); -#else - const float d = ue4m3_to_fp32(a_nvfp4.data[ib].d[sub]) * 0.5; - store_a(col, eff_row, FLOAT_TYPEV2(kvalues_mxfp4[vui & 0xF] * d, - kvalues_mxfp4[vui2 & 0xF] * d)); - store_a(col, eff_row + 4, FLOAT_TYPEV2(kvalues_mxfp4[vui >> 4] * d, - kvalues_mxfp4[vui2 >> 4] * d)); -#endif -} + } #endif } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl index b5936a3493b..c50f14426c9 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl @@ -515,7 +515,7 @@ struct block_iq1_m_packed64 { #define A_TYPE_PACKED32 block_iq1_m_packed32 #endif -#if defined(DATA_A_IQ1_S) || defined(DATA_A_IQ1_M) || defined(MULMAT_QUANT) +#if defined(DATA_A_IQ1_S) || defined(DATA_A_IQ1_M) #define IQ1S_DELTA 0.125f #define IQ1M_DELTA 0.125f @@ -915,17 +915,10 @@ const uint32_t[2048] iq1s_grid_gpu_const = { }; #endif -#ifdef MULMAT_QUANT -shared uint16_t iq1s_grid[(MmTypeA == GGML_TYPE_IQ1_S || MmTypeA == GGML_TYPE_IQ1_M) ? 2048 : 8]; -#if defined(NEEDS_IQ1S_GRID_GPU) -shared uint32_t iq1s_grid_gpu[(MmTypeA == GGML_TYPE_IQ1_S || MmTypeA == GGML_TYPE_IQ1_M) ? 2048 : 8]; -#endif -#else shared uint16_t iq1s_grid[2048]; #if defined(NEEDS_IQ1S_GRID_GPU) shared uint32_t iq1s_grid_gpu[2048]; #endif -#endif #if defined(DATA_A_IQ1_S) || defined(DATA_A_IQ1_M) #define NEEDS_INIT_IQ_SHMEM @@ -953,10 +946,8 @@ void init_iq_shmem(uvec3 wgsize) #endif #endif -#if defined(DATA_A_IQ2_XXS) || defined(DATA_A_IQ2_XS) || defined(DATA_A_IQ2_S) || defined(MULMAT_QUANT) -#ifdef MULMAT_QUANT -shared uvec2 iq2_grid[MmTypeA == GGML_TYPE_IQ2_S ? 1024 : MmTypeA == GGML_TYPE_IQ2_XS ? 512 : MmTypeA == GGML_TYPE_IQ2_XXS ? 256 : 8]; -#elif defined(DATA_A_IQ2_S) +#if defined(DATA_A_IQ2_XXS) || defined(DATA_A_IQ2_XS) || defined(DATA_A_IQ2_S) +#if defined(DATA_A_IQ2_S) shared uvec2 iq2_grid[1024]; #elif defined(DATA_A_IQ2_XS) shared uvec2 iq2_grid[512]; @@ -980,7 +971,7 @@ struct block_iq2_xxs_packed16 uint16_t qs[QUANT_K_IQ2_XXS/8]; }; -#if defined(DATA_A_IQ2_XXS) || defined(MULMAT_QUANT) +#if defined(DATA_A_IQ2_XXS) const uvec2[256] iq2xxs_grid_const = { uvec2(0x08080808, 0x08080808), uvec2(0x0808082b, 0x08080808), uvec2(0x08081919, 0x08080808), uvec2(0x08082b08, 0x08080808), @@ -1088,7 +1079,7 @@ struct block_iq2_xs_packed16 uint16_t scales[QUANT_K_IQ2_XS/64]; }; -#if defined(DATA_A_IQ2_XS) || defined(MULMAT_QUANT) +#if defined(DATA_A_IQ2_XS) const uvec2 iq2xs_grid_const[512] = { uvec2(0x08080808, 0x08080808), uvec2(0x0808082b, 0x08080808), uvec2(0x08081919, 0x08080808), uvec2(0x08082b08, 0x08080808), @@ -1262,7 +1253,7 @@ struct block_iq2_s_packed16 uint16_t scales[QUANT_K_IQ2_S/64]; }; -#if defined(DATA_A_IQ2_S) || defined(MULMAT_QUANT) +#if defined(DATA_A_IQ2_S) const uvec2 iq2s_grid_const[1024] = { uvec2(0x08080808, 0x08080808), uvec2(0x0808082b, 0x08080808), uvec2(0x08081919, 0x08080808), uvec2(0x08082b08, 0x08080808), @@ -1545,10 +1536,8 @@ void init_iq_shmem(uvec3 wgsize) #endif #endif -#if defined(DATA_A_IQ3_XXS) || defined(DATA_A_IQ3_S) || defined(MULMAT_QUANT) -#ifdef MULMAT_QUANT -shared uint32_t iq3_grid[MmTypeA == GGML_TYPE_IQ3_S ? 512 : MmTypeA == GGML_TYPE_IQ3_XXS ? 256 : 8]; -#elif defined(DATA_A_IQ3_S) +#if defined(DATA_A_IQ3_XXS) || defined(DATA_A_IQ3_S) +#if defined(DATA_A_IQ3_S) shared uint32_t iq3_grid[512]; #else shared uint32_t iq3_grid[256]; @@ -1570,7 +1559,7 @@ struct block_iq3_xxs_packed16 uint16_t qs[QUANT_K_IQ3_XXS/8 + QUANT_K_IQ3_XXS/16]; }; -#if defined(DATA_A_IQ3_XXS) || defined(MULMAT_QUANT) +#if defined(DATA_A_IQ3_XXS) const uint32_t iq3xxs_grid_const[256] = { 0x04040404, 0x04040414, 0x04040424, 0x04040c0c, 0x04040c1c, 0x04040c3e, 0x04041404, 0x04041414, @@ -1650,7 +1639,7 @@ struct block_iq3_s_packed16 uint16_t scales[QUANT_K_IQ3_S/64/2]; }; -#if defined(DATA_A_IQ3_S) || defined(MULMAT_QUANT) +#if defined(DATA_A_IQ3_S) const uint32_t iq3s_grid_const[512] = { 0x01010101, 0x01010103, 0x01010105, 0x0101010b, 0x0101010f, 0x01010301, 0x01010303, 0x01010305, @@ -1845,17 +1834,13 @@ struct block_nvfp4_packed32 #define A_TYPE_PACKED32 block_nvfp4_packed32 #endif -#if defined(DATA_A_IQ4_NL) || defined(DATA_A_IQ4_XS) || defined(MULMAT_QUANT) +#if defined(DATA_A_IQ4_NL) || defined(DATA_A_IQ4_XS) const int8_t kvalues_iq4nl_const[16] = { int8_t(-127), int8_t(-104), int8_t(-83), int8_t(-65), int8_t(-49), int8_t(-35), int8_t(-22), int8_t(-10), int8_t(1), int8_t(13), int8_t(25), int8_t(38), int8_t(53), int8_t(69), int8_t(89), int8_t(113) }; -#ifdef MULMAT_QUANT -shared FLOAT_TYPE kvalues_iq4nl[(MmTypeA == GGML_TYPE_IQ4_NL || MmTypeA == GGML_TYPE_IQ4_XS) ? 16 : 8]; -#else shared FLOAT_TYPE kvalues_iq4nl[16]; -#endif #if defined(DATA_A_IQ4_NL) || defined(DATA_A_IQ4_XS) #define NEEDS_INIT_IQ_SHMEM @@ -1870,26 +1855,18 @@ void init_iq_shmem(uvec3 wgsize) #endif #endif -#if defined(DATA_A_MXFP4) || defined(DATA_A_NVFP4) || defined(MULMAT_QUANT) +#if defined(DATA_A_MXFP4) || defined(DATA_A_NVFP4) #if !defined(USE_OCP_FP4) const int8_t kvalues_mxfp4_const[16] = { int8_t(0), int8_t(1), int8_t(2), int8_t(3), int8_t(4), int8_t(6), int8_t(8), int8_t(12), int8_t(0), int8_t(-1), int8_t(-2), int8_t(-3), int8_t(-4), int8_t(-6), int8_t(-8), int8_t(-12), }; -#ifdef MULMAT_QUANT -shared int8_t kvalues_mxfp4[(MmTypeA == GGML_TYPE_MXFP4 || MmTypeA == GGML_TYPE_NVFP4) ? 16 : 8]; -#else shared int8_t kvalues_mxfp4[16]; #endif -#endif -#if (defined(DATA_A_NVFP4) || defined(MULMAT_QUANT)) && !defined(USE_OCP_FP4) -#ifdef MULMAT_QUANT -shared float ue4m3_fp32_lut[MmTypeA == GGML_TYPE_NVFP4 ? 128 : 8]; -#else +#if defined(DATA_A_NVFP4) && !defined(USE_OCP_FP4) shared float ue4m3_fp32_lut[128]; -#endif float ue4m3_to_fp32_build(uint u) { if (u == 0u || u == 127u) { @@ -1955,7 +1932,7 @@ float e8m0_to_fp32(uint8_t x) { return uintBitsToFloat(bits); } -#if defined(DATA_A_NVFP4) || defined(MULMAT_QUANT) +#if defined(DATA_A_NVFP4) #if defined(USE_OCP_FP4) floate4m3_t ue4m3_from_bits(uint8_t x) { if (x == uint8_t(0x7F)) { diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp index 160a1ec1e93..c77401e7928 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -245,6 +245,17 @@ bool is_iq_quant(const std::string& type_name) { return string_starts_with(type_name, "iq"); } +bool is_lut_quant(const std::string& type_name) { + return is_iq_quant(type_name) || type_name == "mxfp4" || type_name == "nvfp4"; +} + +std::string lut_load_vec_a(const std::string& type_name) { + if (type_name == "iq1_s" || type_name == "iq1_m" || type_name == "iq2_xxs" || type_name == "iq2_xs" || type_name == "iq2_s") { + return "8"; + } + return "4"; +} + static const char path_separator = '/'; std::string join_paths(const std::string& path1, const std::string& path2) { @@ -605,7 +616,6 @@ void matmul_shaders(bool fp16, MatMulIdType matmul_id_type, bool coopmat, bool c continue; } - // Quant types: only MMQ (mul_mmq.comp) stays per-type std::string data_a_key = "DATA_A_" + to_uppercase(tname); const std::map float_type_dict = { {"FLOAT_TYPE", FLOAT_TYPE(1, tname)}, @@ -619,6 +629,26 @@ void matmul_shaders(bool fp16, MatMulIdType matmul_id_type, bool coopmat, bool c string_to_spv(shader_name + "_" + tname + "_q8_1", "mul_mmq.comp", merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"D_TYPE", "float"},}), fp16, coopmat, coopmat2, f16acc); } #endif + + if (is_lut_quant(tname)) { + std::string lva = lut_load_vec_a(tname); + + string_to_spv(shader_name + "_" + tname + "_f16" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"LOAD_VEC_A", lva}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f16}, {"B_TYPE_SCALAR", "float16_t"}, {"B_TYPEV4", "f16vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); + + if (!coopmat2) { + string_to_spv(shader_name + "_" + tname + "_f32" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"LOAD_VEC_A", lva}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f32}, {"B_TYPE_SCALAR", "float"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); + } + +#if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) + if ((tname == "mxfp4" || tname == "nvfp4") && (coopmat || coopmat2)) { + string_to_spv(shader_name + "_" + tname + "_f16_ocp" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"USE_OCP_FP4", "1"}, {"LOAD_VEC_A", lva}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f16}, {"B_TYPE_SCALAR", "float16_t"}, {"B_TYPEV4", "f16vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); + if (!coopmat2) { + string_to_spv(shader_name + "_" + tname + "_f32_ocp" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"USE_OCP_FP4", "1"}, {"LOAD_VEC_A", lva}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f32}, {"B_TYPE_SCALAR", "float"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); + } + } +#endif + continue; + } } // Quant shader: one SPIR-V for all quant types, selected via MmTypeA spec constant From f58bd0a249a004e50e12662c08d04b1746c6b4c8 Mon Sep 17 00:00:00 2001 From: Ruben Ortlam Date: Thu, 27 Aug 2026 16:53:41 +0200 Subject: [PATCH 3/9] clean up --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 7 ------- ggml/src/ggml-vulkan/vulkan-shaders/ggml_type_ids.glsl | 4 ++-- ggml/src/ggml-vulkan/vulkan-shaders/types.glsl | 1 + ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp | 6 +----- 4 files changed, 4 insertions(+), 14 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index b5a0eb957e3..4ad15c47a16 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -278,7 +278,6 @@ using matmul_tile_selector_t = std::function& configs)>; - struct vk_device_struct; typedef std::shared_ptr vk_device; typedef std::weak_ptr vk_device_ref; @@ -4754,7 +4753,6 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { return result; }; - // Build tile config vectors std::vector tc_mm = {{s_warptile, s_wg_denoms, s_align}, {m_warptile, m_wg_denoms, m_align}, {l_warptile, l_wg_denoms, l_align}}; std::vector tc_mmq = {{s_warptile_mmq, s_mmq_wg_denoms, s_align}, {m_warptile_mmq, m_mmq_wg_denoms, m_align}, {l_warptile_mmq, l_mmq_wg_denoms, l_align}}; @@ -4811,7 +4809,6 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { GGML_ASSERT(device->subgroup_ballot); - // ID: F16 create_mm_pipelines({GGML_TYPE_F16, GGML_TYPE_F16, true, true}, tc_mm, "matmul_id_subgroup_f16_f16acc", matmul_id_subgroup_f16_f16acc_cm2_len, matmul_id_subgroup_f16_f16acc_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); create_mm_pipelines({GGML_TYPE_F16, GGML_TYPE_F16, true, false}, tc_mm, "matmul_id_subgroup_f16", matmul_id_subgroup_f16_cm2_len, matmul_id_subgroup_f16_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); #if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) @@ -5141,8 +5138,6 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { #endif if (device->subgroup_ballot && device->subgroup_require_full_support && subgroup_min_size_16) { - std::vector tc_id = {{s_warptile_id, s_wg_denoms, s_align}, {m_warptile_id, m_wg_denoms, m_align}, {l_warptile_id, l_wg_denoms, l_align}}; - std::vector tc_mmqid = {{s_warptile_mmqid, s_mmq_wg_denoms, s_align}, {m_warptile_mmqid, m_mmq_wg_denoms, m_align}, {l_warptile_mmqid, l_mmq_wg_denoms, l_align}}; sg_create({GGML_TYPE_F32, GGML_TYPE_F32, true, false}, tc_id, "matmul_id_subgroup_f32_f32", matmul_id_subgroup_f32_f32_fp32_len, matmul_id_subgroup_f32_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); sg_create({GGML_TYPE_F16, GGML_TYPE_F16, true, false}, tc_id, "matmul_id_subgroup_f16", matmul_id_subgroup_f16_fp32_len, matmul_id_subgroup_f16_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); sg_create({GGML_TYPE_F16, GGML_TYPE_F32, true, false}, tc_id, "matmul_id_subgroup_f16_f32", matmul_id_subgroup_f16_f32_fp32_len, matmul_id_subgroup_f16_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); @@ -8737,8 +8732,6 @@ static void ggml_vk_matmul( } - -// New map-based pipeline lookup helpers static bool ggml_vk_get_mul_mat_mat_f16acc(ggml_backend_vk_context * ctx, ggml_type src0_type, ggml_type src1_type, ggml_prec prec) { if (src0_type == GGML_TYPE_F32 || src0_type == GGML_TYPE_BF16) return false; if (src1_type == GGML_TYPE_Q8_1) return false; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/ggml_type_ids.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/ggml_type_ids.glsl index 5f7da985a25..c7485709e28 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/ggml_type_ids.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/ggml_type_ids.glsl @@ -25,9 +25,9 @@ #define GGML_TYPE_IQ1_M 29u #define GGML_TYPE_BF16 30u #define GGML_TYPE_TQ2_0 35u -#define GGML_TYPE_Q1_0 41u -#define GGML_TYPE_Q2_0 42u #define GGML_TYPE_MXFP4 39u #define GGML_TYPE_NVFP4 40u +#define GGML_TYPE_Q1_0 41u +#define GGML_TYPE_Q2_0 42u #endif // !defined(GGML_TYPE_IDS_COMP) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl index c50f14426c9..0b9ac710b59 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl @@ -1866,6 +1866,7 @@ shared int8_t kvalues_mxfp4[16]; #endif #if defined(DATA_A_NVFP4) && !defined(USE_OCP_FP4) +// UE4M3 scale in NVFP4 blocks use only 7 bits; sign (bit 7) is always zero. shared float ue4m3_fp32_lut[128]; float ue4m3_to_fp32_build(uint u) { diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp index c77401e7928..59a362b2983 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -600,7 +600,6 @@ void matmul_shaders(bool fp16, MatMulIdType matmul_id_type, bool coopmat, bool c // Float types keep per-type compilation (different accumulation loop structure) if (tname == "f32" || tname == "f16") { std::string data_a_key = "DATA_A_" + to_uppercase(tname); - std::string load_vec_a = coopmat2 ? load_vec : load_vec; const std::map float_type_dict = { {"FLOAT_TYPE", FLOAT_TYPE(1, tname)}, @@ -609,9 +608,8 @@ void matmul_shaders(bool fp16, MatMulIdType matmul_id_type, bool coopmat, bool c {"FLOAT_TYPEV8", FLOAT_TYPE(8, tname)}, }; - // don't generate f32 variants for coopmat2 if (!coopmat2) { - string_to_spv(shader_name + "_" + tname + "_f32" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"LOAD_VEC_A", load_vec_a}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f32}, {"B_TYPE_SCALAR", "float"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); + string_to_spv(shader_name + "_" + tname + "_f32" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"LOAD_VEC_A", load_vec}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f32}, {"B_TYPE_SCALAR", "float"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); } continue; } @@ -660,10 +658,8 @@ void matmul_shaders(bool fp16, MatMulIdType matmul_id_type, bool coopmat, bool c {"FLOAT_TYPEV8", FLOAT_TYPE(8, "q4_0")}, }; - // f16 B-type (all paths) string_to_spv(shader_name + "_quant_f16" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, quant_float_type_dict), {{"MULMAT_QUANT", "1"}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f16}, {"B_TYPE_SCALAR", "float16_t"}, {"B_TYPEV4", "f16vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); - // f32 B-type (non-coopmat2 only) if (!coopmat2) { string_to_spv(shader_name + "_quant_f32" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, quant_float_type_dict), {{"MULMAT_QUANT", "1"}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f32}, {"B_TYPE_SCALAR", "float"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); } From 6537ccf99df9a0b9e93e76c5f956d69cf87aa530 Mon Sep 17 00:00:00 2001 From: Ruben Ortlam Date: Fri, 28 Aug 2026 13:35:26 +0200 Subject: [PATCH 4/9] restore coopmat2 q4_k/q5_k optimization --- .../vulkan-shaders/dequant_funcs_cm2.glsl | 53 +++++++++++++++---- 1 file changed, 43 insertions(+), 10 deletions(-) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl index d32a30c86f4..0d45443972a 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl @@ -438,14 +438,21 @@ layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufQ4 // function that stores into shared memory. // Q4_K and Q5_K have the same encoding of scales, so everything is shared except // the part that fetches from the structure (which has a different block layout). -#if defined(DATA_A_Q4_K) || defined(DATA_A_Q5_K) +#if defined(DATA_A_Q4_K) || defined(DATA_A_Q5_K) || defined(MULMAT_QUANT) const uint shAscales_stride = (BM + 2); // 1 scale per 32 elements -> 8 scales per block, per row +#if defined(MULMAT_QUANT) +// In the unified shader the quant type is a runtime spec constant, so only reserve +// the full scales tile when Q4_K/Q5_K is actually selected. Otherwise size it down +// to 8 (1 triggers an Nvidia compiler bug) so it costs no meaningful shared memory. +shared vec2 shAscales[(MmTypeA == GGML_TYPE_Q4_K || MmTypeA == GGML_TYPE_Q5_K) ? 8 * shAscales_stride : 8]; +#else shared vec2 shAscales[8 * shAscales_stride]; +#endif uvec4 row_v; #endif -#if defined(DATA_A_Q4_K) +#if defined(DATA_A_Q4_K) || defined(MULMAT_QUANT) layout (binding = 0) readonly buffer A_Q4_K_128 {block_q4_K_packed128 data_a_q4_k_packed128[];}; void fetch_scalesQ4_K(uint ir_BM, uint pos_a, uint stride_a, uint block_k, uint tid, bool in_bounds) @@ -456,13 +463,13 @@ void fetch_scalesQ4_K(uint ir_BM, uint pos_a, uint stride_a, uint block_k, uint uint tid_row = tid / tids_per_row; uint row = ir_BM + tid_row; - uint block_index = pos_a + row * stride_a + (block_k / QUANT_K); + uint block_index = pos_a + row * stride_a + (block_k / QUANT_K_Q4_K); if (in_bounds || row < p.M) { row_v = data_a_q4_k_packed128[block_index].q4k[0]; } } #endif -#if defined(DATA_A_Q5_K) +#if defined(DATA_A_Q5_K) || defined(MULMAT_QUANT) layout (binding = 0) readonly buffer A_Q5_K_128 {block_q5_K_packed128 data_a_q5_k_packed128[];}; void fetch_scalesQ5_K(uint ir_BM, uint pos_a, uint stride_a, uint block_k, uint tid, bool in_bounds) @@ -473,14 +480,14 @@ void fetch_scalesQ5_K(uint ir_BM, uint pos_a, uint stride_a, uint block_k, uint uint tid_row = tid / tids_per_row; uint row = ir_BM + tid_row; - uint block_index = pos_a + row * stride_a + (block_k / QUANT_K); + uint block_index = pos_a + row * stride_a + (block_k / QUANT_K_Q5_K); if (in_bounds || row < p.M) { row_v = data_a_q5_k_packed128[block_index].q5k[0]; } } #endif -#if defined(DATA_A_Q4_K) || defined(DATA_A_Q5_K) +#if defined(DATA_A_Q4_K) || defined(DATA_A_Q5_K) || defined(MULMAT_QUANT) void store_scalesQ4_K(uint tid) { barrier(); @@ -523,6 +530,26 @@ void store_scalesQ4_K(uint tid) } #endif +#if defined(MULMAT_QUANT) +// Runtime dispatch for the unified shader: pick the right fetch based on the spec +// constant type. Only Q4_K/Q5_K use the shared-scales path; other types are no-ops. +void fetch_scales_cm2(uint ir_BM, uint pos_a, uint stride_a, uint block_k, uint tid, bool in_bounds) +{ + if (MmTypeA == GGML_TYPE_Q4_K) { + fetch_scalesQ4_K(ir_BM, pos_a, stride_a, block_k, tid, in_bounds); + } else if (MmTypeA == GGML_TYPE_Q5_K) { + fetch_scalesQ5_K(ir_BM, pos_a, stride_a, block_k, tid, in_bounds); + } +} + +void store_scales_cm2(uint tid) +{ + if (MmTypeA == GGML_TYPE_Q4_K || MmTypeA == GGML_TYPE_Q5_K) { + store_scalesQ4_K(tid); + } +} +#endif + #endif float16_t dequantFuncQ4_K(const in decodeBufQ4_K bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -534,7 +561,7 @@ float16_t dequantFuncQ4_K(const in decodeBufQ4_K bl, const in uint blockCoords[2 const uint b = (idx & 0x20) >> 5; // 0,1 const uint is = (idx & 0xE0) >> 5; // 0..7 -#if defined(IS_MUL_MM2) && defined(DATA_A_Q4_K) +#if defined(IS_MUL_MM2) && (defined(DATA_A_Q4_K) || defined(MULMAT_QUANT)) vec2 v = shAscales[is * shAscales_stride + (blockCoords[0] % BM)]; float d = v.x; float m = v.y; @@ -581,7 +608,7 @@ f16vec4 dequantFuncQ4_K_v(const in decodeBufQ4_K bl, const in uint blockCoords[2 const uint is = idx >> 5; // 0..7 -#if defined(IS_MUL_MM2) && defined(DATA_A_Q4_K) +#if defined(IS_MUL_MM2) && (defined(DATA_A_Q4_K) || defined(MULMAT_QUANT)) vec2 v = shAscales[is * shAscales_stride + (blockCoords[0] % BM)]; float d = v.x; float m = v.y; @@ -647,7 +674,7 @@ float16_t dequantFuncQ5_K(const in decodeBufQ5_K bl, const in uint blockCoords[2 const uint b = (idx & 0x20) >> 5; // 0,1 const uint is = (idx & 0xE0) >> 5; // 0..7 -#if defined(IS_MUL_MM2) && defined(DATA_A_Q5_K) +#if defined(IS_MUL_MM2) && (defined(DATA_A_Q5_K) || defined(MULMAT_QUANT)) vec2 v = shAscales[is * shAscales_stride + (blockCoords[0] % BM)]; float d = v.x; float m = v.y; @@ -698,7 +725,7 @@ f16vec4 dequantFuncQ5_K_v(const in decodeBufQ5_K bl, const in uint blockCoords[2 const uint idx = coordInBlock[1]; const uint is = idx >> 5; -#if defined(IS_MUL_MM2) && defined(DATA_A_Q5_K) +#if defined(IS_MUL_MM2) && (defined(DATA_A_Q5_K) || defined(MULMAT_QUANT)) vec2 v = shAscales[is * shAscales_stride + (blockCoords[0] % BM)]; float d = v.x; float m = v.y; @@ -1464,3 +1491,9 @@ f16vec4 dequantFuncNVFP4_v(const in decodeBufNVFP4 bl, const in uint blockCoords #elif defined(DATA_A_F32) #define dequantFuncA dequantFuncF32 #endif + +// Unified shader: dispatch the Q4_K/Q5_K shared-scales path at runtime on MmTypeA. +#if defined(MULMAT_QUANT) +#define fetch_scales fetch_scales_cm2 +#define store_scales store_scales_cm2 +#endif From 842771698a247d37dfec6aaeafb69e5c24ef6fd3 Mon Sep 17 00:00:00 2001 From: Ruben Ortlam Date: Fri, 28 Aug 2026 15:43:12 +0200 Subject: [PATCH 5/9] split out q4_k/q5_k cm2 shader to fix Ampere regression --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 15 ++++++ .../vulkan-shaders/dequant_funcs_cm2.glsl | 53 ++++--------------- .../vulkan-shaders/vulkan-shaders-gen.cpp | 5 ++ 3 files changed, 30 insertions(+), 43 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 4ad15c47a16..4e285b074a0 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -4781,11 +4781,19 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { } #endif for (const auto type : non_lut_quant_types) { + // regression in unified shader on Ampere + if (type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K) { + continue; + } auto& tc = ((type >= GGML_TYPE_Q2_K && type <= GGML_TYPE_Q6_K) || type == GGML_TYPE_TQ2_0) ? tc_mmq_k : tc_mmq; spec_fn_t qs = [&, type](const std::vector& wt, bool a) { return ggml_vk_mul_mm_cm2_spec(wt, a, (uint32_t)type); }; create_mm_pipelines({type, GGML_TYPE_F16, false, true}, tc, "matmul_quant_f16_f16acc", matmul_quant_f16_f16acc_cm2_len, matmul_quant_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, qs, true); create_mm_pipelines({type, GGML_TYPE_F16, false, false}, tc, "matmul_quant_f16", matmul_quant_f16_cm2_len, matmul_quant_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, qs, true); } + create_mm_pipelines({GGML_TYPE_Q4_K, GGML_TYPE_F16, false, true}, tc_mmq_k, "matmul_q4_k_f16_f16acc", matmul_q4_k_f16_f16acc_cm2_len, matmul_q4_k_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); + create_mm_pipelines({GGML_TYPE_Q4_K, GGML_TYPE_F16, false, false}, tc_mmq_k, "matmul_q4_k_f16", matmul_q4_k_f16_cm2_len, matmul_q4_k_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); + create_mm_pipelines({GGML_TYPE_Q5_K, GGML_TYPE_F16, false, true}, tc_mmq_k, "matmul_q5_k_f16_f16acc", matmul_q5_k_f16_f16acc_cm2_len, matmul_q5_k_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); + create_mm_pipelines({GGML_TYPE_Q5_K, GGML_TYPE_F16, false, false}, tc_mmq_k, "matmul_q5_k_f16", matmul_q5_k_f16_cm2_len, matmul_q5_k_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); #define X_CM2(TYPE, tstr) \ { auto tc = filter_tc(tc_mmq, TYPE, false); \ if (!tc.empty()) { \ @@ -4817,10 +4825,17 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { } #endif for (const auto type : non_lut_quant_types) { + if (type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K) { + continue; + } spec_fn_t qs_id = [&, type](const std::vector& wt, bool a) { return ggml_vk_mul_mm_cm2_spec(wt, a, (uint32_t)type); }; create_mm_pipelines({type, GGML_TYPE_F16, true, true}, tc_mmqid, "matmul_id_subgroup_quant_f16_f16acc", matmul_id_subgroup_quant_f16_f16acc_cm2_len, matmul_id_subgroup_quant_f16_f16acc_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, qs_id, true); create_mm_pipelines({type, GGML_TYPE_F16, true, false}, tc_mmqid, "matmul_id_subgroup_quant_f16", matmul_id_subgroup_quant_f16_cm2_len, matmul_id_subgroup_quant_f16_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, qs_id, true); } + create_mm_pipelines({GGML_TYPE_Q4_K, GGML_TYPE_F16, true, true}, tc_mmqid, "matmul_id_subgroup_q4_k_f16_f16acc", matmul_id_subgroup_q4_k_f16_f16acc_cm2_len, matmul_id_subgroup_q4_k_f16_f16acc_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); + create_mm_pipelines({GGML_TYPE_Q4_K, GGML_TYPE_F16, true, false}, tc_mmqid, "matmul_id_subgroup_q4_k_f16", matmul_id_subgroup_q4_k_f16_cm2_len, matmul_id_subgroup_q4_k_f16_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); + create_mm_pipelines({GGML_TYPE_Q5_K, GGML_TYPE_F16, true, true}, tc_mmqid, "matmul_id_subgroup_q5_k_f16_f16acc", matmul_id_subgroup_q5_k_f16_f16acc_cm2_len, matmul_id_subgroup_q5_k_f16_f16acc_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); + create_mm_pipelines({GGML_TYPE_Q5_K, GGML_TYPE_F16, true, false}, tc_mmqid, "matmul_id_subgroup_q5_k_f16", matmul_id_subgroup_q5_k_f16_cm2_len, matmul_id_subgroup_q5_k_f16_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); #define X_CM2_ID(TYPE, tstr) \ { auto tc = filter_tc(tc_mmqid, TYPE, true); \ if (!tc.empty()) { \ diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl index 0d45443972a..d32a30c86f4 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl @@ -438,21 +438,14 @@ layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufQ4 // function that stores into shared memory. // Q4_K and Q5_K have the same encoding of scales, so everything is shared except // the part that fetches from the structure (which has a different block layout). -#if defined(DATA_A_Q4_K) || defined(DATA_A_Q5_K) || defined(MULMAT_QUANT) +#if defined(DATA_A_Q4_K) || defined(DATA_A_Q5_K) const uint shAscales_stride = (BM + 2); // 1 scale per 32 elements -> 8 scales per block, per row -#if defined(MULMAT_QUANT) -// In the unified shader the quant type is a runtime spec constant, so only reserve -// the full scales tile when Q4_K/Q5_K is actually selected. Otherwise size it down -// to 8 (1 triggers an Nvidia compiler bug) so it costs no meaningful shared memory. -shared vec2 shAscales[(MmTypeA == GGML_TYPE_Q4_K || MmTypeA == GGML_TYPE_Q5_K) ? 8 * shAscales_stride : 8]; -#else shared vec2 shAscales[8 * shAscales_stride]; -#endif uvec4 row_v; #endif -#if defined(DATA_A_Q4_K) || defined(MULMAT_QUANT) +#if defined(DATA_A_Q4_K) layout (binding = 0) readonly buffer A_Q4_K_128 {block_q4_K_packed128 data_a_q4_k_packed128[];}; void fetch_scalesQ4_K(uint ir_BM, uint pos_a, uint stride_a, uint block_k, uint tid, bool in_bounds) @@ -463,13 +456,13 @@ void fetch_scalesQ4_K(uint ir_BM, uint pos_a, uint stride_a, uint block_k, uint uint tid_row = tid / tids_per_row; uint row = ir_BM + tid_row; - uint block_index = pos_a + row * stride_a + (block_k / QUANT_K_Q4_K); + uint block_index = pos_a + row * stride_a + (block_k / QUANT_K); if (in_bounds || row < p.M) { row_v = data_a_q4_k_packed128[block_index].q4k[0]; } } #endif -#if defined(DATA_A_Q5_K) || defined(MULMAT_QUANT) +#if defined(DATA_A_Q5_K) layout (binding = 0) readonly buffer A_Q5_K_128 {block_q5_K_packed128 data_a_q5_k_packed128[];}; void fetch_scalesQ5_K(uint ir_BM, uint pos_a, uint stride_a, uint block_k, uint tid, bool in_bounds) @@ -480,14 +473,14 @@ void fetch_scalesQ5_K(uint ir_BM, uint pos_a, uint stride_a, uint block_k, uint uint tid_row = tid / tids_per_row; uint row = ir_BM + tid_row; - uint block_index = pos_a + row * stride_a + (block_k / QUANT_K_Q5_K); + uint block_index = pos_a + row * stride_a + (block_k / QUANT_K); if (in_bounds || row < p.M) { row_v = data_a_q5_k_packed128[block_index].q5k[0]; } } #endif -#if defined(DATA_A_Q4_K) || defined(DATA_A_Q5_K) || defined(MULMAT_QUANT) +#if defined(DATA_A_Q4_K) || defined(DATA_A_Q5_K) void store_scalesQ4_K(uint tid) { barrier(); @@ -530,26 +523,6 @@ void store_scalesQ4_K(uint tid) } #endif -#if defined(MULMAT_QUANT) -// Runtime dispatch for the unified shader: pick the right fetch based on the spec -// constant type. Only Q4_K/Q5_K use the shared-scales path; other types are no-ops. -void fetch_scales_cm2(uint ir_BM, uint pos_a, uint stride_a, uint block_k, uint tid, bool in_bounds) -{ - if (MmTypeA == GGML_TYPE_Q4_K) { - fetch_scalesQ4_K(ir_BM, pos_a, stride_a, block_k, tid, in_bounds); - } else if (MmTypeA == GGML_TYPE_Q5_K) { - fetch_scalesQ5_K(ir_BM, pos_a, stride_a, block_k, tid, in_bounds); - } -} - -void store_scales_cm2(uint tid) -{ - if (MmTypeA == GGML_TYPE_Q4_K || MmTypeA == GGML_TYPE_Q5_K) { - store_scalesQ4_K(tid); - } -} -#endif - #endif float16_t dequantFuncQ4_K(const in decodeBufQ4_K bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -561,7 +534,7 @@ float16_t dequantFuncQ4_K(const in decodeBufQ4_K bl, const in uint blockCoords[2 const uint b = (idx & 0x20) >> 5; // 0,1 const uint is = (idx & 0xE0) >> 5; // 0..7 -#if defined(IS_MUL_MM2) && (defined(DATA_A_Q4_K) || defined(MULMAT_QUANT)) +#if defined(IS_MUL_MM2) && defined(DATA_A_Q4_K) vec2 v = shAscales[is * shAscales_stride + (blockCoords[0] % BM)]; float d = v.x; float m = v.y; @@ -608,7 +581,7 @@ f16vec4 dequantFuncQ4_K_v(const in decodeBufQ4_K bl, const in uint blockCoords[2 const uint is = idx >> 5; // 0..7 -#if defined(IS_MUL_MM2) && (defined(DATA_A_Q4_K) || defined(MULMAT_QUANT)) +#if defined(IS_MUL_MM2) && defined(DATA_A_Q4_K) vec2 v = shAscales[is * shAscales_stride + (blockCoords[0] % BM)]; float d = v.x; float m = v.y; @@ -674,7 +647,7 @@ float16_t dequantFuncQ5_K(const in decodeBufQ5_K bl, const in uint blockCoords[2 const uint b = (idx & 0x20) >> 5; // 0,1 const uint is = (idx & 0xE0) >> 5; // 0..7 -#if defined(IS_MUL_MM2) && (defined(DATA_A_Q5_K) || defined(MULMAT_QUANT)) +#if defined(IS_MUL_MM2) && defined(DATA_A_Q5_K) vec2 v = shAscales[is * shAscales_stride + (blockCoords[0] % BM)]; float d = v.x; float m = v.y; @@ -725,7 +698,7 @@ f16vec4 dequantFuncQ5_K_v(const in decodeBufQ5_K bl, const in uint blockCoords[2 const uint idx = coordInBlock[1]; const uint is = idx >> 5; -#if defined(IS_MUL_MM2) && (defined(DATA_A_Q5_K) || defined(MULMAT_QUANT)) +#if defined(IS_MUL_MM2) && defined(DATA_A_Q5_K) vec2 v = shAscales[is * shAscales_stride + (blockCoords[0] % BM)]; float d = v.x; float m = v.y; @@ -1491,9 +1464,3 @@ f16vec4 dequantFuncNVFP4_v(const in decodeBufNVFP4 bl, const in uint blockCoords #elif defined(DATA_A_F32) #define dequantFuncA dequantFuncF32 #endif - -// Unified shader: dispatch the Q4_K/Q5_K shared-scales path at runtime on MmTypeA. -#if defined(MULMAT_QUANT) -#define fetch_scales fetch_scales_cm2 -#define store_scales store_scales_cm2 -#endif diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp index 59a362b2983..100e1d9e703 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -647,6 +647,11 @@ void matmul_shaders(bool fp16, MatMulIdType matmul_id_type, bool coopmat, bool c #endif continue; } + + // dedicated shader needed due to regression on Ampere + if (coopmat2 && (tname == "q4_k" || tname == "q5_k")) { + string_to_spv(shader_name + "_" + tname + "_f16" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"LOAD_VEC_A", load_vec}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f16}, {"B_TYPE_SCALAR", "float16_t"}, {"B_TYPEV4", "f16vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); + } } // Quant shader: one SPIR-V for all quant types, selected via MmTypeA spec constant From 36be26cc08ec540c236bf8f56aa7d3d73e004aec Mon Sep 17 00:00:00 2001 From: Ruben Ortlam Date: Mon, 31 Aug 2026 07:32:55 +0200 Subject: [PATCH 6/9] revert iq shmem table renames --- .../vulkan-shaders/dequant_funcs.glsl | 20 +++++----- .../vulkan-shaders/dequant_funcs_cm2.glsl | 20 +++++----- .../vulkan-shaders/dequant_iq2_s.comp | 2 +- .../vulkan-shaders/dequant_iq2_xs.comp | 2 +- .../vulkan-shaders/dequant_iq2_xxs.comp | 2 +- .../vulkan-shaders/dequant_iq3_s.comp | 2 +- .../vulkan-shaders/dequant_iq3_xxs.comp | 4 +- .../vulkan-shaders/mul_mat_vec_iq2_s.comp | 2 +- .../vulkan-shaders/mul_mat_vec_iq2_xs.comp | 2 +- .../vulkan-shaders/mul_mat_vec_iq2_xxs.comp | 4 +- .../vulkan-shaders/mul_mat_vec_iq3_s.comp | 4 +- .../vulkan-shaders/mul_mat_vec_iq3_xxs.comp | 4 +- .../vulkan-shaders/mul_mm_funcs.glsl | 10 ++--- .../src/ggml-vulkan/vulkan-shaders/types.glsl | 38 +++++++++---------- 14 files changed, 58 insertions(+), 58 deletions(-) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl index 6c852fb4266..627932bd354 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl @@ -244,7 +244,7 @@ vec2 dequantize(uint ib, uint iqs, uint a_offset) { // Add parity bit const uint sign8 = sign7 | (bitCount(sign7) << 7); const uint sign = sign8 >> (iqs % 8); - const u8vec4 grid = unpack8(iq2_grid[qs][(iqs % 8) / 4] >> (8 * (iqs % 4))); + const u8vec4 grid = unpack8(iq2xxs_grid[qs][(iqs % 8) / 4] >> (8 * (iqs % 4))); bool sign0 = (sign & 1) != 0; bool sign1 = (sign & 2) != 0; return db * vec2( @@ -264,7 +264,7 @@ vec4 dequantize4(uint ib, uint iqs, uint a_offset) { // Add parity bit const uint sign8 = sign7 | (bitCount(sign7) << 7); const uint sign = sign8 >> (iqs % 8); - const u8vec4 grid = unpack8(iq2_grid[qs][(iqs % 8) / 4] >> (8 * (iqs % 4))); + const u8vec4 grid = unpack8(iq2xxs_grid[qs][(iqs % 8) / 4] >> (8 * (iqs % 4))); bool sign0 = (sign & 1) != 0; bool sign1 = (sign & 2) != 0; bool sign2 = (sign & 4) != 0; @@ -287,7 +287,7 @@ vec2 dequantize(uint ib, uint iqs, uint a_offset) { // Add parity bit const uint sign8 = sign7 | (bitCount(sign7) << 7); const uint sign = sign8 >> (iqs % 8); - const u8vec4 grid = unpack8(iq2_grid[qs & 511][(iqs % 8) / 4] >> (8 * (iqs % 4))); + const u8vec4 grid = unpack8(iq2xs_grid[qs & 511][(iqs % 8) / 4] >> (8 * (iqs % 4))); bool sign0 = (sign & 1) != 0; bool sign1 = (sign & 2) != 0; return db * vec2( @@ -303,7 +303,7 @@ vec4 dequantize4(uint ib, uint iqs, uint a_offset) { // Add parity bit const uint sign8 = sign7 | (bitCount(sign7) << 7); const uint sign = sign8 >> (iqs % 8); - const u8vec4 grid = unpack8(iq2_grid[qs & 511][(iqs % 8) / 4] >> (8 * (iqs % 4))); + const u8vec4 grid = unpack8(iq2xs_grid[qs & 511][(iqs % 8) / 4] >> (8 * (iqs % 4))); bool sign0 = (sign & 1) != 0; bool sign1 = (sign & 2) != 0; bool sign2 = (sign & 4) != 0; @@ -329,7 +329,7 @@ vec2 dequantize(uint ib, uint iqs, uint a_offset) { const uint sign = data_a[a_offset + ib].qs[QUANT_K / 8 + ib8] >> (iqs % 8); const float db = 0.25 * (0.5 + scale); - const u8vec4 grid = unpack8(iq2_grid[qs | ((qh << (8 - qhshift)) & 0x300)][(iqs % 8) / 4]); + const u8vec4 grid = unpack8(iq2s_grid[qs | ((qh << (8 - qhshift)) & 0x300)][(iqs % 8) / 4]); bool sign0 = (sign & 1) != 0; bool sign1 = (sign & 2) != 0; return db * vec2( @@ -348,7 +348,7 @@ vec4 dequantize4(uint ib, uint iqs, uint a_offset) { const uint sign = data_a[a_offset + ib].qs[QUANT_K / 8 + ib8] >> (iqs % 8); const float db = 0.25 * (0.5 + scale); - const u8vec4 grid = unpack8(iq2_grid[qs | ((qh << (8 - qhshift)) & 0x300)][(iqs % 8) / 4]); + const u8vec4 grid = unpack8(iq2s_grid[qs | ((qh << (8 - qhshift)) & 0x300)][(iqs % 8) / 4]); bool sign0 = (sign & 1) != 0; bool sign1 = (sign & 2) != 0; bool sign2 = (sign & 4) != 0; @@ -376,7 +376,7 @@ vec2 dequantize(uint ib, uint iqs, uint a_offset) { // Add parity bit const uint sign8 = sign7 | (bitCount(sign7) << 7); const uint sign = sign8 >> (iqs % 8); - const u8vec4 grid = unpack8(iq3_grid[qs] >> (8 * (iqs % 4))); + const u8vec4 grid = unpack8(iq3xxs_grid[qs] >> (8 * (iqs % 4))); bool sign0 = (sign & 1) != 0; bool sign1 = (sign & 2) != 0; return db * vec2( @@ -396,7 +396,7 @@ vec4 dequantize4(uint ib, uint iqs, uint a_offset) { // Add parity bit const uint sign8 = sign7 | (bitCount(sign7) << 7); const uint sign = sign8 >> (iqs % 8); - const u8vec4 grid = unpack8(iq3_grid[qs]); + const u8vec4 grid = unpack8(iq3xxs_grid[qs]); bool sign0 = (sign & 1) != 0; bool sign1 = (sign & 2) != 0; bool sign2 = (sign & 4) != 0; @@ -419,7 +419,7 @@ vec2 dequantize(uint ib, uint iqs, uint a_offset) { bool sign0 = (sign & 1) != 0; bool sign1 = (sign & 2) != 0; const float db = 1 + 2 * ((scale >> (4 * ((iqs / 32) & 1))) & 0xf); - const uint32_t grid = iq3_grid[qs | ((qh << (8 - ((iqs / 4) % 8))) & 256)] >> (8 * (iqs % 4)); + const uint32_t grid = iq3s_grid[qs | ((qh << (8 - ((iqs / 4) % 8))) & 256)] >> (8 * (iqs % 4)); return db * vec2( int(grid & 0xFF) * (sign0 ? -1.0 : 1.0), int((grid >> 8) & 0xFF) * (sign1 ? -1.0 : 1.0) @@ -437,7 +437,7 @@ vec4 dequantize4(uint ib, uint iqs, uint a_offset) { bool sign2 = (sign & 4) != 0; bool sign3 = (sign & 8) != 0; const float db = 1 + 2 * ((scale >> (4 * (ib32 & 1))) & 0xf); - const uint32_t grid = iq3_grid[qs | ((qh << (8 - ib4 % 8)) & 256)] >> (8 * (iqs % 4)); + const uint32_t grid = iq3s_grid[qs | ((qh << (8 - ib4 % 8)) & 256)] >> (8 * (iqs % 4)); return db * vec4( int(grid & 0xFF) * (sign0 ? -1.0 : 1.0), int((grid >> 8) & 0xFF) * (sign1 ? -1.0 : 1.0), diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl index d32a30c86f4..4d4bb353791 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl @@ -936,7 +936,7 @@ float16_t dequantFuncIQ2_XXS(const in decodeBufIQ2_XXS bl, const in uint blockCo uint sign = bitfieldExtract(signscale, 7 * int(ib8), 7); sign |= bitCount(sign) << 7; - uint g2 = iq2_grid[qs][(idx & 4) >> 2]; + uint g2 = iq2xxs_grid[qs][(idx & 4) >> 2]; g2 >>= (idx & 2) * 8; const vec2 g = vec2(unpack8(g2)); @@ -961,7 +961,7 @@ f16vec4 dequantFuncIQ2_XXS_v(const in decodeBufIQ2_XXS bl, const in uint blockCo sign |= bitCount(sign) << 7; const uint sb = sign >> (idx & 7u); - const uint g2 = iq2_grid[qs][(idx & 4) >> 2]; + const uint g2 = iq2xxs_grid[qs][(idx & 4) >> 2]; const u8vec4 g = unpack8(g2); return f16vec4( @@ -991,7 +991,7 @@ float16_t dequantFuncIQ2_XS(const in decodeBufIQ2_XS bl, const in uint blockCoor uint sign = uint(qs >> 9); sign |= bitCount(sign) << 7; - uint g2 = iq2_grid[qs & 0x1FF][(idx & 4) >> 2]; + uint g2 = iq2xs_grid[qs & 0x1FF][(idx & 4) >> 2]; g2 >>= (idx & 2) * 8; const vec2 g = vec2(unpack8(g2)); @@ -1014,7 +1014,7 @@ f16vec4 dequantFuncIQ2_XS_v(const in decodeBufIQ2_XS bl, const in uint blockCoor sign |= bitCount(sign) << 7; const uint sb = sign >> (idx & 7u); - const uint g2 = iq2_grid[qs & 0x1FF][(idx & 4) >> 2]; + const uint g2 = iq2xs_grid[qs & 0x1FF][(idx & 4) >> 2]; const u8vec4 g = unpack8(g2); return f16vec4( @@ -1046,7 +1046,7 @@ float16_t dequantFuncIQ2_S(const in decodeBufIQ2_S bl, const in uint blockCoords const float d = float(bl.block.d); const float db = d * 0.25 * (0.5 + scale); const ivec2 sign01 = 1 - (2 & ivec2(sign << 1, sign)); - uint g2 = iq2_grid[qs | ((qh << (8 - qhshift)) & 0x300)][(idx & 4) >> 2]; + uint g2 = iq2s_grid[qs | ((qh << (8 - qhshift)) & 0x300)][(idx & 4) >> 2]; g2 >>= (idx & 2) * 8; const vec2 v = db * vec2(sign01) * vec2(unpack8(g2)); return float16_t(v[idx & 1]); @@ -1068,7 +1068,7 @@ f16vec4 dequantFuncIQ2_S_v(const in decodeBufIQ2_S bl, const in uint blockCoords const float d = float(bl.block.d); const float db = d * 0.25 * (0.5 + scale); - const uint g2 = iq2_grid[qs | ((qh << (8 - qhshift)) & 0x300)][(idx & 4) >> 2]; + const uint g2 = iq2s_grid[qs | ((qh << (8 - qhshift)) & 0x300)][(idx & 4) >> 2]; const u8vec4 g = unpack8(g2); return f16vec4( @@ -1106,7 +1106,7 @@ float16_t dequantFuncIQ3_XXS(const in decodeBufIQ3_XXS bl, const in uint blockCo const uint32_t sign7 = bitfieldExtract(signs, 7 * (int(iqs / 2) % 4), 7); const uint sign = (sign7 | (bitCount(sign7) << 7)) >> (idx & 0x6); const ivec2 sign01 = ivec2(1 - (2 & ivec2(sign << 1, sign))); - const uint grid = iq3_grid[qs] >> (16 * ((idx & 2) >> 1)); + const uint grid = iq3xxs_grid[qs] >> (16 * ((idx & 2) >> 1)); const vec2 v = db * vec2(sign01) * vec2(unpack8(grid).xy); return float16_t(v[idx & 1]); } @@ -1127,7 +1127,7 @@ f16vec4 dequantFuncIQ3_XXS_v(const in decodeBufIQ3_XXS bl, const in uint blockCo const uint sign7 = bitfieldExtract(signs, 7 * (int(iqs / 2) % 4), 7); const uint sb = (sign7 | (bitCount(sign7) << 7)) >> (idx & 0x6u); - const uint grid = iq3_grid[qs]; + const uint grid = iq3xxs_grid[qs]; const u8vec4 g = unpack8(grid); return f16vec4( @@ -1157,7 +1157,7 @@ float16_t dequantFuncIQ3_S(const in decodeBufIQ3_S bl, const in uint blockCoords const uint scale = bl.block.scales[iqs / 16]; const ivec2 sign01 = ivec2(1 - (2 & ivec2(sign << 1, sign))); const float db = d * (1 + 2 * ((scale >> (4 * (iqh & 1))) & 0xf)); - const uint32_t grid = iq3_grid[qs | ((qh << (8 - (iqs % 8))) & 256)] >> ((idx & 2) << 3); + const uint32_t grid = iq3s_grid[qs | ((qh << (8 - (iqs % 8))) & 256)] >> ((idx & 2) << 3); const vec2 v = db * vec2(sign01) * vec2(unpack8(grid).xy); return float16_t(v[idx & 1]); @@ -1177,7 +1177,7 @@ f16vec4 dequantFuncIQ3_S_v(const in decodeBufIQ3_S bl, const in uint blockCoords const uint scale = bl.block.scales[iqs / 16]; const float db = d * (1 + 2 * ((scale >> (4 * (iqh & 1))) & 0xf)); - const uint grid = iq3_grid[qs | ((qh << (8 - (iqs % 8))) & 256)]; + const uint grid = iq3s_grid[qs | ((qh << (8 - (iqs % 8))) & 256)]; const u8vec4 g = unpack8(grid); return f16vec4( diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp index f834b785629..78490162cd1 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp @@ -29,7 +29,7 @@ void main() { uint qs = data_a[ib].qs[4 * ib32 + l]; const uint8_t sign = data_a[ib].qs[QUANT_K / 8 + 4 * ib32 + l]; qs |= (qh << (8 - 2 * l)) & 0x300; - const uvec2 grid = iq2_grid[qs]; + const uvec2 grid = iq2s_grid[qs]; const u8vec4 grid0 = unpack8(grid.x); const u8vec4 grid1 = unpack8(grid.y); data_b[b_idx + 8 * l + 0] = D_TYPE(db[l/2] * grid0.x * ((sign & 1) != 0 ? -1.0 : 1.0)); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp index f5bba538f33..9b8ce0a7f81 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp @@ -28,7 +28,7 @@ void main() { uint16_t qs = data_a[ib].qs[4 * ib32 + l]; const uint sign7 = qs >> 9; const uint sign8 = sign7 | (bitCount(sign7) << 7); // parity bit - const uvec2 grid = iq2_grid[qs & 511]; + const uvec2 grid = iq2xs_grid[qs & 511]; const u8vec4 grid0 = unpack8(grid.x); const u8vec4 grid1 = unpack8(grid.y); data_b[b_idx + 8 * l + 0] = D_TYPE(db[l/2] * grid0.x * ((sign8 & 1) != 0 ? -1.0 : 1.0)); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp index dca5b84c6ac..aacf07d0f8e 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp @@ -34,7 +34,7 @@ void main() { const uint sign7 = bitfieldExtract(signscale, 7 * int(l), 7); const uint sign8 = sign7 | (bitCount(sign7) << 7); // parity bit const uint qs = data_a[ib].qs[8 * is + l]; - const uvec2 grid = iq2_grid[qs]; + const uvec2 grid = iq2xxs_grid[qs]; const u8vec4 grid0 = unpack8(grid.x); const u8vec4 grid1 = unpack8(grid.y); data_b[b_idx + 8 * l + 0] = D_TYPE(db * grid0.x * ((sign8 & 1) != 0 ? -1.0 : 1.0)); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp index b18a9d0cd11..f2c20b1d2c0 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp @@ -31,7 +31,7 @@ void main() { const uint qs = data_a[ib].qs[iqs]; const uint gidx = qs | ((qh << (8 - l)) & 256); const uint8_t signs = data_a[ib].signs[iqs / 2] >> (4 * (l & 1)); - const u8vec4 grid = unpack8(iq3_grid[gidx]); + const u8vec4 grid = unpack8(iq3s_grid[gidx]); data_b[b_idx + 4 * l + 0] = D_TYPE(db * grid.x * ((signs & 1) != 0 ? -1.0 : 1.0)); data_b[b_idx + 4 * l + 1] = D_TYPE(db * grid.y * ((signs & 2) != 0 ? -1.0 : 1.0)); data_b[b_idx + 4 * l + 2] = D_TYPE(db * grid.z * ((signs & 4) != 0 ? -1.0 : 1.0)); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp index 60c5318e3be..671c1f4a0d3 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp @@ -37,8 +37,8 @@ void main() { const uint sign8 = sign7 | (bitCount(sign7) << 7); const uint qs0 = data_a[ib].qs[8 * is + 2 * l]; const uint qs1 = data_a[ib].qs[8 * is + 2 * l + 1]; - const u8vec4 grid0 = unpack8(iq3_grid[qs0]); - const u8vec4 grid1 = unpack8(iq3_grid[qs1]); + const u8vec4 grid0 = unpack8(iq3xxs_grid[qs0]); + const u8vec4 grid1 = unpack8(iq3xxs_grid[qs1]); data_b[b_idx + 8 * l + 0] = D_TYPE(db * grid0.x * ((sign8 & 1) != 0 ? -1.0 : 1.0)); data_b[b_idx + 8 * l + 1] = D_TYPE(db * grid0.y * ((sign8 & 2) != 0 ? -1.0 : 1.0)); data_b[b_idx + 8 * l + 2] = D_TYPE(db * grid0.z * ((sign8 & 4) != 0 ? -1.0 : 1.0)); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp index 27c461d60dc..b4f6d1d6b64 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp @@ -24,7 +24,7 @@ void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, [[unroll]] for (uint l = 0; l < 2; ++l) { const uint8_t sign = sign16[l]; const uint qs = qs16[l] | ((qh << (8 - nibble_shift - 2 * l)) & 0x300); - const uvec2 grid = iq2_grid[qs]; + const uvec2 grid = iq2s_grid[qs]; const vec4 grid0 = vec4(unpack8(grid.x)); const vec4 grid1 = vec4(unpack8(grid.y)); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp index 1a202e1297e..d8dafe5f709 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp @@ -31,7 +31,7 @@ void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, const uint qs = data_a[ibi].qs[2 * itid + l]; sign_vals[l] = qs >> 9; sign7_vals[l] = bitCount(sign_vals[l]); - const uvec2 grid_data = iq2_grid[qs & 511]; + const uvec2 grid_data = iq2xs_grid[qs & 511]; grid0_vals[l] = vec4(unpack8(grid_data.x)); grid1_vals[l] = vec4(unpack8(grid_data.y)); } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp index f55ee10e8a7..f75dcf8331d 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp @@ -22,8 +22,8 @@ void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, const uint qs = data_a[ibi].qs[8 * ib32 + 2 * (itid & 1) + l]; const uint sign = bitfieldExtract(signscale, 7 * int(2 * (itid & 1) + l), 7); const uint sign7 = bitCount(sign); - const vec4 grid0 = vec4(unpack8(iq2_grid[qs].x)); - const vec4 grid1 = vec4(unpack8(iq2_grid[qs].y)); + const vec4 grid0 = vec4(unpack8(iq2xxs_grid[qs].x)); + const vec4 grid1 = vec4(unpack8(iq2xxs_grid[qs].y)); [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { const vec4 b0 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 0]); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp index e41538997f4..42f52b4a127 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp @@ -27,8 +27,8 @@ void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, const uint l = l0 + ll; const u8vec2 qs = unpack8(uint32_t(data_a_packed16[ibi].qs[4 * ib32 + l])).xy; // vec4 used due to #12147 const uint sign = data_a[ibi].signs[4 * ib32 + l]; - const vec4 grid0 = vec4(unpack8(iq3_grid[qs.x | ((qh << (8 - 2*l)) & 0x100)])); - const vec4 grid1 = vec4(unpack8(iq3_grid[qs.y | ((qh << (7 - 2*l)) & 0x100)])); + const vec4 grid0 = vec4(unpack8(iq3s_grid[qs.x | ((qh << (8 - 2*l)) & 0x100)])); + const vec4 grid1 = vec4(unpack8(iq3s_grid[qs.y | ((qh << (7 - 2*l)) & 0x100)])); [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { const vec4 b0 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 0]); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp index 5f7a89bc1c3..a88898109ab 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp @@ -23,8 +23,8 @@ void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, const uint qs1 = data_a[ibi].qs[8 * ib32 + 4 * (itid & 1) + 2 * l + 1]; const uint sign = bitfieldExtract(signscale, 7 * int(2 * (itid & 1) + l), 7); const uint sign7 = bitCount(sign); - const vec4 grid0 = vec4(unpack8(iq3_grid[qs0])); - const vec4 grid1 = vec4(unpack8(iq3_grid[qs1])); + const vec4 grid0 = vec4(unpack8(iq3xxs_grid[qs0])); + const vec4 grid1 = vec4(unpack8(iq3xxs_grid[qs1])); [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { const vec4 b0 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 0]); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl index 2c5d69c1fd3..2d829741fcd 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl @@ -131,7 +131,7 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uin const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + (signs >> 28))); const uint32_t sign7 = bitfieldExtract(signs, 7 * int(ib8), 7); const uint sign = sign7 | (bitCount(sign7) << 7); - const uvec2 grid = iq2_grid[qs]; + const uvec2 grid = iq2xxs_grid[qs]; const vec4 grid0 = vec4(unpack8(grid.x)); const vec4 grid1 = vec4(unpack8(grid.y)); @@ -161,7 +161,7 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uin const uint qs = data_a[ib].qs[4 * ib32 + ib8]; const uint sign7 = qs >> 9; const uint sign = sign7 | (bitCount(sign7) << 7); - const uvec2 grid = iq2_grid[qs & 511]; + const uvec2 grid = iq2xs_grid[qs & 511]; const vec4 grid0 = vec4(unpack8(grid.x)); const vec4 grid1 = vec4(unpack8(grid.y)); @@ -193,7 +193,7 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uin const float d = float(data_a[ib].d); const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + scale)); - const uvec2 grid = iq2_grid[qs | ((qh << (8 - qhshift)) & 0x300)]; + const uvec2 grid = iq2s_grid[qs | ((qh << (8 - qhshift)) & 0x300)]; const vec4 grid0 = vec4(unpack8(grid.x)); const vec4 grid1 = vec4(unpack8(grid.y)); @@ -226,7 +226,7 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uin const float db = d * 0.5 * (0.5 + (signs >> 28)); const uint32_t sign7 = bitfieldExtract(signs, 7 * (int(iqs / 2) % 4), 7); const uint sign = (sign7 | (bitCount(sign7) << 7)) >> (4 * (idx % 2)); - const uint grid = iq3_grid[qs]; + const uint grid = iq3xxs_grid[qs]; const vec4 v = db * vec4(unpack8(grid)); store_a(col, k_pair, FLOAT_TYPEV2((sign & 1) != 0 ? -v.x : v.x, @@ -250,7 +250,7 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uin const uint scale = data_a[ib].scales[iqs / 16]; const i8vec2 sign01 = i8vec2(1 - (2 & i8vec2(sign << 1, sign))); const float db = d * (1 + 2 * ((scale >> (4 * (iqh & 1))) & 0xf)); - const uint32_t grid = iq3_grid[qs | ((qh << (8 - (iqs % 8))) & 256)]; + const uint32_t grid = iq3s_grid[qs | ((qh << (8 - (iqs % 8))) & 256)]; const vec4 v = db * vec4(unpack8(grid)); store_a(col, k_pair, FLOAT_TYPEV2((sign & 1) != 0 ? -v.x : v.x, diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl index 0b9ac710b59..2ea1c6fb1ab 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl @@ -948,11 +948,11 @@ void init_iq_shmem(uvec3 wgsize) #if defined(DATA_A_IQ2_XXS) || defined(DATA_A_IQ2_XS) || defined(DATA_A_IQ2_S) #if defined(DATA_A_IQ2_S) -shared uvec2 iq2_grid[1024]; +shared uvec2 iq2s_grid[1024]; #elif defined(DATA_A_IQ2_XS) -shared uvec2 iq2_grid[512]; +shared uvec2 iq2xs_grid[512]; #else -shared uvec2 iq2_grid[256]; +shared uvec2 iq2xxs_grid[256]; #endif #endif @@ -1045,9 +1045,9 @@ const uvec2[256] iq2xxs_grid_const = { void init_iq_shmem(uvec3 wgsize) { // copy the table into shared memory and sync - [[unroll]] for (uint i = 0; i < iq2_grid.length(); i += wgsize.x) { + [[unroll]] for (uint i = 0; i < iq2xxs_grid.length(); i += wgsize.x) { if (iq2xxs_grid_const.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq2xxs_grid_const.length()) { - iq2_grid[i + gl_LocalInvocationIndex.x] = iq2xxs_grid_const[i + gl_LocalInvocationIndex.x]; + iq2xxs_grid[i + gl_LocalInvocationIndex.x] = iq2xxs_grid_const[i + gl_LocalInvocationIndex.x]; } } barrier(); @@ -1217,9 +1217,9 @@ const uvec2 iq2xs_grid_const[512] = { void init_iq_shmem(uvec3 wgsize) { // copy the table into shared memory and sync - [[unroll]] for (uint i = 0; i < iq2_grid.length(); i += wgsize.x) { - if (iq2_grid.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq2xs_grid_const.length()) { - iq2_grid[i + gl_LocalInvocationIndex.x] = iq2xs_grid_const[i + gl_LocalInvocationIndex.x]; + [[unroll]] for (uint i = 0; i < iq2xs_grid.length(); i += wgsize.x) { + if (iq2xs_grid.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq2xs_grid_const.length()) { + iq2xs_grid[i + gl_LocalInvocationIndex.x] = iq2xs_grid_const[i + gl_LocalInvocationIndex.x]; } } barrier(); @@ -1519,9 +1519,9 @@ const uvec2 iq2s_grid_const[1024] = { void init_iq_shmem(uvec3 wgsize) { // copy the table into shared memory and sync - [[unroll]] for (uint i = 0; i < iq2_grid.length(); i += wgsize.x) { - if (iq2_grid.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq2s_grid_const.length()) { - iq2_grid[i + gl_LocalInvocationIndex.x] = iq2s_grid_const[i + gl_LocalInvocationIndex.x]; + [[unroll]] for (uint i = 0; i < iq2s_grid.length(); i += wgsize.x) { + if (iq2s_grid.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq2s_grid_const.length()) { + iq2s_grid[i + gl_LocalInvocationIndex.x] = iq2s_grid_const[i + gl_LocalInvocationIndex.x]; } } barrier(); @@ -1538,9 +1538,9 @@ void init_iq_shmem(uvec3 wgsize) #if defined(DATA_A_IQ3_XXS) || defined(DATA_A_IQ3_S) #if defined(DATA_A_IQ3_S) -shared uint32_t iq3_grid[512]; +shared uint32_t iq3s_grid[512]; #else -shared uint32_t iq3_grid[256]; +shared uint32_t iq3xxs_grid[256]; #endif #endif @@ -1601,9 +1601,9 @@ const uint32_t iq3xxs_grid_const[256] = { void init_iq_shmem(uvec3 wgsize) { // copy the table into shared memory and sync - [[unroll]] for (uint i = 0; i < iq3_grid.length(); i += wgsize.x) { - if (iq3_grid.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq3_grid.length()) { - iq3_grid[i + gl_LocalInvocationIndex.x] = iq3xxs_grid_const[i + gl_LocalInvocationIndex.x]; + [[unroll]] for (uint i = 0; i < iq3xxs_grid.length(); i += wgsize.x) { + if (iq3xxs_grid.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq3xxs_grid.length()) { + iq3xxs_grid[i + gl_LocalInvocationIndex.x] = iq3xxs_grid_const[i + gl_LocalInvocationIndex.x]; } } barrier(); @@ -1713,9 +1713,9 @@ const uint32_t iq3s_grid_const[512] = { void init_iq_shmem(uvec3 wgsize) { // copy the table into shared memory and sync - [[unroll]] for (uint i = 0; i < iq3_grid.length(); i += wgsize.x) { - if (iq3_grid.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq3_grid.length()) { - iq3_grid[i + gl_LocalInvocationIndex.x] = iq3s_grid_const[i + gl_LocalInvocationIndex.x]; + [[unroll]] for (uint i = 0; i < iq3s_grid.length(); i += wgsize.x) { + if (iq3s_grid.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq3s_grid.length()) { + iq3s_grid[i + gl_LocalInvocationIndex.x] = iq3s_grid_const[i + gl_LocalInvocationIndex.x]; } } barrier(); From c14706c1787bfc04389d345ac8ba255a19ebf01f Mon Sep 17 00:00:00 2001 From: Ruben Ortlam Date: Mon, 31 Aug 2026 08:42:55 +0200 Subject: [PATCH 7/9] simplify cm2 code with single uint8_t buffer --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 1 + .../vulkan-shaders/mul_mm_cm2.comp | 138 ++++++------------ 2 files changed, 47 insertions(+), 92 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 4e285b074a0..02fc8ed3617 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -4763,6 +4763,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { spec.push_back(device->subgroup_size); // subgroup_size if (type != UINT32_MAX) { spec.push_back(type); // MmTypeA + spec.push_back((uint32_t)ggml_type_size((ggml_type)type)); // MmABlockBytes } return spec; }; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp index 35ba6d7b463..6e7c34f3025 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp @@ -25,6 +25,7 @@ #ifdef MULMAT_QUANT layout (constant_id = 7) const uint MmTypeA = 0; +layout (constant_id = 8) const uint MmABlockBytes = 2; #endif #include "types.glsl" @@ -99,19 +100,7 @@ layout (push_constant) uniform parameter #ifndef MULMAT_QUANT layout (binding = 0) readonly buffer A {A_TYPE data_a[];}; #else -layout (binding = 0) readonly buffer BUF_Q1_0 { block_q1_0 data[]; } a_q1_0; -layout (binding = 0) readonly buffer BUF_Q2_0 { block_q2_0 data[]; } a_q2_0; -layout (binding = 0) readonly buffer BUF_Q4_0 { block_q4_0 data[]; } a_q4_0; -layout (binding = 0) readonly buffer BUF_Q4_1 { block_q4_1 data[]; } a_q4_1; -layout (binding = 0) readonly buffer BUF_Q5_0 { block_q5_0 data[]; } a_q5_0; -layout (binding = 0) readonly buffer BUF_Q5_1 { block_q5_1 data[]; } a_q5_1; -layout (binding = 0) readonly buffer BUF_Q8_0 { block_q8_0 data[]; } a_q8_0; -layout (binding = 0) readonly buffer BUF_Q2_K { block_q2_K data[]; } a_q2_k; -layout (binding = 0) readonly buffer BUF_TQ2_0 { block_tq2_0 data[]; } a_tq2_0; -layout (binding = 0) readonly buffer BUF_Q3_K { block_q3_K data[]; } a_q3_k; -layout (binding = 0) readonly buffer BUF_Q4_K { block_q4_K data[]; } a_q4_k; -layout (binding = 0) readonly buffer BUF_Q5_K { block_q5_K data[]; } a_q5_k; -layout (binding = 0) readonly buffer BUF_Q6_K { block_q6_K data[]; } a_q6_k; +layout (binding = 0) readonly buffer A {uint8_t data_a[];}; #endif layout (binding = 1) readonly buffer B {B_TYPE data_b[];}; layout (binding = 2) writeonly buffer D {D_TYPE data_d[];}; @@ -129,39 +118,47 @@ layout (binding = 1) readonly buffer B4 {B_TYPEV4 data_b_v4[];}; #define DECODEFUNCA , dequantFuncA #endif #else -// Dispatch coopMatLoadTensorNV to the right decode function -// based on the MmTypeA spec constant. The compiler will dead-code-eliminate -// all but the selected branch. +layout(buffer_reference, std430, buffer_reference_align = 1) buffer decodeBufA { + uint8_t raw[MmABlockBytes]; +}; + +float16_t mmDecodeA(const in decodeBufA bl_in, const in uint blockCoords[2], const in uint coordInBlock[2]) { + switch (MmTypeA) { + case GGML_TYPE_Q1_0: return dequantFuncQ1_0 (decodeBufQ1_0 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q2_0: return dequantFuncQ2_0 (decodeBufQ2_0 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q4_0: return dequantFuncQ4_0 (decodeBufQ4_0 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q4_1: return dequantFuncQ4_1 (decodeBufQ4_1 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q5_0: return dequantFuncQ5_0 (decodeBufQ5_0 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q5_1: return dequantFuncQ5_1 (decodeBufQ5_1 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q8_0: return dequantFuncQ8_0 (decodeBufQ8_0 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q2_K: return dequantFuncQ2_K (decodeBufQ2_K (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_TQ2_0: return dequantFuncTQ2_0(decodeBufTQ2_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q3_K: return dequantFuncQ3_K (decodeBufQ3_K (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q6_K: return dequantFuncQ6_K (decodeBufQ6_K (bl_in), blockCoords, coordInBlock); + default: return float16_t(0); + } +} + #ifdef GGML_VULKAN_COOPMAT2_DECODE_VECTOR -#define COOPMAT_LOAD_A(mat, tensor_layout) \ - if (MmTypeA == GGML_TYPE_Q1_0) { coopMatLoadTensorNV(mat, a_q1_0.data, pos_a, tensor_layout, dequantFuncQ1_0, dequantFuncQ1_0_v); } \ - else if (MmTypeA == GGML_TYPE_Q2_0) { coopMatLoadTensorNV(mat, a_q2_0.data, pos_a, tensor_layout, dequantFuncQ2_0, dequantFuncQ2_0_v); } \ - else if (MmTypeA == GGML_TYPE_Q4_0) { coopMatLoadTensorNV(mat, a_q4_0.data, pos_a, tensor_layout, dequantFuncQ4_0, dequantFuncQ4_0_v); } \ - else if (MmTypeA == GGML_TYPE_Q4_1) { coopMatLoadTensorNV(mat, a_q4_1.data, pos_a, tensor_layout, dequantFuncQ4_1, dequantFuncQ4_1_v); } \ - else if (MmTypeA == GGML_TYPE_Q5_0) { coopMatLoadTensorNV(mat, a_q5_0.data, pos_a, tensor_layout, dequantFuncQ5_0, dequantFuncQ5_0_v); } \ - else if (MmTypeA == GGML_TYPE_Q5_1) { coopMatLoadTensorNV(mat, a_q5_1.data, pos_a, tensor_layout, dequantFuncQ5_1, dequantFuncQ5_1_v); } \ - else if (MmTypeA == GGML_TYPE_Q8_0) { coopMatLoadTensorNV(mat, a_q8_0.data, pos_a, tensor_layout, dequantFuncQ8_0, dequantFuncQ8_0_v); } \ - else if (MmTypeA == GGML_TYPE_Q2_K) { coopMatLoadTensorNV(mat, a_q2_k.data, pos_a, tensor_layout, dequantFuncQ2_K, dequantFuncQ2_K_v); } \ - else if (MmTypeA == GGML_TYPE_TQ2_0) { coopMatLoadTensorNV(mat, a_tq2_0.data, pos_a, tensor_layout, dequantFuncTQ2_0, dequantFuncTQ2_0_v); } \ - else if (MmTypeA == GGML_TYPE_Q3_K) { coopMatLoadTensorNV(mat, a_q3_k.data, pos_a, tensor_layout, dequantFuncQ3_K, dequantFuncQ3_K_v); } \ - else if (MmTypeA == GGML_TYPE_Q4_K) { coopMatLoadTensorNV(mat, a_q4_k.data, pos_a, tensor_layout, dequantFuncQ4_K, dequantFuncQ4_K_v); } \ - else if (MmTypeA == GGML_TYPE_Q5_K) { coopMatLoadTensorNV(mat, a_q5_k.data, pos_a, tensor_layout, dequantFuncQ5_K, dequantFuncQ5_K_v); } \ - else if (MmTypeA == GGML_TYPE_Q6_K) { coopMatLoadTensorNV(mat, a_q6_k.data, pos_a, tensor_layout, dequantFuncQ6_K, dequantFuncQ6_K_v); } +f16vec4 mmDecodeA_v(const in decodeBufA bl_in, const in uint blockCoords[2], const in uint coordInBlock[2]) { + switch (MmTypeA) { + case GGML_TYPE_Q1_0: return dequantFuncQ1_0_v (decodeBufQ1_0 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q2_0: return dequantFuncQ2_0_v (decodeBufQ2_0 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q4_0: return dequantFuncQ4_0_v (decodeBufQ4_0 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q4_1: return dequantFuncQ4_1_v (decodeBufQ4_1 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q5_0: return dequantFuncQ5_0_v (decodeBufQ5_0 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q5_1: return dequantFuncQ5_1_v (decodeBufQ5_1 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q8_0: return dequantFuncQ8_0_v (decodeBufQ8_0 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q2_K: return dequantFuncQ2_K_v (decodeBufQ2_K (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_TQ2_0: return dequantFuncTQ2_0_v(decodeBufTQ2_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q3_K: return dequantFuncQ3_K_v (decodeBufQ3_K (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q6_K: return dequantFuncQ6_K_v (decodeBufQ6_K (bl_in), blockCoords, coordInBlock); + default: return f16vec4(0); + } +} +#define DECODEFUNCA , mmDecodeA, mmDecodeA_v #else -#define COOPMAT_LOAD_A(mat, tensor_layout) \ - if (MmTypeA == GGML_TYPE_Q1_0) { coopMatLoadTensorNV(mat, a_q1_0.data, pos_a, tensor_layout, dequantFuncQ1_0); } \ - else if (MmTypeA == GGML_TYPE_Q2_0) { coopMatLoadTensorNV(mat, a_q2_0.data, pos_a, tensor_layout, dequantFuncQ2_0); } \ - else if (MmTypeA == GGML_TYPE_Q4_0) { coopMatLoadTensorNV(mat, a_q4_0.data, pos_a, tensor_layout, dequantFuncQ4_0); } \ - else if (MmTypeA == GGML_TYPE_Q4_1) { coopMatLoadTensorNV(mat, a_q4_1.data, pos_a, tensor_layout, dequantFuncQ4_1); } \ - else if (MmTypeA == GGML_TYPE_Q5_0) { coopMatLoadTensorNV(mat, a_q5_0.data, pos_a, tensor_layout, dequantFuncQ5_0); } \ - else if (MmTypeA == GGML_TYPE_Q5_1) { coopMatLoadTensorNV(mat, a_q5_1.data, pos_a, tensor_layout, dequantFuncQ5_1); } \ - else if (MmTypeA == GGML_TYPE_Q8_0) { coopMatLoadTensorNV(mat, a_q8_0.data, pos_a, tensor_layout, dequantFuncQ8_0); } \ - else if (MmTypeA == GGML_TYPE_Q2_K) { coopMatLoadTensorNV(mat, a_q2_k.data, pos_a, tensor_layout, dequantFuncQ2_K); } \ - else if (MmTypeA == GGML_TYPE_TQ2_0) { coopMatLoadTensorNV(mat, a_tq2_0.data, pos_a, tensor_layout, dequantFuncTQ2_0); } \ - else if (MmTypeA == GGML_TYPE_Q3_K) { coopMatLoadTensorNV(mat, a_q3_k.data, pos_a, tensor_layout, dequantFuncQ3_K); } \ - else if (MmTypeA == GGML_TYPE_Q4_K) { coopMatLoadTensorNV(mat, a_q4_k.data, pos_a, tensor_layout, dequantFuncQ4_K); } \ - else if (MmTypeA == GGML_TYPE_Q5_K) { coopMatLoadTensorNV(mat, a_q5_k.data, pos_a, tensor_layout, dequantFuncQ5_K); } \ - else if (MmTypeA == GGML_TYPE_Q6_K) { coopMatLoadTensorNV(mat, a_q6_k.data, pos_a, tensor_layout, dequantFuncQ6_K); } +#define DECODEFUNCA , mmDecodeA #endif #endif #else @@ -401,6 +398,11 @@ void main() { uint pos_d = batch_idx * p.batch_stride_d + ik * p.batch_stride_d * p.num_batches; #endif +#ifdef MULMAT_QUANT + // pos_a is a byte offset into the raw buffer; strides stay in block units + pos_a *= MmABlockBytes; +#endif + uint stride_a = p.stride_a / qk; uint stride_b = p.stride_b; @@ -500,11 +502,7 @@ void main() { coopmat mat_a; coopmat mat_b; -#ifndef MULMAT_QUANT coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA); -#else - COOPMAT_LOAD_A(mat_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK)); -#endif coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, ic * BN, BNover4, block_k, BK), tensorViewTranspose); sum = coopMatMulAdd(mat_a, mat_b, sum); @@ -519,11 +517,7 @@ void main() { coopmat mat_a; coopmat mat_b; -#ifndef MULMAT_QUANT coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA); -#else - COOPMAT_LOAD_A(mat_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK)); -#endif coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, ic * BN, BNover4, block_k, BK), tensorViewTranspose); sum = coopMatMulAdd(mat_a, mat_b, sum); @@ -551,11 +545,7 @@ void main() { coopmat mat_a; coopmat mat_b; -#ifndef MULMAT_QUANT coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA); -#else - COOPMAT_LOAD_A(mat_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK)); -#endif coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, ic * BN, BNover2, block_k, BK), tensorViewTranspose); sum = coopMatMulAdd(mat_a, mat_b, sum); @@ -570,11 +560,7 @@ void main() { coopmat mat_a; coopmat mat_b; -#ifndef MULMAT_QUANT coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA); -#else - COOPMAT_LOAD_A(mat_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK)); -#endif coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, ic * BN, BNover2, block_k, BK), tensorViewTranspose); sum = coopMatMulAdd(mat_a, mat_b, sum); @@ -603,11 +589,7 @@ void main() { coopmat mat_a; coopmat mat_b; -#ifndef MULMAT_QUANT coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA); -#else - COOPMAT_LOAD_A(mat_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK)); -#endif coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, ic * BN, BN, block_k, BK), tensorViewTranspose); sum = coopMatMulAdd(mat_a, mat_b, sum); @@ -622,11 +604,7 @@ void main() { coopmat mat_a; coopmat mat_b; -#ifndef MULMAT_QUANT coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA); -#else - COOPMAT_LOAD_A(mat_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK)); -#endif coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, ic * BN, BN, block_k, BK), tensorViewTranspose); sum = coopMatMulAdd(mat_a, mat_b, sum); @@ -678,11 +656,7 @@ void main() { coopmat mat_a; coopmat mat_b; -#ifndef MULMAT_QUANT coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA); -#else - COOPMAT_LOAD_A(mat_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK)); -#endif coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, 0, BNover4, block_k, BK), tensorViewTranspose DECODEFUNCB); sum = coopMatMulAdd(mat_a, mat_b, sum); @@ -690,11 +664,7 @@ void main() { coopmat mat_a; coopmat mat_b; -#ifndef MULMAT_QUANT coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutAClamp, ir * BM, BM, block_k, BK) DECODEFUNCA); -#else - COOPMAT_LOAD_A(mat_a, sliceTensorLayoutNV(tensorLayoutAClamp, ir * BM, BM, block_k, BK)); -#endif coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, 0, BNover4, block_k, BK), tensorViewTranspose DECODEFUNCB); sum = coopMatMulAdd(mat_a, mat_b, sum); @@ -730,11 +700,7 @@ void main() { coopmat mat_a; coopmat mat_b; -#ifndef MULMAT_QUANT coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA); -#else - COOPMAT_LOAD_A(mat_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK)); -#endif coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, 0, BNover2, block_k, BK), tensorViewTranspose DECODEFUNCB); sum = coopMatMulAdd(mat_a, mat_b, sum); @@ -742,11 +708,7 @@ void main() { coopmat mat_a; coopmat mat_b; -#ifndef MULMAT_QUANT coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutAClamp, ir * BM, BM, block_k, BK) DECODEFUNCA); -#else - COOPMAT_LOAD_A(mat_a, sliceTensorLayoutNV(tensorLayoutAClamp, ir * BM, BM, block_k, BK)); -#endif coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, 0, BNover2, block_k, BK), tensorViewTranspose DECODEFUNCB); sum = coopMatMulAdd(mat_a, mat_b, sum); @@ -782,11 +744,7 @@ void main() { coopmat mat_a; coopmat mat_b; -#ifndef MULMAT_QUANT coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA); -#else - COOPMAT_LOAD_A(mat_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK)); -#endif #ifdef MUL_MAT_ID coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, 0, BN, block_k, BK), tensorViewTranspose DECODEFUNCB); #else @@ -798,11 +756,7 @@ void main() { coopmat mat_a; coopmat mat_b; -#ifndef MULMAT_QUANT coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutAClamp, ir * BM, BM, block_k, BK) DECODEFUNCA); -#else - COOPMAT_LOAD_A(mat_a, sliceTensorLayoutNV(tensorLayoutAClamp, ir * BM, BM, block_k, BK)); -#endif #ifdef MUL_MAT_ID coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, 0, BN, block_k, BK), tensorViewTranspose DECODEFUNCB); #else From 1ac1ad53293b06b426c7705fb551afe3bff110cf Mon Sep 17 00:00:00 2001 From: Ruben Ortlam Date: Mon, 31 Aug 2026 10:36:34 +0200 Subject: [PATCH 8/9] fix fp4 extension use switch being overwritten by generic shader --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 46 ++++++++++++++++++---------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 02fc8ed3617..0cbf2e71e06 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -4675,7 +4675,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { GGML_TYPE_Q2_K, GGML_TYPE_TQ2_0, GGML_TYPE_Q3_K, GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K, }; -#define FOR_EACH_LUT_TYPE(X) \ +#define FOR_EACH_LUT_TYPE_NONFP4(X) \ X(GGML_TYPE_IQ1_S, iq1_s) \ X(GGML_TYPE_IQ1_M, iq1_m) \ X(GGML_TYPE_IQ2_XXS, iq2_xxs) \ @@ -4684,12 +4684,13 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { X(GGML_TYPE_IQ3_XXS, iq3_xxs) \ X(GGML_TYPE_IQ3_S, iq3_s) \ X(GGML_TYPE_IQ4_XS, iq4_xs) \ - X(GGML_TYPE_IQ4_NL, iq4_nl) \ - X(GGML_TYPE_MXFP4, mxfp4) \ - X(GGML_TYPE_NVFP4, nvfp4) + X(GGML_TYPE_IQ4_NL, iq4_nl) #define FOR_EACH_LUT_FP4_TYPE(X) \ X(GGML_TYPE_MXFP4, mxfp4) \ X(GGML_TYPE_NVFP4, nvfp4) +#define FOR_EACH_LUT_TYPE(X) \ + FOR_EACH_LUT_TYPE_NONFP4(X) \ + FOR_EACH_LUT_FP4_TYPE(X) const int mul_mat_id_param_count = 5; @@ -4801,8 +4802,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { create_mm_pipelines({TYPE, GGML_TYPE_F16, false, true}, tc, "matmul_" #tstr "_f16_f16acc", matmul_##tstr##_f16_f16acc_cm2_len, matmul_##tstr##_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); \ create_mm_pipelines({TYPE, GGML_TYPE_F16, false, false}, tc, "matmul_" #tstr "_f16", matmul_##tstr##_f16_cm2_len, matmul_##tstr##_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); \ } } - FOR_EACH_LUT_TYPE(X_CM2) -#undef X_CM2 + FOR_EACH_LUT_TYPE_NONFP4(X_CM2) #if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) if (device->ocp_fp4) { #define X_CM2_OCP(TYPE, tstr) \ @@ -4813,8 +4813,12 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { } } FOR_EACH_LUT_FP4_TYPE(X_CM2_OCP) #undef X_CM2_OCP - } + } else #endif + { + FOR_EACH_LUT_FP4_TYPE(X_CM2) + } +#undef X_CM2 GGML_ASSERT(device->subgroup_ballot); @@ -4843,8 +4847,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { create_mm_pipelines({TYPE, GGML_TYPE_F16, true, true}, tc, "matmul_id_subgroup_" #tstr "_f16_f16acc", matmul_id_subgroup_##tstr##_f16_f16acc_cm2_len, matmul_id_subgroup_##tstr##_f16_f16acc_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); \ create_mm_pipelines({TYPE, GGML_TYPE_F16, true, false}, tc, "matmul_id_subgroup_" #tstr "_f16", matmul_id_subgroup_##tstr##_f16_cm2_len, matmul_id_subgroup_##tstr##_f16_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); \ } } - FOR_EACH_LUT_TYPE(X_CM2_ID) -#undef X_CM2_ID + FOR_EACH_LUT_TYPE_NONFP4(X_CM2_ID) #if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) if (device->ocp_fp4) { #define X_CM2_ID_OCP(TYPE, tstr) \ @@ -4855,8 +4858,12 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { } } FOR_EACH_LUT_FP4_TYPE(X_CM2_ID_OCP) #undef X_CM2_ID_OCP - } + } else #endif + { + FOR_EACH_LUT_FP4_TYPE(X_CM2_ID) + } +#undef X_CM2_ID } else #endif // defined(VK_NV_cooperative_matrix2) && defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT) #if defined(VK_KHR_cooperative_matrix) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) @@ -4905,8 +4912,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { if (device->coopmat_acc_f32_support) { \ cm1_create({TYPE, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_" #tstr "_f32", matmul_##tstr##_f32_cm1_len, matmul_##tstr##_f32_cm1_data, sizeof(vk_mat_mat_push_constants), 3); \ } - FOR_EACH_LUT_TYPE(X_CM1) -#undef X_CM1 + FOR_EACH_LUT_TYPE_NONFP4(X_CM1) #if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) if (device->ocp_fp4) { #define X_CM1_OCP(TYPE, tstr) \ @@ -4918,8 +4924,12 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { } FOR_EACH_LUT_FP4_TYPE(X_CM1_OCP) #undef X_CM1_OCP - } + } else #endif + { + FOR_EACH_LUT_FP4_TYPE(X_CM1) + } +#undef X_CM1 GGML_ASSERT(device->subgroup_ballot); @@ -4952,8 +4962,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { if (device->coopmat_acc_f32_support) { \ cm1_create({TYPE, GGML_TYPE_F32, true, false}, tc_mmq, "matmul_id_subgroup_" #tstr "_f32", matmul_id_subgroup_##tstr##_f32_cm1_len, matmul_id_subgroup_##tstr##_f32_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); \ } - FOR_EACH_LUT_TYPE(X_CM1_ID) -#undef X_CM1_ID + FOR_EACH_LUT_TYPE_NONFP4(X_CM1_ID) #if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) if (device->ocp_fp4) { #define X_CM1_ID_OCP(TYPE, tstr) \ @@ -4965,8 +4974,12 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { } FOR_EACH_LUT_FP4_TYPE(X_CM1_ID_OCP) #undef X_CM1_ID_OCP - } + } else #endif + { + FOR_EACH_LUT_FP4_TYPE(X_CM1_ID) + } +#undef X_CM1_ID } else #endif // defined(VK_KHR_cooperative_matrix) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) { @@ -5181,6 +5194,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { } } #undef FOR_EACH_LUT_TYPE +#undef FOR_EACH_LUT_TYPE_NONFP4 #undef FOR_EACH_LUT_FP4_TYPE // BF16 fallback for coopmat devices without bf16 coopmat support if ((device->coopmat2 || device->coopmat_support) From b7b2b390ae8fa0e65fd13e34c6063454b014493c Mon Sep 17 00:00:00 2001 From: Ruben Ortlam Date: Thu, 3 Sep 2026 12:34:34 +0200 Subject: [PATCH 9/9] clean up --- ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp | 9 +-------- .../ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp | 9 --------- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp index 6e7c34f3025..e31f40789c2 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp @@ -337,7 +337,7 @@ void main() { if (tid < BN) { row_ids[tid] = u16vec4(0); } -#if !defined(NEEDS_INIT_IQ_SHMEM) && !defined(MULMAT_QUANT) +#if !defined(NEEDS_INIT_IQ_SHMEM) barrier(); #endif #endif @@ -424,15 +424,8 @@ void main() { #endif tensorLayoutNV<2, gl_CooperativeMatrixClampModeConstantNV> tensorLayoutD = createTensorLayoutNV(2, gl_CooperativeMatrixClampModeConstantNV); -#ifndef MULMAT_QUANT -#if QUANT_K > 1 - tensorLayoutA = setTensorLayoutBlockSizeNV(tensorLayoutA, 1, QUANT_K); - tensorLayoutAClamp = setTensorLayoutBlockSizeNV(tensorLayoutAClamp, 1, QUANT_K); -#endif -#else tensorLayoutA = setTensorLayoutBlockSizeNV(tensorLayoutA, 1, qk); tensorLayoutAClamp = setTensorLayoutBlockSizeNV(tensorLayoutAClamp, 1, qk); -#endif #if defined(MUL_MAT_ID) && defined(GGML_VULKAN_COOPMAT2_DECODE_VECTOR) tensorLayoutB = setTensorLayoutBlockSizeNV(tensorLayoutB, 1, BK); #endif diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp index 100e1d9e703..872c86f21c8 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -668,15 +668,6 @@ void matmul_shaders(bool fp16, MatMulIdType matmul_id_type, bool coopmat, bool c if (!coopmat2) { string_to_spv(shader_name + "_quant_f32" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, quant_float_type_dict), {{"MULMAT_QUANT", "1"}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f32}, {"B_TYPE_SCALAR", "float"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); } - -#if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) - if (coopmat || coopmat2) { - string_to_spv(shader_name + "_quant_f16_ocp" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, quant_float_type_dict), {{"MULMAT_QUANT", "1"}, {"USE_OCP_FP4", "1"}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f16}, {"B_TYPE_SCALAR", "float16_t"}, {"B_TYPEV4", "f16vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); - if (!coopmat2) { - string_to_spv(shader_name + "_quant_f32_ocp" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, quant_float_type_dict), {{"MULMAT_QUANT", "1"}, {"USE_OCP_FP4", "1"}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f32}, {"B_TYPE_SCALAR", "float"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); - } - } -#endif } }