From 6617683e43f90989430528617a5f55f0f0f4c27d Mon Sep 17 00:00:00 2001 From: Simon Teixidor Date: Mon, 7 Sep 2026 15:57:54 +0200 Subject: [PATCH] HIP: per-type MMVQ batch thresholds on RDNA3.5 Add per-type MMVQ batch-size tuning for RDNA3.5 in ggml_cuda_should_use_mmvq, which speeds up batched generation. --- ggml/src/ggml-cuda/mmvq.cu | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu index 6305230b1f92..f0f821f9ad45 100644 --- a/ggml/src/ggml-cuda/mmvq.cu +++ b/ggml/src/ggml-cuda/mmvq.cu @@ -365,6 +365,33 @@ bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11) { return ne11 <= MMVQ_MAX_BATCH_SIZE; } } + if (GGML_CUDA_CC_IS_RDNA3_5(cc)) { + switch (type) { + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + return ne11 <= 2; + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q6_K: + return ne11 <= 3; + case GGML_TYPE_Q8_0: + case GGML_TYPE_Q2_K: + return ne11 <= 4; + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_1: + case GGML_TYPE_MXFP4: + case GGML_TYPE_IQ4_NL: + case GGML_TYPE_IQ3_XXS: + return ne11 <= 5; + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q5_0: + case GGML_TYPE_IQ3_S: + case GGML_TYPE_IQ4_XS: + case GGML_TYPE_IQ1_S: + return ne11 <= 6; + default: + return ne11 <= MMVQ_MAX_BATCH_SIZE; + } + } if (GGML_CUDA_CC_IS_CDNA(cc)) { if (GGML_CUDA_CC_IS_CDNA1(cc)) { switch (type) {