From d1525719114e397f8d2c7e47d5a77979dd38fa1b Mon Sep 17 00:00:00 2001 From: BK Box Date: Wed, 15 Jul 2026 15:41:42 -0500 Subject: [PATCH] ggml-cpu: enable Q2_0 VNNI kernel on AVX-VNNI-only CPUs The fast path in ggml_vec_dot_q2_0_q8_0 was gated on __AVX512VNNI__ && __AVX512VL__, which is stricter than needed: the kernel body uses only 256-bit registers, so it runs unchanged on CPUs that have AVX-VNNI but no AVX512 (e.g. Intel Alder Lake / Raptor Lake). The only difference is the intrinsic name: _mm256_dpbusd_avx_epi32 instead of _mm256_dpbusd_epi32. Extend the gate with defined(__AVXVNNI__) and select the intrinsic via a local macro. Builds with -DGGML_AVX_VNNI=ON (or -march=native on supporting CPUs) now take the vectorized path. Measured on i7-12650H (Raptor/Alder Lake, no AVX512) with Ternary-Bonsai-27B Q2_g64: ~8x decode speedup vs the scalar fallback, approaching the memory-bandwidth limit. Co-Authored-By: Claude Fable 5 --- ggml/src/ggml-cpu/arch/x86/quants.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/ggml/src/ggml-cpu/arch/x86/quants.c b/ggml/src/ggml-cpu/arch/x86/quants.c index 0130dd555ba5..a2f1102494a9 100644 --- a/ggml/src/ggml-cpu/arch/x86/quants.c +++ b/ggml/src/ggml-cpu/arch/x86/quants.c @@ -568,9 +568,17 @@ void ggml_vec_dot_q2_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi float sumf = 0.0f; -#if defined(__AVX512VNNI__) && defined(__AVX512VL__) - // AVX-512-VNNI: unpack 2-bit codes c in {0,1,2,3} (value = c-1), then +#if (defined(__AVX512VNNI__) && defined(__AVX512VL__)) || defined(__AVXVNNI__) + // VNNI: unpack 2-bit codes c in {0,1,2,3} (value = c-1), then // dot((c-1), qy) = dpbusd(c, qy) - dpbusd(1, qy). + // The kernel only uses 256-bit registers, so it runs unchanged on + // AVX-VNNI-only CPUs (e.g. Intel Alder/Raptor Lake, where AVX512 is + // unavailable); the AVX-VNNI intrinsic differs only in name. +#if defined(__AVX512VNNI__) && defined(__AVX512VL__) +#define GGML_Q2_0_DPBUSD(acc, a, b) _mm256_dpbusd_epi32(acc, a, b) +#else +#define GGML_Q2_0_DPBUSD(acc, a, b) _mm256_dpbusd_avx_epi32(acc, a, b) +#endif const __m256i ones = _mm256_set1_epi8(1); const __m128i idxlo = _mm_setr_epi8(0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3); const __m128i idxhi = _mm_setr_epi8(4,4,4,4,5,5,5,5,6,6,6,6,7,7,7,7); @@ -591,12 +599,13 @@ void ggml_vec_dot_q2_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi r0 = _mm256_and_si256(_mm256_srli_epi16(_mm256_mullo_epi16(r0, mul), 6), three); r1 = _mm256_and_si256(_mm256_srli_epi16(_mm256_mullo_epi16(r1, mul), 6), three); __m256i codes = _mm256_permute4x64_epi64(_mm256_packus_epi16(r0, r1), 0xD8); // 32 codes in order - const int dp = hsum_i32_8(_mm256_dpbusd_epi32(_mm256_setzero_si256(), codes, qy)); - const int sy = hsum_i32_8(_mm256_dpbusd_epi32(_mm256_setzero_si256(), ones, qy)); + const int dp = hsum_i32_8(GGML_Q2_0_DPBUSD(_mm256_setzero_si256(), codes, qy)); + const int sy = hsum_i32_8(GGML_Q2_0_DPBUSD(_mm256_setzero_si256(), ones, qy)); sumi += d1 * (float)(dp - sy); } sumf += d0 * sumi; } +#undef GGML_Q2_0_DPBUSD #else for (int i = 0; i < nb; i++) { const float d0 = GGML_CPU_FP16_TO_FP32(x[i].d);