perf(backend-native-cpu): runtime FEAT_DotProd dispatch for Apple arm64 Q4_K/Q6_K kernels (#958)

CHANGELOG.md

@@ -43,6 +43,18 @@
 - **`DirectCpuExecutionContext.ops` is cached.** The getter previously constructed a fresh ops
   instance on every access, re-running per-instance lazy kernel resolution in the eager hot
   loop ([#949](https://github.com/SKaiNET-developers/SKaiNET/issues/949)).
+- **Apple arm64 runtime FEAT_DotProd dispatch for the Q4_K/Q6_K C kernels**
+  (`skainet-backend-native-cpu`, [#958](https://github.com/SKaiNET-developers/SKaiNET/issues/958),
+  part of the iOS kernel track of [#920](https://github.com/SKaiNET-developers/SKaiNET/issues/920)).
+  Apple builds now compile at the SDK-default arm64 baseline — a Kotlin/Native klib embeds
+  exactly one static archive, and Apple A12 (iPhone XS/XR, still iOS-supported) lacks
+  FEAT_DotProd while A13+/M-series have it — with the dotprod hot bodies compiled twice
+  (baseline + `target("dotprod")`-attributed) and selected once per matmul via a cached
+  `sysctlbyname("hw.optional.arm.FEAT_DotProd")` probe. Non-Apple builds keep the compile-time
+  `-march` guard as the only mechanism; Linux codegen is unchanged (qemu parity green, `sdot`
+  verified in the cross archive). The existing macOS FFM dylib moves from TU-level dotprod to
+  baseline+dispatch — runtime-equivalent on every Apple Silicon Mac. iOS builds are static-only
+  (`SKAINET_STATIC_ONLY`, auto-on for `CMAKE_SYSTEM_NAME=iOS`).
 
 ## [0.39.0] - 2026-08-10
 

skainet-backends/skainet-backend-native-cpu/native/CMakeLists.txt

@@ -11,6 +11,7 @@ endif()
 
 set(SKAINET_KERNEL_SOURCES
     src/skainet_smoke.c
+    src/skainet_cpu_features.c
     src/q4k_matmul.c
     src/q5k_matmul.c
     src/q6k_matmul.c
@@ -27,11 +28,22 @@ set(SKAINET_KERNEL_SOURCES
 #         resource (libskainet_kernels.{so,dylib,dll}).
 # STATIC: consumed by Kotlin/Native via cinterop, linked into the K/N binary
 #         (libskainet_kernels.a). Same sources + flags; both keep -fPIC.
-add_library(skainet_kernels SHARED ${SKAINET_KERNEL_SOURCES})
+# iOS has no dylib story for this library (K/N embeds the static archive into
+# the klib; nothing dlopens on iOS), and building the SHARED target there just
+# adds install_name/signing noise — static-only on iOS, overridable elsewhere.
+option(SKAINET_STATIC_ONLY "Build only the static archive (K/N cinterop embedding)" OFF)
+if(CMAKE_SYSTEM_NAME STREQUAL "iOS")
+    set(SKAINET_STATIC_ONLY ON)
+endif()
+
 add_library(skainet_kernels_static STATIC ${SKAINET_KERNEL_SOURCES})
 set_target_properties(skainet_kernels_static PROPERTIES OUTPUT_NAME skainet_kernels)
+set(SKAINET_KERNEL_TARGETS skainet_kernels_static)
 
-set(SKAINET_KERNEL_TARGETS skainet_kernels skainet_kernels_static)
+if(NOT SKAINET_STATIC_ONLY)
+    add_library(skainet_kernels SHARED ${SKAINET_KERNEL_SOURCES})
+    list(APPEND SKAINET_KERNEL_TARGETS skainet_kernels)
+endif()
 
 foreach(tgt IN LISTS SKAINET_KERNEL_TARGETS)
     target_include_directories(${tgt} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
@@ -60,7 +72,13 @@ foreach(tgt IN LISTS SKAINET_KERNEL_TARGETS)
         # would SIGILL on the board. CONFIRM on-device first: `grep Features
         # /proc/cpuinfo` must list `asimddp` and `fphp`/`asimdhp`. Explicit
         # -march (not -mcpu=native) because this is a cross-compile from x86.
-        if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64")
+        #
+        # NOT APPLE: Apple arm64 builds stay at the SDK-default baseline — a
+        # klib embeds ONE archive and A12 (iPhone XS/XR) lacks FEAT_DotProd.
+        # The q4k/q6k dotprod bodies are per-function multiversioned there and
+        # selected at runtime via sysctl (SKAINET_DOTPROD_DISPATCH in
+        # skainet_simd.h, #920). NEON itself needs no flag on Apple.
+        if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64" AND NOT APPLE)
             target_compile_options(${tgt} PRIVATE
                 -march=armv8.2-a+fp16+dotprod
             )

skainet-backends/skainet-backend-native-cpu/native/include/skainet_cpu_features.h

@@ -0,0 +1,17 @@
+#ifndef SKAINET_CPU_FEATURES_H
+#define SKAINET_CPU_FEATURES_H
+
+/*
+ * Runtime CPU feature probes for the native kernels. Internal — this header
+ * is NOT part of the cinterop surface (skainet_kernels.def headerFilter).
+ *
+ * Returns 1 if the CPU executes FEAT_DotProd (sdot/udot). On non-Apple
+ * builds this is a compile-time constant derived from the -march the TU was
+ * built with; on Apple arm64 it is a cached sysctl probe, because a klib
+ * embeds exactly one archive and Apple A12 (iPhone XS/XR, still supported
+ * by current iOS) lacks the feature while A13+ and all Apple Silicon have
+ * it (#920).
+ */
+int skainet_cpu_has_dotprod(void);
+
+#endif /* SKAINET_CPU_FEATURES_H */

skainet-backends/skainet-backend-native-cpu/native/include/skainet_simd.h

@@ -43,6 +43,29 @@
 #  define SKAINET_HAVE_I8MM 1
 #endif
 
+/*
+ * Apple arm64 runtime dotprod dispatch (#920). Apple archives are compiled at
+ * the SDK-default baseline: a Kotlin/Native klib embeds exactly ONE static
+ * archive, and Apple A12 (iPhone XS/XR, still iOS-supported) lacks
+ * FEAT_DotProd while A13+/M-series have it — so `-march=...+dotprod` at the
+ * TU level is not shippable there. Instead the dotprod hot bodies in
+ * q4k/q6k are compiled twice: a baseline copy and a copy carrying
+ * `__attribute__((target("dotprod")))` (which is what gates vdotq_s32
+ * codegen in clang; attributed functions are never inlined into baseline
+ * callers, keeping sdot out of the baseline path). Selection happens once
+ * per matmul call via the cached sysctl probe in skainet_cpu_features.h.
+ *
+ * Non-Apple builds keep the compile-time guard as the ONLY mechanism:
+ * SKAINET_DOTPROD_DISPATCH stays undefined and the TU-level -march carries
+ * the feature, so Linux codegen is unchanged.
+ */
+#if defined(__APPLE__) && defined(SKAINET_HAVE_NEON) && !defined(SKAINET_HAVE_DOTPROD)
+#  define SKAINET_DOTPROD_DISPATCH 1
+#  define SKAINET_DOTPROD_TARGET __attribute__((target("dotprod")))
+#else
+#  define SKAINET_DOTPROD_TARGET /* TU-level -march carries the feature */
+#endif
+
 #ifdef SKAINET_HAVE_NEON
 /* Horizontal sum of a float32x4 lane vector. AArch64 has vaddvq_f32
  * natively; this wrapper keeps call sites readable. */

skainet-backends/skainet-backend-native-cpu/native/src/q4k_matmul.c

@@ -1,5 +1,6 @@
 #include "skainet_kernels.h"
 #include "skainet_simd.h"
+#include "skainet_cpu_features.h"
 
 #include <stddef.h>
 #include <stdint.h>
@@ -89,6 +90,91 @@ static inline float skainet_q8_quantize_block(const float* SKAINET_RESTRICT in,
     return d_in;
 }
 
+/*
+ * One Q4_K block × one output row: the four 64-byte groups, each covering a
+ * lo/hi sub-block pair. Extracted so the dotprod body can be compiled twice
+ * for the Apple runtime dispatch (see SKAINET_DOTPROD_DISPATCH in
+ * skainet_simd.h) — one call per block × output row, so the call overhead is
+ * amortized over 128 weight bytes + 64 int8 dots.
+ */
+#if defined(SKAINET_HAVE_DOTPROD) || defined(SKAINET_DOTPROD_DISPATCH)
+SKAINET_DOTPROD_TARGET
+static void skainet_q4k_block_dot_dp(
+    const uint8_t* SKAINET_RESTRICT qs,
+    const int8_t* SKAINET_RESTRICT q8_block,
+    const int* SKAINET_RESTRICT scale_idx,
+    const int* SKAINET_RESTRICT min_idx,
+    int64_t* SKAINET_RESTRICT block_scale_dot,
+    int64_t* SKAINET_RESTRICT block_min_sum
+) {
+    for (int group_j = 0; group_j < 4; ++group_j) {
+        const uint8_t* qs_group = qs + group_j * Q4K_SUB_BLOCK_SIZE;
+        const int sb_lo = 2 * group_j;
+        const int sb_hi = sb_lo + 1;
+        const int8_t* q8_lo = q8_block + sb_lo * Q4K_SUB_BLOCK_SIZE;
+        const int8_t* q8_hi = q8_block + sb_hi * Q4K_SUB_BLOCK_SIZE;
+
+        int32x4_t acc_dot_lo = vdupq_n_s32(0), acc_dot_hi = vdupq_n_s32(0);
+        int32_t acc_sum_lo = 0, acc_sum_hi = 0;
+        for (int off = 0; off < Q4K_SUB_BLOCK_SIZE; off += 16) {
+            const uint8x16_t packed = vld1q_u8(qs_group + off);
+            const int8x16_t code_lo = vreinterpretq_s8_u8(vandq_u8(packed, vdupq_n_u8(0x0F)));
+            const int8x16_t code_hi = vreinterpretq_s8_u8(vshrq_n_u8(packed, 4));
+            const int8x16_t a_lo = vld1q_s8(q8_lo + off);
+            const int8x16_t a_hi = vld1q_s8(q8_hi + off);
+            acc_dot_lo = vdotq_s32(acc_dot_lo, code_lo, a_lo);
+            acc_dot_hi = vdotq_s32(acc_dot_hi, code_hi, a_hi);
+            acc_sum_lo += vaddlvq_s8(a_lo);
+            acc_sum_hi += vaddlvq_s8(a_hi);
+        }
+        const int32_t dot_lo = vaddvq_s32(acc_dot_lo);
+        const int32_t dot_hi = vaddvq_s32(acc_dot_hi);
+
+        *block_scale_dot += (int64_t) scale_idx[sb_lo] * dot_lo
+                          + (int64_t) scale_idx[sb_hi] * dot_hi;
+        *block_min_sum   += (int64_t) min_idx[sb_lo] * acc_sum_lo
+                          + (int64_t) min_idx[sb_hi] * acc_sum_hi;
+    }
+}
+#endif
+
+#if !defined(SKAINET_HAVE_DOTPROD) || defined(SKAINET_DOTPROD_DISPATCH)
+static void skainet_q4k_block_dot_generic(
+    const uint8_t* SKAINET_RESTRICT qs,
+    const int8_t* SKAINET_RESTRICT q8_block,
+    const int* SKAINET_RESTRICT scale_idx,
+    const int* SKAINET_RESTRICT min_idx,
+    int64_t* SKAINET_RESTRICT block_scale_dot,
+    int64_t* SKAINET_RESTRICT block_min_sum
+) {
+    for (int group_j = 0; group_j < 4; ++group_j) {
+        const uint8_t* qs_group = qs + group_j * Q4K_SUB_BLOCK_SIZE;
+        const int sb_lo = 2 * group_j;
+        const int sb_hi = sb_lo + 1;
+        const int8_t* q8_lo = q8_block + sb_lo * Q4K_SUB_BLOCK_SIZE;
+        const int8_t* q8_hi = q8_block + sb_hi * Q4K_SUB_BLOCK_SIZE;
+
+        int32_t dot_lo = 0, sum_lo = 0, dot_hi = 0, sum_hi = 0;
+        for (int i = 0; i < Q4K_SUB_BLOCK_SIZE; ++i) {
+            const uint8_t pb = qs_group[i];
+            const int code_lo = (int)(pb & 0x0F);
+            const int code_hi = (int)(pb >> 4);
+            const int a_lo = (int) q8_lo[i];
+            const int a_hi = (int) q8_hi[i];
+            dot_lo += a_lo * code_lo;
+            sum_lo += a_lo;
+            dot_hi += a_hi * code_hi;
+            sum_hi += a_hi;
+        }
+
+        *block_scale_dot += (int64_t) scale_idx[sb_lo] * dot_lo
+                          + (int64_t) scale_idx[sb_hi] * dot_hi;
+        *block_min_sum   += (int64_t) min_idx[sb_lo] * sum_lo
+                          + (int64_t) min_idx[sb_hi] * sum_hi;
+    }
+}
+#endif
+
 /*
  * Native Q4_K matrix-vector multiply matching the
  * sk.ainet.backend.api.kernel.Q4KMatmulKernel SPI contract. Single input row
@@ -117,6 +203,11 @@ SKAINET_API void skainet_q4k_matmul(
 ) {
     if (output_dim <= 0 || input_dim <= 0) return;
 
+#ifdef SKAINET_DOTPROD_DISPATCH
+    /* One probe per matmul call; cached in skainet_cpu_has_dotprod. */
+    const int use_dp = skainet_cpu_has_dotprod();
+#endif
+
     const int32_t blocks_per_input_dim = input_dim / Q4K_BLOCK_SIZE;
     const float* in_base = input + input_offset;
     float* out_base = output + output_offset;
@@ -166,53 +257,22 @@ SKAINET_API void skainet_q4k_matmul(
             int64_t block_scale_dot = 0;
             int64_t block_min_sum = 0;
 
-            for (int group_j = 0; group_j < 4; ++group_j) {
-                const uint8_t* qs_group = qs + group_j * Q4K_SUB_BLOCK_SIZE;
-                const int sb_lo = 2 * group_j;
-                const int sb_hi = sb_lo + 1;
-                const int8_t* q8_lo = q8_block + sb_lo * Q4K_SUB_BLOCK_SIZE;
-                const int8_t* q8_hi = q8_block + sb_hi * Q4K_SUB_BLOCK_SIZE;
-
-                int32_t dot_lo = 0, sum_lo = 0, dot_hi = 0, sum_hi = 0;
-
-#ifdef SKAINET_HAVE_DOTPROD
-                int32x4_t acc_dot_lo = vdupq_n_s32(0), acc_dot_hi = vdupq_n_s32(0);
-                int32_t acc_sum_lo = 0, acc_sum_hi = 0;
-                for (int off = 0; off < Q4K_SUB_BLOCK_SIZE; off += 16) {
-                    const uint8x16_t packed = vld1q_u8(qs_group + off);
-                    const int8x16_t code_lo = vreinterpretq_s8_u8(vandq_u8(packed, vdupq_n_u8(0x0F)));
-                    const int8x16_t code_hi = vreinterpretq_s8_u8(vshrq_n_u8(packed, 4));
-                    const int8x16_t a_lo = vld1q_s8(q8_lo + off);
-                    const int8x16_t a_hi = vld1q_s8(q8_hi + off);
-                    acc_dot_lo = vdotq_s32(acc_dot_lo, code_lo, a_lo);
-                    acc_dot_hi = vdotq_s32(acc_dot_hi, code_hi, a_hi);
-                    acc_sum_lo += vaddlvq_s8(a_lo);
-                    acc_sum_hi += vaddlvq_s8(a_hi);
-                }
-                dot_lo = vaddvq_s32(acc_dot_lo);
-                dot_hi = vaddvq_s32(acc_dot_hi);
-                sum_lo = acc_sum_lo;
-                sum_hi = acc_sum_hi;
+#if defined(SKAINET_HAVE_DOTPROD)
+            skainet_q4k_block_dot_dp(qs, q8_block, scale_idx, min_idx,
+                                     &block_scale_dot, &block_min_sum);
+#elif defined(SKAINET_DOTPROD_DISPATCH)
+            if (use_dp) {
+                skainet_q4k_block_dot_dp(qs, q8_block, scale_idx, min_idx,
+                                         &block_scale_dot, &block_min_sum);
+            } else {
+                skainet_q4k_block_dot_generic(qs, q8_block, scale_idx, min_idx,
+                                              &block_scale_dot, &block_min_sum);
+            }
 #else
-                for (int i = 0; i < Q4K_SUB_BLOCK_SIZE; ++i) {
-                    const uint8_t pb = qs_group[i];
-                    const int code_lo = (int)(pb & 0x0F);
-                    const int code_hi = (int)(pb >> 4);
-                    const int a_lo = (int) q8_lo[i];
-                    const int a_hi = (int) q8_hi[i];
-                    dot_lo += a_lo * code_lo;
-                    sum_lo += a_lo;
-                    dot_hi += a_hi * code_hi;
-                    sum_hi += a_hi;
-                }
+            skainet_q4k_block_dot_generic(qs, q8_block, scale_idx, min_idx,
+                                          &block_scale_dot, &block_min_sum);
 #endif
 
-                block_scale_dot += (int64_t) scale_idx[sb_lo] * dot_lo
-                                 + (int64_t) scale_idx[sb_hi] * dot_hi;
-                block_min_sum   += (int64_t) min_idx[sb_lo] * sum_lo
-                                 + (int64_t) min_idx[sb_hi] * sum_hi;
-            }
-
             out_base[o] += di * (d * (float) block_scale_dot - d_min * (float) block_min_sum);
         }
     }

skainet-backends/skainet-backend-native-cpu/native/src/q6k_matmul.c

@@ -1,5 +1,6 @@
 #include "skainet_kernels.h"
 #include "skainet_simd.h"
+#include "skainet_cpu_features.h"
 
 #include <stddef.h>
 #include <stdint.h>
@@ -109,30 +110,46 @@ static inline void skainet_q6k_unpack_codes(const uint8_t* SKAINET_RESTRICT bloc
  * scale sc[half*8 + is + 2*k]. On AArch64 with dotprod each 16-element dot is a
  * single vdotq_s32; otherwise a scalar fallback (auto-vectorizes under -O3).
  */
-static inline int64_t skainet_q6k_weighted_dot(const int8_t* SKAINET_RESTRICT q8,
-                                               const int8_t* SKAINET_RESTRICT codes,
-                                               const int8_t* SKAINET_RESTRICT sc) {
+#if defined(SKAINET_HAVE_DOTPROD) || defined(SKAINET_DOTPROD_DISPATCH)
+SKAINET_DOTPROD_TARGET
+static int64_t skainet_q6k_weighted_dot_dp(const int8_t* SKAINET_RESTRICT q8,
+                                           const int8_t* SKAINET_RESTRICT codes,
+                                           const int8_t* SKAINET_RESTRICT sc) {
     int64_t sum = 0;
     for (int half = 0; half < 2; ++half) {
         for (int k = 0; k < 4; ++k) {
             for (int is = 0; is < 2; ++is) {
                 const int start = half * 128 + 32 * k + is * 16;
                 const int gs = half * 8 + is + 2 * k;
-                int32_t dot;
-#ifdef SKAINET_HAVE_DOTPROD
                 const int32x4_t acc = vdotq_s32(vdupq_n_s32(0),
                     vld1q_s8(codes + start), vld1q_s8(q8 + start));
-                dot = vaddvq_s32(acc);
-#else
-                dot = 0;
-                for (int j = 0; j < 16; ++j) dot += (int) q8[start + j] * (int) codes[start + j];
+                sum += (int64_t) sc[gs] * vaddvq_s32(acc);
+            }
+        }
+    }
+    return sum;
+}
 #endif
+
+#if !defined(SKAINET_HAVE_DOTPROD) || defined(SKAINET_DOTPROD_DISPATCH)
+static int64_t skainet_q6k_weighted_dot_generic(const int8_t* SKAINET_RESTRICT q8,
+                                                const int8_t* SKAINET_RESTRICT codes,
+                                                const int8_t* SKAINET_RESTRICT sc) {
+    int64_t sum = 0;
+    for (int half = 0; half < 2; ++half) {
+        for (int k = 0; k < 4; ++k) {
+            for (int is = 0; is < 2; ++is) {
+                const int start = half * 128 + 32 * k + is * 16;
+                const int gs = half * 8 + is + 2 * k;
+                int32_t dot = 0;
+                for (int j = 0; j < 16; ++j) dot += (int) q8[start + j] * (int) codes[start + j];
                 sum += (int64_t) sc[gs] * dot;
             }
         }
     }
     return sum;
 }
+#endif
 
 /*
  * Native Q6_K matrix-vector multiply matching the
@@ -158,6 +175,11 @@ SKAINET_API void skainet_q6k_matmul(
 ) {
     if (output_dim <= 0 || input_dim <= 0) return;
 
+#ifdef SKAINET_DOTPROD_DISPATCH
+    /* One probe per matmul call; cached in skainet_cpu_has_dotprod. */
+    const int use_dp = skainet_cpu_has_dotprod();
+#endif
+
     const int32_t blocks_per_input_dim = input_dim / Q6K_BLOCK_SIZE;
     const float* in_base = input + input_offset;
     float* out_base = output + output_offset;
@@ -196,7 +218,15 @@ SKAINET_API void skainet_q6k_matmul(
             const int8_t* sc = (const int8_t*)(block + Q6K_SCALES_OFFSET);
 
             skainet_q6k_unpack_codes(block, codes);
-            const int64_t wdot = skainet_q6k_weighted_dot(q8_block, codes, sc);
+#if defined(SKAINET_HAVE_DOTPROD)
+            const int64_t wdot = skainet_q6k_weighted_dot_dp(q8_block, codes, sc);
+#elif defined(SKAINET_DOTPROD_DISPATCH)
+            const int64_t wdot = use_dp
+                ? skainet_q6k_weighted_dot_dp(q8_block, codes, sc)
+                : skainet_q6k_weighted_dot_generic(q8_block, codes, sc);
+#else
+            const int64_t wdot = skainet_q6k_weighted_dot_generic(q8_block, codes, sc);
+#endif
 
             out_base[o] += d * di * (float) wdot;
         }