From 11cd1777293d54be9af9511b0b23956fc5bc3b04 Mon Sep 17 00:00:00 2001 From: Michal Harakal Date: Tue, 28 Apr 2026 15:38:06 +0200 Subject: [PATCH] bench(kernel): add KernelMatmulBench for scalar vs Panama parity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Direct Fp32MatmulKernel.matmul JMH harness, sizes 256/512/1024, provider param toggles ScalarMatmulKernel vs PanamaVectorMatmulKernel. Used to validate the M5 milestone target (Panama ≥1.5× scalar) without entanglement from the rest of the op pipeline. Local run on JDK 21.0.10 (M-series macOS) clears the target comfortably: size scalar panama speedup 256 9.454ms 1.356ms 6.97x 512 79.68ms 13.62ms 5.85x 1024 862.8ms 118.2ms 7.30x Adds skainet-backend-api as a direct dep on the bench module so the JMH source set can see the kernel SPI types, and documents the new bench in docs/.../perf/jvm-cpu.adoc. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../ROOT/pages/explanation/perf/jvm-cpu.adoc | 11 ++- .../benchmarks/jvm-cpu-jmh/build.gradle.kts | 1 + .../sk/ainet/bench/KernelMatmulBench.kt | 70 +++++++++++++++++++ 3 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 skainet-backends/benchmarks/jvm-cpu-jmh/src/jmh/kotlin/sk/ainet/bench/KernelMatmulBench.kt diff --git a/docs/modules/ROOT/pages/explanation/perf/jvm-cpu.adoc b/docs/modules/ROOT/pages/explanation/perf/jvm-cpu.adoc index 36bcc71dd..71060e1b5 100644 --- a/docs/modules/ROOT/pages/explanation/perf/jvm-cpu.adoc +++ b/docs/modules/ROOT/pages/explanation/perf/jvm-cpu.adoc @@ -6,7 +6,8 @@ This page explains how to run the JMH benchmarks for the JVM CPU backend and how * Elementwise: FP32 `add` on 1,000,000 elements * Reductions: FP32 `sum` and `mean` on 1,000,000 elements -* Matmul: FP32 square `matmul` with sizes 256, 512, and 1024 +* Matmul (op-level): FP32 square `matmul` with sizes 256, 512, and 1024 — exercises `ctx.ops.matmul`, i.e. the production routing path +* Matmul (kernel-level): direct `Fp32MatmulKernel.matmul` invocation, scalar vs Panama Vector, sizes 256/512/1024 — used to validate the M5 milestone target (Panama ≥1.5× scalar) without entanglement from the rest of the op pipeline Benchmarks are implemented in module: @@ -17,6 +18,7 @@ Source files: * `src/jmh/kotlin/sk/ainet/bench/ElementwiseAdd1MBench.kt` * `src/jmh/kotlin/sk/ainet/bench/Reductions1MBench.kt` * `src/jmh/kotlin/sk/ainet/bench/MatmulBench.kt` +* `src/jmh/kotlin/sk/ainet/bench/KernelMatmulBench.kt` ===== Prerequisites @@ -75,6 +77,13 @@ This will build and execute all JMH benchmarks with the default parameters defin -Pjmh.param.blasEnabled=true .... +* Kernel-level scalar vs Panama at all sizes (M5 ≥1.5× target): + +.... +./gradlew :skainet-backends:benchmarks:jvm-cpu-jmh:jmh \ + -Pjmh.include=KernelMatmulBench +.... + * Matmul at 512 only, comparing BLAS on/off with vector on: .... diff --git a/skainet-backends/benchmarks/jvm-cpu-jmh/build.gradle.kts b/skainet-backends/benchmarks/jvm-cpu-jmh/build.gradle.kts index 5c1a9b5ea..370f78921 100644 --- a/skainet-backends/benchmarks/jvm-cpu-jmh/build.gradle.kts +++ b/skainet-backends/benchmarks/jvm-cpu-jmh/build.gradle.kts @@ -5,6 +5,7 @@ plugins { dependencies { implementation(project(":skainet-lang:skainet-lang-core")) + implementation(project(":skainet-backends:skainet-backend-api")) implementation(project(":skainet-backends:skainet-backend-cpu")) } diff --git a/skainet-backends/benchmarks/jvm-cpu-jmh/src/jmh/kotlin/sk/ainet/bench/KernelMatmulBench.kt b/skainet-backends/benchmarks/jvm-cpu-jmh/src/jmh/kotlin/sk/ainet/bench/KernelMatmulBench.kt new file mode 100644 index 000000000..8ddf64318 --- /dev/null +++ b/skainet-backends/benchmarks/jvm-cpu-jmh/src/jmh/kotlin/sk/ainet/bench/KernelMatmulBench.kt @@ -0,0 +1,70 @@ +package sk.ainet.bench + +import java.util.concurrent.TimeUnit +import org.openjdk.jmh.annotations.Benchmark +import org.openjdk.jmh.annotations.BenchmarkMode +import org.openjdk.jmh.annotations.Level +import org.openjdk.jmh.annotations.Mode +import org.openjdk.jmh.annotations.OutputTimeUnit +import org.openjdk.jmh.annotations.Param +import org.openjdk.jmh.annotations.Scope +import org.openjdk.jmh.annotations.Setup +import org.openjdk.jmh.annotations.State +import sk.ainet.backend.api.kernel.Fp32MatmulKernel +import sk.ainet.exec.kernel.PanamaVectorMatmulKernel +import sk.ainet.exec.kernel.ScalarMatmulKernel + +/** + * Direct kernel-level matmul bench: `Fp32MatmulKernel.matmul` only, + * with no `TensorOps` wrapper / dispatch / context allocation in the + * timed region. Used to validate the M5 milestone target — Panama + * Vector kernel ≥ 1.5× scalar — independent of the rest of the op + * pipeline. + * + * Compare against `MatmulBench`, which exercises the same operation + * through `ctx.ops.matmul` (production routing). Until + * `DefaultCpuOpsJvm.matmul` is wired through `KernelRegistry`, only + * this bench reflects pure kernel-vs-kernel performance. + */ +@State(Scope.Benchmark) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +open class KernelMatmulBench { + + @Param("256", "512", "1024") + var size: Int = 512 + + @Param("scalar", "panama") + var provider: String = "panama" + + private lateinit var kernel: Fp32MatmulKernel + private lateinit var a: FloatArray + private lateinit var b: FloatArray + private lateinit var out: FloatArray + + @Setup(Level.Trial) + fun setup() { + kernel = when (provider) { + "scalar" -> ScalarMatmulKernel + "panama" -> PanamaVectorMatmulKernel + else -> error("unknown provider: $provider") + } + val n = size + // Same input seeding as MatmulBench so numbers compare cleanly. + a = FloatArray(n * n) { ((it % 251) - 125).toFloat() / 127f } + b = FloatArray(n * n) { ((it * 13 % 257) - 128).toFloat() / 127f } + out = FloatArray(n * n) + } + + @Benchmark + fun matmul_fp32_square(): FloatArray { + val n = size + kernel.matmul( + a, 0, n, + b, 0, n, + out, 0, n, + n, n, n, + ) + return out + } +}