From 325dde1b00c3a2e97a72e3dc0b19ebf9214b125a Mon Sep 17 00:00:00 2001 From: Michal Harakal Date: Wed, 26 Aug 2026 14:22:35 +0200 Subject: [PATCH] feat(bench): ternary f32 gemv scenario + the loaded-weight memory evidence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit engine-ternary-f32-gemv is the first ternary scenario, and deliberately benches the REAL dispatch path rather than a raw kernel loop — what a decode step pays, adapters included. Three providers for the same operands: ffm-lut (the vendored NeoGPU kernel behind the exact FP32×b1.58 key), int8 (per-call I8-absmax requantize + the portable bitnet_gemv reference), f32-reference (the Kotlin oracle on the exact key). BitNet-2B FFN dims (k=2560, n=6912 — crosses the LUT kernel's internal pthread threshold). Measured on an Apple-arm64 dev host through real dispatch: ffm-lut 51.8 GOPS vs int8 0.43 GOPS vs reference 0.30 GOPS — exact math, two orders of magnitude over the portable path (#1136 proof point 1). TernaryWeightMemoryTest is the evidence artifact for proof point 2: the same I2_S GGUF tensor loaded keep-packed vs FP32-widened, asserting the >=15x ratio (measured 15.98x: 0.25 B/weight + scale vs 4 B/weight). jvm-cpu-publish gains the backend-native-cpu dependency so the benchmark runs the bundled libskainet_kernels exactly as a consumer would. Closes #1141 together with the tutorial that landed in #1163. The OpenBenchmarking profile dir and a JMH twin stay optional follow-ups. Refs #1141, #1136 Co-Authored-By: Claude Fable 5 --- .../jvm-cpu-publish/build.gradle.kts | 4 + .../publish/scenarios/ScenarioRegistry.kt | 2 + .../scenarios/TernaryF32GemvScenario.kt | 96 +++++++++++++++++++ .../ainet/io/gguf/TernaryWeightMemoryTest.kt | 77 +++++++++++++++ 4 files changed, 179 insertions(+) create mode 100644 skainet-backends/benchmarks/jvm-cpu-publish/src/main/kotlin/sk/ainet/bench/publish/scenarios/TernaryF32GemvScenario.kt create mode 100644 skainet-io/skainet-io-gguf/src/jvmTest/kotlin/sk/ainet/io/gguf/TernaryWeightMemoryTest.kt diff --git a/skainet-backends/benchmarks/jvm-cpu-publish/build.gradle.kts b/skainet-backends/benchmarks/jvm-cpu-publish/build.gradle.kts index 2402d9025..18b34818e 100644 --- a/skainet-backends/benchmarks/jvm-cpu-publish/build.gradle.kts +++ b/skainet-backends/benchmarks/jvm-cpu-publish/build.gradle.kts @@ -13,6 +13,10 @@ dependencies { implementation(project(":skainet-lang:skainet-lang-core")) implementation(project(":skainet-backends:skainet-backend-api")) implementation(project(":skainet-backends:skainet-backend-cpu")) + // The ternary scenario benches the vendored NeoGPU LUT kernel through its + // FFM face (#1141) — the native module bundles libskainet_kernels as a + // jar resource, so the benchmark runs it exactly as a consumer would. + implementation(project(":skainet-backends:skainet-backend-native-cpu")) testImplementation(kotlin("test")) testImplementation(libs.kotest.runner.junit5) diff --git a/skainet-backends/benchmarks/jvm-cpu-publish/src/main/kotlin/sk/ainet/bench/publish/scenarios/ScenarioRegistry.kt b/skainet-backends/benchmarks/jvm-cpu-publish/src/main/kotlin/sk/ainet/bench/publish/scenarios/ScenarioRegistry.kt index b4b747cc3..74a595fea 100644 --- a/skainet-backends/benchmarks/jvm-cpu-publish/src/main/kotlin/sk/ainet/bench/publish/scenarios/ScenarioRegistry.kt +++ b/skainet-backends/benchmarks/jvm-cpu-publish/src/main/kotlin/sk/ainet/bench/publish/scenarios/ScenarioRegistry.kt @@ -10,6 +10,7 @@ public object ScenarioRegistry { "engine-kernel-matmul", "engine-bf16-matmul", "engine-q8-matmul", + "engine-ternary-f32-gemv", "engine-elementwise-add", "engine-reductions-sum", "engine-reductions-mean", @@ -21,6 +22,7 @@ public object ScenarioRegistry { "engine-kernel-matmul" -> KernelMatmulScenario(smoke = smoke, providerName = provider) "engine-bf16-matmul" -> Bf16MatmulScenario(smoke = smoke, providerName = provider) "engine-q8-matmul" -> Q8MatmulScenario(smoke = smoke, providerName = provider) + "engine-ternary-f32-gemv" -> TernaryF32GemvScenario(smoke = smoke, providerName = provider) "engine-elementwise-add" -> ElementwiseAddScenario(smoke = smoke, providerName = provider) "engine-reductions-sum" -> ReductionsScenario(op = ReductionOp.SUM, smoke = smoke, providerName = provider) "engine-reductions-mean" -> ReductionsScenario(op = ReductionOp.MEAN, smoke = smoke, providerName = provider) diff --git a/skainet-backends/benchmarks/jvm-cpu-publish/src/main/kotlin/sk/ainet/bench/publish/scenarios/TernaryF32GemvScenario.kt b/skainet-backends/benchmarks/jvm-cpu-publish/src/main/kotlin/sk/ainet/bench/publish/scenarios/TernaryF32GemvScenario.kt new file mode 100644 index 000000000..627ae8675 --- /dev/null +++ b/skainet-backends/benchmarks/jvm-cpu-publish/src/main/kotlin/sk/ainet/bench/publish/scenarios/TernaryF32GemvScenario.kt @@ -0,0 +1,96 @@ +package sk.ainet.bench.publish.scenarios + +import sk.ainet.backend.api.kernel.KernelDispatch +import sk.ainet.backend.api.kernel.TernaryF32GemvKernel +import sk.ainet.backend.api.kernel.TernaryKernelPacks +import sk.ainet.bench.publish.runner.Scenario +import sk.ainet.exec.kernel.NativeTernaryF32GemvKernel +import sk.ainet.lang.memory.ExperimentalMemoryApi +import sk.ainet.lang.memory.Scope +import sk.ainet.lang.memory.Storage +import sk.ainet.lang.memory.TensorView +import sk.ainet.lang.tensor.Shape +import sk.ainet.lang.tensor.data.BitNetB158TensorData +import sk.ainet.lang.types.FP32 +import kotlin.random.Random + +/** + * FP32 activation × `BITNET_B1_58` ternary weight through the REAL dispatch path (#1141) — the + * first ternary scenario, and deliberately not a raw kernel loop: what is measured is what a + * decode step pays, adapters included. + * + * Providers select the path `KernelDispatch.matmul` takes for the same operands: + * + * - `ffm-lut` — the vendored NeoGPU LUT kernel (#1137) behind the exact FP32×b1.58 key (#1138): + * exact math, no requantization step. The kernel threads internally above 512 output rows. + * - `int8` — the pre-existing W1.58A8 path: per-call I8-absmax requantization adapter + the + * portable `bitnet_gemv` reference (the JVM has no native `bitnet_gemv`), ~1.5 % quant error. + * - `f32-reference` — the portable Kotlin f32 reference registered on the exact key: exact math + * at reference speed, the LUT kernel's correctness oracle. + * + * Dims default to the BitNet-2B FFN projection (k=2560, n=6912); the n=6912 regime also crosses + * the LUT kernel's internal pthread threshold. Primary metric: GOP/s over `2·k·n` ops. + */ +@OptIn(ExperimentalMemoryApi::class) +internal class TernaryF32GemvScenario( + smoke: Boolean, + private val providerName: String, +) : Scenario { + override val id: String = "engine-ternary-f32-gemv" + override val suite: String = "skainet-engine" + override val primaryMetric: String = "gops" + override val unit: String = "gops" + override val higherIsBetter: Boolean = true + override val kernelProvider: String = providerName + + private val inputDim: Int = 2560 + private val outputDim: Int = if (smoke) 256 else 6912 + override val parameters: Map = mapOf( + "input_dim" to inputDim.toString(), + "output_dim" to outputDim.toString(), + "kernel" to providerName, + ) + + private lateinit var activation: TensorView + private lateinit var weight: TensorView + private lateinit var out: TensorView + + override fun setup() { + KernelDispatch.clearForTesting() + when (providerName.lowercase()) { + "ffm-lut", "ffm", "native", "lut" -> { + TernaryKernelPacks.install(native = null, warn = {}) + check(NativeTernaryF32GemvKernel.isAvailable()) { + "bundled libskainet_kernels missing — the ffm-lut provider cannot run" + } + NativeTernaryF32GemvKernel.install() + } + "int8", "requant" -> TernaryKernelPacks.install(native = null, warn = {}) + "f32-reference", "reference", "scalar" -> + KernelDispatch.register(TernaryF32GemvKernel(TernaryF32GemvKernel.keyFor())) + else -> error("unknown kernel provider: $providerName (use 'ffm-lut', 'int8' or 'f32-reference')") + } + + val rng = Random((inputDim + outputDim).toLong()) + val values = FloatArray(outputDim * inputDim) { (rng.nextInt(3) - 1) * 0.5f } + weight = BitNetB158TensorData.fromFloats(Shape(outputDim, inputDim), values).packedView + activation = TensorView.dense( + Storage.Heap.wrap(FloatArray(inputDim) { rng.nextFloat() - 0.5f }), + Shape(1, inputDim), FP32, + ) + out = TensorView.dense(Storage.Heap.floats(outputDim), Shape(1, outputDim), FP32) + } + + override fun runOnce(): Double { + val start = System.nanoTime() + KernelDispatch.matmul(activation, weight, out, Scope.Ambient) + val elapsedNs = System.nanoTime() - start + @Suppress("UNUSED_VARIABLE") val sink = out.get(0, 0) + val ops = 2.0 * inputDim.toDouble() * outputDim.toDouble() + return (ops / (elapsedNs / 1_000_000_000.0)) / 1e9 + } + + override fun teardown() { + KernelDispatch.clearForTesting() + } +} diff --git a/skainet-io/skainet-io-gguf/src/jvmTest/kotlin/sk/ainet/io/gguf/TernaryWeightMemoryTest.kt b/skainet-io/skainet-io-gguf/src/jvmTest/kotlin/sk/ainet/io/gguf/TernaryWeightMemoryTest.kt new file mode 100644 index 000000000..ecfbd80eb --- /dev/null +++ b/skainet-io/skainet-io-gguf/src/jvmTest/kotlin/sk/ainet/io/gguf/TernaryWeightMemoryTest.kt @@ -0,0 +1,77 @@ +package sk.ainet.io.gguf + +import kotlinx.coroutines.runBlocking +import sk.ainet.context.DefaultDataExecutionContext +import sk.ainet.io.JvmRandomAccessSource +import sk.ainet.lang.memory.ExperimentalMemoryApi +import sk.ainet.lang.memory.plan.EncodingRequest +import sk.ainet.lang.memory.plan.WeightForm +import sk.ainet.lang.tensor.Tensor +import sk.ainet.lang.tensor.data.BitNetB158TensorData +import sk.ainet.lang.tensor.data.FloatArrayTensorData +import sk.ainet.lang.types.FP32 +import java.io.File +import java.nio.ByteBuffer +import java.nio.ByteOrder +import kotlin.random.Random +import kotlin.test.Test +import kotlin.test.assertIs +import kotlin.test.assertTrue + +/** + * The memory proof point of #1136 (evidence artifact for #1141): the same I2_S GGUF tensor loaded + * keep-packed vs FP32-widened, byte counts compared. The packed path must hold the ~16× ratio — + * 0.25 bytes per weight plus one FP32 scale against 4 bytes per weight. + */ +@OptIn(ExperimentalMemoryApi::class) +class TernaryWeightMemoryTest { + + private fun i2sTensor(name: String, elements: Int, seed: Int): SyntheticGguf.TestTensor { + val rng = Random(seed) + val qk = 128 + val bytesPerBlock = qk / 4 + val payload = ByteArray(elements / 4) + for (j in 0 until elements) { + val jb = j % qk + val idx = (j / qk) * bytesPerBlock + jb % bytesPerBlock + payload[idx] = (payload[idx].toInt() or (rng.nextInt(3) shl (6 - 2 * (jb / bytesPerBlock)))).toByte() + } + val trailer = ByteBuffer.allocate(32).order(ByteOrder.LITTLE_ENDIAN) + repeat(8) { trailer.putFloat(0.5f) } + return SyntheticGguf.TestTensor(name, GGMLQuantizationType.I2_S, elements.toLong(), payload + trailer.array()) + } + + private fun load(file: File, form: WeightForm?): Tensor { + val ctx = DefaultDataExecutionContext() + var tensor: Tensor? = null + runBlocking { + StreamingGgufParametersLoader( + sourceProvider = { JvmRandomAccessSource.open(file) }, + weightForm = form, + ).load(ctx, FP32::class) { _, t -> tensor = t } + } + return tensor!! + } + + @Test + fun packedLoadIsAtLeastFifteenTimesSmallerThanTheWidenedLoad() { + val elements = 2560 * 64 // one BitNet-2B-ish projection slice + val file = SyntheticGguf.write(i2sTensor("w", elements, seed = 7)) + try { + val packed = assertIs(load(file, form = null).data) + val widened = assertIs>( + load(file, WeightForm(encoding = EncodingRequest.DequantizeTo(FP32))).data, + ) + val packedBytes = packed.packedData.size.toLong() + val widenedBytes = widened.buffer.size.toLong() * 4 + val ratio = widenedBytes.toDouble() / packedBytes + println( + "[ternary-memory] $elements weights: packed=$packedBytes B, " + + "fp32-widened=$widenedBytes B, ratio=%.1fx".format(ratio), + ) + assertTrue(ratio >= 15.0, "expected ~16x memory saving, measured %.1fx".format(ratio)) + } finally { + file.delete() + } + } +}