From f41d84423545437cf66acb34214efcff6e7457ac Mon Sep 17 00:00:00 2001 From: michal harakal Date: Thu, 13 Aug 2026 12:51:21 +0200 Subject: [PATCH] test(kernel): q8-activation reference kernels for tight Q4_K/Q6_K parity (#944) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The native/FFM/Kotlin-Native/JNI Q4_K and Q6_K kernels quantize the input activation to int8 first (ggml's block_q8_K fast path) before an integer dot product, while ScalarQ4_KMatmulKernel/ScalarQ6_KMatmulKernel do exact float math. These are genuinely different algorithms (#944's bisection result) — the existing parity tests already work around this correctly via an aggregate RMS-energy tolerance, but that gate can't distinguish a real kernel bug from the expected, bounded quantization loss it's designed to absorb. Add Q4_KQ8ActivationReferenceKernel / Q6_KQ8ActivationReferenceKernel (skainet-backend-cpu commonMain): faithful Kotlin transcriptions of the C kernels' int8 activation-quant algorithm (same block_q8_K quantize, same integer dot, same scale/min application), test support only, not registered with any KernelRegistry. Comparing a native kernel against these instead of the exact-float scalar references isolates genuine kernel bugs (wrong offsets/layout/scale-decode/dispatch) from the intended quantization loss — agreement should be tight, not RMS-energy-bounded. Add tight per-row parity tests using the new references alongside the existing RMS-gated ones (kept as-is) in: - NativeQ4KMatmulKernelParityTest / NativeQ6KMatmulKernelParityTest (skainet-backend-native-cpu jvmTest, FFM) - NativeKnQ4KMatmulKernelParityTest / NativeKnQ6KMatmulKernelParityTest (skainet-backend-native-cpu nativeTest, Kotlin/Native cinterop) - JniKernelParityTest (skainet-backend-jni-cpu androidTest, JNI bridge) Verified: all new tests pass on JVM FFM and Kotlin/Native linuxX64 (8 new tests each, tight 1e-2 abs / 1e-4 rel tolerance — much tighter than the previous 3% RMS-relative gate), confirming the native kernels are correct and the divergence really is exactly the activation-quant loss #944 identified, nothing more. linuxArm64 has no runnable test task in this environment (cross-compiled target, no QEMU runner configured) and the JNI androidTest changes compile but need a device/emulator to run — not verified here, same C algorithm and same reference kernel as the two verified paths. Regenerated skainet-backend-cpu's jvm API dump for the two new public reference-kernel objects. Co-Authored-By: Claude Sonnet 5 --- .../api/jvm/skainet-backend-cpu.api | 10 ++ .../kernel/Q4_KQ8ActivationReferenceKernel.kt | 141 ++++++++++++++++++ .../kernel/Q6_KQ8ActivationReferenceKernel.kt | 138 +++++++++++++++++ .../exec/kernel/jni/JniKernelParityTest.kt | 34 +++++ .../kernel/NativeQ4KMatmulKernelParityTest.kt | 52 +++++++ .../kernel/NativeQ6KMatmulKernelParityTest.kt | 41 +++++ .../NativeKnQ4KMatmulKernelParityTest.kt | 41 +++++ .../NativeKnQ6KMatmulKernelParityTest.kt | 40 +++++ 8 files changed, 497 insertions(+) create mode 100644 skainet-backends/skainet-backend-cpu/src/commonMain/kotlin/sk/ainet/exec/kernel/Q4_KQ8ActivationReferenceKernel.kt create mode 100644 skainet-backends/skainet-backend-cpu/src/commonMain/kotlin/sk/ainet/exec/kernel/Q6_KQ8ActivationReferenceKernel.kt diff --git a/skainet-backends/skainet-backend-cpu/api/jvm/skainet-backend-cpu.api b/skainet-backends/skainet-backend-cpu/api/jvm/skainet-backend-cpu.api index 2ce08b899..6425c257a 100644 --- a/skainet-backends/skainet-backend-cpu/api/jvm/skainet-backend-cpu.api +++ b/skainet-backends/skainet-backend-cpu/api/jvm/skainet-backend-cpu.api @@ -129,6 +129,16 @@ public final class sk/ainet/exec/kernel/PanamaVectorQ8_0MatmulKernel : sk/ainet/ public fun matmul ([FI[BIII[FI)V } +public final class sk/ainet/exec/kernel/Q4_KQ8ActivationReferenceKernel : sk/ainet/backend/api/kernel/Q4KMatmulKernel { + public static final field INSTANCE Lsk/ainet/exec/kernel/Q4_KQ8ActivationReferenceKernel; + public fun matmul ([FI[BIII[FI)V +} + +public final class sk/ainet/exec/kernel/Q6_KQ8ActivationReferenceKernel : sk/ainet/backend/api/kernel/Q6KMatmulKernel { + public static final field INSTANCE Lsk/ainet/exec/kernel/Q6_KQ8ActivationReferenceKernel; + public fun matmul ([FI[BIII[FI)V +} + public final class sk/ainet/exec/kernel/ScalarBf16MatmulKernel : sk/ainet/backend/api/kernel/Bf16MatmulKernel { public static final field INSTANCE Lsk/ainet/exec/kernel/ScalarBf16MatmulKernel; public fun getCodec ()Lsk/ainet/lang/types/NarrowFloatCodec; diff --git a/skainet-backends/skainet-backend-cpu/src/commonMain/kotlin/sk/ainet/exec/kernel/Q4_KQ8ActivationReferenceKernel.kt b/skainet-backends/skainet-backend-cpu/src/commonMain/kotlin/sk/ainet/exec/kernel/Q4_KQ8ActivationReferenceKernel.kt new file mode 100644 index 000000000..ead0e6784 --- /dev/null +++ b/skainet-backends/skainet-backend-cpu/src/commonMain/kotlin/sk/ainet/exec/kernel/Q4_KQ8ActivationReferenceKernel.kt @@ -0,0 +1,141 @@ +package sk.ainet.exec.kernel + +import sk.ainet.backend.api.kernel.Q4KMatmulKernel +import kotlin.math.abs + +/** + * Test-support reference for the native/FFM/Kotlin-Native/JNI Q4_K kernels' + * ggml-style int8 activation-quantization fast path (`skainet_q4k_matmul` in + * `q4k_matmul.c`) — deliberately NOT [ScalarQ4_KMatmulKernel]'s exact-float + * algorithm. + * + * [ScalarQ4_KMatmulKernel] and the native-family kernels compute genuinely + * different things (#944): the native side quantizes the input activation to + * int8 first (ggml's `block_q8_K`, `d_in = maxabs/127`, symmetric round + + * clamp to [-127,127]) before an integer dot product against the 4-bit + * weight codes, then applies the block's `d`/`dMin` scale/min — a small, + * deliberate, expected source of divergence against the exact-float scalar + * kernel that per-row or RMS-energy tolerances have to absorb. Comparing a + * native kernel's output against *this* kernel instead — same activation + * quantization, same integer dot, same scale/min application, transcribed + * byte-for-byte from the C — isolates genuine kernel bugs (wrong offsets, + * layout, scale decode, dispatch) from that expected loss: agreement should + * be tight (float-accumulation-order / rounding-tie noise only), not + * activation-quant noise. Use a tight tolerance against this kernel; keep + * using the RMS-energy gate against [ScalarQ4_KMatmulKernel] for the + * "is the intended lossy path still within its expected budget" check. + * + * Not registered with any [sk.ainet.backend.api.kernel.KernelRegistry] and + * not tuned for speed (allocates per call) — test support only, never a + * production dispatch target. + */ +public object Q4_KQ8ActivationReferenceKernel : Q4KMatmulKernel { + + private const val BLOCK_SIZE = 256 + private const val SUB_BLOCK = 32 + private const val BYTES_PER_BLOCK = 144 + + override fun matmul( + input: FloatArray, inputOffset: Int, + weight: ByteArray, weightByteOffset: Int, + inputDim: Int, outputDim: Int, + output: FloatArray, outputOffset: Int, + ) { + require(inputDim % BLOCK_SIZE == 0) { + "Q4_KQ8ActivationReferenceKernel: inputDim must be a multiple of $BLOCK_SIZE; got $inputDim" + } + if (outputDim == 0) return + if (inputDim == 0) { for (o in 0 until outputDim) output[outputOffset + o] = 0f; return } + + val blocksPerInputDim = inputDim / BLOCK_SIZE + // Pre-quantize the whole input row to Q8 once, mirroring the C + // kernel's single quantization pass reused across every output row. + val q8 = IntArray(inputDim) + val dIn = FloatArray(blocksPerInputDim) + for (b in 0 until blocksPerInputDim) { + val base = inputOffset + b * BLOCK_SIZE + var maxAbs = 0f + for (i in 0 until BLOCK_SIZE) { + val a = abs(input[base + i]) + if (a > maxAbs) maxAbs = a + } + if (maxAbs == 0f) { + dIn[b] = 0f + for (i in 0 until BLOCK_SIZE) q8[b * BLOCK_SIZE + i] = 0 + continue + } + val inv = 127f / maxAbs + dIn[b] = maxAbs / 127f + for (i in 0 until BLOCK_SIZE) { + var v = roundHalfAwayFromZero(input[base + i] * inv) + if (v > 127) v = 127 else if (v < -127) v = -127 + q8[b * BLOCK_SIZE + i] = v + } + } + + val scaleIdx = IntArray(8) + val minIdx = IntArray(8) + for (o in 0 until outputDim) output[outputOffset + o] = 0f + + for (blockIdx in 0 until blocksPerInputDim) { + val di = dIn[blockIdx] + val q8Base = blockIdx * BLOCK_SIZE + + for (o in 0 until outputDim) { + val blockBase = weightByteOffset + (blockIdx * outputDim + o) * BYTES_PER_BLOCK + val d = decodeHalf(((weight[blockBase + 1].toInt() and 0xFF) shl 8) or (weight[blockBase].toInt() and 0xFF)) + val dMin = decodeHalf(((weight[blockBase + 3].toInt() and 0xFF) shl 8) or (weight[blockBase + 2].toInt() and 0xFF)) + + // ggml get_scale_min_k4 over the 12 scale bytes — identical to + // ScalarQ4_KMatmulKernel and skainet_q4k_decode_scales. + val sc = blockBase + 4 + for (sb in 0 until 4) { + scaleIdx[sb] = weight[sc + sb].toInt() and 0x3F + minIdx[sb] = weight[sc + sb + 4].toInt() and 0x3F + } + for (sb in 4 until 8) { + val low4S = weight[sc + sb + 4].toInt() and 0x0F + val high2S = (weight[sc + sb - 4].toInt() and 0xFF) ushr 6 + scaleIdx[sb] = low4S or (high2S shl 4) + val low4M = (weight[sc + sb + 4].toInt() and 0xFF) ushr 4 + val high2M = (weight[sc + sb].toInt() and 0xFF) ushr 6 + minIdx[sb] = low4M or (high2M shl 4) + } + + val codesOffset = blockBase + 16 + var blockScaleDot = 0L + var blockMinSum = 0L + for (groupJ in 0 until 4) { + val qsRegion = codesOffset + groupJ * 32 + val sbLo = 2 * groupJ + val sbHi = sbLo + 1 + val q8LoBase = q8Base + sbLo * SUB_BLOCK + val q8HiBase = q8Base + sbHi * SUB_BLOCK + var dotLo = 0 + var sumLo = 0 + var dotHi = 0 + var sumHi = 0 + for (i in 0 until SUB_BLOCK) { + val b = weight[qsRegion + i].toInt() and 0xFF + val codeLo = b and 0x0F + val codeHi = b ushr 4 + val aLo = q8[q8LoBase + i] + val aHi = q8[q8HiBase + i] + dotLo += aLo * codeLo + sumLo += aLo + dotHi += aHi * codeHi + sumHi += aHi + } + blockScaleDot += scaleIdx[sbLo].toLong() * dotLo + scaleIdx[sbHi].toLong() * dotHi + blockMinSum += minIdx[sbLo].toLong() * sumLo + minIdx[sbHi].toLong() * sumHi + } + + output[outputOffset + o] += di * (d * blockScaleDot.toFloat() - dMin * blockMinSum.toFloat()) + } + } + } + + /** Round-half-away-from-zero, matching `lrintf`'s effect for the non-tie floats real inputs produce. */ + private fun roundHalfAwayFromZero(x: Float): Int = + if (x >= 0f) (x + 0.5f).toInt() else -(-x + 0.5f).toInt() +} diff --git a/skainet-backends/skainet-backend-cpu/src/commonMain/kotlin/sk/ainet/exec/kernel/Q6_KQ8ActivationReferenceKernel.kt b/skainet-backends/skainet-backend-cpu/src/commonMain/kotlin/sk/ainet/exec/kernel/Q6_KQ8ActivationReferenceKernel.kt new file mode 100644 index 000000000..89ca3cf2e --- /dev/null +++ b/skainet-backends/skainet-backend-cpu/src/commonMain/kotlin/sk/ainet/exec/kernel/Q6_KQ8ActivationReferenceKernel.kt @@ -0,0 +1,138 @@ +package sk.ainet.exec.kernel + +import sk.ainet.backend.api.kernel.Q6KMatmulKernel + +/** + * Test-support reference for the native/FFM/Kotlin-Native/JNI Q6_K kernels' + * ggml-style int8 activation-quantization fast path (`skainet_q6k_matmul` in + * `q6k_matmul.c`) — deliberately NOT [ScalarQ6_KMatmulKernel]'s exact-float + * algorithm. + * + * Same rationale as [Q4_KQ8ActivationReferenceKernel] (see its kdoc and + * #944): the native side quantizes the activation to int8 first + * (`d_in = maxabs/127`), unpacks the 6-bit weight code to a centered int8 + * `code - 32`, and does 16 int8 dot-products per block (one per scale + * group) instead of an exact float dequant-then-dot. Comparing a native + * kernel against this transcription instead of the exact-float scalar + * reference isolates genuine kernel bugs from the expected, small + * activation-quantization loss — use a tight tolerance here, keep the + * RMS-energy gate against [ScalarQ6_KMatmulKernel]. + * + * Not registered with any [sk.ainet.backend.api.kernel.KernelRegistry] and + * not tuned for speed — test support only. + */ +public object Q6_KQ8ActivationReferenceKernel : Q6KMatmulKernel { + + private const val BLOCK_SIZE = 256 + private const val BYTES_PER_BLOCK = 210 + private const val QL_OFFSET = 0 + private const val QH_OFFSET = 128 + private const val SCALES_OFFSET = 192 + private const val D_OFFSET = 208 + + override fun matmul( + input: FloatArray, inputOffset: Int, + weight: ByteArray, weightByteOffset: Int, + inputDim: Int, outputDim: Int, + output: FloatArray, outputOffset: Int, + ) { + require(inputDim % BLOCK_SIZE == 0) { + "Q6_KQ8ActivationReferenceKernel: inputDim must be a multiple of $BLOCK_SIZE; got $inputDim" + } + if (outputDim == 0) return + if (inputDim == 0) { for (o in 0 until outputDim) output[outputOffset + o] = 0f; return } + + val blocksPerInputDim = inputDim / BLOCK_SIZE + val q8 = IntArray(inputDim) + val dIn = FloatArray(blocksPerInputDim) + for (b in 0 until blocksPerInputDim) { + val base = inputOffset + b * BLOCK_SIZE + var maxAbs = 0f + for (i in 0 until BLOCK_SIZE) { + val a = kotlin.math.abs(input[base + i]) + if (a > maxAbs) maxAbs = a + } + if (maxAbs == 0f) { + dIn[b] = 0f + for (i in 0 until BLOCK_SIZE) q8[b * BLOCK_SIZE + i] = 0 + continue + } + val inv = 127f / maxAbs + dIn[b] = maxAbs / 127f + for (i in 0 until BLOCK_SIZE) { + var v = roundHalfAwayFromZero(input[base + i] * inv) + if (v > 127) v = 127 else if (v < -127) v = -127 + q8[b * BLOCK_SIZE + i] = v + } + } + + val codes = IntArray(BLOCK_SIZE) + for (o in 0 until outputDim) output[outputOffset + o] = 0f + + for (blockIdx in 0 until blocksPerInputDim) { + val di = dIn[blockIdx] + val q8Base = blockIdx * BLOCK_SIZE + + for (o in 0 until outputDim) { + val blockBase = weightByteOffset + (blockIdx * outputDim + o) * BYTES_PER_BLOCK + val d = decodeHalf( + ((weight[blockBase + D_OFFSET + 1].toInt() and 0xFF) shl 8) or + (weight[blockBase + D_OFFSET].toInt() and 0xFF), + ) + + unpackCodes(weight, blockBase, codes) + + // Σ_g sc[g] · Σ_{i∈g} q8[i]·codes[i], over 16 contiguous 16-element + // scale groups — matches skainet_q6k_weighted_dot_generic exactly + // (same group start/scale-index formula, same accumulation order). + var wdot = 0L + for (half in 0 until 2) { + for (k in 0 until 4) { + for (isb in 0 until 2) { + val start = half * 128 + 32 * k + isb * 16 + val gs = half * 8 + isb + 2 * k + val sc = weight[blockBase + SCALES_OFFSET + gs].toInt() // signed int8 + var dot = 0 + for (j in 0 until 16) { + dot += q8[q8Base + start + j] * codes[start + j] + } + wdot += sc.toLong() * dot + } + } + } + + output[outputOffset + o] += d * di * wdot.toFloat() + } + } + } + + /** + * Unpack one 256-element Q6_K block into centered int8 codes (`code - 32`, + * range [-32,31]) in natural element order. Byte-for-byte transcription of + * `skainet_q6k_unpack_codes`. + */ + private fun unpackCodes(weight: ByteArray, blockBase: Int, codes: IntArray) { + val ql0 = blockBase + QL_OFFSET + val qh0 = blockBase + QH_OFFSET + for (half in 0 until 2) { + val ql = ql0 + half * 64 + val qh = qh0 + half * 32 + val out = half * 128 + for (isb in 0 until 2) { + val lStart = isb * 16 + for (l in lStart until lStart + 16) { + val qL0 = weight[ql + l].toInt() and 0xFF + val qL32 = weight[ql + l + 32].toInt() and 0xFF + val qH = weight[qh + l].toInt() and 0xFF + codes[out + l + 0] = ((qL0 and 0x0F) or ((qH and 0x03) shl 4)) - 32 + codes[out + l + 32] = ((qL32 and 0x0F) or (((qH ushr 2) and 0x03) shl 4)) - 32 + codes[out + l + 64] = ((qL0 ushr 4) or (((qH ushr 4) and 0x03) shl 4)) - 32 + codes[out + l + 96] = ((qL32 ushr 4) or (((qH ushr 6) and 0x03) shl 4)) - 32 + } + } + } + } + + private fun roundHalfAwayFromZero(x: Float): Int = + if (x >= 0f) (x + 0.5f).toInt() else -(-x + 0.5f).toInt() +} diff --git a/skainet-backends/skainet-backend-jni-cpu/src/androidTest/kotlin/sk/ainet/exec/kernel/jni/JniKernelParityTest.kt b/skainet-backends/skainet-backend-jni-cpu/src/androidTest/kotlin/sk/ainet/exec/kernel/jni/JniKernelParityTest.kt index 355559bff..aeaa331cb 100644 --- a/skainet-backends/skainet-backend-jni-cpu/src/androidTest/kotlin/sk/ainet/exec/kernel/jni/JniKernelParityTest.kt +++ b/skainet-backends/skainet-backend-jni-cpu/src/androidTest/kotlin/sk/ainet/exec/kernel/jni/JniKernelParityTest.kt @@ -6,6 +6,8 @@ import org.junit.Assert.assertNotNull import org.junit.Assert.assertTrue import org.junit.Test import org.junit.runner.RunWith +import sk.ainet.exec.kernel.Q4_KQ8ActivationReferenceKernel +import sk.ainet.exec.kernel.Q6_KQ8ActivationReferenceKernel import sk.ainet.exec.kernel.ScalarQ4_0MatmulKernel import sk.ainet.exec.kernel.ScalarQ4_KMatmulKernel import sk.ainet.exec.kernel.ScalarQ5_0MatmulKernel @@ -221,6 +223,38 @@ class JniKernelParityTest { reference = ScalarQ6_KMatmulKernel::matmul, jni = JniKernels::q6kMatmul, ) + /** + * Tight per-row parity against [Q4_KQ8ActivationReferenceKernel] / + * [Q6_KQ8ActivationReferenceKernel] (#944): these references perform the + * same int8 activation quantization the JNI kernel does, so agreement + * should be tight like the exact-float formats above — a wide + * divergence here indicates a real kernel/bridge bug, not the expected + * quantization loss the RMS-gated tests above absorb. + */ + @Test + fun q4k_parity_single_block_q8ActivationReference() = assertExactParity( + 256, 16, 42, 1e-2f, 256, 144, conditionQ4K, + reference = Q4_KQ8ActivationReferenceKernel::matmul, jni = JniKernels::q4kMatmul, + ) + + @Test + fun q4k_parity_q8ActivationReference() = assertExactParity( + 1024, 64, 123, 1e-2f, 256, 144, conditionQ4K, + reference = Q4_KQ8ActivationReferenceKernel::matmul, jni = JniKernels::q4kMatmul, + ) + + @Test + fun q6k_parity_single_block_q8ActivationReference() = assertExactParity( + 256, 16, 42, 1e-2f, 256, 210, conditionQ6K, + reference = Q6_KQ8ActivationReferenceKernel::matmul, jni = JniKernels::q6kMatmul, + ) + + @Test + fun q6k_parity_q8ActivationReference() = assertExactParity( + 1024, 64, 999, 1e-2f, 256, 210, conditionQ6K, + reference = Q6_KQ8ActivationReferenceKernel::matmul, jni = JniKernels::q6kMatmul, + ) + @Test fun smoke_roundtrip() { val input = floatArrayOf(1f, 2f, 3f, 4f) diff --git a/skainet-backends/skainet-backend-native-cpu/src/jvmTest/kotlin/sk/ainet/exec/kernel/NativeQ4KMatmulKernelParityTest.kt b/skainet-backends/skainet-backend-native-cpu/src/jvmTest/kotlin/sk/ainet/exec/kernel/NativeQ4KMatmulKernelParityTest.kt index 7cc278cf9..6a46d537f 100644 --- a/skainet-backends/skainet-backend-native-cpu/src/jvmTest/kotlin/sk/ainet/exec/kernel/NativeQ4KMatmulKernelParityTest.kt +++ b/skainet-backends/skainet-backend-native-cpu/src/jvmTest/kotlin/sk/ainet/exec/kernel/NativeQ4KMatmulKernelParityTest.kt @@ -102,28 +102,80 @@ class NativeQ4KMatmulKernelParityTest { const val AGG_REL_TOL = 0.03 } + /** + * Tight per-row parity against [Q4_KQ8ActivationReferenceKernel] (#944): + * unlike [assertParity] above, this reference performs the SAME int8 + * activation quantization the native kernel does, so agreement should be + * tight (float-accumulation-order noise only) — a wide divergence here, + * unlike against the exact-float Panama reference, indicates a genuine + * kernel bug (wrong offset/layout/scale-decode/dispatch), not the + * expected quantization loss. + */ + private fun assertQ8ActivationParity(inputDim: Int, outputDim: Int, seed: Int) { + val numBlocks = (inputDim / blockSize) * outputDim + val packed = randomQ4KBytes(numBlocks, seed) + val input = FloatArray(inputDim) { Random(seed + it).nextFloat() - 0.5f } + + val refOut = FloatArray(outputDim) + Q4_KQ8ActivationReferenceKernel.matmul(input, 0, packed, 0, inputDim, outputDim, refOut, 0) + + val nativeOut = FloatArray(outputDim) + NativeQ4KMatmulKernel.matmul(input, 0, packed, 0, inputDim, outputDim, nativeOut, 0) + + for (o in 0 until outputDim) { + val diff = abs(refOut[o] - nativeOut[o]) + val rel = diff / (abs(refOut[o]) + 1e-6f) + assertTrue( + diff <= 1e-2f || rel < 1e-4f, + "row $o diverged vs Q8-activation reference: ref=${refOut[o]} native=${nativeOut[o]} " + + "diff=$diff rel=$rel — this reference already accounts for int8 activation " + + "quantization, so this points at a real kernel bug, not expected quant loss (#944)", + ) + } + } + @Test fun single_block_single_row() { assertParity(inputDim = 256, outputDim = 1, seed = 42, tol = 1e-2f) } + @Test + fun single_block_single_row_q8ActivationReference() { + assertQ8ActivationParity(inputDim = 256, outputDim = 1, seed = 42) + } + @Test fun single_block_multi_row() { assertParity(inputDim = 256, outputDim = 16, seed = 7, tol = 1e-2f) } + @Test + fun single_block_multi_row_q8ActivationReference() { + assertQ8ActivationParity(inputDim = 256, outputDim = 16, seed = 7) + } + @Test fun multi_block_multi_row() { // 4 super-blocks × 1024 elements; outputs 64 cells. assertParity(inputDim = 1024, outputDim = 64, seed = 123, tol = 5e-2f) } + @Test + fun multi_block_multi_row_q8ActivationReference() { + assertQ8ActivationParity(inputDim = 1024, outputDim = 64, seed = 123) + } + @Test fun llm_typical_shape_4096_outputDim_64() { // 4096 inputs × 64 outputs — slice of an LLM hidden→ffn matrix. assertParity(inputDim = 4096, outputDim = 64, seed = 999, tol = 5e-1f) } + @Test + fun llm_typical_shape_4096_outputDim_64_q8ActivationReference() { + assertQ8ActivationParity(inputDim = 4096, outputDim = 64, seed = 999) + } + @Test fun rejects_inputDim_not_multiple_of_block() { val packed = randomQ4KBytes(numBlocks = 2, seed = 1) diff --git a/skainet-backends/skainet-backend-native-cpu/src/jvmTest/kotlin/sk/ainet/exec/kernel/NativeQ6KMatmulKernelParityTest.kt b/skainet-backends/skainet-backend-native-cpu/src/jvmTest/kotlin/sk/ainet/exec/kernel/NativeQ6KMatmulKernelParityTest.kt index 572286c9f..b2112c54e 100644 --- a/skainet-backends/skainet-backend-native-cpu/src/jvmTest/kotlin/sk/ainet/exec/kernel/NativeQ6KMatmulKernelParityTest.kt +++ b/skainet-backends/skainet-backend-native-cpu/src/jvmTest/kotlin/sk/ainet/exec/kernel/NativeQ6KMatmulKernelParityTest.kt @@ -86,18 +86,59 @@ class NativeQ6KMatmulKernelParityTest { const val AGG_REL_TOL = 0.03 } + /** + * Tight per-row parity against [Q6_KQ8ActivationReferenceKernel] (#944) — + * see [NativeQ4KMatmulKernelParityTest.assertQ8ActivationParity] for the + * rationale: this reference performs the same int8 activation + * quantization the native kernel does, so a wide divergence here points + * at a real kernel bug, not the expected quantization loss. + */ + private fun assertQ8ActivationParity(inputDim: Int, outputDim: Int, seed: Int) { + val numBlocks = (inputDim / blockSize) * outputDim + val packed = randomQ6KBytes(numBlocks, seed) + val input = FloatArray(inputDim) { Random(seed + it).nextFloat() - 0.5f } + + val refOut = FloatArray(outputDim) + Q6_KQ8ActivationReferenceKernel.matmul(input, 0, packed, 0, inputDim, outputDim, refOut, 0) + + val nativeOut = FloatArray(outputDim) + NativeQ6KMatmulKernel.matmul(input, 0, packed, 0, inputDim, outputDim, nativeOut, 0) + + for (o in 0 until outputDim) { + val diff = abs(refOut[o] - nativeOut[o]) + val rel = diff / (abs(refOut[o]) + 1e-6f) + assertTrue( + diff <= 1e-2f || rel < 1e-4f, + "row $o diverged vs Q8-activation reference: ref=${refOut[o]} native=${nativeOut[o]} " + + "diff=$diff rel=$rel (#944)", + ) + } + } + @Test fun single_block_single_row() = assertParity(256, 1, 42, 1e-2f) + @Test + fun single_block_single_row_q8ActivationReference() = assertQ8ActivationParity(256, 1, 42) + @Test fun single_block_multi_row() = assertParity(256, 16, 7, 5e-2f) + @Test + fun single_block_multi_row_q8ActivationReference() = assertQ8ActivationParity(256, 16, 7) + @Test fun multi_block_multi_row() = assertParity(1024, 64, 123, 2e-1f) + @Test + fun multi_block_multi_row_q8ActivationReference() = assertQ8ActivationParity(1024, 64, 123) + @Test fun llm_typical_shape_4096_outputDim_64() = assertParity(4096, 64, 999, 2e0f) + @Test + fun llm_typical_shape_4096_outputDim_64_q8ActivationReference() = assertQ8ActivationParity(4096, 64, 999) + @Test fun rejects_inputDim_not_multiple_of_block() { val packed = randomQ6KBytes(numBlocks = 2, seed = 1) diff --git a/skainet-backends/skainet-backend-native-cpu/src/nativeTest/kotlin/sk/ainet/exec/kernel/NativeKnQ4KMatmulKernelParityTest.kt b/skainet-backends/skainet-backend-native-cpu/src/nativeTest/kotlin/sk/ainet/exec/kernel/NativeKnQ4KMatmulKernelParityTest.kt index 4b6e1171d..4d08a904c 100644 --- a/skainet-backends/skainet-backend-native-cpu/src/nativeTest/kotlin/sk/ainet/exec/kernel/NativeKnQ4KMatmulKernelParityTest.kt +++ b/skainet-backends/skainet-backend-native-cpu/src/nativeTest/kotlin/sk/ainet/exec/kernel/NativeKnQ4KMatmulKernelParityTest.kt @@ -1,5 +1,6 @@ package sk.ainet.exec.kernel +import kotlin.math.abs import kotlin.math.sqrt import kotlin.random.Random import kotlin.test.Test @@ -76,15 +77,55 @@ class NativeKnQ4KMatmulKernelParityTest { const val AGG_REL_TOL = 0.03 } + /** + * Tight per-row parity against [Q4_KQ8ActivationReferenceKernel] (#944): + * this reference performs the same int8 activation quantization the + * native kernel does, so agreement should be tight — a wide divergence + * here indicates a real kernel bug, not the expected quantization loss. + */ + private fun assertQ8ActivationParity(inputDim: Int, outputDim: Int, seed: Int) { + val numBlocks = (inputDim / blockSize) * outputDim + val packed = randomQ4KBytes(numBlocks, seed) + val input = FloatArray(inputDim) { Random(seed + it).nextFloat() - 0.5f } + + val refOut = FloatArray(outputDim) + Q4_KQ8ActivationReferenceKernel.matmul(input, 0, packed, 0, inputDim, outputDim, refOut, 0) + + val knOut = FloatArray(outputDim) + NativeKnQ4KMatmulKernel.matmul(input, 0, packed, 0, inputDim, outputDim, knOut, 0) + + for (o in 0 until outputDim) { + val diff = abs(refOut[o] - knOut[o]) + val rel = diff / (abs(refOut[o]) + 1e-6f) + assertTrue( + diff <= 1e-2f || rel < 1e-4f, + "row $o diverged vs Q8-activation reference: ref=${refOut[o]} kn=${knOut[o]} " + + "diff=$diff rel=$rel (#944)", + ) + } + } + @Test fun single_block_single_row() = assertParity(256, 1, 42, 1e-2f) + @Test + fun single_block_single_row_q8ActivationReference() = assertQ8ActivationParity(256, 1, 42) + @Test fun single_block_multi_row() = assertParity(256, 16, 7, 1e-2f) + @Test + fun single_block_multi_row_q8ActivationReference() = assertQ8ActivationParity(256, 16, 7) + @Test fun multi_block_multi_row() = assertParity(1024, 64, 123, 5e-2f) + @Test + fun multi_block_multi_row_q8ActivationReference() = assertQ8ActivationParity(1024, 64, 123) + @Test fun llm_typical_shape() = assertParity(4096, 64, 999, 5e-1f) + + @Test + fun llm_typical_shape_q8ActivationReference() = assertQ8ActivationParity(4096, 64, 999) } diff --git a/skainet-backends/skainet-backend-native-cpu/src/nativeTest/kotlin/sk/ainet/exec/kernel/NativeKnQ6KMatmulKernelParityTest.kt b/skainet-backends/skainet-backend-native-cpu/src/nativeTest/kotlin/sk/ainet/exec/kernel/NativeKnQ6KMatmulKernelParityTest.kt index a6bf2c192..35fb154d8 100644 --- a/skainet-backends/skainet-backend-native-cpu/src/nativeTest/kotlin/sk/ainet/exec/kernel/NativeKnQ6KMatmulKernelParityTest.kt +++ b/skainet-backends/skainet-backend-native-cpu/src/nativeTest/kotlin/sk/ainet/exec/kernel/NativeKnQ6KMatmulKernelParityTest.kt @@ -69,15 +69,55 @@ class NativeKnQ6KMatmulKernelParityTest { const val AGG_REL_TOL = 0.03 } + /** + * Tight per-row parity against [Q6_KQ8ActivationReferenceKernel] (#944): + * this reference performs the same int8 activation quantization the + * native kernel does, so agreement should be tight — a wide divergence + * here indicates a real kernel bug, not the expected quantization loss. + */ + private fun assertQ8ActivationParity(inputDim: Int, outputDim: Int, seed: Int) { + val numBlocks = (inputDim / blockSize) * outputDim + val packed = randomQ6KBytes(numBlocks, seed) + val input = FloatArray(inputDim) { Random(seed + it).nextFloat() - 0.5f } + + val refOut = FloatArray(outputDim) + Q6_KQ8ActivationReferenceKernel.matmul(input, 0, packed, 0, inputDim, outputDim, refOut, 0) + + val knOut = FloatArray(outputDim) + NativeKnQ6KMatmulKernel.matmul(input, 0, packed, 0, inputDim, outputDim, knOut, 0) + + for (o in 0 until outputDim) { + val diff = abs(refOut[o] - knOut[o]) + val rel = diff / (abs(refOut[o]) + 1e-6f) + assertTrue( + diff <= 1e-2f || rel < 1e-4f, + "row $o diverged vs Q8-activation reference: ref=${refOut[o]} kn=${knOut[o]} " + + "diff=$diff rel=$rel (#944)", + ) + } + } + @Test fun single_block_single_row() = assertParity(256, 1, 42, 1e-2f) + @Test + fun single_block_single_row_q8ActivationReference() = assertQ8ActivationParity(256, 1, 42) + @Test fun single_block_multi_row() = assertParity(256, 16, 7, 5e-2f) + @Test + fun single_block_multi_row_q8ActivationReference() = assertQ8ActivationParity(256, 16, 7) + @Test fun multi_block_multi_row() = assertParity(1024, 64, 123, 2e-1f) + @Test + fun multi_block_multi_row_q8ActivationReference() = assertQ8ActivationParity(1024, 64, 123) + @Test fun llm_typical_shape() = assertParity(4096, 64, 999, 2e0f) + + @Test + fun llm_typical_shape_q8ActivationReference() = assertQ8ActivationParity(4096, 64, 999) }