diff --git a/skainet-io/skainet-io-gguf/build.gradle.kts b/skainet-io/skainet-io-gguf/build.gradle.kts index 8fd5c2bf4..854c04052 100644 --- a/skainet-io/skainet-io-gguf/build.gradle.kts +++ b/skainet-io/skainet-io-gguf/build.gradle.kts @@ -84,6 +84,11 @@ kotlin { implementation(libs.junit) implementation(libs.kotlinx.coroutines) implementation(libs.kotlinx.coroutines.test) + // #1118's acceptance test loads a model and *runs* it, so it needs a backend that + // computes: this module's own DefaultDataExecutionContext carries VoidTensorOps. + // Test-only, and not a cycle — the CPU backend does not know about GGUF. + implementation(project(":skainet-backends:skainet-backend-cpu")) + implementation(project(":skainet-backends:skainet-backend-api")) } } diff --git a/skainet-io/skainet-io-gguf/src/jvmTest/kotlin/sk/ainet/io/gguf/WeightFormAcceptanceTest.kt b/skainet-io/skainet-io-gguf/src/jvmTest/kotlin/sk/ainet/io/gguf/WeightFormAcceptanceTest.kt new file mode 100644 index 000000000..e6c839a1f --- /dev/null +++ b/skainet-io/skainet-io-gguf/src/jvmTest/kotlin/sk/ainet/io/gguf/WeightFormAcceptanceTest.kt @@ -0,0 +1,185 @@ +package sk.ainet.io.gguf + +import kotlinx.coroutines.runBlocking +import sk.ainet.backend.api.kernel.KernelRegistry +import sk.ainet.context.DirectCpuExecutionContext +import sk.ainet.exec.kernel.ScalarKernelProvider +import sk.ainet.io.JvmRandomAccessSource +import sk.ainet.lang.memory.ExperimentalMemoryApi +import sk.ainet.lang.memory.plan.EncodingRequest +import sk.ainet.lang.memory.plan.KernelCapabilities +import sk.ainet.lang.memory.plan.MemoryPlans +import sk.ainet.lang.memory.plan.PlannerProfile +import sk.ainet.lang.memory.plan.WeightForm +import sk.ainet.lang.memory.plan.WeightFormResolver +import sk.ainet.lang.memory.plan.WeightResidency +import sk.ainet.lang.memory.plan.WeightShapeOrientation +import sk.ainet.lang.memory.plan.resolveWeightForms +import sk.ainet.lang.tensor.Shape +import sk.ainet.lang.tensor.Tensor +import sk.ainet.lang.tensor.matmulWeightTransposed +import sk.ainet.lang.tensor.storage.TensorEncoding +import sk.ainet.lang.types.FP32 +import java.io.File +import kotlin.math.abs +import kotlin.test.BeforeTest +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNotEquals +import kotlin.test.assertTrue + +/** + * #1118, the acceptance criterion for #1109: **the model author writes nothing about weight forms, + * and the same code is correct on two very different devices.** + * + * Everything else in #1109 is machinery for this one claim. So the test is arranged to make the + * claim falsifiable rather than to exercise the machinery: [userCode] below is written once, takes + * no policy, no profile and no form, and is called identically on both paths. If honouring a device + * ever required the caller to say something, this file would have to change to keep passing. + */ +@OptIn(ExperimentalMemoryApi::class) +class WeightFormAcceptanceTest { + + @BeforeTest + fun registerKernels() { + // What every PlatformCpuOpsFactory does at startup, and what this module does not get for + // free: consumed as a dependency, `DirectCpuExecutionContext` resolves to the *common* + // `DefaultCpuOps` with an empty registry, and packed matmul is silently wrong in exactly + // that configuration (#1124, found by this test). Registering the scalar provider puts the + // test in the configuration a real application runs in. + KernelRegistry.register(ScalarKernelProvider) + } + + // Three Q8_0 blocks per row, so block order is discriminable (#968), and an output dimension + // that is a whole number of blocks, so the relayouted weight is row-block-aligned. + private val outDim = 32 + private val inDim = 96 + + /** + * The unchanged snippet. No profile, no form, no policy — a weight and an activation. + * + * This is the whole point of #1109: whether `w` arrived packed, dequantized, heap or mapped is + * decided elsewhere, and none of it appears here. + */ + private fun userCode(x: Tensor, w: Tensor): FloatArray = + x.matmulWeightTransposed(w).data.copyToFloatArray() + + private fun modelFile(): File = SyntheticGguf.write( + SyntheticGguf.tensor("blk.0.attn_q.weight", GGMLQuantizationType.Q8_0, elements = outDim * inDim) + .copy(dims = listOf(inDim.toLong(), outDim.toLong())), + ) + + /** Load [f] the way a device described by [profile] and [capabilities] calls for. */ + private fun loadFor( + f: File, + profile: PlannerProfile, + capabilities: KernelCapabilities, + ): Pair> { + // The device-dependent axes come from the resolver. The shape axis does not, by design: + // which way round a checkpoint labels its dimensions is a property of the *format*, not of + // the machine, so it is stated once here and is identical on both paths. GGUF writes `ne` + // order, the engine means [out, in]. + val form = WeightFormResolver.resolve(TensorEncoding.Q8_0, profile, capabilities) + .copy(shape = WeightShapeOrientation.OUT_IN) + val ctx = DirectCpuExecutionContext() + var weight: Tensor? = null + runBlocking { + StreamingGgufParametersLoader( + sourceProvider = { JvmRandomAccessSource.open(f) }, + weightForm = form, + ).load(ctx, FP32::class) { _, tensor -> weight = tensor } + } + return form to weight!! + } + + private fun activation(): Tensor { + val ctx = DirectCpuExecutionContext() + return ctx.fromFloatArray( + Shape(1, inDim), FP32::class, FloatArray(inDim) { (it % 13) * 0.0625f }, + ) + } + + @Test + fun `one model and one snippet of code run correctly on a desktop and on a 2 GB board`() { + val f = modelFile() + try { + // A workstation with the packed kernels SKaiNET ships. + val (desktopForm, desktopWeight) = loadFor(f, PlannerProfile.DESKTOP, KernelCapabilities.EVERYTHING) + // A 2 GB board: weights mapped, and — for the sake of the contrast — a build whose + // kernels cannot feed Q8_0, so the resolver must dequantize rather than dequantize + // per forward pass. + val (mobileForm, mobileWeight) = loadFor(f, PlannerProfile.MOBILE_2GB, KernelCapabilities.DENSE_ONLY) + + // 1. The two devices resolved to different forms — asserted, not assumed. + assertNotEquals(desktopForm, mobileForm, "if both devices got the same form this test proves nothing") + assertEquals(EncodingRequest.KeepAsStored, desktopForm.encoding, "the desktop can feed Q8_0, so it keeps it") + assertEquals( + EncodingRequest.DequantizeTo(FP32), mobileForm.encoding, + "this board cannot feed Q8_0, so it pays once at load rather than every forward pass", + ) + assertEquals(WeightResidency.HEAP, desktopForm.residency) + assertEquals(WeightResidency.MAPPED, mobileForm.residency, "a 2 GB board maps its weights") + + // 2. And the same code, unchanged, is correct on both. + val x = activation() + val desktopOut = userCode(x, desktopWeight) + val mobileOut = userCode(x, mobileWeight) + + assertEquals(desktopOut.size, mobileOut.size) + for (o in desktopOut.indices) { + // Not bit-identical, and should not be claimed to be: one path multiplies through a + // Q8_0 kernel and the other through dequantized floats, so they differ by + // quantization error, not by disagreement about the matrix. + val tolerance = 1e-2f * maxOf(1.0f, abs(desktopOut[o])) + assertTrue( + abs(desktopOut[o] - mobileOut[o]) <= tolerance, + "output[$o]: desktop ${desktopOut[o]} vs mobile ${mobileOut[o]}", + ) + } + } finally { + f.delete() + } + } + + @Test + fun `the mobile plan knows what its form costs before the load happens`() { + val f = modelFile() + try { + val stored = JvmRandomAccessSource.open(f).use { src -> + StreamingGGUFReader.open(src).planInput(ctx = 512) + } + + val kept = stored.resolveWeightForms(PlannerProfile.MOBILE_2GB, KernelCapabilities.EVERYTHING) + val dequantized = stored.resolveWeightForms(PlannerProfile.MOBILE_2GB, KernelCapabilities.DENSE_ONLY) + + val keptPlan = MemoryPlans.plan(kept) + val dequantizedPlan = MemoryPlans.plan(dequantized) + + assertEquals(0L, keptPlan.formConversionBytes, "keeping the stored encoding converts nothing") + assertTrue( + dequantizedPlan.formConversionBytes > 0, + "dequantizing costs something and the plan must say so before the load, not after", + ) + assertEquals( + dequantizedPlan.weightsBytes - keptPlan.weightsBytes, + dequantizedPlan.formConversionBytes, + "and the number it reports is exactly the difference between the two plans", + ) + } finally { + f.delete() + } + } + + @Test + fun `a strict board is told about a missing kernel instead of quietly paying for it`() { + // MOBILE_2GB's own documentation calls dispatcher-inserted dequantization "the defect it + // is". With strict set, the resolver refuses rather than resolving to a 4x load. + val strict = PlannerProfile.MOBILE_2GB.copy(strict = true) + val failure = kotlin.runCatching { + WeightFormResolver.resolve(TensorEncoding.Q8_0, strict, KernelCapabilities.DENSE_ONLY) + }.exceptionOrNull() + + assertTrue(failure is IllegalStateException, "expected a refusal, got $failure") + assertTrue(failure.message!!.contains("Q8_0"), failure.message!!) + } +} diff --git a/skainet-lang/skainet-lang-core/api/jvm/skainet-lang-core.api b/skainet-lang/skainet-lang-core/api/jvm/skainet-lang-core.api index 2c542fba2..4efe7e2c0 100644 --- a/skainet-lang/skainet-lang-core/api/jvm/skainet-lang-core.api +++ b/skainet-lang/skainet-lang-core/api/jvm/skainet-lang-core.api @@ -1848,7 +1848,8 @@ public final class sk/ainet/lang/memory/plan/WeightForm$Companion { public final class sk/ainet/lang/memory/plan/WeightFormResolver { public static final field INSTANCE Lsk/ainet/lang/memory/plan/WeightFormResolver; - public final fun resolve (Lsk/ainet/lang/tensor/storage/TensorEncoding;Lsk/ainet/lang/memory/plan/PlannerProfile;Lsk/ainet/lang/memory/plan/KernelCapabilities;)Lsk/ainet/lang/memory/plan/WeightForm; + public final fun resolve (Lsk/ainet/lang/tensor/storage/TensorEncoding;Lsk/ainet/lang/memory/plan/PlannerProfile;Lsk/ainet/lang/memory/plan/KernelCapabilities;Z)Lsk/ainet/lang/memory/plan/WeightForm; + public static synthetic fun resolve$default (Lsk/ainet/lang/memory/plan/WeightFormResolver;Lsk/ainet/lang/tensor/storage/TensorEncoding;Lsk/ainet/lang/memory/plan/PlannerProfile;Lsk/ainet/lang/memory/plan/KernelCapabilities;ZILjava/lang/Object;)Lsk/ainet/lang/memory/plan/WeightForm; } public final class sk/ainet/lang/memory/plan/WeightFormResolverKt { diff --git a/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/memory/plan/WeightFormResolver.kt b/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/memory/plan/WeightFormResolver.kt index aa207e19e..1c5442650 100644 --- a/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/memory/plan/WeightFormResolver.kt +++ b/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/memory/plan/WeightFormResolver.kt @@ -39,12 +39,28 @@ public object WeightFormResolver { * cost — FP32 is roughly eight times a Q4_K tensor — and a profile that says [PlannerProfile.strict] * means a missing kernel is a bug to surface, not a slow path to take quietly. * + * ## Why [canProduceKernelFeedOrder] exists + * + * *Wanting* feed order and being able to *produce* it are different facts about different + * components. [KernelCapabilities.wantsKernelFeedOrder] answers the first — a property of the + * kernel. The second is a property of whoever materializes the bytes, and today no loader can: + * packed `TensorData` addresses its payload as canonical row-major, so feed-order bytes would + * decode the wrong elements without failing (#1120, and #973/#968 before it). + * + * So the resolver does not ask for what nothing can deliver. The default is `false`, which + * makes every resolved form loadable; #1120 flips it by passing `true` from a pipeline that can + * honour it. Collapsing the two facts into one is what let slice 1 hand slice 2 a form it had + * to reject — caught by the end-to-end test in #1118 and not by either slice's own tests. + * + * @param canProduceKernelFeedOrder whether the caller's pipeline can actually write feed-order + * bytes; `false` until #1120 * @throws IllegalStateException when nothing can feed [stored] and the profile is strict */ public fun resolve( stored: TensorEncoding?, profile: PlannerProfile, capabilities: KernelCapabilities, + canProduceKernelFeedOrder: Boolean = false, ): WeightForm { val residency = if (profile.weightsMapped) WeightResidency.MAPPED else WeightResidency.HEAP @@ -54,7 +70,7 @@ public object WeightFormResolver { if (capabilities.canFeedMatmul(stored)) { val order = - if (capabilities.wantsKernelFeedOrder(stored)) WeightByteOrder.KERNEL_FEED + if (capabilities.wantsKernelFeedOrder(stored) && canProduceKernelFeedOrder) WeightByteOrder.KERNEL_FEED else WeightByteOrder.AS_STORED return WeightForm(EncodingRequest.KeepAsStored, order, residency = residency) } diff --git a/skainet-lang/skainet-lang-core/src/commonTest/kotlin/sk/ainet/lang/memory/plan/WeightFormResolverTest.kt b/skainet-lang/skainet-lang-core/src/commonTest/kotlin/sk/ainet/lang/memory/plan/WeightFormResolverTest.kt index 1becccebf..fbf6ee1b0 100644 --- a/skainet-lang/skainet-lang-core/src/commonTest/kotlin/sk/ainet/lang/memory/plan/WeightFormResolverTest.kt +++ b/skainet-lang/skainet-lang-core/src/commonTest/kotlin/sk/ainet/lang/memory/plan/WeightFormResolverTest.kt @@ -32,10 +32,12 @@ class WeightFormResolverTest { } @Test - fun `a weight whose kernel exists keeps its encoding and gets feed order`() { + fun `a weight whose kernel exists keeps its encoding and gets feed order when it can be produced`() { for (encoding in packed) { for (profile in listOf(PlannerProfile.DESKTOP, PlannerProfile.MOBILE_2GB)) { - val form = WeightFormResolver.resolve(encoding, profile, capableOf(encoding)) + val form = WeightFormResolver.resolve( + encoding, profile, capableOf(encoding), canProduceKernelFeedOrder = true, + ) assertEquals( EncodingRequest.KeepAsStored, form.encoding, "${encoding.name} on ${profile.name}: a feedable weight must not be re-encoded", @@ -48,6 +50,22 @@ class WeightFormResolverTest { } } + @Test + fun `feed order is not asked for by default because nothing can produce it yet`() { + // The gap #1118's end-to-end test found: the resolver asked for KERNEL_FEED and the loader + // had to reject it, for the *common* case of a target that has kernels. Wanting feed order + // is a fact about the kernel; producing it is a fact about the loader, and until #1120 the + // answer to the second is no. Neither slice's own tests could see this — only running them + // together could. + for (encoding in packed) { + val form = WeightFormResolver.resolve(encoding, PlannerProfile.DESKTOP, capableOf(encoding)) + assertEquals( + WeightByteOrder.AS_STORED, form.order, + "${encoding.name}: the default resolution must be one a loader can actually honour", + ) + } + } + @Test fun `a weight with no kernel is dequantized once at load rather than every forward pass`() { for (encoding in packed) {