Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -70,10 +70,17 @@ public fun main(args: Array<String>) {
// for rather than detected. `all` keeps the plan exactly as it was before #1116.
val input = when {
profile == null -> stored
else -> stored.resolveWeightForms(
profile,
if (kernels == "dense") KernelCapabilities.DENSE_ONLY else KernelCapabilities.EVERYTHING,
)
else -> try {
stored.resolveWeightForms(
profile,
if (kernels == "dense") KernelCapabilities.DENSE_ONLY else KernelCapabilities.EVERYTHING,
)
} catch (e: IllegalStateException) {
// A strict profile refuses a weight nothing on the target can feed. That is an
// answer to the question the planner was asked, not a crash, so it reads as one.
System.err.println("skainet plan: ${e.message}")
exitProcess(1)
}
}
val available = budget?.let { parseBytes(it) } ?: Runtime.getRuntime().maxMemory()
if (profile != null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -103,20 +103,19 @@ class WeightFormAcceptanceTest {
fun `one model and one snippet of code run correctly on a desktop and on a 2 GB board`() {
val f = modelFile()
try {
// A workstation with the packed kernels SKaiNET ships.
val (desktopForm, desktopWeight) = loadFor(f, PlannerProfile.DESKTOP, KernelCapabilities.EVERYTHING)
// A 2 GB board: weights mapped, and — for the sake of the contrast — a build whose
// kernels cannot feed Q8_0, so the resolver must dequantize rather than dequantize
// per forward pass.
val (mobileForm, mobileWeight) = loadFor(f, PlannerProfile.MOBILE_2GB, KernelCapabilities.DENSE_ONLY)
// A desktop build without a Q8_0 kernel: it has the memory to absorb a widening, so
// the resolver dequantizes once at load rather than on every forward pass.
val (desktopForm, desktopWeight) = loadFor(f, PlannerProfile.DESKTOP, KernelCapabilities.DENSE_ONLY)
// A 2 GB board with the packed kernels SKaiNET ships: weights mapped, encoding kept.
val (mobileForm, mobileWeight) = loadFor(f, PlannerProfile.MOBILE_2GB, KernelCapabilities.EVERYTHING)

// 1. The two devices resolved to different forms — asserted, not assumed.
assertNotEquals(desktopForm, mobileForm, "if both devices got the same form this test proves nothing")
assertEquals(EncodingRequest.KeepAsStored, desktopForm.encoding, "the desktop can feed Q8_0, so it keeps it")
assertEquals(
EncodingRequest.DequantizeTo(FP32), mobileForm.encoding,
"this board cannot feed Q8_0, so it pays once at load rather than every forward pass",
EncodingRequest.DequantizeTo(FP32), desktopForm.encoding,
"no kernel for Q8_0 here, and a desktop can afford to pay once at load",
)
assertEquals(EncodingRequest.KeepAsStored, mobileForm.encoding, "the board can feed Q8_0, so it keeps it")
assertEquals(WeightResidency.HEAP, desktopForm.residency)
assertEquals(WeightResidency.MAPPED, mobileForm.residency, "a 2 GB board maps its weights")

Expand Down Expand Up @@ -150,7 +149,10 @@ class WeightFormAcceptanceTest {
}

val kept = stored.resolveWeightForms(PlannerProfile.MOBILE_2GB, KernelCapabilities.EVERYTHING)
val dequantized = stored.resolveWeightForms(PlannerProfile.MOBILE_2GB, KernelCapabilities.DENSE_ONLY)
// Strict is the point of MOBILE_2GB, so pricing a widening on it means opting out of
// the refusal first — which is exactly the decision the number is meant to inform.
val lenientMobile = PlannerProfile.MOBILE_2GB.copy(strict = false)
val dequantized = stored.resolveWeightForms(lenientMobile, KernelCapabilities.DENSE_ONLY)

val keptPlan = MemoryPlans.plan(kept)
val dequantizedPlan = MemoryPlans.plan(dequantized)
Expand All @@ -174,12 +176,17 @@ class WeightFormAcceptanceTest {
fun `a strict board is told about a missing kernel instead of quietly paying for it`() {
// MOBILE_2GB's own documentation calls dispatcher-inserted dequantization "the defect it
// is". With strict set, the resolver refuses rather than resolving to a 4x load.
val strict = PlannerProfile.MOBILE_2GB.copy(strict = true)
val failure = kotlin.runCatching {
WeightFormResolver.resolve(TensorEncoding.Q8_0, strict, KernelCapabilities.DENSE_ONLY)
WeightFormResolver.resolve(TensorEncoding.Q8_0, PlannerProfile.MOBILE_2GB, KernelCapabilities.DENSE_ONLY)
}.exceptionOrNull()

assertTrue(failure is IllegalStateException, "expected a refusal, got $failure")
assertTrue(failure is IllegalStateException, "MOBILE_2GB is strict by default, so this must refuse: $failure")
assertTrue(failure.message!!.contains("Q8_0"), failure.message!!)

// A build that would rather load slowly than not at all opts out explicitly.
val lenient = WeightFormResolver.resolve(
TensorEncoding.Q8_0, PlannerProfile.MOBILE_2GB.copy(strict = false), KernelCapabilities.DENSE_ONLY,
)
assertEquals(EncodingRequest.DequantizeTo(FP32), lenient.encoding)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,10 @@ public data class PlannerProfile(
/**
* A 2 GB-class phone: 700 MB reserved for the OS and the app, weights mapped, KV
* automatically quantized once the plan passes 80 % of the budget, and dequantization
* treated as the defect it is. The default on Android.
* treated as the defect it is — [strict], so a missing kernel fails rather than quietly
* costing several times the weight's size. The default on Android.
*
* Use `copy(strict = false)` for a build that would rather load slowly than not at all.
*/
public val MOBILE_2GB: PlannerProfile = PlannerProfile(
name = "mobile-2gb",
Expand All @@ -115,6 +118,11 @@ public data class PlannerProfile(
kvMode = KvCacheMode.BF16,
kvAutoQuantizeAbove = 0.80,
weightsMapped = true,
// This profile has always *said* dequantization is "the defect it is"; the flag said
// otherwise. On a 2 GB board a missing kernel is not a slow path to take quietly — it
// is a weight arriving several times its size on the device least able to hold it, and
// the honest moment to say so is before the load rather than at the OOM.
strict = true,
)

/** A desktop or server JVM: the same reserve, no automatic KV quantization, heap staging. */
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ class PlannerProfileTest {
assertEquals(0.80, m.kvAutoQuantizeAbove, "KV auto-quantizes over 80 % of the budget")
assertEquals(0.05, m.dequantWarnFraction, "dispatcher dequant warns over 5 %")
assertTrue(m.weightsMapped, "on a phone the weights are mapped")
assertFalse(m.strict)
assertTrue(m.strict, "and a missing kernel fails rather than costing several times the weight")

val d = PlannerProfile.DESKTOP
assertEquals(1.0, d.kvAutoQuantizeAbove, "a desktop never silently re-quantizes the cache")
Expand Down Expand Up @@ -155,17 +155,22 @@ class PlannerProfileTest {

@Test
fun dequantizationOverTheLimitWarnsAndFailsUnderStrict() {
val lenient = PlannerProfile.MOBILE_2GB.checkDequant(0.31)
// A desktop has the memory to absorb a widening, so it is told and carries on.
val lenient = PlannerProfile.DESKTOP.checkDequant(0.31)
assertEquals(DequantSeverity.WARN, lenient.severity)
assertTrue(lenient.message.contains("31.0%"), lenient.message)
assertTrue(lenient.message.contains("kernel for the on-disk format is missing"), lenient.message)
lenient.requireAcceptable() // a warning does not stop a desktop run

val strict = PlannerProfile.MOBILE_2GB.strict().checkDequant(0.31)
assertEquals(DequantSeverity.ERROR, strict.severity)
assertTrue(strict.profile.name.contains("strict"))
// A 2 GB board does not, so the same share is an error there without asking for strict.
val strict = PlannerProfile.MOBILE_2GB.checkDequant(0.31)
assertEquals(DequantSeverity.ERROR, strict.severity, "MOBILE_2GB is strict by default")
val failure = assertFailsWith<IllegalStateException> { strict.requireAcceptable() }
assertTrue(failure.message!!.contains("over the 5.0%"), failure.message!!)

// strict() stays available for turning a lenient profile into a failing one in CI.
assertTrue(PlannerProfile.DESKTOP.strict().checkDequant(0.31).profile.name.contains("strict"))
assertEquals(DequantSeverity.ERROR, PlannerProfile.DESKTOP.strict().checkDequant(0.31).severity)
}

// --- picking a profile ---------------------------------------------------------------------
Expand Down
Loading