diff --git a/build.gradle.kts b/build.gradle.kts index 1faad0e8..74c2aa10 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -5,6 +5,9 @@ plugins { // :llm-runtime:iree-android, which bundles prebuilt JNI .so's via jniLibs, matching // the engine repo's skainet-backend-jni-cpu precedent. alias(libs.plugins.androidLibrary) apply false + // com.android.application: :llm-apps:skainet-decode-android, the on-device decode/metrics + // sample (SKaiNET#1244) — first application module in the repo. + alias(libs.plugins.androidApplication) apply false alias(libs.plugins.jetbrainsKotlinJvm) apply false alias(libs.plugins.vanniktech.mavenPublish) apply false alias(libs.plugins.kover) diff --git a/docs/modules/ROOT/pages/reference/architecture.adoc b/docs/modules/ROOT/pages/reference/architecture.adoc index 6682c5ad..321c7372 100644 --- a/docs/modules/ROOT/pages/reference/architecture.adoc +++ b/docs/modules/ROOT/pages/reference/architecture.adoc @@ -51,7 +51,9 @@ llm-apps/ skainet-cli/ Unified CLI (auto-detects architecture, routes every GGUF family) kllama-cli/ Llama-family CLI with agent/tool-calling mode kbert-cli/ BERT CLI - skainet-decode/ Shared decode entry points + skainet-decode/ GGUF decode + GenerationMetrics CLI (JVM leg, SKaiNET#1129) + skainet-decode-core/ Shared DecodeSession flow behind both decode legs (KMP: jvm + android) + skainet-decode-android/ On-device decode/metrics activity (Android leg, SKaiNET#1244) kllama-java-sample/ Java interop sample llm-performance/ Benchmarking module ---- diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index efddb1b4..d524afb0 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -108,6 +108,7 @@ skainet-io-safetensors = { module = "sk.ainet.core:skainet-io-safetensors" } skainet-data-source = { module = "sk.ainet.core:skainet-data-source" } [plugins] +androidApplication = { id = "com.android.application", version.ref = "agp" } androidLibrary = { id = "com.android.library", version.ref = "agp" } androidMultiplatformLibrary = { id = "com.android.kotlin.multiplatform.library", version.ref = "agp" } kotlinMultiplatform = { id = "org.jetbrains.kotlin.multiplatform", version.ref = "kotlin" } diff --git a/llm-apps/skainet-decode-android/README.md b/llm-apps/skainet-decode-android/README.md new file mode 100644 index 00000000..e19fe1ff --- /dev/null +++ b/llm-apps/skainet-decode-android/README.md @@ -0,0 +1,53 @@ +# skainet-decode-android + +The Android activity leg of `skainet-decode` (SKaiNET#1244): load a GGUF, decode, and report the +same `GenerationMetrics` block as the JVM CLI — on the platform the 2 GB memory arc actually +targets. The decode flow lives in `:llm-apps:skainet-decode-core` (`DecodeSession`), so the JVM +and Android legs cannot diverge; this module adds only the rows a device can answer: + +- **major-fault rate for mapped weights** — the loader maps the GGUF (`MappedRandomAccessSource`), + so cold weight pages fault in from storage; `MemoryProbe` is sampled inside every decode span + and `GenerationMetrics` derives faults/s from the counters. +- **RSS on a constrained device** — sampled per decode step, with a first/last footer. + +## Running it + +There is no model picker and no download path — the audience is measurement, and the app's own +external-files dir needs no permissions: + +```sh +./gradlew :llm-apps:skainet-decode-android:installDebug +adb push smollm2-135m-q8_0.gguf \ + /sdcard/Android/data/sk.ainet.apps.decode/files/model.gguf +``` + +Launch **skainet-decode**, adjust prompt/steps, press **Run**. The pre-flight renders the +device-fit verdict (`AndroidGguf.fits`, header-only — the refusal happens before a byte of +payload is read) and the run ends with `GenerationMetrics.render()` plus the device footer. + +The full report mirrors to logcat (`adb logcat -s SkDecode`) and lands beside the model: + +```sh +adb pull /sdcard/Android/data/sk.ainet.apps.decode/files/decode-report.md +``` + +## The numbers this lane owns (SKEEP-002) + +The engine repo's off-heap storage docs cite two device measurements that belong to this sample: + +- **≤ 40 MB managed heap for SmolLM2-135M Q8_0** — measured as shipped (`largeHeap="true"`; + the heap *cap* is larger, the claim is about heap *use* with mapped, keep-packed weights). +- **a ~600 MB Q4_K model loads on a 256 MB heap** — measured with `largeHeap` **off**: flip the + manifest attribute to `false` for this run. Llama-family 135M models need `largeHeap` on only + because their dense FP32 `token_embd` (~113 MiB) lands on the managed heap (transformers#272). + +## Caveats + +- **Kernel fallback skews tok/s, not correctness.** Kernels arrive via the self-healing + ServiceLoader SPI in the `skainet-backend-jni-cpu` AAR (0.52.0). If a quant/view combination + has no pack, dispatch falls back to the decoding reference kernel — the run is correct but + slow, and the *kernel/adapter share* rows in the report are how you notice. Packaging merges + `META-INF/services/**`; if a minified build ever strips them, everything silently slows. +- x86_64 emulator runs keep the lane executable but prove nothing about performance. +- `RecordingTraceSink` is single-threaded by design: the whole session runs on one worker + thread; the UI only receives strings. diff --git a/llm-apps/skainet-decode-android/build.gradle.kts b/llm-apps/skainet-decode-android/build.gradle.kts new file mode 100644 index 00000000..3bce5d74 --- /dev/null +++ b/llm-apps/skainet-decode-android/build.gradle.kts @@ -0,0 +1,67 @@ +import org.jetbrains.kotlin.gradle.dsl.JvmTarget + +// The Android activity leg of skainet-decode (SKaiNET#1244): load a GGUF, decode, and report +// the same GenerationMetrics as the JVM CLI — on the platform the 2 GB memory arc actually +// targets. The rows the JVM leg cannot show (page-fault rate for mapped weights, RSS on a +// constrained device) are the whole point; the decode flow itself lives in +// :llm-apps:skainet-decode-core so the two legs cannot diverge. +plugins { + // AGP 9 ships built-in Kotlin support — applying kotlin("android") is an error since 9.0 + // (see :llm-runtime:iree-android, the precedent module). First com.android.application in + // the repo; not published, so no maven-publish and invisible to bom-coverage by design. + alias(libs.plugins.androidApplication) +} + +android { + namespace = "sk.ainet.apps.decode" + compileSdk = libs.versions.android.compileSdk.get().toInt() + + defaultConfig { + applicationId = "sk.ainet.apps.decode" + minSdk = libs.versions.android.minSdk.get().toInt() + targetSdk = libs.versions.android.compileSdk.get().toInt() + versionCode = 1 + versionName = "0.1" + + ndk { + // Real numbers come from arm64 hardware; x86_64 keeps the emulator lane runnable. + // Matches the engine's skainet-backend-jni-cpu AAR ABI set. + abiFilters += listOf("arm64-v8a", "x86_64") + } + } + + packaging { + resources { + // The 0.52.0 self-healing kernel dispatch is ServiceLoader-driven + // (SKaiNET#1240): losing META-INF/services entries silently drops every + // matmul to the slow decoding reference kernel. Merge, never exclude. + merges += "META-INF/services/**" + } + } + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 + } + + kotlin { + compilerOptions { + jvmTarget.set(JvmTarget.JVM_11) + optIn.add("sk.ainet.lang.memory.ExperimentalMemoryApi") + } + } +} + +dependencies { + implementation(platform(project(":llm-bom"))) + implementation(project(":llm-apps:skainet-decode-core")) + // AndroidGguf (device-fit pre-flight, mapped loader) and MappedRandomAccessSource. + implementation(libs.skainet.io.gguf) + implementation(libs.skainet.io.core) + // AndroidTraceSink (Perfetto spans), MemoryProbe, GenerationMetrics. + implementation(libs.skainet.lang.core) + implementation(libs.kotlinx.coroutines) + // The NEON kernel AAR; registers itself via ServiceLoader on ART — no bootstrap code. + // skainet-backend-native-cpu (FFM) must NOT appear here: FFM does not exist on ART. + runtimeOnly(libs.skainet.backend.jniCpu) +} diff --git a/llm-apps/skainet-decode-android/src/main/AndroidManifest.xml b/llm-apps/skainet-decode-android/src/main/AndroidManifest.xml new file mode 100644 index 00000000..8439b3f8 --- /dev/null +++ b/llm-apps/skainet-decode-android/src/main/AndroidManifest.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + diff --git a/llm-apps/skainet-decode-android/src/main/kotlin/sk/ainet/apps/decode/MainActivity.kt b/llm-apps/skainet-decode-android/src/main/kotlin/sk/ainet/apps/decode/MainActivity.kt new file mode 100644 index 00000000..883d831f --- /dev/null +++ b/llm-apps/skainet-decode-android/src/main/kotlin/sk/ainet/apps/decode/MainActivity.kt @@ -0,0 +1,205 @@ +package sk.ainet.apps.decode + +import android.app.Activity +import android.graphics.Typeface +import android.os.Bundle +import android.text.method.ScrollingMovementMethod +import android.util.Log +import android.view.ViewGroup.LayoutParams.MATCH_PARENT +import android.view.ViewGroup.LayoutParams.WRAP_CONTENT +import android.widget.Button +import android.widget.EditText +import android.widget.LinearLayout +import android.widget.ScrollView +import android.widget.TextView +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.asCoroutineDispatcher +import kotlinx.coroutines.cancel +import kotlinx.coroutines.launch +import sk.ainet.apps.decode.core.DecodeSession +import sk.ainet.io.MappedRandomAccessSource +import sk.ainet.io.gguf.AndroidGguf +import sk.ainet.lang.memory.MemoryProbe +import sk.ainet.lang.memory.sample +import sk.ainet.lang.memory.trace.AndroidTraceSink +import java.io.File +import java.util.concurrent.Executors + +/** + * The on-device skainet-decode leg (SKaiNET#1244): pick a pushed GGUF, decode, and render the + * same `GenerationMetrics.render()` block as the JVM CLI — plus the rows only a device can + * answer: RSS and major-fault rate for mapped weights. + * + * Everything measurement-relevant runs on ONE background thread ([worker]): + * `RecordingTraceSink` is not thread-safe and `KernelDispatch.defaultSink` is assigned in the + * `DecodeSession` constructor, so session construction, load, and the whole traced loop stay on + * that thread. The UI only ever receives strings via [Activity.runOnUiThread]. + * + * The full report is mirrored to logcat (tag `SkDecode`) and written to + * `getExternalFilesDir(null)/decode-report.md` for `adb pull` — the M2-A5 harness pattern. + */ +class MainActivity : Activity() { + + private companion object { + const val TAG = "SkDecode" + + /** KV-sizing context for the header-only pre-flight plan; prompt+steps stay well under it. */ + const val PREFLIGHT_CTX = 512 + } + + private val workerExecutor = Executors.newSingleThreadExecutor { r -> Thread(r, "skainet-decode") } + private val worker = workerExecutor.asCoroutineDispatcher() + private val scope = CoroutineScope(worker) + + private lateinit var pathField: EditText + private lateinit var promptField: EditText + private lateinit var stepsField: EditText + private lateinit var runButton: Button + private lateinit var output: TextView + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + + val modelsDir = getExternalFilesDir(null) + val defaultModel = File(modelsDir, "model.gguf") + val ggufs = modelsDir?.listFiles { f -> f.name.endsWith(".gguf") }?.map { it.name }.orEmpty() + + val pad = (8 * resources.displayMetrics.density).toInt() + val root = LinearLayout(this).apply { + orientation = LinearLayout.VERTICAL + setPadding(pad, pad, pad, pad) + } + + root.addView(TextView(this).apply { + text = if (ggufs.isEmpty()) { + "No .gguf in ${modelsDir?.absolutePath} — push one:\nadb push model.gguf ${modelsDir?.absolutePath}/model.gguf" + } else { + "Models in ${modelsDir?.absolutePath}:\n" + ggufs.joinToString("\n") { " $it" } + } + typeface = Typeface.MONOSPACE + textSize = 11f + }) + + pathField = EditText(this).apply { + hint = "model path" + setText(defaultModel.absolutePath) + maxLines = 2 + } + promptField = EditText(this).apply { + hint = "prompt" + setText("Once upon a time") + } + stepsField = EditText(this).apply { + hint = "steps" + setText("32") + } + runButton = Button(this).apply { + text = "Run" + setOnClickListener { runDecode() } + } + output = TextView(this).apply { + typeface = Typeface.MONOSPACE + textSize = 11f + movementMethod = ScrollingMovementMethod() + } + + root.addView(pathField) + root.addView(promptField) + root.addView(stepsField) + root.addView(runButton) + root.addView(output, LinearLayout.LayoutParams(MATCH_PARENT, WRAP_CONTENT)) + + setContentView(ScrollView(this).apply { addView(root) }) + } + + override fun onDestroy() { + super.onDestroy() + scope.cancel() + workerExecutor.shutdownNow() + } + + private fun runDecode() { + val path = pathField.text.toString().trim() + val prompt = promptField.text.toString() + val steps = stepsField.text.toString().trim().toIntOrNull() ?: 32 + runButton.isEnabled = false + output.text = "" + + val report = StringBuilder() + fun line(s: String) { + Log.i(TAG, s) + report.appendLine(s) + runOnUiThread { output.append(s + "\n") } + } + + scope.launch { + try { + if (!File(path).canRead()) { + line("cannot read $path — push a model first (see the hint above)") + return@launch + } + + // Pre-flight: refuse before a byte of payload is read (M2-F6). + val device = AndroidGguf.deviceMemory(this@MainActivity) + line("device: ram ${device.totalRamBytes.mib()} MiB (avail ${device.availableRamBytes.mib()} MiB), " + + "heap cap ${device.heapMaxBytes.mib()} MiB" + if (device.lowMemory) " [LOW MEMORY]" else "") + val fit = AndroidGguf.fits(this@MainActivity, path, ctx = PREFLIGHT_CTX, weightsMapped = true) + line("fit @ctx=$PREFLIGHT_CTX: " + if (fit.fits) "OK" else "REFUSED (pool: ${fit.blockingPool})") + if (!fit.fits) { + fit.suggestions.forEach { line(" suggestion: $it") } + return@launch + } + + val before = MemoryProbe.sample() + line("rss before: $before") + + // Session construction assigns KernelDispatch.defaultSink — must happen here, + // on the same single thread that runs the traced loop. + val session = DecodeSession(extraSink = AndroidTraceSink()) + val result = session.run( + sourceProvider = { MappedRandomAccessSource.open(path) }, + prompt = prompt, + steps = steps, + onModelInfo = { info -> + line("model: ${info.architecture} (${info.family}), ${info.blockCount} blocks, " + + "vocab ${info.vocabSize}, embd ${info.embeddingLength}") + }, + onGenerationStart = { line("generating…") }, + onToken = { piece -> runOnUiThread { output.append(piece) } }, + ) + runOnUiThread { output.append("\n") } + Log.i(TAG, "text: ${result.text}") + report.appendLine().appendLine("```").appendLine(result.text).appendLine("```") + + line("") + line(result.metrics.render()) + if (result.droppedTraceEvents > 0) { + line("WARNING: trace ring dropped ${result.droppedTraceEvents} events — early prefill spans may be undercounted") + } + + val after = MemoryProbe.sample() + line("") + line("--- device footer ---") + line("rss after: $after (peak not tracked; last decode-span RSS is in the metrics above)") + line("major faults over run: ${after.majorFaultsSince(before) ?: "—"}") + line("minor faults over run: ${minorDelta(before.minorFaults, after.minorFaults)}") + line("heap: max ${Runtime.getRuntime().maxMemory().mib()} MiB, " + + "used ${(Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()).mib()} MiB") + + val reportFile = File(getExternalFilesDir(null), "decode-report.md") + reportFile.writeText(report.toString()) + line("report written: ${reportFile.absolutePath}") + } catch (t: Throwable) { + Log.e(TAG, "decode failed", t) + line("FAILED: $t") + } finally { + runOnUiThread { runButton.isEnabled = true } + } + } + } + + private fun Long.mib(): Long = this / (1024 * 1024) + + private fun minorDelta(before: Long?, after: Long?): String = + if (before != null && after != null) (after - before).toString() else "—" +} diff --git a/llm-apps/skainet-decode-core/build.gradle.kts b/llm-apps/skainet-decode-core/build.gradle.kts new file mode 100644 index 00000000..3b3d3470 --- /dev/null +++ b/llm-apps/skainet-decode-core/build.gradle.kts @@ -0,0 +1,42 @@ +import org.jetbrains.kotlin.gradle.dsl.JvmTarget + +plugins { + alias(libs.plugins.kotlinMultiplatform) + alias(libs.plugins.androidMultiplatformLibrary) +} + +kotlin { + compilerOptions { + optIn.add("sk.ainet.lang.memory.ExperimentalMemoryApi") + } + + android { + namespace = "sk.ainet.apps.decode.core" + compileSdk = libs.versions.android.compileSdk.get().toInt() + minSdk = libs.versions.android.minSdk.get().toInt() + compilerOptions { + jvmTarget.set(JvmTarget.JVM_11) + } + } + + jvm() + + sourceSets { + commonMain.dependencies { + implementation(project.dependencies.platform(project(":llm-bom"))) + // api, not implementation: DecodeSession.run's onModelInfo exposes llm-core's + // GGUFModelInfo in a public signature (same pattern as kllama, see #226). + api(project(":llm-core")) + implementation(project(":llm-inference:llama")) + implementation(project(":llm-inference:qwen")) + implementation(project(":llm-inference:bitnet")) + implementation(libs.skainet.lang.core) + implementation(libs.skainet.backend.api) + implementation(libs.skainet.backend.cpu) + implementation(libs.skainet.io.core) + implementation(libs.skainet.io.gguf) + implementation(libs.kotlinx.io.core) + implementation(libs.kotlinx.coroutines) + } + } +} diff --git a/llm-apps/skainet-decode-core/src/commonMain/kotlin/sk/ainet/apps/decode/core/DecodeSession.kt b/llm-apps/skainet-decode-core/src/commonMain/kotlin/sk/ainet/apps/decode/core/DecodeSession.kt new file mode 100644 index 00000000..a546437d --- /dev/null +++ b/llm-apps/skainet-decode-core/src/commonMain/kotlin/sk/ainet/apps/decode/core/DecodeSession.kt @@ -0,0 +1,164 @@ +package sk.ainet.apps.decode.core + +import sk.ainet.apps.llm.GGUFModelInfo +import sk.ainet.apps.llm.ModelFamily +import sk.ainet.apps.llm.OptimizedLLMMode +import sk.ainet.apps.llm.OptimizedLLMRuntime +import sk.ainet.apps.llm.UnifiedModelLoader +import sk.ainet.apps.llm.sampleFromTensor +import sk.ainet.apps.llm.tokenizer.TokenizerFactory +import sk.ainet.backend.api.kernel.KernelDispatch +import sk.ainet.context.DirectCpuExecutionContext +import sk.ainet.context.ExecutionContext +import sk.ainet.io.RandomAccessSource +import sk.ainet.lang.memory.MemoryProbe +import sk.ainet.lang.memory.sample +import sk.ainet.lang.memory.trace.CompositeTraceSink +import sk.ainet.lang.memory.trace.GenerationMetrics +import sk.ainet.lang.memory.trace.RecordingTraceSink +import sk.ainet.lang.memory.trace.TraceSink +import sk.ainet.lang.memory.trace.decodeStep +import sk.ainet.lang.memory.trace.prefill +import sk.ainet.lang.memory.trace.sample +import sk.ainet.lang.types.FP32 +import sk.ainet.models.bitnet.BitNetWeightLoader +import sk.ainet.lang.nn.dsl.decoder.DecoderGgufWeightLoader +import sk.ainet.models.llama.LlamaNetworkLoader +import sk.ainet.models.qwen.QwenNetworkLoader + +/** What one [DecodeSession.run] measured: the metrics, the generated text, and trace-ring health. */ +public data class DecodeReport( + val metrics: GenerationMetrics, + /** The decoded continuation (prompt not included). */ + val text: String, + /** Events the recording ring dropped; > 0 means early prefill spans may be undercounted. */ + val droppedTraceEvents: Long, +) + +/** + * The shared `skainet-decode` flow (SKaiNET#1129/#1244): load a GGUF, decode, and report + * [GenerationMetrics] — TTFT, prefill and decode tok/s, effective memory bandwidth, + * kernel/adapter shares, page-fault rate. The JVM CLI and the Android activity are both thin + * callers of [run], so the two legs cannot diverge. + * + * Two deliberate properties, straight from the issue: + * + * - **Weight forms are resolved, not configured.** The family loaders' defaults do the deciding + * (keep-packed, `MAPPED` where the file can be served zero-copy); this flow carries no policy + * flags at all. + * - **A traced run is a reportable run.** The generation loop opens the `prefill` / `decode` / + * `sample` spans on a [RecordingTraceSink]; kernel runs, adapter insertions and byte counters + * arrive through `KernelDispatch.defaultSink`; [GenerationMetrics.from] reads the stream back. + * [MemoryProbe] is sampled inside every decode span, so the RSS / page-fault rows light up on + * any platform with a `/proc` answer (Android, Linux) and render "—" elsewhere. + * + * Families: BitNet (packed I2_S path) and the shared decoder families (Llama / Mistral / Qwen). + * + * Threading: [RecordingTraceSink] is not thread-safe — construct the session and call [run] on + * one thread, and only hand the returned [DecodeReport] across threads. + * + * @param extraSink an additional sink composed beside the recording one (e.g. an + * `AndroidTraceSink` for Perfetto spans); metrics always come from the recording sink. + */ +public class DecodeSession( + capacity: Int = 1 shl 20, + extraSink: TraceSink? = null, +) { + /** The metrics source; exposed so callers can inspect or export the raw events. */ + public val recording: RecordingTraceSink = RecordingTraceSink(capacity = capacity) + + private val sink: TraceSink = + if (extraSink != null) CompositeTraceSink(listOf(recording, extraSink)) else recording + + /** The execution context the whole session runs under; its trace sink is [sink]. */ + public val ctx: ExecutionContext + + init { + // One sink sees everything: the loop's phase spans (via ctx.traceSink) and — through the + // 0.52 diagnostic hook — every kernel run, adapter insertion and byte counter the + // dispatcher emits. That is what lights up the bandwidth / kernel-share rows. + KernelDispatch.defaultSink = sink + ctx = object : ExecutionContext by DirectCpuExecutionContext() { + override val traceSink: TraceSink get() = sink + } + // Self-healing dispatch (ternary packs included, SKaiNET#1240) — no per-pack bootstrap. + KernelDispatch.ensureInstalled() + } + + /** + * Load the model behind [sourceProvider], ingest [prompt], decode [steps] tokens, and return + * the [DecodeReport]. + * + * @param onModelInfo fires after the GGUF header peek, before weights load. + * @param onGenerationStart fires once the runtime is built, right before the prefill span. + * @param onToken fires per decoded token with its text. + */ + public suspend fun run( + sourceProvider: () -> RandomAccessSource, + prompt: String, + steps: Int, + temperature: Float = 0f, + peakBytesPerSecond: Long? = null, + onModelInfo: (GGUFModelInfo) -> Unit = {}, + onGenerationStart: () -> Unit = {}, + onToken: (String) -> Unit = {}, + ): DecodeReport { + val modelInfo = UnifiedModelLoader.peek(sourceProvider) + onModelInfo(modelInfo) + val tokenizer = TokenizerFactory.fromGgufFields(modelInfo.fields) + + val (module, bos) = when (modelInfo.family) { + ModelFamily.BITNET -> { + val loaded = BitNetWeightLoader.loadWithMetadata(ctx, sourceProvider) + loaded.model to loaded.metadata.bosTokenId + } + else -> { + val weights = DecoderGgufWeightLoader( + randomAccessProvider = sourceProvider, + acceptedArchitectures = modelInfo.family.architectures + setOf(modelInfo.architecture), + ).loadToMapStreaming(ctx) + val m = when (modelInfo.family) { + ModelFamily.QWEN -> QwenNetworkLoader.fromWeights(weights) + else -> LlamaNetworkLoader.fromWeights(weights) + } + m to weights.metadata.bosTokenId + } + } + val runtime = OptimizedLLMRuntime(module, ctx, OptimizedLLMMode.DIRECT, FP32::class, bos = bos) + + val raw = tokenizer.encode(prompt) + val promptTokens = + if (raw.isNotEmpty() && raw[0] == tokenizer.bosTokenId) raw else intArrayOf(tokenizer.bosTokenId) + raw + + onGenerationStart() + + // Prompt ingestion under one prefill span — logits of all but the last prompt token are + // discarded, so their forwards are pure ingestion. + sink.prefill(tokens = promptTokens.size - 1) { + for (p in 0 until promptTokens.size - 1) runtime.forward(promptTokens[p]) + } + + val text = StringBuilder() + var token = promptTokens.last() + repeat(steps) { step -> + val logits = sink.decodeStep(step) { + val l = runtime.forward(token) + // Sampled inside the open decode span: GenerationMetrics derives the page-fault + // rate from the first-vs-last counter values it sees between decode boundaries. + MemoryProbe.sample().emitTo(sink) + l + } + val next = sink.sample(step) { sampleFromTensor(logits, temperature) } + val piece = tokenizer.decode(next) + text.append(piece) + onToken(piece) + token = next + } + + return DecodeReport( + metrics = GenerationMetrics.from(recording.events(), peakBytesPerSecond), + text = text.toString(), + droppedTraceEvents = recording.dropped, + ) + } +} diff --git a/llm-apps/skainet-decode/build.gradle.kts b/llm-apps/skainet-decode/build.gradle.kts index 787a590a..f7616cfb 100644 --- a/llm-apps/skainet-decode/build.gradle.kts +++ b/llm-apps/skainet-decode/build.gradle.kts @@ -10,18 +10,14 @@ application { dependencies { implementation(platform(project(":llm-bom"))) - implementation(project(":llm-core")) - implementation(project(":llm-inference:llama")) - implementation(project(":llm-inference:qwen")) - implementation(project(":llm-inference:bitnet")) + implementation(project(":llm-apps:skainet-decode-core")) implementation(libs.skainet.lang.core) - implementation(libs.skainet.backend.api) - implementation(libs.skainet.backend.cpu) + // KernelPacks + FfmRowMajorKernelPack: without the FFM row-major pack on the runtime + // classpath, MAPPED/keep-packed weights fall to the decoding reference kernel (see the + // kllama jvmMain note) — JVM-only, so it stays here rather than in -core. implementation(libs.skainet.backend.nativeCpu) implementation(libs.skainet.io.core) - implementation(libs.skainet.io.gguf) - implementation(libs.kotlinx.io.core) implementation(libs.kotlinx.coroutines) } diff --git a/llm-apps/skainet-decode/src/main/kotlin/sk/ainet/apps/decode/Main.kt b/llm-apps/skainet-decode/src/main/kotlin/sk/ainet/apps/decode/Main.kt index 05a7db6b..cea21fcd 100644 --- a/llm-apps/skainet-decode/src/main/kotlin/sk/ainet/apps/decode/Main.kt +++ b/llm-apps/skainet-decode/src/main/kotlin/sk/ainet/apps/decode/Main.kt @@ -1,49 +1,20 @@ package sk.ainet.apps.decode import kotlinx.coroutines.runBlocking -import sk.ainet.apps.llm.ModelFamily -import sk.ainet.apps.llm.OptimizedLLMMode -import sk.ainet.apps.llm.OptimizedLLMRuntime -import sk.ainet.apps.llm.UnifiedModelLoader -import sk.ainet.apps.llm.sampleFromTensor -import sk.ainet.apps.llm.tokenizer.TokenizerFactory -import sk.ainet.backend.api.kernel.KernelDispatch -import sk.ainet.context.DirectCpuExecutionContext -import sk.ainet.context.ExecutionContext +import sk.ainet.apps.decode.core.DecodeSession import sk.ainet.io.JvmRandomAccessSource -import sk.ainet.lang.memory.ExperimentalMemoryApi -import sk.ainet.lang.memory.trace.GenerationMetrics -import sk.ainet.lang.memory.trace.RecordingTraceSink -import sk.ainet.lang.memory.trace.TraceSink -import sk.ainet.lang.memory.trace.decodeStep -import sk.ainet.lang.memory.trace.prefill -import sk.ainet.lang.memory.trace.sample -import sk.ainet.lang.types.FP32 -import sk.ainet.models.bitnet.BitNetWeightLoader -import sk.ainet.lang.nn.dsl.decoder.DecoderGgufWeightLoader -import sk.ainet.models.llama.LlamaNetworkLoader -import sk.ainet.models.qwen.QwenNetworkLoader import kotlin.system.exitProcess /** - * `skainet-decode` (SKaiNET#1129): load a GGUF, decode, and report [GenerationMetrics] — TTFT, + * `skainet-decode` (SKaiNET#1129): load a GGUF, decode, and report `GenerationMetrics` — TTFT, * prefill and decode tok/s, **effective memory bandwidth**, kernel/adapter shares, page-fault * rate. These are the numbers the SKEEP-003 memory work exists to move, measured on a real * model instead of a microbench. * - * Two deliberate properties, straight from the issue: - * - * - **Weight forms are resolved, not configured.** The family loaders' defaults do the deciding - * (keep-packed, `MAPPED` where the file can be served zero-copy); this sample carries no - * policy flags at all. - * - **A traced run is a reportable run.** The generation loop opens the `prefill` / - * `decode` / `sample` spans on a [RecordingTraceSink]; kernel runs, adapter insertions and - * byte counters arrive through `KernelDispatch.defaultSink`; [GenerationMetrics.from] reads - * the stream back. - * - * Families: BitNet (packed I2_S path) and the shared decoder families (Llama / Mistral / Qwen). + * The flow itself lives in [DecodeSession] (`llm-apps:skainet-decode-core`, SKaiNET#1244) so the + * Android activity leg reports the same numbers from the same loop; this CLI is a thin caller + * that owns only argument parsing, the JVM source, and stdout. */ -@OptIn(ExperimentalMemoryApi::class) fun main(args: Array) { var model: String? = null var prompt = "The capital of France is" @@ -65,69 +36,29 @@ fun main(args: Array) { exitProcess(1) } - // One sink sees everything: the loop's phase spans (via ctx.traceSink) and — through the - // 0.52 diagnostic hook — every kernel run, adapter insertion and byte counter the - // dispatcher emits. That is what lights up the bandwidth / kernel-share rows below. - val sink = RecordingTraceSink(capacity = 1 shl 20) - KernelDispatch.defaultSink = sink - val ctx: ExecutionContext = object : ExecutionContext by DirectCpuExecutionContext() { - override val traceSink: TraceSink get() = sink - } - // Self-healing dispatch (ternary packs included, SKaiNET#1240) — no per-pack bootstrap. - KernelDispatch.ensureInstalled() + val session = DecodeSession() runBlocking { - val modelInfo = UnifiedModelLoader.peek { JvmRandomAccessSource.open(modelPath) } - println("Model: $modelPath (${modelInfo.family.displayName}, ${modelInfo.blockCount} layers, vocab=${modelInfo.vocabSize})") - val tokenizer = TokenizerFactory.fromGgufFields(modelInfo.fields) - - val (module, bos) = when (modelInfo.family) { - ModelFamily.BITNET -> { - val loaded = BitNetWeightLoader.loadWithMetadata( - ctx, { JvmRandomAccessSource.open(modelPath) }, - ) - loaded.model to loaded.metadata.bosTokenId - } - else -> { - val weights = DecoderGgufWeightLoader( - randomAccessProvider = { JvmRandomAccessSource.open(modelPath) }, - acceptedArchitectures = modelInfo.family.architectures + setOf(modelInfo.architecture), - ).loadToMapStreaming(ctx) - val m = when (modelInfo.family) { - ModelFamily.QWEN -> QwenNetworkLoader.fromWeights(weights) - else -> LlamaNetworkLoader.fromWeights(weights) - } - m to weights.metadata.bosTokenId - } - } - val runtime = OptimizedLLMRuntime(module, ctx, OptimizedLLMMode.DIRECT, FP32::class, bos = bos) - - val raw = tokenizer.encode(prompt) - val promptTokens = if (raw.isNotEmpty() && raw[0] == tokenizer.bosTokenId) raw else intArrayOf(tokenizer.bosTokenId) + raw - - println("Generating $steps tokens (temperature=$temperature)...") - print(prompt) - - // Prompt ingestion under one prefill span — logits of all but the last prompt token are - // discarded, so their forwards are pure ingestion. - sink.prefill(tokens = promptTokens.size - 1) { - for (p in 0 until promptTokens.size - 1) runtime.forward(promptTokens[p]) - } - - var token = promptTokens.last() - repeat(steps) { step -> - val logits = sink.decodeStep(step) { runtime.forward(token) } - val next = sink.sample(step) { sampleFromTensor(logits, temperature) } - print(tokenizer.decode(next)) - token = next - } + val report = session.run( + sourceProvider = { JvmRandomAccessSource.open(modelPath) }, + prompt = prompt, + steps = steps, + temperature = temperature, + onModelInfo = { info -> + println("Model: $modelPath (${info.family.displayName}, ${info.blockCount} layers, vocab=${info.vocabSize})") + }, + onGenerationStart = { + println("Generating $steps tokens (temperature=$temperature)...") + print(prompt) + }, + onToken = ::print, + ) println() println() - val metrics = GenerationMetrics.from(sink.events()) - println(metrics.render()) - if (sink.dropped > 0) { - println("(trace ring dropped ${sink.dropped} events — early prefill spans may be undercounted)") + println(report.metrics.render()) + if (report.droppedTraceEvents > 0) { + println("(trace ring dropped ${report.droppedTraceEvents} events — early prefill spans may be undercounted)") } } } diff --git a/settings.gradle.kts b/settings.gradle.kts index 4508e39c..98685fdc 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -76,6 +76,8 @@ include("llm-runtime:iree-android") include("llm-performance") include("llm-apps:skainet-cli") include("llm-apps:skainet-decode") +include("llm-apps:skainet-decode-core") +include("llm-apps:skainet-decode-android") include("llm-apps:kllama-cli") include("llm-apps:kllama-java-sample") include("llm-apps:kbert-cli")