From 8f52909147745a43b719847b9942eaaeb045ceb5 Mon Sep 17 00:00:00 2001 From: Michal Harakal Date: Fri, 14 Aug 2026 07:50:27 +0200 Subject: [PATCH] Add androidNativeArm32 eager FunctionGemma: NativeFunctionGemma + CLI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ports FunctionGemma.kt's eager facade (fromGguf().call()) to a target that didn't exist before: androidNativeArm32. Same gemmaNetwork() DSL, DirectCpuExecutionContext, chat template, and tool-call parser as every other path — only the file-access layer differs (createRandomAccessSource + SystemFileSystem/kotlinx-io instead of JvmRandomAccessSource/java.io). Needed androidNativeArm32 added to llm-inference:gemma and llm-runtime:gemma-iree first (both had it missing entirely). CLI entry point lives in its own package (cli.android, not cli) so it doesn't collide with the existing nativeMain stub shared by linux/macos. Board-verified on the actual device (SKaiNET-functiongemma's docs/DEVICE-BRINGUP.md): linked ELF 32-bit ARM EABI5 executable, real 260MB checkpoint, correct output (set_lights), no host involvement at runtime. Eager throughput is a known, root-caused separate gap (DIRECT mode cost + a graph-tracing bug in OPTIMIZED mode) — not blocking here. --- llm-agent/build.gradle.kts | 1 + llm-core/build.gradle.kts | 2 + llm-inference/gemma/build.gradle.kts | 1 + llm-runtime/gemma-iree/build.gradle.kts | 4 + llm-runtime/kgemma/build.gradle.kts | 22 +++++ .../ainet/apps/kgemma/NativeFunctionGemma.kt | 90 +++++++++++++++++++ .../sk/ainet/apps/kgemma/cli/android/Main.kt | 26 ++++++ 7 files changed, 146 insertions(+) create mode 100644 llm-runtime/kgemma/src/androidNativeArm32Main/kotlin/sk/ainet/apps/kgemma/NativeFunctionGemma.kt create mode 100644 llm-runtime/kgemma/src/androidNativeArm32Main/kotlin/sk/ainet/apps/kgemma/cli/android/Main.kt diff --git a/llm-agent/build.gradle.kts b/llm-agent/build.gradle.kts index ac63b566..66a2da05 100644 --- a/llm-agent/build.gradle.kts +++ b/llm-agent/build.gradle.kts @@ -25,6 +25,7 @@ kotlin { linuxX64() linuxArm64() macosArm64() + androidNativeArm32() jvm() js { diff --git a/llm-core/build.gradle.kts b/llm-core/build.gradle.kts index 1780a82b..71fa553f 100644 --- a/llm-core/build.gradle.kts +++ b/llm-core/build.gradle.kts @@ -24,6 +24,7 @@ kotlin { linuxX64() linuxArm64() macosArm64() + androidNativeArm32() jvm() @@ -93,6 +94,7 @@ kotlin { val linuxX64Main by getting { dependsOn(nativeMain) } val linuxArm64Main by getting { dependsOn(nativeMain) } val macosArm64Main by getting { dependsOn(nativeMain) } + val androidNativeArm32Main by getting { dependsOn(nativeMain) } if (!project.hasProperty("buildFatJar")) { val androidMain by getting { dependsOn(registryBasedMain) } diff --git a/llm-inference/gemma/build.gradle.kts b/llm-inference/gemma/build.gradle.kts index 4d2a9c92..19804c4f 100644 --- a/llm-inference/gemma/build.gradle.kts +++ b/llm-inference/gemma/build.gradle.kts @@ -24,6 +24,7 @@ kotlin { macosArm64() linuxX64() linuxArm64() + androidNativeArm32() jvm() diff --git a/llm-runtime/gemma-iree/build.gradle.kts b/llm-runtime/gemma-iree/build.gradle.kts index cc0f556e..375c932b 100644 --- a/llm-runtime/gemma-iree/build.gradle.kts +++ b/llm-runtime/gemma-iree/build.gradle.kts @@ -20,6 +20,10 @@ kotlin { linuxX64() linuxArm64() macosArm64() // Apple Silicon host dev (mirrors :llm-runtime:kgemma); uses the same nativeMain sources + // ToolCall/CompactCodec/FunctionGemmaChatTemplate for :llm-runtime:kgemma's + // androidNativeArm32 eager CLI — IreeRuntime/GemmaDecoder (the on-device + // vmfb decode loop) aren't used by that path, only the commonMain grammar. + androidNativeArm32() sourceSets { commonMain.dependencies { diff --git a/llm-runtime/kgemma/build.gradle.kts b/llm-runtime/kgemma/build.gradle.kts index 4c8cc816..60e02300 100644 --- a/llm-runtime/kgemma/build.gradle.kts +++ b/llm-runtime/kgemma/build.gradle.kts @@ -52,6 +52,18 @@ kotlin { iosArm64() iosSimulatorArm64() + // Real EAGER FunctionGemma CLI, unlike the shared nativeMain stub above — + // entry point in its own package (cli.android, not cli) so it doesn't + // collide with nativeMain's `cli.main` in the same compilation. + androidNativeArm32 { + binaries { + executable { + entryPoint = "sk.ainet.apps.kgemma.cli.android.main" + baseName = "kgemma" + } + } + } + jvm { mainRun { mainClass.set("sk.ainet.apps.kgemma.cli.MainKt") @@ -156,6 +168,16 @@ kotlin { val macosArm64Main by getting { dependsOn(macosMain) } val iosArm64Main by getting { dependsOn(iosMain) } val iosSimulatorArm64Main by getting { dependsOn(iosMain) } + val androidNativeArm32Main by getting { + dependsOn(nativeMain) + dependencies { + // ToolCall/CompactCodec/FunctionGemmaChatTemplate — the compile-leg + // export (FunctionGemmaExportHarness) isn't needed on-device, so + // unlike jvmMain this doesn't pull in :llm-inference:functiongemma. + implementation(project(":llm-runtime:gemma-iree")) + implementation(project(":llm-agent")) + } + } } } diff --git a/llm-runtime/kgemma/src/androidNativeArm32Main/kotlin/sk/ainet/apps/kgemma/NativeFunctionGemma.kt b/llm-runtime/kgemma/src/androidNativeArm32Main/kotlin/sk/ainet/apps/kgemma/NativeFunctionGemma.kt new file mode 100644 index 00000000..244f34d8 --- /dev/null +++ b/llm-runtime/kgemma/src/androidNativeArm32Main/kotlin/sk/ainet/apps/kgemma/NativeFunctionGemma.kt @@ -0,0 +1,90 @@ +package sk.ainet.apps.kgemma + +import kotlinx.coroutines.runBlocking +import kotlinx.io.buffered +import kotlinx.io.files.Path +import kotlinx.io.files.SystemFileSystem +import sk.ainet.apps.llm.InferenceRuntime +import sk.ainet.apps.llm.OptimizedLLMMode +import sk.ainet.apps.llm.OptimizedLLMRuntime +import sk.ainet.apps.llm.generate +import sk.ainet.apps.llm.tokenizer.GGUFTokenizer +import sk.ainet.context.DirectCpuExecutionContext +import sk.ainet.io.gguf.createRandomAccessSource +import sk.ainet.io.model.QuantPolicy +import sk.ainet.lang.types.FP32 +import sk.ainet.models.gemma.Gemma4WeightLoader +import sk.ainet.models.gemma.GemmaNetworkLoader +import sk.ainet.apps.kllama.chat.ChatMessage +import sk.ainet.apps.kllama.chat.ChatRole +import sk.ainet.transformers.gemma.iree.CompactCodec +import sk.ainet.transformers.gemma.iree.FunctionGemmaChatTemplate +import sk.ainet.transformers.gemma.iree.ToolCall +import kotlin.random.Random + +/** + * androidNativeArm32's [FunctionGemma]: identical contract (`fromGguf(path).call(text)`), + * same `gemmaNetwork()` DSL + `DirectCpuExecutionContext` as the JVM facade — only the + * file-access layer differs, because `JvmRandomAccessSource`/`java.io.File` don't exist + * here. Tokenizer loading mirrors [sk.ainet.transformers.gemma.iree.GemmaKvDecoder]'s own + * native GGUF read (`SystemFileSystem`/`kotlinx-io`); weight loading uses + * `sk.ainet.io.gguf.createRandomAccessSource` — POSIX `pread` via + * `Posix32PreadRandomAccessSource` on this 32-bit target (see that class's own doc for why + * it isn't the same implementation the 64-bit natives share). + * + * No compile-leg export here (unlike the JVM facade's `exportCompiled`) — that's host + * tooling (`:llm-inference:functiongemma`), out of scope for an on-device eager CLI. + */ +public class NativeFunctionGemma private constructor( + private val runtime: InferenceRuntime, + private val tokenizer: GGUFTokenizer, + private val bos: Int, + private val eot: Int, + private val eos: Int, +) { + public data class Turn(val text: String, val calls: List) + + private val chatTemplate = FunctionGemmaChatTemplate() + + public fun call(userText: String, maxTokens: Int = 24): Turn { + val prompt = chatTemplate.apply(listOf(ChatMessage(ChatRole.USER, userText))) + val ptoks = tokenizer.encode(prompt) + val gen = ArrayList(maxTokens) + var stopped = false + runtime.generate(prompt = ptoks, steps = maxTokens, temperature = 0f, bosToken = bos) { id -> + if (!stopped) { + if (id == eot || id == eos) stopped = true else gen.add(id) + } + } + val text = tokenizer.decode(gen.toIntArray()) + return Turn(text, CompactCodec.parse(text)) + } + + public companion object { + public fun fromGguf(gguf: String, partialRotary: Float = 1.0f): NativeFunctionGemma = runBlocking { + val tok = GGUFTokenizer.fromSource(SystemFileSystem.source(Path(gguf)).buffered()) + val ctx = DirectCpuExecutionContext.create() + val weights = Gemma4WeightLoader( + randomAccessProvider = { + createRandomAccessSource(gguf) + ?: error("could not open $gguf for random access (pread failed?)") + }, + quantPolicy = QuantPolicy.DEQUANTIZE_TO_FP32, + ).loadToMapStreaming(ctx, FP32::class) + val patched = weights.copy( + metadata = weights.metadata.copy( + ropeParametersFull = weights.metadata.ropeParametersFull.copy(partialRotaryFactor = partialRotary), + ), + ) + val model = GemmaNetworkLoader.fromWeights(ctx, patched, FP32::class) + val runtime = OptimizedLLMRuntime(model, ctx, OptimizedLLMMode.DIRECT, FP32::class, random = Random.Default) + NativeFunctionGemma( + runtime = runtime, + tokenizer = tok, + bos = tok.bosTokenId, + eot = tok.encode("").single(), + eos = tok.eosTokenId, + ) + } + } +} diff --git a/llm-runtime/kgemma/src/androidNativeArm32Main/kotlin/sk/ainet/apps/kgemma/cli/android/Main.kt b/llm-runtime/kgemma/src/androidNativeArm32Main/kotlin/sk/ainet/apps/kgemma/cli/android/Main.kt new file mode 100644 index 00000000..caa32dea --- /dev/null +++ b/llm-runtime/kgemma/src/androidNativeArm32Main/kotlin/sk/ainet/apps/kgemma/cli/android/Main.kt @@ -0,0 +1,26 @@ +package sk.ainet.apps.kgemma.cli.android + +import sk.ainet.apps.kgemma.NativeFunctionGemma + +/** + * Real (non-stub) native CLI entry point, androidNativeArm32-only — unlike the shared + * `sk.ainet.apps.kgemma.cli.main` stub (nativeMain, used by linux/macos), this one + * actually runs [NativeFunctionGemma]. Own package so both can coexist without a + * redeclaration clash (Main.kt's `main` would otherwise collide with nativeMain's). + * + * Usage: kgemma + */ +public fun main(args: Array) { + val gguf = args.getOrNull(0) + ?: run { println("usage: kgemma "); return } + val instruction = args.drop(1).joinToString(" ") + if (instruction.isBlank()) { + println("usage: kgemma ") + return + } + + val fg = NativeFunctionGemma.fromGguf(gguf) + val turn = fg.call(instruction) + println("text: ${turn.text}") + turn.calls.forEach { println("call: ${it.tool}(${it.args})") } +}