AndroidCraft — on-device generative AI studio (chat, image, voice, video) + CI/CD APK

.github/workflows/android-build.yml

@@ -36,6 +36,13 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+        with:
+          submodules: false
+
+      - name: Init native submodules
+        run: |
+          git submodule update --init app/src/main/cpp/stable-diffusion.cpp
+          git -C app/src/main/cpp/stable-diffusion.cpp submodule update --init ggml
 
       - name: Set up JDK 17
         uses: actions/setup-java@v4
@@ -48,7 +55,8 @@ jobs:
 
       - name: Install SDK packages
         run: |
-          sdkmanager --install "platforms;android-35" "build-tools;35.0.0" "platform-tools" > /dev/null
+          sdkmanager --install "platforms;android-35" "build-tools;35.0.0" "platform-tools" \
+            "ndk;27.0.12077973" "cmake;3.22.1" > /dev/null
 
       - name: Set up Gradle
         uses: gradle/actions/setup-gradle@v4

.github/workflows/release.yml

@@ -19,6 +19,13 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+        with:
+          submodules: false
+
+      - name: Init native submodules
+        run: |
+          git submodule update --init app/src/main/cpp/stable-diffusion.cpp
+          git -C app/src/main/cpp/stable-diffusion.cpp submodule update --init ggml
 
       - name: Set up JDK 17
         uses: actions/setup-java@v4
@@ -31,7 +38,8 @@ jobs:
 
       - name: Install SDK packages
         run: |
-          sdkmanager --install "platforms;android-35" "build-tools;35.0.0" "platform-tools" > /dev/null
+          sdkmanager --install "platforms;android-35" "build-tools;35.0.0" "platform-tools" \
+            "ndk;27.0.12077973" "cmake;3.22.1" > /dev/null
 
       - name: Set up Gradle
         uses: gradle/actions/setup-gradle@v4

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "app/src/main/cpp/stable-diffusion.cpp"]
+	path = app/src/main/cpp/stable-diffusion.cpp
+	url = https://github.com/leejet/stable-diffusion.cpp

README.md

@@ -14,10 +14,14 @@ download and manage yourself. Every APK is built in CI, never on a device or lap
 | Tab | Capability | Engine |
 | --- | --- | --- |
 | **Chat** | Fully offline LLM chat | Google AI Edge / MediaPipe GenAI (LiteRT `.task`) |
-| **Image** | Text-to-image | Bundled on-device procedural renderer (diffusion-ready) |
+| **Image** | Text-to-image, **selectable engine** | Procedural (instant) **or real diffusion via stable-diffusion.cpp** |
 | **Voice** | Text-to-speech to a WAV file | Android on-device neural TTS |
-| **Video** | Prompt-seeded motion clip | Experimental on-device frame renderer |
-| **Models** | Download / delete models, device-fit checks, HF token | Download manager + `ModelRepository` |
+| **Video** | Prompt-seeded motion clip → **MP4** | On-device frame renderer + MediaCodec encoder |
+| **Models** | Download / delete, device-fit checks, **add any model by URL** | Download manager + `ModelRepository` |
+
+Every generated image, voice clip, and video can be **saved to the gallery/Music/Movies
+and shared**. Model downloads need **no login** by default (an optional Hugging Face token
+covers gated repos).
 
 Everything runs on the device. Nothing is sent to a server for inference.
 
@@ -43,23 +47,33 @@ com.androidcraft.studio
 
 ## Models
 
-The catalog (`ModelCatalog.kt`) ships curated on-device models:
+Everything is **one-tap from inside the app** — no account, no token, no links to find,
+and no manual URL entry. The catalog (`ModelCatalog.kt`) ships only vetted, **ungated,
+direct-download** models, and the **Models** screen sorts them by how well they fit your
+phone's RAM and badges the best pick with **"Best for your device."**
 
-- **Text:** Gemma 3 1B Instruct (int4 / int8) and Hammer 2.1 1.5B — LiteRT `.task`
-  bundles the MediaPipe GenAI runtime executes directly.
+- **Text (chat):** Gemma 3 1B (int4) and Qwen 2.5 1.5B (int8) — LiteRT `.task` bundles the
+  MediaPipe GenAI runtime executes directly (the same models Google's AI Edge Gallery ships).
+- **Image:** Stable Diffusion 1.5 as single-file **GGUF** weights (Q8) for the native
+  diffusion engine.
 - **Voice:** the system TTS engine (no download).
-- **Image / Video:** bundled renderers (no download).
+- **Image/Video previews:** bundled procedural renderers (no download).
+
+### Image generation engines
 
-Some text models are **gated** on Hugging Face. Paste a read token on the **Models**
-screen (stored only on-device) to enable those downloads. You can also change or extend
-the catalog by editing `ModelCatalog.kt` — any direct-download URL works.
+The Image tab has a **method selector**:
 
-### Making image generation photoreal
+- **Procedural** — instant, no download, abstract art (always available).
+- **Diffusion · SD.cpp** — real Stable Diffusion running fully on-device via a native
+  [stable-diffusion.cpp](https://github.com/leejet/stable-diffusion.cpp) library (arm64,
+  compiled in CI). One-tap download a Stable Diffusion GGUF model from the Models tab, then
+  generate offline. CPU-bound, so expect tens of seconds per image on a flagship.
+- **Diffusion · ONNX** — a second diffusion backend (ONNX Runtime, full SD 1.5 pipeline). Only
+  shown when an ONNX bundle is present, so the picker stays limited to engines you can use now.
 
-`ImageEngine.render()` is a deterministic procedural stand-in with the same signature a
-real diffusion call would have. To enable photoreal output, drop in Stable Diffusion
-LiteRT weights and replace the body of `render()` with a MediaPipe/LiteRT image-generation
-call — the UI, prompt box, and gallery save all keep working unchanged.
+The native engine lives in `app/src/main/cpp` (`sdjni.cpp` JNI bridge + the
+`stable-diffusion.cpp` submodule); the whole pipeline — tokenizer, scheduler, UNet, VAE —
+runs in C++.
 
 ## Building
 
@@ -68,10 +82,14 @@ call — the UI, prompt box, and gallery save all keep working unchanged.
 Pushing a commit that touches the app or its build config triggers
 [`.github/workflows/android-build.yml`](.github/workflows/android-build.yml), which:
 
-1. sets up JDK 17 + the Android SDK (platform 35, build-tools 35),
-2. runs the unit tests,
-3. assembles the **debug APK**, and
-4. uploads it as the `androidcraft-debug-apk` build artifact.
+1. initialises the native submodules (`stable-diffusion.cpp` + `ggml`),
+2. sets up JDK 17 + the Android SDK (platform 35, build-tools 35, **NDK 27 + CMake**),
+3. runs the unit tests,
+4. compiles the native diffusion library and assembles the **debug APK**, and
+5. uploads it as the `androidcraft-debug-apk` build artifact.
+
+> Cloning the repo for local builds needs `git clone --recurse-submodules` (or
+> `git submodule update --init` after cloning) so the native sources are present.
 
 Tagging a release (`git tag v0.1.0 && git push origin v0.1.0`) runs
 [`.github/workflows/release.yml`](.github/workflows/release.yml), which builds the APKs and
@@ -107,6 +125,8 @@ compileSdk 35 / minSdk 26.
 ## Roadmap
 
 - Streaming token output for chat (MediaPipe async session API).
-- Real on-device diffusion for the Image tab.
-- MP4 export for the Video tab via `MediaCodec` / `MediaMuxer`.
+- Second diffusion backend: ONNX Runtime SD 1.5 pipeline behind the existing selector.
 - `llama.cpp` GGUF engine as an alternate text backend for broader model support.
+
+Done: real on-device diffusion (stable-diffusion.cpp), MP4 video export, save/share for
+all media, add-model-by-URL, optional (not required) Hugging Face token.

app/build.gradle.kts

@@ -7,6 +7,7 @@ plugins {
 android {
     namespace = "com.androidcraft.studio"
     compileSdk = 35
+    ndkVersion = "27.0.12077973"
 
     defaultConfig {
         applicationId = "com.androidcraft.studio"
@@ -18,6 +19,24 @@ android {
         vectorDrawables {
             useSupportLibrary = true
         }
+
+        // Native on-device diffusion (stable-diffusion.cpp). arm64 only — the target for
+        // modern phones and the only ABI the diffusion engine is expected to run on.
+        ndk {
+            abiFilters += "arm64-v8a"
+        }
+        externalNativeBuild {
+            cmake {
+                cppFlags += "-O3"
+            }
+        }
+    }
+
+    externalNativeBuild {
+        cmake {
+            path = file("src/main/cpp/CMakeLists.txt")
+            version = "3.22.1"
+        }
     }
 
     buildTypes {
@@ -76,6 +95,9 @@ dependencies {
     // On-device LLM inference (Google AI Edge / MediaPipe GenAI).
     implementation(libs.mediapipe.tasks.genai)
 
+    // On-device ONNX Runtime for the ONNX diffusion image engine.
+    implementation(libs.onnxruntime.android)
+
     debugImplementation(libs.androidx.ui.tooling)
 
     testImplementation(libs.junit)

app/src/main/cpp/CMakeLists.txt

@@ -0,0 +1,24 @@
+cmake_minimum_required(VERSION 3.22.1)
+project(sdjni CXX C)
+
+# Build only the core stable-diffusion library — no CLI examples, no WebP/WebM I/O,
+# CPU backend only (no CUDA/Vulkan/OpenCL on this target).
+set(SD_BUILD_EXAMPLES     OFF CACHE BOOL "" FORCE)
+set(SD_BUILD_SHARED_LIBS  OFF CACHE BOOL "" FORCE)
+set(SD_WEBP               OFF CACHE BOOL "" FORCE)
+set(SD_WEBM               OFF CACHE BOOL "" FORCE)
+
+# ggml: cross-compiling for Android, so disable host-native tuning; NEON is enabled
+# automatically for arm64. Keep OpenMP off for a self-contained library.
+set(GGML_NATIVE   OFF CACHE BOOL "" FORCE)
+set(GGML_OPENMP   OFF CACHE BOOL "" FORCE)
+set(GGML_LLAMAFILE OFF CACHE BOOL "" FORCE)
+
+add_subdirectory(stable-diffusion.cpp)
+
+add_library(sdjni SHARED sdjni.cpp)
+target_compile_features(sdjni PRIVATE cxx_std_17)
+
+find_library(log-lib log)
+# stable-diffusion PUBLIC-exports its include dirs, so sdjni sees stable-diffusion.h.
+target_link_libraries(sdjni PRIVATE stable-diffusion ${log-lib})

app/src/main/cpp/sdjni.cpp

@@ -0,0 +1,90 @@
+// JNI bridge from AndroidCraft's NativeDiffusionGenerator to stable-diffusion.cpp.
+// The entire diffusion pipeline (tokenizer, scheduler, UNet, VAE) runs in the C++ library;
+// this file only marshals a prompt in and ARGB pixels out. Runs fully on-device (CPU).
+
+#include <jni.h>
+#include <android/log.h>
+#include <cstdint>
+#include <cstdlib>
+#include <vector>
+
+#include "stable-diffusion.h"
+
+#define LOG_TAG "sdjni"
+#define LOGI(...) __android_log_print(ANDROID_LOG_INFO, LOG_TAG, __VA_ARGS__)
+#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR, LOG_TAG, __VA_ARGS__)
+
+extern "C" JNIEXPORT jintArray JNICALL
+Java_com_androidcraft_studio_engine_image_NativeDiffusionGenerator_nativeTxt2Img(
+    JNIEnv* env,
+    jobject /* thiz */,
+    jstring jModelPath,
+    jstring jPrompt,
+    jint steps,
+    jlong seed,
+    jint width,
+    jint height) {
+
+    const char* modelPath = env->GetStringUTFChars(jModelPath, nullptr);
+    const char* prompt = env->GetStringUTFChars(jPrompt, nullptr);
+
+    sd_ctx_params_t ctxParams;
+    sd_ctx_params_init(&ctxParams);
+    ctxParams.model_path = modelPath;
+
+    sd_ctx_t* ctx = new_sd_ctx(&ctxParams);
+    env->ReleaseStringUTFChars(jModelPath, modelPath);
+    if (ctx == nullptr) {
+        LOGE("new_sd_ctx failed (model load)");
+        env->ReleaseStringUTFChars(jPrompt, prompt);
+        return nullptr;
+    }
+
+    sd_img_gen_params_t genParams;
+    sd_img_gen_params_init(&genParams);
+    genParams.prompt = prompt;
+    genParams.negative_prompt = "";
+    genParams.width = width;
+    genParams.height = height;
+    genParams.seed = static_cast<int64_t>(seed);
+    genParams.batch_count = 1;
+    genParams.sample_params.sample_steps = steps;
+
+    sd_image_t* images = nullptr;
+    int numImages = 0;
+    bool ok = generate_image(ctx, &genParams, &images, &numImages);
+    env->ReleaseStringUTFChars(jPrompt, prompt);
+
+    if (!ok || images == nullptr || numImages < 1) {
+        LOGE("generate_image failed");
+        free_sd_ctx(ctx);
+        return nullptr;
+    }
+
+    const sd_image_t& img = images[0];
+    const uint32_t w = img.width;
+    const uint32_t h = img.height;
+    const uint32_t c = img.channel;
+    const int count = static_cast<int>(w * h);
+
+    jintArray result = env->NewIntArray(count);
+    if (result == nullptr) {
+        free_sd_images(images, numImages);
+        free_sd_ctx(ctx);
+        return nullptr;
+    }
+
+    std::vector<jint> pixels(count);
+    for (int i = 0; i < count; ++i) {
+        const uint8_t r = img.data[i * c + 0];
+        const uint8_t g = c > 1 ? img.data[i * c + 1] : r;
+        const uint8_t b = c > 2 ? img.data[i * c + 2] : r;
+        pixels[i] = (0xFF << 24) | (r << 16) | (g << 8) | b;
+    }
+    env->SetIntArrayRegion(result, 0, count, pixels.data());
+
+    free_sd_images(images, numImages);
+    free_sd_ctx(ctx);
+    LOGI("generated %dx%d image", w, h);
+    return result;
+}

app/src/main/java/com/androidcraft/studio/AndroidCraftApp.kt

@@ -13,7 +13,7 @@ class AppContainer(context: Context) {
     val modelRepository = ModelRepository.get(context)
     val textEngine = TextEngine(context.applicationContext)
     val audioEngine = AudioEngine(context.applicationContext)
-    val imageEngine = ImageEngine()
+    val imageEngine = ImageEngine(context)
     val videoEngine = VideoEngine()
 }