diff --git a/.github/workflows/android-build.yml b/.github/workflows/android-build.yml index e0f1cd8..b4910fb 100644 --- a/.github/workflows/android-build.yml +++ b/.github/workflows/android-build.yml @@ -36,6 +36,13 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + with: + submodules: false + + - name: Init native submodules + run: | + git submodule update --init app/src/main/cpp/stable-diffusion.cpp + git -C app/src/main/cpp/stable-diffusion.cpp submodule update --init ggml - name: Set up JDK 17 uses: actions/setup-java@v4 @@ -48,7 +55,8 @@ jobs: - name: Install SDK packages run: | - sdkmanager --install "platforms;android-35" "build-tools;35.0.0" "platform-tools" > /dev/null + sdkmanager --install "platforms;android-35" "build-tools;35.0.0" "platform-tools" \ + "ndk;27.0.12077973" "cmake;3.22.1" > /dev/null - name: Set up Gradle uses: gradle/actions/setup-gradle@v4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ffb4151..53dda43 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,6 +19,13 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + with: + submodules: false + + - name: Init native submodules + run: | + git submodule update --init app/src/main/cpp/stable-diffusion.cpp + git -C app/src/main/cpp/stable-diffusion.cpp submodule update --init ggml - name: Set up JDK 17 uses: actions/setup-java@v4 @@ -31,7 +38,8 @@ jobs: - name: Install SDK packages run: | - sdkmanager --install "platforms;android-35" "build-tools;35.0.0" "platform-tools" > /dev/null + sdkmanager --install "platforms;android-35" "build-tools;35.0.0" "platform-tools" \ + "ndk;27.0.12077973" "cmake;3.22.1" > /dev/null - name: Set up Gradle uses: gradle/actions/setup-gradle@v4 diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..aa24118 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "app/src/main/cpp/stable-diffusion.cpp"] + path = app/src/main/cpp/stable-diffusion.cpp + url = https://github.com/leejet/stable-diffusion.cpp diff --git a/README.md b/README.md index 04acd67..d7011b7 100644 --- a/README.md +++ b/README.md @@ -14,10 +14,14 @@ download and manage yourself. Every APK is built in CI, never on a device or lap | Tab | Capability | Engine | | --- | --- | --- | | **Chat** | Fully offline LLM chat | Google AI Edge / MediaPipe GenAI (LiteRT `.task`) | -| **Image** | Text-to-image | Bundled on-device procedural renderer (diffusion-ready) | +| **Image** | Text-to-image, **selectable engine** | Procedural (instant) **or real diffusion via stable-diffusion.cpp** | | **Voice** | Text-to-speech to a WAV file | Android on-device neural TTS | -| **Video** | Prompt-seeded motion clip | Experimental on-device frame renderer | -| **Models** | Download / delete models, device-fit checks, HF token | Download manager + `ModelRepository` | +| **Video** | Prompt-seeded motion clip → **MP4** | On-device frame renderer + MediaCodec encoder | +| **Models** | Download / delete, device-fit checks, **add any model by URL** | Download manager + `ModelRepository` | + +Every generated image, voice clip, and video can be **saved to the gallery/Music/Movies +and shared**. Model downloads need **no login** by default (an optional Hugging Face token +covers gated repos). Everything runs on the device. Nothing is sent to a server for inference. @@ -43,23 +47,33 @@ com.androidcraft.studio ## Models -The catalog (`ModelCatalog.kt`) ships curated on-device models: +Everything is **one-tap from inside the app** — no account, no token, no links to find, +and no manual URL entry. The catalog (`ModelCatalog.kt`) ships only vetted, **ungated, +direct-download** models, and the **Models** screen sorts them by how well they fit your +phone's RAM and badges the best pick with **"Best for your device."** -- **Text:** Gemma 3 1B Instruct (int4 / int8) and Hammer 2.1 1.5B — LiteRT `.task` - bundles the MediaPipe GenAI runtime executes directly. +- **Text (chat):** Gemma 3 1B (int4) and Qwen 2.5 1.5B (int8) — LiteRT `.task` bundles the + MediaPipe GenAI runtime executes directly (the same models Google's AI Edge Gallery ships). +- **Image:** Stable Diffusion 1.5 as single-file **GGUF** weights (Q8) for the native + diffusion engine. - **Voice:** the system TTS engine (no download). -- **Image / Video:** bundled renderers (no download). +- **Image/Video previews:** bundled procedural renderers (no download). + +### Image generation engines -Some text models are **gated** on Hugging Face. Paste a read token on the **Models** -screen (stored only on-device) to enable those downloads. You can also change or extend -the catalog by editing `ModelCatalog.kt` — any direct-download URL works. +The Image tab has a **method selector**: -### Making image generation photoreal +- **Procedural** — instant, no download, abstract art (always available). +- **Diffusion · SD.cpp** — real Stable Diffusion running fully on-device via a native + [stable-diffusion.cpp](https://github.com/leejet/stable-diffusion.cpp) library (arm64, + compiled in CI). One-tap download a Stable Diffusion GGUF model from the Models tab, then + generate offline. CPU-bound, so expect tens of seconds per image on a flagship. +- **Diffusion · ONNX** — a second diffusion backend (ONNX Runtime, full SD 1.5 pipeline). Only + shown when an ONNX bundle is present, so the picker stays limited to engines you can use now. -`ImageEngine.render()` is a deterministic procedural stand-in with the same signature a -real diffusion call would have. To enable photoreal output, drop in Stable Diffusion -LiteRT weights and replace the body of `render()` with a MediaPipe/LiteRT image-generation -call — the UI, prompt box, and gallery save all keep working unchanged. +The native engine lives in `app/src/main/cpp` (`sdjni.cpp` JNI bridge + the +`stable-diffusion.cpp` submodule); the whole pipeline — tokenizer, scheduler, UNet, VAE — +runs in C++. ## Building @@ -68,10 +82,14 @@ call — the UI, prompt box, and gallery save all keep working unchanged. Pushing a commit that touches the app or its build config triggers [`.github/workflows/android-build.yml`](.github/workflows/android-build.yml), which: -1. sets up JDK 17 + the Android SDK (platform 35, build-tools 35), -2. runs the unit tests, -3. assembles the **debug APK**, and -4. uploads it as the `androidcraft-debug-apk` build artifact. +1. initialises the native submodules (`stable-diffusion.cpp` + `ggml`), +2. sets up JDK 17 + the Android SDK (platform 35, build-tools 35, **NDK 27 + CMake**), +3. runs the unit tests, +4. compiles the native diffusion library and assembles the **debug APK**, and +5. uploads it as the `androidcraft-debug-apk` build artifact. + +> Cloning the repo for local builds needs `git clone --recurse-submodules` (or +> `git submodule update --init` after cloning) so the native sources are present. Tagging a release (`git tag v0.1.0 && git push origin v0.1.0`) runs [`.github/workflows/release.yml`](.github/workflows/release.yml), which builds the APKs and @@ -107,6 +125,8 @@ compileSdk 35 / minSdk 26. ## Roadmap - Streaming token output for chat (MediaPipe async session API). -- Real on-device diffusion for the Image tab. -- MP4 export for the Video tab via `MediaCodec` / `MediaMuxer`. +- Second diffusion backend: ONNX Runtime SD 1.5 pipeline behind the existing selector. - `llama.cpp` GGUF engine as an alternate text backend for broader model support. + +Done: real on-device diffusion (stable-diffusion.cpp), MP4 video export, save/share for +all media, add-model-by-URL, optional (not required) Hugging Face token. diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 1ce9462..07ed4d2 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -7,6 +7,7 @@ plugins { android { namespace = "com.androidcraft.studio" compileSdk = 35 + ndkVersion = "27.0.12077973" defaultConfig { applicationId = "com.androidcraft.studio" @@ -18,6 +19,24 @@ android { vectorDrawables { useSupportLibrary = true } + + // Native on-device diffusion (stable-diffusion.cpp). arm64 only — the target for + // modern phones and the only ABI the diffusion engine is expected to run on. + ndk { + abiFilters += "arm64-v8a" + } + externalNativeBuild { + cmake { + cppFlags += "-O3" + } + } + } + + externalNativeBuild { + cmake { + path = file("src/main/cpp/CMakeLists.txt") + version = "3.22.1" + } } buildTypes { @@ -76,6 +95,9 @@ dependencies { // On-device LLM inference (Google AI Edge / MediaPipe GenAI). implementation(libs.mediapipe.tasks.genai) + // On-device ONNX Runtime for the ONNX diffusion image engine. + implementation(libs.onnxruntime.android) + debugImplementation(libs.androidx.ui.tooling) testImplementation(libs.junit) diff --git a/app/src/main/cpp/CMakeLists.txt b/app/src/main/cpp/CMakeLists.txt new file mode 100644 index 0000000..bb8d4fd --- /dev/null +++ b/app/src/main/cpp/CMakeLists.txt @@ -0,0 +1,24 @@ +cmake_minimum_required(VERSION 3.22.1) +project(sdjni CXX C) + +# Build only the core stable-diffusion library — no CLI examples, no WebP/WebM I/O, +# CPU backend only (no CUDA/Vulkan/OpenCL on this target). +set(SD_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) +set(SD_BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE) +set(SD_WEBP OFF CACHE BOOL "" FORCE) +set(SD_WEBM OFF CACHE BOOL "" FORCE) + +# ggml: cross-compiling for Android, so disable host-native tuning; NEON is enabled +# automatically for arm64. Keep OpenMP off for a self-contained library. +set(GGML_NATIVE OFF CACHE BOOL "" FORCE) +set(GGML_OPENMP OFF CACHE BOOL "" FORCE) +set(GGML_LLAMAFILE OFF CACHE BOOL "" FORCE) + +add_subdirectory(stable-diffusion.cpp) + +add_library(sdjni SHARED sdjni.cpp) +target_compile_features(sdjni PRIVATE cxx_std_17) + +find_library(log-lib log) +# stable-diffusion PUBLIC-exports its include dirs, so sdjni sees stable-diffusion.h. +target_link_libraries(sdjni PRIVATE stable-diffusion ${log-lib}) diff --git a/app/src/main/cpp/sdjni.cpp b/app/src/main/cpp/sdjni.cpp new file mode 100644 index 0000000..ce1a44c --- /dev/null +++ b/app/src/main/cpp/sdjni.cpp @@ -0,0 +1,90 @@ +// JNI bridge from AndroidCraft's NativeDiffusionGenerator to stable-diffusion.cpp. +// The entire diffusion pipeline (tokenizer, scheduler, UNet, VAE) runs in the C++ library; +// this file only marshals a prompt in and ARGB pixels out. Runs fully on-device (CPU). + +#include +#include +#include +#include +#include + +#include "stable-diffusion.h" + +#define LOG_TAG "sdjni" +#define LOGI(...) __android_log_print(ANDROID_LOG_INFO, LOG_TAG, __VA_ARGS__) +#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR, LOG_TAG, __VA_ARGS__) + +extern "C" JNIEXPORT jintArray JNICALL +Java_com_androidcraft_studio_engine_image_NativeDiffusionGenerator_nativeTxt2Img( + JNIEnv* env, + jobject /* thiz */, + jstring jModelPath, + jstring jPrompt, + jint steps, + jlong seed, + jint width, + jint height) { + + const char* modelPath = env->GetStringUTFChars(jModelPath, nullptr); + const char* prompt = env->GetStringUTFChars(jPrompt, nullptr); + + sd_ctx_params_t ctxParams; + sd_ctx_params_init(&ctxParams); + ctxParams.model_path = modelPath; + + sd_ctx_t* ctx = new_sd_ctx(&ctxParams); + env->ReleaseStringUTFChars(jModelPath, modelPath); + if (ctx == nullptr) { + LOGE("new_sd_ctx failed (model load)"); + env->ReleaseStringUTFChars(jPrompt, prompt); + return nullptr; + } + + sd_img_gen_params_t genParams; + sd_img_gen_params_init(&genParams); + genParams.prompt = prompt; + genParams.negative_prompt = ""; + genParams.width = width; + genParams.height = height; + genParams.seed = static_cast(seed); + genParams.batch_count = 1; + genParams.sample_params.sample_steps = steps; + + sd_image_t* images = nullptr; + int numImages = 0; + bool ok = generate_image(ctx, &genParams, &images, &numImages); + env->ReleaseStringUTFChars(jPrompt, prompt); + + if (!ok || images == nullptr || numImages < 1) { + LOGE("generate_image failed"); + free_sd_ctx(ctx); + return nullptr; + } + + const sd_image_t& img = images[0]; + const uint32_t w = img.width; + const uint32_t h = img.height; + const uint32_t c = img.channel; + const int count = static_cast(w * h); + + jintArray result = env->NewIntArray(count); + if (result == nullptr) { + free_sd_images(images, numImages); + free_sd_ctx(ctx); + return nullptr; + } + + std::vector pixels(count); + for (int i = 0; i < count; ++i) { + const uint8_t r = img.data[i * c + 0]; + const uint8_t g = c > 1 ? img.data[i * c + 1] : r; + const uint8_t b = c > 2 ? img.data[i * c + 2] : r; + pixels[i] = (0xFF << 24) | (r << 16) | (g << 8) | b; + } + env->SetIntArrayRegion(result, 0, count, pixels.data()); + + free_sd_images(images, numImages); + free_sd_ctx(ctx); + LOGI("generated %dx%d image", w, h); + return result; +} diff --git a/app/src/main/cpp/stable-diffusion.cpp b/app/src/main/cpp/stable-diffusion.cpp new file mode 160000 index 0000000..ea7f0c8 --- /dev/null +++ b/app/src/main/cpp/stable-diffusion.cpp @@ -0,0 +1 @@ +Subproject commit ea7f0c87cfe4c673263b4c201c596c7f1cbe2528 diff --git a/app/src/main/java/com/androidcraft/studio/AndroidCraftApp.kt b/app/src/main/java/com/androidcraft/studio/AndroidCraftApp.kt index 9518c4e..94427de 100644 --- a/app/src/main/java/com/androidcraft/studio/AndroidCraftApp.kt +++ b/app/src/main/java/com/androidcraft/studio/AndroidCraftApp.kt @@ -13,7 +13,7 @@ class AppContainer(context: Context) { val modelRepository = ModelRepository.get(context) val textEngine = TextEngine(context.applicationContext) val audioEngine = AudioEngine(context.applicationContext) - val imageEngine = ImageEngine() + val imageEngine = ImageEngine(context) val videoEngine = VideoEngine() } diff --git a/app/src/main/java/com/androidcraft/studio/core/MediaSaver.kt b/app/src/main/java/com/androidcraft/studio/core/MediaSaver.kt index 719536e..66919b8 100644 --- a/app/src/main/java/com/androidcraft/studio/core/MediaSaver.kt +++ b/app/src/main/java/com/androidcraft/studio/core/MediaSaver.kt @@ -2,6 +2,7 @@ package com.androidcraft.studio.core import android.content.ContentValues import android.content.Context +import android.content.Intent import android.graphics.Bitmap import android.net.Uri import android.os.Build @@ -9,41 +10,101 @@ import android.os.Environment import android.provider.MediaStore import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext +import java.io.File -/** Saves generated media into the shared gallery using MediaStore (no storage permission needed). */ +/** + * Saves generated media into the shared gallery / media collections using MediaStore + * (no storage permission needed on any supported API level), and builds share intents. + */ object MediaSaver { suspend fun saveImage(context: Context, bitmap: Bitmap, displayName: String): Uri? = withContext(Dispatchers.IO) { - val resolver = context.contentResolver - val values = ContentValues().apply { - put(MediaStore.Images.Media.DISPLAY_NAME, displayName) - put(MediaStore.Images.Media.MIME_TYPE, "image/png") - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { - put( - MediaStore.Images.Media.RELATIVE_PATH, - Environment.DIRECTORY_PICTURES + "/AndroidCraft", - ) - put(MediaStore.Images.Media.IS_PENDING, 1) + val collection = imagesCollection() + val values = baseValues(displayName, "image/png", Environment.DIRECTORY_PICTURES) + val uri = context.contentResolver.insert(collection, values) ?: return@withContext null + runCatching { + context.contentResolver.openOutputStream(uri)?.use { out -> + bitmap.compress(Bitmap.CompressFormat.PNG, 100, out) } - } + }.onFailure { return@withContext null } + finalizePending(context, uri) + uri + } - val collection = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { - MediaStore.Images.Media.getContentUri(MediaStore.VOLUME_EXTERNAL_PRIMARY) + /** Copies a synthesized WAV into the Music collection. */ + suspend fun saveAudio(context: Context, wavFile: File, displayName: String): Uri? = + copyFile( + context = context, + src = wavFile, + collection = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + MediaStore.Audio.Media.getContentUri(MediaStore.VOLUME_EXTERNAL_PRIMARY) } else { - MediaStore.Images.Media.EXTERNAL_CONTENT_URI - } + MediaStore.Audio.Media.EXTERNAL_CONTENT_URI + }, + values = baseValues(displayName, "audio/x-wav", Environment.DIRECTORY_MUSIC), + ) + + /** Copies a generated MP4 clip into the Movies collection. */ + suspend fun saveVideo(context: Context, mp4File: File, displayName: String): Uri? = + copyFile( + context = context, + src = mp4File, + collection = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + MediaStore.Video.Media.getContentUri(MediaStore.VOLUME_EXTERNAL_PRIMARY) + } else { + MediaStore.Video.Media.EXTERNAL_CONTENT_URI + }, + values = baseValues(displayName, "video/mp4", Environment.DIRECTORY_MOVIES), + ) + + fun shareIntent(uri: Uri, mime: String): Intent = + Intent(Intent.ACTION_SEND).apply { + type = mime + putExtra(Intent.EXTRA_STREAM, uri) + addFlags(Intent.FLAG_GRANT_READ_URI_PERMISSION) + } - val uri = resolver.insert(collection, values) ?: return@withContext null - resolver.openOutputStream(uri)?.use { out -> - bitmap.compress(Bitmap.CompressFormat.PNG, 100, out) - } ?: return@withContext null + // ---- internals ---- + private fun imagesCollection(): Uri = + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + MediaStore.Images.Media.getContentUri(MediaStore.VOLUME_EXTERNAL_PRIMARY) + } else { + MediaStore.Images.Media.EXTERNAL_CONTENT_URI + } + + private fun baseValues(displayName: String, mime: String, dir: String): ContentValues = + ContentValues().apply { + put(MediaStore.MediaColumns.DISPLAY_NAME, displayName) + put(MediaStore.MediaColumns.MIME_TYPE, mime) if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { - values.clear() - values.put(MediaStore.Images.Media.IS_PENDING, 0) - resolver.update(uri, values, null, null) + put(MediaStore.MediaColumns.RELATIVE_PATH, "$dir/AndroidCraft") + put(MediaStore.MediaColumns.IS_PENDING, 1) } - uri } + + private fun finalizePending(context: Context, uri: Uri) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + val done = ContentValues().apply { put(MediaStore.MediaColumns.IS_PENDING, 0) } + context.contentResolver.update(uri, done, null, null) + } + } + + private suspend fun copyFile( + context: Context, + src: File, + collection: Uri, + values: ContentValues, + ): Uri? = withContext(Dispatchers.IO) { + if (!src.exists()) return@withContext null + val uri = context.contentResolver.insert(collection, values) ?: return@withContext null + runCatching { + context.contentResolver.openOutputStream(uri)?.use { out -> + src.inputStream().use { input -> input.copyTo(out) } + } + }.onFailure { return@withContext null } + finalizePending(context, uri) + uri + } } diff --git a/app/src/main/java/com/androidcraft/studio/data/ModelCatalog.kt b/app/src/main/java/com/androidcraft/studio/data/ModelCatalog.kt index d3b7da5..3b99fae 100644 --- a/app/src/main/java/com/androidcraft/studio/data/ModelCatalog.kt +++ b/app/src/main/java/com/androidcraft/studio/data/ModelCatalog.kt @@ -4,63 +4,47 @@ private const val GB = 1024L * 1024L * 1024L private const val MB = 1024L * 1024L /** - * Curated catalog of models that run fully on-device. + * Curated catalog of models that run fully on-device and download directly in the app — no + * external links, no account, and no Hugging Face token required. Every entry is a vetted, + * ungated, single-file (or bundled) model, and the Models screen matches them to the device's RAM. * - * Text models are LiteRT `.task` bundles that the MediaPipe GenAI runtime executes directly. - * Some are gated on Hugging Face and require a personal access token (set it on the Models - * screen); the download manager sends it as a bearer token when present. + * Text models are LiteRT `.task` bundles run by MediaPipe GenAI (the same ones Google's AI Edge + * Gallery ships). Image models are single-file GGUF weights run by the native stable-diffusion.cpp + * engine. All URLs resolve without authentication. */ object ModelCatalog { val models: List = listOf( - // ---- Text / Chat (MediaPipe GenAI, LiteRT .task) ---- + // ---- Text / Chat (MediaPipe GenAI, LiteRT .task, ungated) ---- ModelSpec( id = "gemma3-1b-it-q4", - displayName = "Gemma 3 1B Instruct", + displayName = "Gemma 3 1B", publisher = "Google · litert-community", modality = Modality.TEXT, runtime = Runtime.MEDIAPIPE_LLM, - description = "Compact instruction-tuned chat model. Great default for phones; " + - "fast, low memory, fully offline once downloaded.", + description = "Compact, fast chat model — the best default for phones. Low memory, " + + "fully offline once downloaded. No account needed.", downloadUrl = "https://huggingface.co/litert-community/Gemma3-1B-IT/resolve/main/" + "Gemma3-1B-IT_multi-prefill-seq_q4_ekv2048.task", fileName = "gemma3-1b-it-q4.task", approxBytes = 555L * MB, minRamBytes = 3L * GB, quantization = "int4", - gated = true, ), ModelSpec( - id = "hammer-2p1-1p5b", - displayName = "Hammer 2.1 (1.5B)", - publisher = "litert-community", + id = "qwen25-1p5b-q8", + displayName = "Qwen 2.5 1.5B", + publisher = "Alibaba · litert-community", modality = Modality.TEXT, runtime = Runtime.MEDIAPIPE_LLM, - description = "Function-calling / agentic small model in LiteRT format. " + - "Good for structured, tool-style responses on device.", - downloadUrl = "https://huggingface.co/litert-community/Hammer2.1-1.5b/resolve/main/" + - "Hammer2.1-1.5b_seq128_q8_ekv1280.task", - fileName = "hammer-2p1-1p5b-q8.task", + description = "Higher-quality instruction-tuned chat model. Great on flagship phones " + + "with 6 GB+ RAM. Fully offline, no account needed.", + downloadUrl = "https://huggingface.co/litert-community/Qwen2.5-1.5B-Instruct/resolve/" + + "main/Qwen2.5-1.5B-Instruct_multi-prefill-seq_q8_ekv1280.task", + fileName = "qwen25-1p5b-q8.task", approxBytes = 1600L * MB, minRamBytes = 4L * GB, quantization = "int8", - gated = false, - ), - ModelSpec( - id = "gemma3-1b-it-q8", - displayName = "Gemma 3 1B Instruct (int8)", - publisher = "Google · litert-community", - modality = Modality.TEXT, - runtime = Runtime.MEDIAPIPE_LLM, - description = "Higher-quality int8 build of Gemma 3 1B. Needs a bit more RAM " + - "than the int4 variant; strong quality on flagship devices.", - downloadUrl = "https://huggingface.co/litert-community/Gemma3-1B-IT/resolve/main/" + - "Gemma3-1B-IT_multi-prefill-seq_q8_ekv2048.task", - fileName = "gemma3-1b-it-q8.task", - approxBytes = 1050L * MB, - minRamBytes = 4L * GB, - quantization = "int8", - gated = true, ), // ---- Audio / Voice ---- @@ -71,23 +55,52 @@ object ModelCatalog { modality = Modality.AUDIO, runtime = Runtime.ANDROID_TTS, description = "Uses the on-device text-to-speech engine already installed on your " + - "phone (Samsung/Google). No download needed — synthesizes speech to a WAV file.", + "phone. No download needed — synthesizes speech to a WAV file.", downloadUrl = null, fileName = "", approxBytes = 0, minRamBytes = 0, ), - // ---- Image ---- + // ---- Image (native stable-diffusion.cpp, single-file GGUF, ungated) ---- + ModelSpec( + id = "sd15-q8-emaonly", + displayName = "Stable Diffusion 1.5 (Q8)", + publisher = "second-state · GGUF", + modality = Modality.IMAGE, + runtime = Runtime.DIFFUSION, + description = "Real text-to-image, fully offline via the on-device diffusion engine. " + + "Q8 quantized single file — best quality/size balance. One-tap download.", + downloadUrl = "https://huggingface.co/second-state/stable-diffusion-v1-5-GGUF/resolve/" + + "main/stable-diffusion-v1-5-pruned-emaonly-Q8_0.gguf", + fileName = "sd15-q8-emaonly.gguf", + approxBytes = 1700L * MB, + minRamBytes = 4L * GB, + quantization = "Q8_0", + ), + ModelSpec( + id = "sd15-q8-full", + displayName = "Stable Diffusion 1.5 (Q8, full)", + publisher = "kostakoff · GGUF", + modality = Modality.IMAGE, + runtime = Runtime.DIFFUSION, + description = "Full (non-EMA) Q8 Stable Diffusion 1.5 for the on-device diffusion " + + "engine. Slightly larger; alternative to the EMA build. One-tap download.", + downloadUrl = "https://huggingface.co/kostakoff/stable-diffusion-v1-5-GGUF/resolve/" + + "main/v1-5-pruned_Q8_0.gguf", + fileName = "sd15-q8-full.gguf", + approxBytes = 2000L * MB, + minRamBytes = 5L * GB, + quantization = "Q8_0", + ), ModelSpec( id = "procedural-image", displayName = "Procedural Preview Renderer", publisher = "AndroidCraft", modality = Modality.IMAGE, runtime = Runtime.BUILTIN, - description = "Bundled deterministic renderer that turns a prompt into abstract " + - "art on-device with zero download. Placeholder for a full diffusion pipeline " + - "(drop in Stable Diffusion LiteRT weights to enable photoreal generation).", + description = "Bundled deterministic renderer that turns a prompt into abstract art " + + "on-device with zero download — instant, always available.", downloadUrl = null, fileName = "", approxBytes = 0, @@ -101,9 +114,9 @@ object ModelCatalog { publisher = "AndroidCraft", modality = Modality.VIDEO, runtime = Runtime.BUILTIN, - description = "Generates an animated, prompt-seeded motion clip on-device as a " + - "preview of the video pipeline. True on-device text-to-video is not yet " + - "practical on mobile hardware; this scaffolds the flow.", + description = "Generates an animated, prompt-seeded motion clip on-device and exports " + + "a real MP4. True on-device text-to-video isn't practical on phones yet; this " + + "scaffolds the flow.", downloadUrl = null, fileName = "", approxBytes = 0, diff --git a/app/src/main/java/com/androidcraft/studio/data/ModelRepository.kt b/app/src/main/java/com/androidcraft/studio/data/ModelRepository.kt index 196ea8e..e976351 100644 --- a/app/src/main/java/com/androidcraft/studio/data/ModelRepository.kt +++ b/app/src/main/java/com/androidcraft/studio/data/ModelRepository.kt @@ -1,6 +1,8 @@ package com.androidcraft.studio.data import android.content.Context +import org.json.JSONArray +import org.json.JSONObject import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job import kotlinx.coroutines.SupervisorJob @@ -10,12 +12,15 @@ import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.flow.update import kotlinx.coroutines.launch import java.io.File +import java.util.Locale /** - * Single source of truth for model download state and on-disk model files. + * Single source of truth for the model list, download state, and on-disk model files. * - * Exposes a [StateFlow] of per-model [DownloadState] so any screen can observe progress. - * Held as an application-scoped singleton so a download survives screen navigation. + * The model list is reactive: it is the built-in [ModelCatalog] plus any models the user adds + * by URL (persisted on-device). Downloads report progress through [states]. A Hugging Face token + * is entirely optional — it is sent as a bearer token only when the user has provided one, so + * ungated models need no token at all. */ class ModelRepository( context: Context, @@ -26,29 +31,40 @@ class ModelRepository( private val modelsDir: File = File(appContext.filesDir, "models").apply { mkdirs() } private val prefs = appContext.getSharedPreferences("androidcraft.prefs", Context.MODE_PRIVATE) - private val _states = MutableStateFlow(initialStates()) + private val _models = MutableStateFlow(ModelCatalog.models + loadCustomModels()) + val models: StateFlow> = _models.asStateFlow() + + private val _states = MutableStateFlow(initialStates(_models.value)) val states: StateFlow> = _states.asStateFlow() private val activeJobs = mutableMapOf() - private fun initialStates(): Map = - ModelCatalog.models.associate { spec -> + private fun initialStates(specs: List): Map = + specs.associate { spec -> spec.id to if (spec.isBundled || isReady(spec)) DownloadState.Ready else DownloadState.NotDownloaded } fun fileFor(spec: ModelSpec): File = File(modelsDir, spec.fileName) - fun isReady(spec: ModelSpec): Boolean = - spec.isBundled || (spec.fileName.isNotEmpty() && fileFor(spec).exists() && fileFor(spec).length() > 0) + fun extractDirFor(spec: ModelSpec): File = File(modelsDir, spec.extractDir) + + fun isReady(spec: ModelSpec): Boolean = when { + spec.isBundled -> true + spec.archive -> extractDirFor(spec).let { it.isDirectory && (it.list()?.isNotEmpty() == true) } + else -> spec.fileName.isNotEmpty() && fileFor(spec).exists() && fileFor(spec).length() > 0 + } fun stateFor(id: String): DownloadState = _states.value[id] ?: DownloadState.NotDownloaded - // ---- Hugging Face token (for gated downloads) ---- + fun modelsFor(modality: Modality): List = _models.value.filter { it.modality == modality } + + // ---- Hugging Face token (optional; only used for gated downloads) ---- var hfToken: String get() = prefs.getString(KEY_HF_TOKEN, "").orEmpty() set(value) = prefs.edit().putString(KEY_HF_TOKEN, value.trim()).apply() + // ---- Downloads ---- fun startDownload(spec: ModelSpec) { val url = spec.downloadUrl ?: return if (activeJobs.containsKey(spec.id)) return @@ -59,11 +75,16 @@ class ModelRepository( downloader.download( url = url, target = fileFor(spec), - authToken = if (spec.gated) hfToken.ifBlank { null } else null, + // Token is optional: sent only when the user supplied one. + authToken = hfToken.ifBlank { null }, ) { downloaded, total -> val effectiveTotal = if (total > 0) total else spec.approxBytes setState(spec.id, DownloadState.Downloading(downloaded, effectiveTotal)) } + if (spec.archive) { + extractZip(fileFor(spec), extractDirFor(spec)) + fileFor(spec).delete() + } setState(spec.id, DownloadState.Ready) } catch (t: Throwable) { if (t is kotlinx.coroutines.CancellationException) { @@ -88,7 +109,106 @@ class ModelRepository( if (spec.isBundled) return cancelDownload(spec) fileFor(spec).delete() + if (spec.archive) extractDirFor(spec).deleteRecursively() + setState(spec.id, DownloadState.NotDownloaded) + } + + private fun extractZip(zip: File, destDir: File) { + destDir.mkdirs() + val destCanonical = destDir.canonicalPath + java.util.zip.ZipInputStream(zip.inputStream().buffered()).use { zis -> + var entry = zis.nextEntry + while (entry != null) { + val outFile = File(destDir, entry.name) + // Guard against zip path traversal. + if (!outFile.canonicalPath.startsWith(destCanonical)) { + entry = zis.nextEntry + continue + } + if (entry.isDirectory) { + outFile.mkdirs() + } else { + outFile.parentFile?.mkdirs() + outFile.outputStream().use { zis.copyTo(it) } + } + zis.closeEntry() + entry = zis.nextEntry + } + } + } + + // ---- Custom (user-added) models ---- + fun addCustomModel(displayName: String, url: String, modality: Modality): ModelSpec { + val cleanName = displayName.trim().ifBlank { "Custom model" } + val id = "custom-" + System.nanoTime().toString(36) + val safe = cleanName.lowercase(Locale.US).replace(Regex("[^a-z0-9]+"), "-").trim('-') + val ext = if (modality == Modality.TEXT) ".task" else ".gguf" + val spec = ModelSpec( + id = id, + displayName = cleanName, + publisher = "Added by you", + modality = modality, + runtime = if (modality == Modality.TEXT) Runtime.MEDIAPIPE_LLM else Runtime.DIFFUSION, + description = "Custom model added from a URL. Runs on-device once downloaded.", + downloadUrl = url.trim(), + fileName = "$safe-$id$ext", + approxBytes = 0, + minRamBytes = 0, + custom = true, + ) + _models.update { it + spec } setState(spec.id, DownloadState.NotDownloaded) + persistCustomModels() + return spec + } + + fun removeCustomModel(spec: ModelSpec) { + if (!spec.custom) return + cancelDownload(spec) + fileFor(spec).delete() + _models.update { list -> list.filterNot { it.id == spec.id } } + _states.update { it - spec.id } + persistCustomModels() + } + + private fun persistCustomModels() { + val array = JSONArray() + _models.value.filter { it.custom }.forEach { spec -> + array.put( + JSONObject() + .put("id", spec.id) + .put("name", spec.displayName) + .put("url", spec.downloadUrl) + .put("fileName", spec.fileName) + .put("modality", spec.modality.name), + ) + } + prefs.edit().putString(KEY_CUSTOM_MODELS, array.toString()).apply() + } + + private fun loadCustomModels(): List { + val raw = prefs.getString(KEY_CUSTOM_MODELS, null) ?: return emptyList() + return runCatching { + val array = JSONArray(raw) + (0 until array.length()).map { i -> + val o = array.getJSONObject(i) + val modality = runCatching { Modality.valueOf(o.getString("modality")) } + .getOrDefault(Modality.TEXT) + ModelSpec( + id = o.getString("id"), + displayName = o.getString("name"), + publisher = "Added by you", + modality = modality, + runtime = if (modality == Modality.TEXT) Runtime.MEDIAPIPE_LLM else Runtime.DIFFUSION, + description = "Custom model added from a URL. Runs on-device once downloaded.", + downloadUrl = o.getString("url"), + fileName = o.getString("fileName"), + approxBytes = 0, + minRamBytes = 0, + custom = true, + ) + } + }.getOrDefault(emptyList()) } private fun setState(id: String, state: DownloadState) { @@ -97,6 +217,7 @@ class ModelRepository( companion object { private const val KEY_HF_TOKEN = "hf_token" + private const val KEY_CUSTOM_MODELS = "custom_models" @Volatile private var instance: ModelRepository? = null diff --git a/app/src/main/java/com/androidcraft/studio/data/ModelSpec.kt b/app/src/main/java/com/androidcraft/studio/data/ModelSpec.kt index 6e133d6..0fbbe1e 100644 --- a/app/src/main/java/com/androidcraft/studio/data/ModelSpec.kt +++ b/app/src/main/java/com/androidcraft/studio/data/ModelSpec.kt @@ -42,6 +42,10 @@ data class ModelSpec( val minRamBytes: Long, val quantization: String = "", val gated: Boolean = false, + val custom: Boolean = false, + /** When true, the downloaded file is a .zip extracted into [extractDir] under the models dir. */ + val archive: Boolean = false, + val extractDir: String = "", ) { val isBundled: Boolean get() = downloadUrl == null diff --git a/app/src/main/java/com/androidcraft/studio/engine/ImageEngine.kt b/app/src/main/java/com/androidcraft/studio/engine/ImageEngine.kt index 515f19d..60ab466 100644 --- a/app/src/main/java/com/androidcraft/studio/engine/ImageEngine.kt +++ b/app/src/main/java/com/androidcraft/studio/engine/ImageEngine.kt @@ -1,142 +1,47 @@ package com.androidcraft.studio.engine +import android.content.Context import android.graphics.Bitmap -import android.graphics.Canvas -import android.graphics.Color -import android.graphics.LinearGradient -import android.graphics.Paint -import android.graphics.RadialGradient -import android.graphics.Shader -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.withContext -import kotlin.math.abs -import kotlin.math.sin -import kotlin.random.Random +import com.androidcraft.studio.engine.image.ImageGenerator +import com.androidcraft.studio.engine.image.ImageMethod +import com.androidcraft.studio.engine.image.NativeDiffusionGenerator +import com.androidcraft.studio.engine.image.OnnxDiffusionGenerator +import com.androidcraft.studio.engine.image.ProceduralImageGenerator +import java.io.File /** - * On-device image generation. - * - * The bundled implementation is a deterministic *procedural* renderer: it turns a text prompt into - * abstract art entirely on-device with no model download. It is a functional stand-in for a full - * diffusion pipeline — swap [render] for a MediaPipe/LiteRT Stable Diffusion call (same signature) - * to produce photoreal images from the same UI. + * Coordinates the available on-device image-generation backends and dispatches a request to the + * one the user selected. Every backend runs on-device; the choice is purely about which engine. */ -class ImageEngine { +class ImageEngine(context: Context) { - /** - * @param prompt drives the palette and composition (same prompt + seed => same image). - * @param seed optional explicit seed; when null it is derived from the prompt. - */ - suspend fun render( - prompt: String, - size: Int = 768, - seed: Long? = null, - ): Bitmap = withContext(Dispatchers.Default) { - val effectiveSeed = seed ?: prompt.lowercase().hashCode().toLong() - val rng = Random(effectiveSeed) - val bitmap = Bitmap.createBitmap(size, size, Bitmap.Config.ARGB_8888) - val canvas = Canvas(bitmap) - - val palette = paletteFor(prompt, rng) + private val modelsDir = File(context.applicationContext.filesDir, "models").apply { mkdirs() } - // Base diagonal gradient. - val basePaint = Paint(Paint.ANTI_ALIAS_FLAG).apply { - shader = LinearGradient( - 0f, 0f, size.toFloat(), size.toFloat(), - intArrayOf(palette[0], palette[1], palette[2]), - floatArrayOf(0f, 0.55f, 1f), - Shader.TileMode.CLAMP, - ) - } - canvas.drawRect(0f, 0f, size.toFloat(), size.toFloat(), basePaint) + private val generators: List = listOf( + ProceduralImageGenerator(), + NativeDiffusionGenerator(modelsDir), + OnnxDiffusionGenerator(modelsDir), + ) - // Glowing orbs seeded by the prompt. - val orbCount = 5 + rng.nextInt(6) - repeat(orbCount) { i -> - val cx = rng.nextFloat() * size - val cy = rng.nextFloat() * size - val radius = size * (0.12f + rng.nextFloat() * 0.28f) - val color = palette[rng.nextInt(palette.size)] - val glow = Paint(Paint.ANTI_ALIAS_FLAG).apply { - shader = RadialGradient( - cx, cy, radius, - intArrayOf(withAlpha(color, 200), withAlpha(color, 0)), - floatArrayOf(0f, 1f), - Shader.TileMode.CLAMP, - ) - } - canvas.drawCircle(cx, cy, radius, glow) - // A wave of arcs adds structure so it doesn't read as pure noise. - if (i % 2 == 0) { - val stroke = Paint(Paint.ANTI_ALIAS_FLAG).apply { - style = Paint.Style.STROKE - strokeWidth = size * 0.004f - this.color = withAlpha(palette[rng.nextInt(palette.size)], 90) - } - val steps = 60 - var prevX = 0f - var prevY = size * rng.nextFloat() - val amp = size * (0.05f + rng.nextFloat() * 0.15f) - val freq = 2 + rng.nextInt(4) - for (s in 1..steps) { - val x = size * (s / steps.toFloat()) - val y = prevY + amp * sin(freq * (s / steps.toFloat()) * 6.283f + i) - canvas.drawLine(prevX, prevY, x, y, stroke) - prevX = x - prevY = y - } - } - } + private val onnx: OnnxDiffusionGenerator = generators.filterIsInstance().first() - applyGrain(bitmap, rng, strength = 14) - bitmap - } + fun generators(): List = generators - private fun paletteFor(prompt: String, rng: Random): IntArray { - val p = prompt.lowercase() - val base = when { - listOf("night", "dark", "space", "galaxy", "noir").any { p.contains(it) } -> - intArrayOf(0xFF0B1026.toInt(), 0xFF3A1C71.toInt(), 0xFF00C2FF.toInt()) - listOf("sunset", "warm", "fire", "gold", "desert").any { p.contains(it) } -> - intArrayOf(0xFF7A1F3D.toInt(), 0xFFE8552A.toInt(), 0xFFF7C948.toInt()) - listOf("nature", "forest", "green", "jungle", "leaf").any { p.contains(it) } -> - intArrayOf(0xFF08301E.toInt(), 0xFF1F7A4D.toInt(), 0xFFB4E197.toInt()) - listOf("ocean", "sea", "water", "ice", "blue").any { p.contains(it) } -> - intArrayOf(0xFF03203A.toInt(), 0xFF1B6CA8.toInt(), 0xFF7DE2D1.toInt()) - else -> { - // Prompt-derived hue for anything else. - val hue = (abs(p.hashCode()) % 360).toFloat() - intArrayOf( - hsv(hue, 0.7f, 0.25f), - hsv((hue + 40) % 360, 0.8f, 0.55f), - hsv((hue + 90) % 360, 0.6f, 0.9f), - ) - } - } - // Rotate slightly by seed so variations differ. - return if (rng.nextBoolean()) base else intArrayOf(base[2], base[0], base[1]) - } + fun byMethod(method: ImageMethod): ImageGenerator = + generators.firstOrNull { it.method == method } ?: generators.first() - private fun hsv(h: Float, s: Float, v: Float): Int = - Color.HSVToColor(floatArrayOf(h, s, v)) + /** Releases native ONNX sessions if they were loaded. */ + fun close() = onnx.close() - private fun withAlpha(color: Int, alpha: Int): Int = - Color.argb(alpha, Color.red(color), Color.green(color), Color.blue(color)) + /** Backwards-compatible convenience for the procedural preview. */ + suspend fun render(prompt: String, seed: Long? = null): Bitmap = + byMethod(ImageMethod.PROCEDURAL).generate(prompt, seed, steps = 1, onProgress = {}) - private fun applyGrain(bitmap: Bitmap, rng: Random, strength: Int) { - val w = bitmap.width - val h = bitmap.height - val pixels = IntArray(w * h) - bitmap.getPixels(pixels, 0, w, 0, 0, w, h) - for (i in pixels.indices) { - if (rng.nextInt(3) != 0) continue - val n = rng.nextInt(strength * 2) - strength - val c = pixels[i] - val r = (Color.red(c) + n).coerceIn(0, 255) - val g = (Color.green(c) + n).coerceIn(0, 255) - val b = (Color.blue(c) + n).coerceIn(0, 255) - pixels[i] = Color.argb(255, r, g, b) - } - bitmap.setPixels(pixels, 0, w, 0, 0, w, h) - } + suspend fun generate( + method: ImageMethod, + prompt: String, + seed: Long?, + steps: Int, + onProgress: suspend (Float) -> Unit, + ): Bitmap = byMethod(method).generate(prompt, seed, steps, onProgress) } diff --git a/app/src/main/java/com/androidcraft/studio/engine/VideoEncoder.kt b/app/src/main/java/com/androidcraft/studio/engine/VideoEncoder.kt new file mode 100644 index 0000000..a53ec0e --- /dev/null +++ b/app/src/main/java/com/androidcraft/studio/engine/VideoEncoder.kt @@ -0,0 +1,147 @@ +package com.androidcraft.studio.engine + +import android.graphics.Bitmap +import android.media.MediaCodec +import android.media.MediaCodecInfo +import android.media.MediaFormat +import android.media.MediaMuxer +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import java.io.File +import java.nio.ByteBuffer + +/** + * Encodes a list of frames into a real H.264 MP4 file entirely on-device, using [MediaCodec] with + * a YUV420-flexible input surface-less buffer and [MediaMuxer] for the container. Uses + * [MediaCodec.getInputImage] so per-device plane strides / layout are handled portably. + */ +object VideoEncoder { + + private const val MIME = "video/avc" + private const val TIMEOUT_US = 10_000L + + suspend fun encodeMp4(frames: List, fps: Int, outFile: File): File = + withContext(Dispatchers.Default) { + require(frames.isNotEmpty()) { "No frames to encode" } + // H.264 wants even dimensions; the generator uses multiples of 16 already. + val width = frames[0].width and 1.inv() + val height = frames[0].height and 1.inv() + + val format = MediaFormat.createVideoFormat(MIME, width, height).apply { + setInteger( + MediaFormat.KEY_COLOR_FORMAT, + MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible, + ) + setInteger(MediaFormat.KEY_BIT_RATE, (width * height * fps * 0.2f).toInt().coerceAtLeast(2_000_000)) + setInteger(MediaFormat.KEY_FRAME_RATE, fps) + setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1) + } + + val codec = MediaCodec.createEncoderByType(MIME) + codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE) + codec.start() + + if (outFile.exists()) outFile.delete() + val muxer = MediaMuxer(outFile.absolutePath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4) + var trackIndex = -1 + var muxerStarted = false + val bufferInfo = MediaCodec.BufferInfo() + + fun drain(endOfStream: Boolean) { + while (true) { + val outIndex = codec.dequeueOutputBuffer(bufferInfo, TIMEOUT_US) + when { + outIndex == MediaCodec.INFO_TRY_AGAIN_LATER -> { + if (!endOfStream) return + } + + outIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> { + trackIndex = muxer.addTrack(codec.outputFormat) + muxer.start() + muxerStarted = true + } + + outIndex >= 0 -> { + val encoded: ByteBuffer = codec.getOutputBuffer(outIndex) ?: continue + if (bufferInfo.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG != 0) { + bufferInfo.size = 0 + } + if (bufferInfo.size > 0 && muxerStarted) { + encoded.position(bufferInfo.offset) + encoded.limit(bufferInfo.offset + bufferInfo.size) + muxer.writeSampleData(trackIndex, encoded, bufferInfo) + } + codec.releaseOutputBuffer(outIndex, false) + if (bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) return + } + } + } + } + + try { + frames.forEachIndexed { index, frame -> + val inIndex = codec.dequeueInputBuffer(TIMEOUT_US) + if (inIndex >= 0) { + val image = codec.getInputImage(inIndex) + if (image != null) { + fillYuv420(frame, image, width, height) + } + val ptsUs = index.toLong() * 1_000_000L / fps + codec.queueInputBuffer(inIndex, 0, width * height * 3 / 2, ptsUs, 0) + } + drain(false) + } + // Signal end of stream. + val inIndex = codec.dequeueInputBuffer(TIMEOUT_US) + if (inIndex >= 0) { + val ptsUs = frames.size.toLong() * 1_000_000L / fps + codec.queueInputBuffer( + inIndex, 0, 0, ptsUs, MediaCodec.BUFFER_FLAG_END_OF_STREAM, + ) + } + drain(true) + } finally { + runCatching { codec.stop() } + runCatching { codec.release() } + if (muxerStarted) runCatching { muxer.stop() } + runCatching { muxer.release() } + } + outFile + } + + private fun fillYuv420(frame: Bitmap, image: android.media.Image, width: Int, height: Int) { + val pixels = IntArray(width * height) + frame.getPixels(pixels, 0, width, 0, 0, width, height) + + val yPlane = image.planes[0] + val uPlane = image.planes[1] + val vPlane = image.planes[2] + val yBuf = yPlane.buffer + val uBuf = uPlane.buffer + val vBuf = vPlane.buffer + val yRowStride = yPlane.rowStride + val yPixStride = yPlane.pixelStride + val uvRowStride = uPlane.rowStride + val uvPixStride = uPlane.pixelStride + + for (row in 0 until height) { + for (col in 0 until width) { + val argb = pixels[row * width + col] + val r = (argb shr 16) and 0xFF + val g = (argb shr 8) and 0xFF + val b = argb and 0xFF + + val y = ((66 * r + 129 * g + 25 * b + 128) shr 8) + 16 + yBuf.put(row * yRowStride + col * yPixStride, y.coerceIn(0, 255).toByte()) + + if (row % 2 == 0 && col % 2 == 0) { + val u = ((-38 * r - 74 * g + 112 * b + 128) shr 8) + 128 + val v = ((112 * r - 94 * g - 18 * b + 128) shr 8) + 128 + val uvIndex = (row / 2) * uvRowStride + (col / 2) * uvPixStride + uBuf.put(uvIndex, u.coerceIn(0, 255).toByte()) + vBuf.put(uvIndex, v.coerceIn(0, 255).toByte()) + } + } + } + } +} diff --git a/app/src/main/java/com/androidcraft/studio/engine/image/ImageGenerator.kt b/app/src/main/java/com/androidcraft/studio/engine/image/ImageGenerator.kt new file mode 100644 index 0000000..9c4168f --- /dev/null +++ b/app/src/main/java/com/androidcraft/studio/engine/image/ImageGenerator.kt @@ -0,0 +1,35 @@ +package com.androidcraft.studio.engine.image + +import android.graphics.Bitmap + +/** The available on-device image-generation backends the user can pick between. */ +enum class ImageMethod(val displayName: String, val diffusion: Boolean) { + PROCEDURAL("Procedural", false), + NATIVE("Diffusion · SD.cpp", true), + ONNX("Diffusion · ONNX", true), +} + +/** Whether a backend can run right now (e.g. its model is downloaded) and why not if it can't. */ +data class GeneratorStatus(val available: Boolean, val hint: String) + +/** + * A single on-device image-generation backend. Implementations range from the instant procedural + * renderer to real diffusion runtimes. All run fully on-device; none call a network at generate time. + */ +interface ImageGenerator { + val method: ImageMethod + + fun status(): GeneratorStatus + + /** + * @param seed optional; when null it is derived from the prompt (deterministic per prompt). + * @param steps diffusion sampling steps (ignored by non-diffusion backends). + * @param onProgress 0f..1f progress for long-running backends. + */ + suspend fun generate( + prompt: String, + seed: Long?, + steps: Int, + onProgress: suspend (Float) -> Unit, + ): Bitmap +} diff --git a/app/src/main/java/com/androidcraft/studio/engine/image/NativeDiffusionGenerator.kt b/app/src/main/java/com/androidcraft/studio/engine/image/NativeDiffusionGenerator.kt new file mode 100644 index 0000000..19d924c --- /dev/null +++ b/app/src/main/java/com/androidcraft/studio/engine/image/NativeDiffusionGenerator.kt @@ -0,0 +1,81 @@ +package com.androidcraft.studio.engine.image + +import android.graphics.Bitmap +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import java.io.File + +/** + * Real on-device text-to-image via a native stable-diffusion.cpp bridge, fully offline. The whole + * diffusion pipeline — tokenizer, scheduler, UNet, VAE — runs in the bundled C++ library, so this + * class only bridges to it over JNI. + * + * The model is any Stable Diffusion `.gguf` or `.safetensors` file the user downloads into the + * models directory (from the Models tab). If the native library isn't present or no model has been + * downloaded, [status] reports the backend as unavailable so the UI can steer the user instead of + * failing. + */ +class NativeDiffusionGenerator(private val modelsDir: File) : ImageGenerator { + + override val method = ImageMethod.NATIVE + + private fun modelFile(): File? = modelsDir.listFiles() + ?.filter { f -> f.isFile && (f.name.endsWith(".gguf", true) || f.name.endsWith(".safetensors", true)) } + ?.maxByOrNull { it.lastModified() } + + override fun status(): GeneratorStatus = when { + !libAvailable -> GeneratorStatus( + available = false, + hint = "Native diffusion engine isn't included in this build.", + ) + modelFile() == null -> GeneratorStatus( + available = false, + hint = "Download a Stable Diffusion model (.gguf/.safetensors) from the Models tab.", + ) + else -> GeneratorStatus(available = true, hint = "On-device diffusion · CPU") + } + + override suspend fun generate( + prompt: String, + seed: Long?, + steps: Int, + onProgress: suspend (Float) -> Unit, + ): Bitmap = withContext(Dispatchers.Default) { + check(libAvailable) { "Native diffusion library not available" } + val model = modelFile() ?: error("No Stable Diffusion model downloaded") + onProgress(0.02f) + val effectiveSeed = seed ?: prompt.lowercase().hashCode().toLong() + val pixels = nativeTxt2Img( + model.absolutePath, + prompt, + steps.coerceIn(1, 50), + effectiveSeed, + SIZE, + SIZE, + ) + onProgress(0.98f) + checkNotNull(pixels) { "Diffusion returned no image" } + val bitmap = Bitmap.createBitmap(SIZE, SIZE, Bitmap.Config.ARGB_8888) + bitmap.setPixels(pixels, 0, SIZE, 0, 0, SIZE, SIZE) + onProgress(1f) + bitmap + } + + /** Implemented in the native `sdjni` library. Returns SIZE*SIZE ARGB pixels, or null on failure. */ + private external fun nativeTxt2Img( + modelPath: String, + prompt: String, + steps: Int, + seed: Long, + width: Int, + height: Int, + ): IntArray? + + companion object { + private const val SIZE = 512 + + private val libAvailable: Boolean = runCatching { + System.loadLibrary("sdjni") + }.isSuccess + } +} diff --git a/app/src/main/java/com/androidcraft/studio/engine/image/OnnxDiffusionGenerator.kt b/app/src/main/java/com/androidcraft/studio/engine/image/OnnxDiffusionGenerator.kt new file mode 100644 index 0000000..e6c84ad --- /dev/null +++ b/app/src/main/java/com/androidcraft/studio/engine/image/OnnxDiffusionGenerator.kt @@ -0,0 +1,203 @@ +package com.androidcraft.studio.engine.image + +import android.graphics.Bitmap +import ai.onnxruntime.OnnxTensor +import ai.onnxruntime.OrtEnvironment +import ai.onnxruntime.OrtSession +import com.androidcraft.studio.engine.image.onnx.ClipTokenizer +import com.androidcraft.studio.engine.image.onnx.DdimScheduler +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import java.io.File +import java.nio.FloatBuffer +import java.nio.IntBuffer +import java.nio.LongBuffer +import java.util.Random + +/** + * Real on-device text-to-image via ONNX Runtime executing a Stable Diffusion v1.5 export + * (CLIP text encoder + UNet + VAE decoder), fully offline. Expects an Optimum-style ONNX bundle + * extracted into [bundleDir]: + * + * ``` + * sd15-onnx/ + * text_encoder/model.onnx + * unet/model.onnx + * vae_decoder/model.onnx + * tokenizer/vocab.json + * tokenizer/merges.txt + * ``` + * + * Reported unavailable until that bundle is present. Latent size is fixed to 512×512 (64×64×4). + */ +class OnnxDiffusionGenerator(modelsDir: File) : ImageGenerator { + + override val method = ImageMethod.ONNX + + private val bundleDir = File(modelsDir, "sd15-onnx") + private val textEncoderFile = File(bundleDir, "text_encoder/model.onnx") + private val unetFile = File(bundleDir, "unet/model.onnx") + private val vaeFile = File(bundleDir, "vae_decoder/model.onnx") + private val vocabFile = File(bundleDir, "tokenizer/vocab.json") + private val mergesFile = File(bundleDir, "tokenizer/merges.txt") + + @Volatile private var env: OrtEnvironment? = null + @Volatile private var textEncoder: OrtSession? = null + @Volatile private var unet: OrtSession? = null + @Volatile private var vae: OrtSession? = null + @Volatile private var tokenizer: ClipTokenizer? = null + + private fun bundleReady(): Boolean = + textEncoderFile.exists() && unetFile.exists() && vaeFile.exists() && + vocabFile.exists() && mergesFile.exists() + + override fun status(): GeneratorStatus = if (bundleReady()) { + GeneratorStatus(available = true, hint = "ONNX Runtime · on-device · CPU") + } else { + GeneratorStatus( + available = false, + hint = "Add an SD 1.5 ONNX bundle (sd15-onnx/) — download it in the Models tab.", + ) + } + + override suspend fun generate( + prompt: String, + seed: Long?, + steps: Int, + onProgress: suspend (Float) -> Unit, + ): Bitmap = withContext(Dispatchers.Default) { + check(bundleReady()) { "ONNX SD bundle not present" } + ensureLoaded() + val ortEnv = env!! + val tok = tokenizer!! + + val guidance = 7.5f + val height = 512 + val width = 512 + val latentH = height / 8 + val latentW = width / 8 + val latentSize = 4 * latentH * latentW + + // 1) Text embeddings for prompt (cond) and empty prompt (uncond). + val condEmb = encodeText(ortEnv, tok, prompt) + val uncondEmb = encodeText(ortEnv, tok, "") + onProgress(0.05f) + + // 2) Initial latents ~ N(0,1). + val rng = Random(seed ?: prompt.lowercase().hashCode().toLong()) + val scheduler = DdimScheduler() + var latents = FloatArray(latentSize) { (rng.nextGaussian() * scheduler.initNoiseSigma).toFloat() } + + // 3) Denoising loop. + val timesteps = scheduler.timesteps(steps.coerceIn(1, 50)) + val latentShape = longArrayOf(1, 4, latentH.toLong(), latentW.toLong()) + for ((index, t) in timesteps.withIndex()) { + val epsUncond = runUnet(ortEnv, latents, latentShape, t, uncondEmb) + val epsCond = runUnet(ortEnv, latents, latentShape, t, condEmb) + val eps = FloatArray(latentSize) { i -> + epsUncond[i] + guidance * (epsCond[i] - epsUncond[i]) + } + latents = scheduler.step(eps, t, latents) + onProgress(0.05f + 0.85f * (index + 1) / timesteps.size) + } + + // 4) VAE decode (latents are scaled by 1/0.18215 first). + val scaled = FloatArray(latentSize) { latents[it] / 0.18215f } + val decoded = runVae(ortEnv, scaled, latentShape) + onProgress(0.98f) + val bitmap = toBitmap(decoded, width, height) + onProgress(1f) + bitmap + } + + private fun encodeText(ortEnv: OrtEnvironment, tok: ClipTokenizer, text: String): FloatArray { + val ids = tok.encode(text) + val input = OnnxTensor.createTensor( + ortEnv, IntBuffer.wrap(ids), longArrayOf(1, ids.size.toLong()), + ) + input.use { + textEncoder!!.run(mapOf(textEncoder!!.inputNames.first() to it)).use { res -> + return readFloats(res.get(0) as OnnxTensor) + } + } + } + + private fun runUnet( + ortEnv: OrtEnvironment, + latents: FloatArray, + latentShape: LongArray, + t: Int, + emb: FloatArray, + ): FloatArray { + val sample = OnnxTensor.createTensor(ortEnv, FloatBuffer.wrap(latents), latentShape) + val timestep = OnnxTensor.createTensor(ortEnv, LongBuffer.wrap(longArrayOf(t.toLong())), longArrayOf(1)) + val hidden = OnnxTensor.createTensor(ortEnv, FloatBuffer.wrap(emb), longArrayOf(1, 77, 768)) + val inputs = HashMap() + val names = unet!!.inputNames.toList() + // Optimum SD UNet order: sample, timestep, encoder_hidden_states. + inputs[names.getOrElse(0) { "sample" }] = sample + inputs[names.getOrElse(1) { "timestep" }] = timestep + inputs[names.getOrElse(2) { "encoder_hidden_states" }] = hidden + try { + unet!!.run(inputs).use { res -> return readFloats(res.get(0) as OnnxTensor) } + } finally { + sample.close(); timestep.close(); hidden.close() + } + } + + private fun runVae(ortEnv: OrtEnvironment, latents: FloatArray, latentShape: LongArray): FloatArray { + val input = OnnxTensor.createTensor(ortEnv, FloatBuffer.wrap(latents), latentShape) + input.use { + vae!!.run(mapOf(vae!!.inputNames.first() to it)).use { res -> + return readFloats(res.get(0) as OnnxTensor) + } + } + } + + private fun readFloats(tensor: OnnxTensor): FloatArray { + val fb = tensor.floatBuffer + val arr = FloatArray(fb.remaining()) + fb.get(arr) + return arr + } + + /** Decoded VAE output is [1,3,H,W] in [-1,1], channel-major. */ + private fun toBitmap(decoded: FloatArray, width: Int, height: Int): Bitmap { + val pixels = IntArray(width * height) + val plane = width * height + for (y in 0 until height) { + for (x in 0 until width) { + val idx = y * width + x + val r = channelToByte(decoded[0 * plane + idx]) + val g = channelToByte(decoded[1 * plane + idx]) + val b = channelToByte(decoded[2 * plane + idx]) + pixels[idx] = (0xFF shl 24) or (r shl 16) or (g shl 8) or b + } + } + val bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888) + bitmap.setPixels(pixels, 0, width, 0, 0, width, height) + return bitmap + } + + private fun channelToByte(v: Float): Int = + (((v / 2f) + 0.5f) * 255f).toInt().coerceIn(0, 255) + + @Synchronized + private fun ensureLoaded() { + if (unet != null) return + val ortEnv = OrtEnvironment.getEnvironment() + val opts = OrtSession.SessionOptions() + env = ortEnv + textEncoder = ortEnv.createSession(textEncoderFile.absolutePath, opts) + unet = ortEnv.createSession(unetFile.absolutePath, opts) + vae = ortEnv.createSession(vaeFile.absolutePath, opts) + tokenizer = ClipTokenizer(vocabFile, mergesFile) + } + + fun close() { + runCatching { textEncoder?.close() } + runCatching { unet?.close() } + runCatching { vae?.close() } + textEncoder = null; unet = null; vae = null; tokenizer = null + } +} diff --git a/app/src/main/java/com/androidcraft/studio/engine/image/ProceduralImageGenerator.kt b/app/src/main/java/com/androidcraft/studio/engine/image/ProceduralImageGenerator.kt new file mode 100644 index 0000000..b7cbec6 --- /dev/null +++ b/app/src/main/java/com/androidcraft/studio/engine/image/ProceduralImageGenerator.kt @@ -0,0 +1,134 @@ +package com.androidcraft.studio.engine.image + +import android.graphics.Bitmap +import android.graphics.Canvas +import android.graphics.Color +import android.graphics.LinearGradient +import android.graphics.Paint +import android.graphics.RadialGradient +import android.graphics.Shader +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import kotlin.math.abs +import kotlin.math.sin +import kotlin.random.Random + +/** + * Deterministic procedural renderer: turns a prompt into abstract art entirely on-device with no + * model download. Always available; a good instant preview and a zero-dependency fallback. + */ +class ProceduralImageGenerator : ImageGenerator { + + override val method = ImageMethod.PROCEDURAL + + override fun status() = GeneratorStatus(available = true, hint = "Instant · no download") + + override suspend fun generate( + prompt: String, + seed: Long?, + steps: Int, + onProgress: suspend (Float) -> Unit, + ): Bitmap = withContext(Dispatchers.Default) { + val size = 768 + val effectiveSeed = seed ?: prompt.lowercase().hashCode().toLong() + val rng = Random(effectiveSeed) + val bitmap = Bitmap.createBitmap(size, size, Bitmap.Config.ARGB_8888) + val canvas = Canvas(bitmap) + val palette = paletteFor(prompt, rng) + + val basePaint = Paint(Paint.ANTI_ALIAS_FLAG).apply { + shader = LinearGradient( + 0f, 0f, size.toFloat(), size.toFloat(), + intArrayOf(palette[0], palette[1], palette[2]), + floatArrayOf(0f, 0.55f, 1f), + Shader.TileMode.CLAMP, + ) + } + canvas.drawRect(0f, 0f, size.toFloat(), size.toFloat(), basePaint) + + val orbCount = 5 + rng.nextInt(6) + repeat(orbCount) { i -> + val cx = rng.nextFloat() * size + val cy = rng.nextFloat() * size + val radius = size * (0.12f + rng.nextFloat() * 0.28f) + val color = palette[rng.nextInt(palette.size)] + val glow = Paint(Paint.ANTI_ALIAS_FLAG).apply { + shader = RadialGradient( + cx, cy, radius, + intArrayOf(withAlpha(color, 200), withAlpha(color, 0)), + floatArrayOf(0f, 1f), + Shader.TileMode.CLAMP, + ) + } + canvas.drawCircle(cx, cy, radius, glow) + if (i % 2 == 0) { + val stroke = Paint(Paint.ANTI_ALIAS_FLAG).apply { + style = Paint.Style.STROKE + strokeWidth = size * 0.004f + this.color = withAlpha(palette[rng.nextInt(palette.size)], 90) + } + val steps2 = 60 + var prevX = 0f + var prevY = size * rng.nextFloat() + val amp = size * (0.05f + rng.nextFloat() * 0.15f) + val freq = 2 + rng.nextInt(4) + for (s in 1..steps2) { + val x = size * (s / steps2.toFloat()) + val y = prevY + amp * sin(freq * (s / steps2.toFloat()) * 6.283f + i) + canvas.drawLine(prevX, prevY, x, y, stroke) + prevX = x + prevY = y + } + } + } + + applyGrain(bitmap, rng, strength = 14) + onProgress(1f) + bitmap + } + + private fun paletteFor(prompt: String, rng: Random): IntArray { + val p = prompt.lowercase() + val base = when { + listOf("night", "dark", "space", "galaxy", "noir").any { p.contains(it) } -> + intArrayOf(0xFF0B1026.toInt(), 0xFF3A1C71.toInt(), 0xFF00C2FF.toInt()) + listOf("sunset", "warm", "fire", "gold", "desert").any { p.contains(it) } -> + intArrayOf(0xFF7A1F3D.toInt(), 0xFFE8552A.toInt(), 0xFFF7C948.toInt()) + listOf("nature", "forest", "green", "jungle", "leaf").any { p.contains(it) } -> + intArrayOf(0xFF08301E.toInt(), 0xFF1F7A4D.toInt(), 0xFFB4E197.toInt()) + listOf("ocean", "sea", "water", "ice", "blue").any { p.contains(it) } -> + intArrayOf(0xFF03203A.toInt(), 0xFF1B6CA8.toInt(), 0xFF7DE2D1.toInt()) + else -> { + val hue = (abs(p.hashCode()) % 360).toFloat() + intArrayOf( + hsv(hue, 0.7f, 0.25f), + hsv((hue + 40) % 360, 0.8f, 0.55f), + hsv((hue + 90) % 360, 0.6f, 0.9f), + ) + } + } + return if (rng.nextBoolean()) base else intArrayOf(base[2], base[0], base[1]) + } + + private fun hsv(h: Float, s: Float, v: Float): Int = Color.HSVToColor(floatArrayOf(h, s, v)) + + private fun withAlpha(color: Int, alpha: Int): Int = + Color.argb(alpha, Color.red(color), Color.green(color), Color.blue(color)) + + private fun applyGrain(bitmap: Bitmap, rng: Random, strength: Int) { + val w = bitmap.width + val h = bitmap.height + val pixels = IntArray(w * h) + bitmap.getPixels(pixels, 0, w, 0, 0, w, h) + for (i in pixels.indices) { + if (rng.nextInt(3) != 0) continue + val n = rng.nextInt(strength * 2) - strength + val c = pixels[i] + val r = (Color.red(c) + n).coerceIn(0, 255) + val g = (Color.green(c) + n).coerceIn(0, 255) + val b = (Color.blue(c) + n).coerceIn(0, 255) + pixels[i] = Color.argb(255, r, g, b) + } + bitmap.setPixels(pixels, 0, w, 0, 0, w, h) + } +} diff --git a/app/src/main/java/com/androidcraft/studio/engine/image/onnx/ClipTokenizer.kt b/app/src/main/java/com/androidcraft/studio/engine/image/onnx/ClipTokenizer.kt new file mode 100644 index 0000000..6e88f1b --- /dev/null +++ b/app/src/main/java/com/androidcraft/studio/engine/image/onnx/ClipTokenizer.kt @@ -0,0 +1,112 @@ +package com.androidcraft.studio.engine.image.onnx + +import org.json.JSONObject +import java.io.File + +/** + * CLIP byte-level BPE tokenizer (the one Stable Diffusion 1.x uses), loaded from a standard + * `vocab.json` + `merges.txt` pair. Produces the 77-length `input_ids` the CLIP text encoder wants. + */ +class ClipTokenizer(vocabJson: File, mergesTxt: File) { + + private val encoder: Map + private val bpeRanks: Map, Int> + private val byteEncoder: Map = bytesToUnicode() + + private val bos: Int + private val eos: Int + private val maxLen = 77 + + // CLIP's tokenization regex. + private val pattern = Regex( + "<\\|startoftext\\|>|<\\|endoftext\\|>|'s|'t|'re|'ve|'m|'ll|'d|" + + "\\p{L}+|\\p{N}|[^\\s\\p{L}\\p{N}]+", + RegexOption.IGNORE_CASE, + ) + + init { + val obj = JSONObject(vocabJson.readText()) + val map = HashMap(obj.length()) + val keys = obj.keys() + while (keys.hasNext()) { + val k = keys.next() + map[k] = obj.getInt(k) + } + encoder = map + bos = encoder["<|startoftext|>"] ?: 49406 + eos = encoder["<|endoftext|>"] ?: 49407 + + val ranks = HashMap, Int>() + mergesTxt.useLines { lines -> + lines.drop(1).forEachIndexed { i, line -> + val parts = line.trim().split(" ") + if (parts.size == 2) ranks[parts[0] to parts[1]] = i + } + } + bpeRanks = ranks + } + + /** Returns exactly [maxLen] token ids (bos … eos, eos-padded). */ + fun encode(text: String): IntArray { + val tokens = ArrayList() + tokens.add(bos) + for (match in pattern.findAll(text.lowercase().trim())) { + val token = match.value + val mapped = token.toByteArray(Charsets.UTF_8) + .joinToString("") { byteEncoder[it.toInt() and 0xFF].toString() } + for (piece in bpe(mapped).split(" ")) { + encoder[piece]?.let { tokens.add(it) } + } + } + val ids = IntArray(maxLen) { eos } + val limit = minOf(tokens.size, maxLen - 1) + for (i in 0 until limit) ids[i] = tokens[i] + ids[minOf(tokens.size, maxLen - 1)] = eos + return ids + } + + private fun bpe(token: String): String { + if (token.isEmpty()) return token + var word = token.map { it.toString() }.toMutableList() + word[word.lastIndex] = word.last() + "" + + while (word.size > 1) { + var minRank = Int.MAX_VALUE + var minPair = -1 + for (i in 0 until word.size - 1) { + val rank = bpeRanks[word[i] to word[i + 1]] ?: continue + if (rank < minRank) { + minRank = rank + minPair = i + } + } + if (minPair == -1) break + val merged = word[minPair] + word[minPair + 1] + val next = ArrayList(word.size - 1) + next.addAll(word.subList(0, minPair)) + next.add(merged) + if (minPair + 2 <= word.lastIndex) next.addAll(word.subList(minPair + 2, word.size)) + word = next + } + return word.joinToString(" ") + } + + private fun bytesToUnicode(): Map { + val bs = ArrayList() + (('!'.code)..('~'.code)).forEach { bs.add(it) } + (('¡'.code)..('¬'.code)).forEach { bs.add(it) } + (('®'.code)..('ÿ'.code)).forEach { bs.add(it) } + val cs = ArrayList(bs) + var n = 0 + for (b in 0 until 256) { + if (b !in bs) { + bs.add(b) + cs.add(256 + n) + n++ + } + } + val map = HashMap() + for (i in bs.indices) map[bs[i]] = cs[i].toChar() + return map + } +} diff --git a/app/src/main/java/com/androidcraft/studio/engine/image/onnx/DdimScheduler.kt b/app/src/main/java/com/androidcraft/studio/engine/image/onnx/DdimScheduler.kt new file mode 100644 index 0000000..1a14864 --- /dev/null +++ b/app/src/main/java/com/androidcraft/studio/engine/image/onnx/DdimScheduler.kt @@ -0,0 +1,60 @@ +package com.androidcraft.studio.engine.image.onnx + +import kotlin.math.sqrt + +/** + * Minimal deterministic DDIM scheduler (eta = 0) with the standard Stable Diffusion 1.x config + * (scaled-linear betas, 1000 train steps). Enough to drive the ONNX UNet denoising loop. + */ +class DdimScheduler( + private val numTrain: Int = 1000, + betaStart: Double = 0.00085, + betaEnd: Double = 0.012, +) { + private val alphasCumprod: DoubleArray + private var stepRatio: Int = 1 + + /** DDIM starts from unit-variance noise. */ + val initNoiseSigma: Double = 1.0 + + init { + val betas = DoubleArray(numTrain) + val start = sqrt(betaStart) + val end = sqrt(betaEnd) + for (i in 0 until numTrain) { + val b = start + (end - start) * i / (numTrain - 1) + betas[i] = b * b // scaled_linear + } + alphasCumprod = DoubleArray(numTrain) + var cum = 1.0 + for (i in 0 until numTrain) { + cum *= (1.0 - betas[i]) + alphasCumprod[i] = cum + } + } + + /** Returns the descending list of train timesteps to visit. */ + fun timesteps(steps: Int): IntArray { + stepRatio = numTrain / steps + return IntArray(steps) { i -> (steps - 1 - i) * stepRatio } + } + + /** DDIM update: given the predicted noise at [t], return the previous latent. */ + fun step(modelOutput: FloatArray, t: Int, sample: FloatArray): FloatArray { + val prevT = t - stepRatio + val aT = alphasCumprod[t] + val aPrev = if (prevT >= 0) alphasCumprod[prevT] else 1.0 + val sqrtAT = sqrt(aT) + val sqrtOneMinusAT = sqrt(1.0 - aT) + val sqrtAPrev = sqrt(aPrev) + val sqrtOneMinusAPrev = sqrt(1.0 - aPrev) + + val out = FloatArray(sample.size) + for (i in sample.indices) { + val eps = modelOutput[i].toDouble() + val predX0 = (sample[i] - sqrtOneMinusAT * eps) / sqrtAT + out[i] = (sqrtAPrev * predX0 + sqrtOneMinusAPrev * eps).toFloat() + } + return out + } +} diff --git a/app/src/main/java/com/androidcraft/studio/ui/audio/AudioScreen.kt b/app/src/main/java/com/androidcraft/studio/ui/audio/AudioScreen.kt index 12f11e1..81a384c 100644 --- a/app/src/main/java/com/androidcraft/studio/ui/audio/AudioScreen.kt +++ b/app/src/main/java/com/androidcraft/studio/ui/audio/AudioScreen.kt @@ -10,9 +10,11 @@ import androidx.compose.foundation.rememberScrollState import androidx.compose.foundation.shape.RoundedCornerShape import androidx.compose.foundation.verticalScroll import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.Download import androidx.compose.material.icons.filled.GraphicEq import androidx.compose.material.icons.filled.Pause import androidx.compose.material.icons.filled.PlayArrow +import androidx.compose.material.icons.filled.Share import androidx.compose.material3.Button import androidx.compose.material3.CircularProgressIndicator import androidx.compose.material3.ExperimentalMaterial3Api @@ -27,19 +29,31 @@ import androidx.compose.runtime.Composable import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.getValue import androidx.compose.ui.Modifier +import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.text.font.FontWeight import androidx.compose.ui.unit.dp import androidx.lifecycle.compose.collectAsStateWithLifecycle import androidx.lifecycle.viewmodel.compose.viewModel +import com.androidcraft.studio.core.MediaSaver import com.androidcraft.studio.ui.components.SectionCard @OptIn(ExperimentalMaterial3Api::class) @Composable fun AudioScreen(viewModel: AudioViewModel = viewModel()) { val state by viewModel.state.collectAsStateWithLifecycle() + val context = LocalContext.current LaunchedEffect(Unit) { viewModel.loadVoices() } + LaunchedEffect(state.shareUri) { + state.shareUri?.let { uri -> + context.startActivity( + android.content.Intent.createChooser(MediaSaver.shareIntent(uri, "audio/x-wav"), "Share audio"), + ) + viewModel.onShareConsumed() + } + } + Column( modifier = Modifier .fillMaxSize() @@ -118,8 +132,17 @@ fun AudioScreen(viewModel: AudioViewModel = viewModel()) { contentDescription = "Play", ) } + OutlinedButton(onClick = viewModel::save, enabled = state.hasAudio) { + Icon(Icons.Filled.Download, contentDescription = "Save") + } + OutlinedButton(onClick = viewModel::share, enabled = state.hasAudio) { + Icon(Icons.Filled.Share, contentDescription = "Share") + } } + state.statusMessage?.let { + Text(it, style = MaterialTheme.typography.bodySmall, color = MaterialTheme.colorScheme.secondary) + } state.error?.let { Text(it, style = MaterialTheme.typography.bodySmall, color = MaterialTheme.colorScheme.error) } diff --git a/app/src/main/java/com/androidcraft/studio/ui/audio/AudioViewModel.kt b/app/src/main/java/com/androidcraft/studio/ui/audio/AudioViewModel.kt index 6bd324e..744c665 100644 --- a/app/src/main/java/com/androidcraft/studio/ui/audio/AudioViewModel.kt +++ b/app/src/main/java/com/androidcraft/studio/ui/audio/AudioViewModel.kt @@ -2,9 +2,11 @@ package com.androidcraft.studio.ui.audio import android.app.Application import android.media.MediaPlayer +import android.net.Uri import androidx.lifecycle.AndroidViewModel import androidx.lifecycle.viewModelScope import com.androidcraft.studio.AndroidCraftApp +import com.androidcraft.studio.core.MediaSaver import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow @@ -22,6 +24,8 @@ data class AudioUiState( val speed: Float = 1.0f, val pitch: Float = 1.0f, val error: String? = null, + val statusMessage: String? = null, + val shareUri: Uri? = null, ) class AudioViewModel(app: Application) : AndroidViewModel(app) { @@ -104,6 +108,36 @@ class AudioViewModel(app: Application) : AndroidViewModel(app) { } } + private suspend fun export(): Uri? { + val file = lastFile ?: return null + return MediaSaver.saveAudio( + getApplication(), + file, + "androidcraft_${System.currentTimeMillis()}.wav", + ) + } + + fun save() { + if (lastFile == null) return + viewModelScope.launch { + val uri = export() + _state.update { + it.copy(statusMessage = if (uri != null) "Saved to Music" else "Save failed") + } + } + } + + fun share() { + if (lastFile == null) return + viewModelScope.launch { + val uri = export() + if (uri != null) _state.update { it.copy(shareUri = uri) } + else _state.update { it.copy(statusMessage = "Save failed") } + } + } + + fun onShareConsumed() = _state.update { it.copy(shareUri = null) } + override fun onCleared() { player?.release() player = null diff --git a/app/src/main/java/com/androidcraft/studio/ui/chat/ChatViewModel.kt b/app/src/main/java/com/androidcraft/studio/ui/chat/ChatViewModel.kt index eda83c7..436d2f7 100644 --- a/app/src/main/java/com/androidcraft/studio/ui/chat/ChatViewModel.kt +++ b/app/src/main/java/com/androidcraft/studio/ui/chat/ChatViewModel.kt @@ -39,7 +39,7 @@ class ChatViewModel(app: Application) : AndroidViewModel(app) { } fun refreshModels() { - val ready = ModelCatalog.forModality(Modality.TEXT) + val ready = repo.modelsFor(Modality.TEXT) .filter { repo.states.value[it.id] is DownloadState.Ready || repo.isReady(it) } _state.update { current -> current.copy( diff --git a/app/src/main/java/com/androidcraft/studio/ui/image/ImageScreen.kt b/app/src/main/java/com/androidcraft/studio/ui/image/ImageScreen.kt index f136c92..b667ea8 100644 --- a/app/src/main/java/com/androidcraft/studio/ui/image/ImageScreen.kt +++ b/app/src/main/java/com/androidcraft/studio/ui/image/ImageScreen.kt @@ -15,30 +15,54 @@ import androidx.compose.foundation.verticalScroll import androidx.compose.material.icons.Icons import androidx.compose.material.icons.filled.Casino import androidx.compose.material.icons.filled.Download +import androidx.compose.material.icons.filled.Share import androidx.compose.material3.Button import androidx.compose.material3.CircularProgressIndicator +import androidx.compose.material3.ExperimentalMaterial3Api +import androidx.compose.material3.FilterChip import androidx.compose.material3.Icon +import androidx.compose.material3.LinearProgressIndicator import androidx.compose.material3.MaterialTheme import androidx.compose.material3.OutlinedButton import androidx.compose.material3.OutlinedTextField +import androidx.compose.material3.Slider import androidx.compose.material3.Surface import androidx.compose.material3.Text import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.getValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.draw.clip import androidx.compose.ui.graphics.asImageBitmap import androidx.compose.ui.layout.ContentScale +import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.text.font.FontWeight import androidx.compose.ui.unit.dp import androidx.lifecycle.compose.collectAsStateWithLifecycle import androidx.lifecycle.viewmodel.compose.viewModel +import com.androidcraft.studio.core.MediaSaver import com.androidcraft.studio.ui.components.SectionCard +@OptIn(ExperimentalMaterial3Api::class) @Composable fun ImageScreen(viewModel: ImageViewModel = viewModel()) { val state by viewModel.state.collectAsStateWithLifecycle() + val context = LocalContext.current + + LaunchedEffect(Unit) { viewModel.refreshMethods() } + + LaunchedEffect(state.shareUri) { + state.shareUri?.let { uri -> + context.startActivity( + android.content.Intent.createChooser(MediaSaver.shareIntent(uri, "image/png"), "Share image"), + ) + viewModel.onShareConsumed() + } + } + + val selectedInfo = state.methods.firstOrNull { it.method == state.selected } + val diffusion = state.selected.diffusion Column( modifier = Modifier @@ -53,11 +77,29 @@ fun ImageScreen(viewModel: ImageViewModel = viewModel()) { fontWeight = FontWeight.Bold, ) Text( - "Runs on-device with the bundled renderer — no download required.", + "Choose a generation method — all run fully on-device.", style = MaterialTheme.typography.bodyMedium, color = MaterialTheme.colorScheme.onSurfaceVariant, ) + // ---- Method selector ---- + Row(horizontalArrangement = Arrangement.spacedBy(8.dp)) { + state.methods.forEach { info -> + FilterChip( + selected = state.selected == info.method, + onClick = { viewModel.selectMethod(info.method) }, + label = { Text(info.method.displayName) }, + ) + } + } + selectedInfo?.let { + Text( + it.hint, + style = MaterialTheme.typography.labelSmall, + color = if (it.available) MaterialTheme.colorScheme.secondary else MaterialTheme.colorScheme.error, + ) + } + Surface( shape = RoundedCornerShape(24.dp), color = MaterialTheme.colorScheme.surface, @@ -67,13 +109,22 @@ fun ImageScreen(viewModel: ImageViewModel = viewModel()) { val bmp = state.bitmap when { state.isGenerating -> { - Column(horizontalAlignment = Alignment.CenterHorizontally) { + Column( + horizontalAlignment = Alignment.CenterHorizontally, + modifier = Modifier.padding(24.dp), + ) { CircularProgressIndicator() Text( - "Rendering…", + if (diffusion) "Diffusing… ${(state.progress * 100).toInt()}%" else "Rendering…", modifier = Modifier.padding(top = 12.dp), style = MaterialTheme.typography.bodySmall, ) + if (diffusion) { + LinearProgressIndicator( + progress = { state.progress }, + modifier = Modifier.fillMaxWidth().padding(top = 10.dp), + ) + } } } @@ -107,16 +158,32 @@ fun ImageScreen(viewModel: ImageViewModel = viewModel()) { minLines = 2, ) + if (diffusion) { + SectionCard { + Text("Sampling steps ${state.steps}", style = MaterialTheme.typography.labelLarge) + Slider( + value = state.steps.toFloat(), + onValueChange = { viewModel.onStepsChange(it.toInt()) }, + valueRange = 4f..40f, + ) + Text( + "More steps = higher quality, slower. On-device diffusion is CPU-bound.", + style = MaterialTheme.typography.labelSmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } + } + Row(horizontalArrangement = Arrangement.spacedBy(10.dp)) { Button( - onClick = viewModel::generate, + onClick = { viewModel.generate() }, enabled = state.prompt.isNotBlank() && !state.isGenerating, modifier = Modifier.weight(1f), ) { Text("Generate") } OutlinedButton( - onClick = viewModel::regenerateVariation, + onClick = { viewModel.generate(newSeed = true) }, enabled = state.prompt.isNotBlank() && !state.isGenerating, ) { Icon(Icons.Filled.Casino, contentDescription = "Variation") @@ -127,6 +194,12 @@ fun ImageScreen(viewModel: ImageViewModel = viewModel()) { ) { Icon(Icons.Filled.Download, contentDescription = "Save") } + OutlinedButton( + onClick = viewModel::share, + enabled = state.bitmap != null && !state.isGenerating, + ) { + Icon(Icons.Filled.Share, contentDescription = "Share") + } } state.statusMessage?.let { @@ -137,10 +210,11 @@ fun ImageScreen(viewModel: ImageViewModel = viewModel()) { } SectionCard { - Text("Upgrade path", style = MaterialTheme.typography.titleMedium, fontWeight = FontWeight.SemiBold) + Text("On-device engines", style = MaterialTheme.typography.titleMedium, fontWeight = FontWeight.SemiBold) Text( - "The renderer is a drop-in stand-in. Add Stable Diffusion LiteRT weights and point " + - "the image engine at them to produce photoreal results from the same prompt box.", + "Procedural is instant and always available. The diffusion engines (stable-diffusion.cpp " + + "and ONNX Runtime) produce real text-to-image results from a model you download in the " + + "Models tab, and run fully offline on the device.", style = MaterialTheme.typography.bodySmall, color = MaterialTheme.colorScheme.onSurfaceVariant, modifier = Modifier.padding(top = 4.dp), diff --git a/app/src/main/java/com/androidcraft/studio/ui/image/ImageViewModel.kt b/app/src/main/java/com/androidcraft/studio/ui/image/ImageViewModel.kt index e5f27ff..7dcd60a 100644 --- a/app/src/main/java/com/androidcraft/studio/ui/image/ImageViewModel.kt +++ b/app/src/main/java/com/androidcraft/studio/ui/image/ImageViewModel.kt @@ -2,73 +2,112 @@ package com.androidcraft.studio.ui.image import android.app.Application import android.graphics.Bitmap +import android.net.Uri import androidx.lifecycle.AndroidViewModel import androidx.lifecycle.viewModelScope import com.androidcraft.studio.AndroidCraftApp import com.androidcraft.studio.core.MediaSaver +import com.androidcraft.studio.engine.image.ImageMethod import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.flow.update import kotlinx.coroutines.launch +data class ImageMethodInfo(val method: ImageMethod, val available: Boolean, val hint: String) + data class ImageUiState( val prompt: String = "", val isGenerating: Boolean = false, val bitmap: Bitmap? = null, val statusMessage: String? = null, val error: String? = null, + val shareUri: Uri? = null, + val methods: List = emptyList(), + val selected: ImageMethod = ImageMethod.PROCEDURAL, + val steps: Int = 20, + val progress: Float = 0f, ) class ImageViewModel(app: Application) : AndroidViewModel(app) { - private val container = (app as AndroidCraftApp).container - private val engine = container.imageEngine + private val engine = (app as AndroidCraftApp).container.imageEngine private val _state = MutableStateFlow(ImageUiState()) val state: StateFlow = _state.asStateFlow() - fun onPromptChange(value: String) = _state.update { it.copy(prompt = value) } + init { + refreshMethods() + } - fun generate() { - val prompt = _state.value.prompt.trim() - if (prompt.isEmpty() || _state.value.isGenerating) return - _state.update { it.copy(isGenerating = true, error = null, statusMessage = null) } - viewModelScope.launch { - try { - val bmp = engine.render(prompt) - _state.update { it.copy(bitmap = bmp, isGenerating = false) } - } catch (t: Throwable) { - _state.update { it.copy(isGenerating = false, error = t.message) } + fun refreshMethods() { + val methods = engine.generators() + .map { gen -> + val status = gen.status() + ImageMethodInfo(gen.method, status.available, status.hint) } + // Only surface the ONNX backend once a bundle is actually present, to keep the + // picker to engines the user can use right now. + .filter { it.method != ImageMethod.ONNX || it.available } + _state.update { current -> + val stillValid = methods.any { it.method == current.selected } + current.copy( + methods = methods, + selected = if (stillValid) current.selected else ImageMethod.PROCEDURAL, + ) } } - fun regenerateVariation() { - val prompt = _state.value.prompt.trim() - if (prompt.isEmpty() || _state.value.isGenerating) return - _state.update { it.copy(isGenerating = true, error = null, statusMessage = null) } + fun onPromptChange(value: String) = _state.update { it.copy(prompt = value) } + fun onStepsChange(value: Int) = _state.update { it.copy(steps = value) } + fun selectMethod(method: ImageMethod) = _state.update { it.copy(selected = method, error = null) } + + fun generate(newSeed: Boolean = false) { + val current = _state.value + val prompt = current.prompt.trim() + if (prompt.isEmpty() || current.isGenerating) return + + val info = current.methods.firstOrNull { it.method == current.selected } + if (info != null && !info.available) { + _state.update { it.copy(error = info.hint) } + return + } + + _state.update { it.copy(isGenerating = true, error = null, statusMessage = null, progress = 0f) } viewModelScope.launch { try { - val bmp = engine.render(prompt, seed = System.nanoTime()) - _state.update { it.copy(bitmap = bmp, isGenerating = false) } + val seed = if (newSeed) System.nanoTime() else null + val bmp = engine.generate(current.selected, prompt, seed, current.steps) { p -> + _state.update { it.copy(progress = p) } + } + _state.update { it.copy(bitmap = bmp, isGenerating = false, progress = 1f) } } catch (t: Throwable) { - _state.update { it.copy(isGenerating = false, error = t.message) } + _state.update { it.copy(isGenerating = false, error = t.message ?: "Generation failed") } } } } + private suspend fun export(): Uri? { + val bmp = _state.value.bitmap ?: return null + return MediaSaver.saveImage(getApplication(), bmp, "androidcraft_${System.currentTimeMillis()}.png") + } + fun save() { - val bmp = _state.value.bitmap ?: return + if (_state.value.bitmap == null) return viewModelScope.launch { - val uri = MediaSaver.saveImage( - getApplication(), - bmp, - "androidcraft_${System.currentTimeMillis()}.png", - ) - _state.update { - it.copy(statusMessage = if (uri != null) "Saved to gallery" else "Save failed") - } + val uri = export() + _state.update { it.copy(statusMessage = if (uri != null) "Saved to gallery" else "Save failed") } } } + + fun share() { + if (_state.value.bitmap == null) return + viewModelScope.launch { + val uri = export() + if (uri != null) _state.update { it.copy(shareUri = uri) } + else _state.update { it.copy(statusMessage = "Save failed") } + } + } + + fun onShareConsumed() = _state.update { it.copy(shareUri = null) } } diff --git a/app/src/main/java/com/androidcraft/studio/ui/models/ModelsScreen.kt b/app/src/main/java/com/androidcraft/studio/ui/models/ModelsScreen.kt index 0b2b15d..b83d38b 100644 --- a/app/src/main/java/com/androidcraft/studio/ui/models/ModelsScreen.kt +++ b/app/src/main/java/com/androidcraft/studio/ui/models/ModelsScreen.kt @@ -6,21 +6,16 @@ import androidx.compose.foundation.layout.Row import androidx.compose.foundation.layout.fillMaxWidth import androidx.compose.foundation.layout.padding import androidx.compose.foundation.lazy.LazyColumn -import androidx.compose.foundation.lazy.items import androidx.compose.foundation.shape.RoundedCornerShape import androidx.compose.material.icons.Icons import androidx.compose.material.icons.filled.CheckCircle import androidx.compose.material.icons.filled.Close import androidx.compose.material.icons.filled.Delete import androidx.compose.material.icons.filled.Download -import androidx.compose.material3.CircularProgressIndicator -import androidx.compose.material3.ExperimentalMaterial3Api import androidx.compose.material3.Icon import androidx.compose.material3.IconButton import androidx.compose.material3.LinearProgressIndicator import androidx.compose.material3.MaterialTheme -import androidx.compose.material3.OutlinedButton -import androidx.compose.material3.OutlinedTextField import androidx.compose.material3.Surface import androidx.compose.material3.Text import androidx.compose.runtime.Composable @@ -35,17 +30,16 @@ import androidx.lifecycle.viewmodel.compose.viewModel import com.androidcraft.studio.core.CapabilityVerdict import com.androidcraft.studio.core.DeviceCapabilities import com.androidcraft.studio.data.DownloadState -import com.androidcraft.studio.data.Modality import com.androidcraft.studio.data.ModelSpec import com.androidcraft.studio.ui.components.SectionCard import com.androidcraft.studio.ui.theme.CraftError import com.androidcraft.studio.ui.theme.CraftSuccess -@OptIn(ExperimentalMaterial3Api::class) @Composable fun ModelsScreen(viewModel: ModelsViewModel = viewModel()) { val states by viewModel.downloadStates.collectAsStateWithLifecycle() - val token by viewModel.hfToken.collectAsStateWithLifecycle() + val grouped by viewModel.grouped.collectAsStateWithLifecycle() + val caps = viewModel.deviceCapabilities LazyColumn( modifier = Modifier.fillMaxWidth().padding(horizontal = 16.dp), @@ -53,36 +47,32 @@ fun ModelsScreen(viewModel: ModelsViewModel = viewModel()) { contentPadding = androidx.compose.foundation.layout.PaddingValues(vertical = 16.dp), ) { item { + Text("Models", style = MaterialTheme.typography.headlineMedium, fontWeight = FontWeight.Bold) Text( - "Models", - style = MaterialTheme.typography.headlineMedium, - fontWeight = FontWeight.Bold, + "Pick a model and tap download — everything runs on your device. No account, no " + + "links to find. Models are matched to your phone's memory.", + style = MaterialTheme.typography.bodyMedium, + color = MaterialTheme.colorScheme.onSurfaceVariant, + modifier = Modifier.padding(top = 4.dp), ) } - item { DeviceCard(viewModel.deviceCapabilities) } + item { DeviceCard(caps) } - item { - SectionCard { - Text("Hugging Face token", style = MaterialTheme.typography.titleMedium, fontWeight = FontWeight.SemiBold) - Text( - "Some models are gated. Paste a read token to enable their download. Stored only on this device.", - style = MaterialTheme.typography.bodySmall, - color = MaterialTheme.colorScheme.onSurfaceVariant, - modifier = Modifier.padding(vertical = 6.dp), - ) - OutlinedTextField( - value = token, - onValueChange = viewModel::onTokenChange, - modifier = Modifier.fillMaxWidth(), - label = { Text("hf_… (optional)") }, - singleLine = true, - shape = RoundedCornerShape(14.dp), - ) - } - } + grouped.forEach { (modality, specs) -> + // Order by device fit (best first), largest-quality first; built-ins last. + val ordered = specs.sortedWith( + compareBy( + { it.isBundled }, + { caps.canRun(it).ordinal }, + { -it.approxBytes }, + ), + ) + // Recommend the highest-quality downloadable model that still runs well. + val recommendedId = ordered.firstOrNull { + !it.isBundled && caps.canRun(it) == CapabilityVerdict.GOOD + }?.id - viewModel.grouped.forEach { (modality, specs) -> item { Text( modality.label, @@ -91,12 +81,13 @@ fun ModelsScreen(viewModel: ModelsViewModel = viewModel()) { modifier = Modifier.padding(top = 4.dp), ) } - items(specs.size) { index -> - val spec = specs[index] + items(ordered.size) { index -> + val spec = ordered[index] ModelCard( spec = spec, state = states[spec.id] ?: DownloadState.NotDownloaded, - verdict = viewModel.deviceCapabilities.canRun(spec), + verdict = caps.canRun(spec), + recommended = spec.id == recommendedId, onDownload = { viewModel.download(spec) }, onCancel = { viewModel.cancel(spec) }, onDelete = { viewModel.delete(spec) }, @@ -126,6 +117,7 @@ private fun ModelCard( spec: ModelSpec, state: DownloadState, verdict: CapabilityVerdict, + recommended: Boolean, onDownload: () -> Unit, onCancel: () -> Unit, onDelete: () -> Unit, @@ -137,7 +129,10 @@ private fun ModelCard( verticalAlignment = Alignment.Top, ) { Column(modifier = Modifier.weight(1f)) { - Text(spec.displayName, style = MaterialTheme.typography.titleMedium, fontWeight = FontWeight.SemiBold) + Row(verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(8.dp)) { + Text(spec.displayName, style = MaterialTheme.typography.titleMedium, fontWeight = FontWeight.SemiBold) + if (recommended) RecommendedBadge() + } Text( spec.publisher, style = MaterialTheme.typography.labelSmall, @@ -154,12 +149,9 @@ private fun ModelCard( modifier = Modifier.padding(top = 8.dp), ) - Row( - modifier = Modifier.padding(top = 10.dp), - horizontalArrangement = Arrangement.spacedBy(8.dp), - ) { + Row(modifier = Modifier.padding(top = 10.dp), horizontalArrangement = Arrangement.spacedBy(8.dp)) { if (!spec.isBundled) { - MetaChip("${spec.approxSizeLabel()}") + MetaChip(spec.approxSizeLabel()) if (spec.quantization.isNotEmpty()) MetaChip(spec.quantization) MetaChip(spec.runtime.label) if (spec.minRamBytes > 0) VerdictChip(verdict) @@ -171,10 +163,7 @@ private fun ModelCard( if (state is DownloadState.Downloading) { Column(modifier = Modifier.padding(top = 12.dp)) { - LinearProgressIndicator( - progress = { state.fraction }, - modifier = Modifier.fillMaxWidth(), - ) + LinearProgressIndicator(progress = { state.fraction }, modifier = Modifier.fillMaxWidth()) Text( "${(state.fraction * 100).toInt()}%", style = MaterialTheme.typography.labelSmall, @@ -221,12 +210,22 @@ private fun ActionSlot( } } +@Composable +private fun RecommendedBadge() { + Surface(color = CraftSuccess.copy(alpha = 0.18f), shape = RoundedCornerShape(50)) { + Text( + "Best for your device", + style = MaterialTheme.typography.labelSmall, + color = CraftSuccess, + fontWeight = FontWeight.SemiBold, + modifier = Modifier.padding(horizontal = 10.dp, vertical = 3.dp), + ) + } +} + @Composable private fun MetaChip(text: String) { - Surface( - color = MaterialTheme.colorScheme.surfaceVariant, - shape = RoundedCornerShape(50), - ) { + Surface(color = MaterialTheme.colorScheme.surfaceVariant, shape = RoundedCornerShape(50)) { Text( text, style = MaterialTheme.typography.labelSmall, diff --git a/app/src/main/java/com/androidcraft/studio/ui/models/ModelsViewModel.kt b/app/src/main/java/com/androidcraft/studio/ui/models/ModelsViewModel.kt index 656a166..04528fb 100644 --- a/app/src/main/java/com/androidcraft/studio/ui/models/ModelsViewModel.kt +++ b/app/src/main/java/com/androidcraft/studio/ui/models/ModelsViewModel.kt @@ -6,11 +6,14 @@ import com.androidcraft.studio.AndroidCraftApp import com.androidcraft.studio.core.DeviceCapabilities import com.androidcraft.studio.data.DownloadState import com.androidcraft.studio.data.Modality -import com.androidcraft.studio.data.ModelCatalog import com.androidcraft.studio.data.ModelSpec import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.stateIn +import kotlinx.coroutines.flow.SharingStarted +import androidx.lifecycle.viewModelScope import kotlinx.coroutines.flow.update class ModelsViewModel(app: Application) : AndroidViewModel(app) { @@ -21,12 +24,14 @@ class ModelsViewModel(app: Application) : AndroidViewModel(app) { val deviceCapabilities: DeviceCapabilities = DeviceCapabilities.read(app) + /** Reactive grouping so user-added models appear immediately. */ + val grouped: StateFlow>>> = repo.models + .map { list -> Modality.entries.map { m -> m to list.filter { it.modality == m } }.filter { it.second.isNotEmpty() } } + .stateIn(viewModelScope, SharingStarted.WhileSubscribed(5000), emptyList()) + private val _hfToken = MutableStateFlow(repo.hfToken) val hfToken: StateFlow = _hfToken.asStateFlow() - val grouped: List>> = - Modality.entries.map { it to ModelCatalog.forModality(it) }.filter { it.second.isNotEmpty() } - fun onTokenChange(value: String) { _hfToken.update { value } repo.hfToken = value @@ -34,5 +39,12 @@ class ModelsViewModel(app: Application) : AndroidViewModel(app) { fun download(spec: ModelSpec) = repo.startDownload(spec) fun cancel(spec: ModelSpec) = repo.cancelDownload(spec) - fun delete(spec: ModelSpec) = repo.deleteModel(spec) + fun delete(spec: ModelSpec) { + if (spec.custom) repo.removeCustomModel(spec) else repo.deleteModel(spec) + } + + fun addCustomModel(name: String, url: String, modality: Modality) { + if (url.isBlank()) return + repo.addCustomModel(name, url, modality) + } } diff --git a/app/src/main/java/com/androidcraft/studio/ui/video/VideoScreen.kt b/app/src/main/java/com/androidcraft/studio/ui/video/VideoScreen.kt index 7f2b1bb..19ab26d 100644 --- a/app/src/main/java/com/androidcraft/studio/ui/video/VideoScreen.kt +++ b/app/src/main/java/com/androidcraft/studio/ui/video/VideoScreen.kt @@ -13,8 +13,10 @@ import androidx.compose.foundation.rememberScrollState import androidx.compose.foundation.shape.RoundedCornerShape import androidx.compose.foundation.verticalScroll import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.Download import androidx.compose.material.icons.filled.Pause import androidx.compose.material.icons.filled.PlayArrow +import androidx.compose.material.icons.filled.Share import androidx.compose.material3.Button import androidx.compose.material3.Icon import androidx.compose.material3.LinearProgressIndicator @@ -24,21 +26,34 @@ import androidx.compose.material3.OutlinedTextField import androidx.compose.material3.Surface import androidx.compose.material3.Text import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.getValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.draw.clip import androidx.compose.ui.graphics.asImageBitmap import androidx.compose.ui.layout.ContentScale +import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.text.font.FontWeight import androidx.compose.ui.unit.dp import androidx.lifecycle.compose.collectAsStateWithLifecycle import androidx.lifecycle.viewmodel.compose.viewModel +import com.androidcraft.studio.core.MediaSaver import com.androidcraft.studio.ui.components.SectionCard @Composable fun VideoScreen(viewModel: VideoViewModel = viewModel()) { val state by viewModel.state.collectAsStateWithLifecycle() + val context = LocalContext.current + + LaunchedEffect(state.shareUri) { + state.shareUri?.let { uri -> + context.startActivity( + android.content.Intent.createChooser(MediaSaver.shareIntent(uri, "video/mp4"), "Share clip"), + ) + viewModel.onShareConsumed() + } + } Column( modifier = Modifier @@ -140,8 +155,30 @@ fun VideoScreen(viewModel: VideoViewModel = viewModel()) { contentDescription = "Play/Pause", ) } + OutlinedButton( + onClick = viewModel::save, + enabled = state.frames.isNotEmpty() && !state.isGenerating && !state.isExporting, + ) { + Icon(Icons.Filled.Download, contentDescription = "Save MP4") + } + OutlinedButton( + onClick = viewModel::share, + enabled = state.frames.isNotEmpty() && !state.isGenerating && !state.isExporting, + ) { + Icon(Icons.Filled.Share, contentDescription = "Share") + } } + if (state.isExporting) { + Text( + "Encoding MP4…", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } + state.statusMessage?.let { + Text(it, style = MaterialTheme.typography.bodySmall, color = MaterialTheme.colorScheme.secondary) + } state.error?.let { Text(it, style = MaterialTheme.typography.bodySmall, color = MaterialTheme.colorScheme.error) } diff --git a/app/src/main/java/com/androidcraft/studio/ui/video/VideoViewModel.kt b/app/src/main/java/com/androidcraft/studio/ui/video/VideoViewModel.kt index 3ecc2b3..a78a749 100644 --- a/app/src/main/java/com/androidcraft/studio/ui/video/VideoViewModel.kt +++ b/app/src/main/java/com/androidcraft/studio/ui/video/VideoViewModel.kt @@ -2,9 +2,12 @@ package com.androidcraft.studio.ui.video import android.app.Application import android.graphics.Bitmap +import android.net.Uri import androidx.lifecycle.AndroidViewModel import androidx.lifecycle.viewModelScope import com.androidcraft.studio.AndroidCraftApp +import com.androidcraft.studio.core.MediaSaver +import com.androidcraft.studio.engine.VideoEncoder import kotlinx.coroutines.Job import kotlinx.coroutines.delay import kotlinx.coroutines.flow.MutableStateFlow @@ -13,6 +16,7 @@ import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.flow.update import kotlinx.coroutines.isActive import kotlinx.coroutines.launch +import java.io.File data class VideoUiState( val prompt: String = "", @@ -23,6 +27,9 @@ data class VideoUiState( val isPlaying: Boolean = false, val fps: Int = 12, val error: String? = null, + val isExporting: Boolean = false, + val statusMessage: String? = null, + val shareUri: Uri? = null, ) class VideoViewModel(app: Application) : AndroidViewModel(app) { @@ -84,6 +91,51 @@ class VideoViewModel(app: Application) : AndroidViewModel(app) { } } + private suspend fun exportToFile(): Uri? { + val frames = _state.value.frames + if (frames.isEmpty()) return null + val app = getApplication() + val out = File(app.cacheDir, "androidcraft_clip.mp4") + VideoEncoder.encodeMp4(frames, _state.value.fps, out) + return MediaSaver.saveVideo(app, out, "androidcraft_${System.currentTimeMillis()}.mp4") + } + + fun save() { + if (_state.value.frames.isEmpty() || _state.value.isExporting) return + _state.update { it.copy(isExporting = true, statusMessage = null, error = null) } + viewModelScope.launch { + try { + val uri = exportToFile() + _state.update { + it.copy( + isExporting = false, + statusMessage = if (uri != null) "Saved to Movies" else "Export failed", + ) + } + } catch (t: Throwable) { + _state.update { it.copy(isExporting = false, error = "Export failed: ${t.message}") } + } + } + } + + fun share() { + if (_state.value.frames.isEmpty() || _state.value.isExporting) return + _state.update { it.copy(isExporting = true, statusMessage = null, error = null) } + viewModelScope.launch { + try { + val uri = exportToFile() + _state.update { + if (uri != null) it.copy(isExporting = false, shareUri = uri) + else it.copy(isExporting = false, statusMessage = "Export failed") + } + } catch (t: Throwable) { + _state.update { it.copy(isExporting = false, error = "Export failed: ${t.message}") } + } + } + } + + fun onShareConsumed() = _state.update { it.copy(shareUri = null) } + override fun onCleared() { playbackJob?.cancel() super.onCleared() diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 239b8f8..7a7eace 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -9,6 +9,7 @@ navigationCompose = "2.8.4" coroutines = "1.9.0" okhttp = "4.12.0" mediapipeGenai = "0.10.24" +onnxruntime = "1.20.0" junit = "4.13.2" [libraries] @@ -28,6 +29,7 @@ androidx-navigation-compose = { group = "androidx.navigation", name = "navigatio kotlinx-coroutines-android = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-android", version.ref = "coroutines" } okhttp = { group = "com.squareup.okhttp3", name = "okhttp", version.ref = "okhttp" } mediapipe-tasks-genai = { group = "com.google.mediapipe", name = "tasks-genai", version.ref = "mediapipeGenai" } +onnxruntime-android = { group = "com.microsoft.onnxruntime", name = "onnxruntime-android", version.ref = "onnxruntime" } junit = { group = "junit", name = "junit", version.ref = "junit" } [plugins]