diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 32e1e8e2e..b333b2944 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -151,6 +151,12 @@ jobs: - name: Install ffmpeg (homebrew-core) and verify headers/libs run: | brew update + # macos-14 images can retain an orphaned openssl@1.1 symlink. + # `brew unlink` removes zero links there, so remove this one stale + # symlink only when it still points at 1.1 before installing ffmpeg. + if [[ -L /opt/homebrew/bin/openssl && "$(readlink /opt/homebrew/bin/openssl)" == *openssl@1.1* ]]; then + rm /opt/homebrew/bin/openssl + fi brew install ffmpeg ls /opt/homebrew/opt/ffmpeg/include | head -5 ls /opt/homebrew/opt/ffmpeg/lib | head -5 diff --git a/FORK-CHANGELOG.md b/FORK-CHANGELOG.md index b55d5ed1b..8022b9f43 100644 --- a/FORK-CHANGELOG.md +++ b/FORK-CHANGELOG.md @@ -17,6 +17,21 @@ see [FORK.md](FORK.md). For the dated review of newer upstream changes, see [FORK-UPSTREAM-REVIEW.md](FORK-UPSTREAM-REVIEW.md). +## Unreleased — ChatGPT subscription image scenes + +- Adds an explicit image-scene action to the AI session whenever ChatGPT + subscription (Codex) is connected, even if conversation turns use Claude or + another provider. The user confirms each generation because it consumes + subscription limits. No API key is passed to the CLI, and the active chat + provider/model is not changed. +- Copies the generated original image into the app-owned project media folder, + previews it in the conversation, and creates a five-second MP4 still clip for + the existing video-only Media panel. The original is kept beside the clip so + later scene changes can start from the full-quality image. +- Keeps the Codex run read-only with shell, web, apps and editor tools disabled. + The workflow does not yet animate images or integrate Blender, Hyperframe or + the PurplePrint Design Coach. It is not part of the published `.11` build. + ## 1.10.0-subscription.11 — 2026-09-28 (Windows prerelease) - Adopts the PurplePrint GraphOS family's neutral surfaces, restrained violet diff --git a/electron/ai-edition/chat-service.test.ts b/electron/ai-edition/chat-service.test.ts index 081aa9a85..42bf9526b 100644 --- a/electron/ai-edition/chat-service.test.ts +++ b/electron/ai-edition/chat-service.test.ts @@ -3,6 +3,7 @@ import path from "node:path"; import { beforeEach, describe, expect, it } from "vitest"; import { type AxcutDocument, createEmptyDocument } from "../../src/lib/ai-edition/schema"; import { + appendGeneratedScene, createSession, deleteSession, listSessions, @@ -45,6 +46,26 @@ describe("chat-service sessions", () => { expect(got2?.messages).toEqual([]); }); + it("records a generated image in its own session", () => { + const project = "proj_generated_scene"; + const session = createSession(project); + const messages = appendGeneratedScene( + project, + session.id, + "A violet sphere", + "/tmp/source.png", + ); + expect(selectSession(project, session.id)?.messages).toEqual([ + messages.userMessage, + messages.assistantMessage, + ]); + expect(messages.assistantMessage.generatedImagePath).toBe("/tmp/source.png"); + expect(listSessions(project)[0].messageCount).toBe(2); + expect(() => appendGeneratedScene(project, "other-session", "x", "/tmp/x.png")).toThrow( + "unavailable", + ); + }); + it("selectSession returns null for an unknown sessionId", () => { expect(selectSession(projectId, "nope")).toBeNull(); }); diff --git a/electron/ai-edition/chat-service.ts b/electron/ai-edition/chat-service.ts index 00c8f0fab..48de88dec 100644 --- a/electron/ai-edition/chat-service.ts +++ b/electron/ai-edition/chat-service.ts @@ -230,6 +230,34 @@ export function selectSession(projectId: string, sessionId: string): ChatSession }; } +/** Record a generated artifact in the current in-memory conversation. */ +export function appendGeneratedScene( + projectId: string, + sessionId: string, + prompt: string, + imagePath: string, +): { userMessage: AiEditionChatMessage; assistantMessage: AiEditionChatMessage } { + const session = sessionsByProject.get(projectId)?.get(sessionId); + if (!session) throw new Error("Chat session unavailable."); + const now = new Date().toISOString(); + const userMessage: AiEditionChatMessage = { + id: randomUUID(), + role: "user", + content: prompt, + createdAt: now, + }; + const assistantMessage: AiEditionChatMessage = { + id: randomUUID(), + role: "assistant", + content: + "Generated a five-second still-image scene. The original image is saved beside the video clip.", + createdAt: new Date().toISOString(), + generatedImagePath: imagePath, + }; + session.messages.push(userMessage, assistantMessage); + return { userMessage, assistantMessage }; +} + export function renameSession( projectId: string, sessionId: string, diff --git a/electron/ai-edition/codex/app-server.test.ts b/electron/ai-edition/codex/app-server.test.ts index 50621fcc7..a2483fda2 100644 --- a/electron/ai-edition/codex/app-server.test.ts +++ b/electron/ai-edition/codex/app-server.test.ts @@ -279,6 +279,51 @@ describe("Codex app-server subscription transport", () => { expect(f.sent.some((packet) => packet.method === "turn/interrupt")).toBe(false); }); + it("generates one image through the subscription capability without editor tools", async () => { + const f = fixture((packet, emit) => { + if (packet.method === "modelProvider/capabilities/read") { + emit({ id: packet.id, result: { imageGeneration: true } }); + return true; + } + if (packet.method === "turn/start") { + emit({ id: packet.id, result: { turn: { id: "image-turn" } } }); + emit({ + method: "item/completed", + params: { + threadId: "thread-1", + item: { type: "imageGeneration", status: "completed", savedPath: "C:/image.png" }, + }, + }); + emit({ + method: "turn/completed", + params: { threadId: "thread-1", turn: { id: "image-turn", status: "completed" } }, + }); + return true; + } + return false; + }); + expect(await f.runtime.generateImage("A violet sphere")).toEqual({ + savedPath: "C:/image.png", + }); + expect(f.sent.find((packet) => packet.method === "thread/start")?.params).toMatchObject({ + sandbox: "read-only", + dynamicTools: [], + }); + expect(f.sent.find((packet) => packet.method === "turn/start")?.params).toMatchObject({ + input: [{ type: "text", text: "$imagegen A violet sphere", text_elements: [] }], + }); + }); + + it("does not start an image turn when the subscription capability is absent", async () => { + const f = fixture((packet, emit) => { + if (packet.method !== "modelProvider/capabilities/read") return false; + emit({ id: packet.id, result: { imageGeneration: false } }); + return true; + }); + await expect(f.runtime.generateImage("A violet sphere")).rejects.toThrow("unavailable"); + expect(f.sent.some((packet) => packet.method === "turn/start")).toBe(false); + }); + it("interrupts timed-out turns and releases their thread", async () => { const f = fixture((packet, emit) => { if (packet.method !== "turn/start") return false; diff --git a/electron/ai-edition/codex/app-server.ts b/electron/ai-edition/codex/app-server.ts index a2a2a78e2..c1e5e8d56 100644 --- a/electron/ai-edition/codex/app-server.ts +++ b/electron/ai-edition/codex/app-server.ts @@ -30,6 +30,8 @@ export interface CodexRun { input: string; tools?: CodexTool[]; onText?: (text: string) => void; + onImageGeneration?: (item: ObjectValue) => void; + timeoutMs?: number; signal?: AbortSignal; } interface PendingRequest { @@ -40,6 +42,7 @@ interface PendingRequest { interface ActiveTurn { tools: Map; onText?: (text: string) => void; + onImageGeneration?: (item: ObjectValue) => void; text: string; turnId?: string; toolCalls: number; @@ -260,6 +263,9 @@ export class CodexAppServer { if (payload.method === "item/agentMessage/delta" && typeof params.delta === "string") { turn.text += params.delta; turn.onText?.(params.delta); + } else if (payload.method === "item/completed") { + const item = object(params.item); + if (item.type === "imageGeneration") turn.onImageGeneration?.(item); } else if (payload.method === "turn/started") { turn.turnId = String(object(params.turn).id ?? ""); } else if (payload.method === "turn/completed") { @@ -438,6 +444,7 @@ export class CodexAppServer { const turn: ActiveTurn = { tools: new Map((input.tools ?? []).map((tool) => [tool.name, tool])), onText: input.onText, + onImageGeneration: input.onImageGeneration, text: "", toolCalls: 0, resolve, @@ -452,7 +459,7 @@ export class CodexAppServer { } timer = setTimeout( () => reject(new Error("Codex AI request timed out. Please retry.")), - this.turnTimeoutMs, + input.timeoutMs ?? this.turnTimeoutMs, ); void this.request("turn/start", { threadId, @@ -477,6 +484,36 @@ export class CodexAppServer { }); } } + + /** Built-in Codex image generation, using only the app-owned ChatGPT subscription profile. */ + async generateImage(prompt: string): Promise<{ savedPath?: string; result?: string }> { + const trimmed = prompt.trim(); + if (!trimmed || trimmed.length > 4_000) + throw new Error("Image prompt must be 1–4,000 characters."); + const status = await this.status(); + if (!status.connected) + throw new Error(status.error ?? "Connect your ChatGPT account in AI settings first."); + const capabilities = await this.request("modelProvider/capabilities/read"); + if (capabilities.imageGeneration !== true) + throw new Error("Image generation is unavailable for this ChatGPT subscription."); + const images: ObjectValue[] = []; + await this.run({ + instructions: + "Create exactly one still image for a video scene. Use only built-in image generation. " + + "Do not use shell, code, web, files, apps, or editor tools. Do not edit the project.", + input: `$imagegen ${trimmed}`, + tools: [], + timeoutMs: 360_000, + onImageGeneration: (item) => images.push(item), + }); + if (images.length !== 1 || images[0].status !== "completed") + throw new Error("Codex did not return one completed image."); + const image = images[0]; + return { + ...(typeof image.savedPath === "string" ? { savedPath: image.savedPath } : {}), + ...(typeof image.result === "string" ? { result: image.result } : {}), + }; + } } let singleton: Promise | undefined; diff --git a/electron/ai-edition/document-service.test.ts b/electron/ai-edition/document-service.test.ts index 6cbdb97c9..c70763fc2 100644 --- a/electron/ai-edition/document-service.test.ts +++ b/electron/ai-edition/document-service.test.ts @@ -260,6 +260,18 @@ describe("DocumentService", () => { expect(updated.project.primaryAssetId).toBe(asset?.id); }); + it("stores a verified duration supplied for a generated clip", async () => { + const doc = await service.createProject("P"); + const updated = await service.addAsset(doc.project.id, { + path: "/tmp/scene.mp4", + durationSec: 5, + }); + expect(updated.assets[0]?.durationSec).toBe(5); + await expect( + service.addAsset(doc.project.id, { path: "/tmp/bad.mp4", durationSec: -1 }), + ).rejects.toThrow("duration must be positive"); + }); + it("resolves relative paths against the cwd", async () => { const doc = await service.createProject("P"); const updated = await service.addAsset(doc.project.id, { path: "recording.webm" }); diff --git a/electron/ai-edition/document-service.ts b/electron/ai-edition/document-service.ts index 3c93e3bc0..406c89dc2 100644 --- a/electron/ai-edition/document-service.ts +++ b/electron/ai-edition/document-service.ts @@ -38,6 +38,7 @@ export interface ProjectSummary { export interface AddAssetInput { path: string; label?: string; + durationSec?: number; } export class DocumentNotFoundError extends Error { @@ -289,6 +290,11 @@ export class DocumentService { projectId, ); } + if ( + input.durationSec !== undefined && + (!Number.isFinite(input.durationSec) || input.durationSec <= 0) + ) + throw new ProjectFileError("Asset duration must be positive and finite.", projectId); const absolutePath = path.isAbsolute(input.path) ? input.path : path.resolve(input.path); // P3.1 — capture the file size at import. Non-fatal: a stat failure // (network drive, permissions) just leaves sizeBytes undefined. @@ -303,6 +309,7 @@ export class DocumentService { kind: "video", label: input.label?.trim() || path.basename(absolutePath), originalPath: absolutePath, + ...(input.durationSec !== undefined ? { durationSec: input.durationSec } : {}), sizeBytes, cameraTrack: null, }; diff --git a/electron/ai-edition/generated-scene.test.ts b/electron/ai-edition/generated-scene.test.ts new file mode 100644 index 000000000..3ee6e4122 --- /dev/null +++ b/electron/ai-edition/generated-scene.test.ts @@ -0,0 +1,71 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { encodeStillScene, saveGeneratedImage } from "./generated-scene"; + +const roots: string[] = []; +afterEach(async () => { + for (const root of roots.splice(0)) { + if (path.dirname(root) !== os.tmpdir() || !path.basename(root).startsWith("studio-scene-test-")) + throw new Error("Unsafe test cleanup path"); + await fs.rm(root, { recursive: true, force: true }); + } +}); + +async function temporaryRoot(): Promise { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "studio-scene-test-")); + roots.push(root); + return root; +} + +function pngBytes(): Buffer { + const bytes = Buffer.alloc(256); + Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]).copy(bytes); + return bytes; +} + +describe("generated still scene assets", () => { + it("copies a Codex image to app-owned project storage and retains the source", async () => { + const root = await temporaryRoot(); + const source = path.join(root, "codex-output.png"); + await fs.writeFile(source, pngBytes()); + const saved = await saveGeneratedImage(root, "proj_123", { savedPath: source }); + expect(saved.imagePath).toContain(path.join("generated-scenes", "proj_123")); + expect(saved.videoPath).toBe(saved.imagePath.replace(/\.png$/, ".mp4")); + expect(await fs.readFile(saved.imagePath)).toEqual(pngBytes()); + expect(await fs.readFile(source)).toEqual(pngBytes()); + }); + + it("accepts an inline image but rejects traversal and non-image data", async () => { + const root = await temporaryRoot(); + const inline = `data:image/png;base64,${pngBytes().toString("base64")}`; + expect(await saveGeneratedImage(root, "proj_123", { result: inline })).toMatchObject({ + imagePath: expect.stringMatching(/\.png$/), + }); + await expect(saveGeneratedImage(root, "../other", { result: inline })).rejects.toThrow( + "Invalid project id", + ); + await expect( + saveGeneratedImage(root, "proj_123", { result: Buffer.alloc(256).toString("base64") }), + ).rejects.toThrow("unsupported or invalid"); + }); + + it("encodes a five-second clip and falls back when OpenH264 is missing", async () => { + const root = await temporaryRoot(); + const imagePath = path.join(root, "source.png"); + const videoPath = path.join(root, "scene.mp4"); + await fs.writeFile(imagePath, pngBytes()); + const run = vi.fn(async (_exe: string, args: string[]) => { + if (args.includes("libopenh264")) throw new Error("Unknown encoder 'libopenh264'"); + await fs.writeFile(args.at(-1)!, Buffer.alloc(2_000)); + }); + await encodeStillScene(imagePath, videoPath, "ffmpeg", run); + expect(run).toHaveBeenCalledTimes(2); + expect(run.mock.calls[0][1]).toContain("libopenh264"); + expect(run.mock.calls[1][1]).toContain("mpeg4"); + expect(run.mock.calls[1][1]).toContain("120"); + expect((await fs.stat(videoPath)).size).toBe(2_000); + expect(await fs.readFile(imagePath)).toEqual(pngBytes()); + }); +}); diff --git a/electron/ai-edition/generated-scene.ts b/electron/ai-edition/generated-scene.ts new file mode 100644 index 000000000..0cd0e6baf --- /dev/null +++ b/electron/ai-edition/generated-scene.ts @@ -0,0 +1,143 @@ +import { spawn } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { resolveFfmpeg } from "../media/audioPeaks"; + +const MAX_IMAGE_BYTES = 40 * 1024 * 1024; +const VIDEO_FRAMES = 120; // Five seconds at 24 fps. + +export interface CodexGeneratedImage { + savedPath?: string; + result?: string; +} + +function imageExtension(bytes: Buffer): ".png" | ".jpg" | ".webp" | null { + if (bytes.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]))) return ".png"; + if (bytes.subarray(0, 3).equals(Buffer.from([255, 216, 255]))) return ".jpg"; + if (bytes.toString("ascii", 0, 4) === "RIFF" && bytes.toString("ascii", 8, 12) === "WEBP") + return ".webp"; + return null; +} + +async function imageBytes(image: CodexGeneratedImage): Promise { + if (image.savedPath && path.isAbsolute(image.savedPath)) { + try { + const stat = await fs.stat(image.savedPath); + if (stat.isFile() && stat.size > 0 && stat.size <= MAX_IMAGE_BYTES) + return await fs.readFile(image.savedPath); + } catch { + // Some Codex versions return only an inline image. Try that below. + } + } + const inline = image.result; + if (!inline) throw new Error("Codex returned no readable image file."); + const base64 = inline.replace(/^data:image\/(?:png|jpeg|webp);base64,/i, "").replace(/\s/g, ""); + if ( + base64.length > Math.ceil((MAX_IMAGE_BYTES * 4) / 3) + 8 || + !/^[A-Za-z0-9+/]+={0,2}$/.test(base64) + ) + throw new Error("Codex returned invalid image data."); + return Buffer.from(base64, "base64"); +} + +/** Copy the image into Studio-owned project storage before the CLI removes its temporary file. */ +export async function saveGeneratedImage( + userData: string, + projectId: string, + image: CodexGeneratedImage, +): Promise<{ imagePath: string; videoPath: string }> { + if (!/^[A-Za-z0-9_-]+$/.test(projectId)) throw new Error("Invalid project id."); + const bytes = await imageBytes(image); + const extension = imageExtension(bytes); + if (!extension || bytes.length < 100 || bytes.length > MAX_IMAGE_BYTES) + throw new Error("Codex returned an unsupported or invalid image."); + const folder = path.join(userData, "generated-scenes", projectId); + await fs.mkdir(folder, { recursive: true }); + const id = randomUUID(); + const imagePath = path.join(folder, `${id}${extension}`); + const videoPath = path.join(folder, `${id}.mp4`); + await fs.writeFile(imagePath, bytes, { flag: "wx" }); + return { imagePath, videoPath }; +} + +function runFfmpeg(executable: string, args: string[]): Promise { + return new Promise((resolve, reject) => { + const child = spawn(executable, args, { + windowsHide: true, + shell: false, + stdio: ["ignore", "ignore", "pipe"], + }); + let stderr = ""; + let settled = false; + const timer = setTimeout(() => { + if (settled) return; + settled = true; + child.kill(); + reject(new Error("Video encoding timed out.")); + }, 120_000); + child.stderr?.on("data", (chunk: Buffer) => { + stderr = (stderr + chunk.toString()).slice(-2_000); + }); + child.once("error", (error) => { + if (settled) return; + settled = true; + clearTimeout(timer); + reject(error); + }); + child.once("close", (code) => { + if (settled) return; + settled = true; + clearTimeout(timer); + if (code === 0) resolve(); + else reject(new Error(`Video encoding failed${stderr ? `: ${stderr}` : "."}`)); + }); + }); +} + +/** A reusable timeline clip; the untouched source image remains beside it. */ +export async function encodeStillScene( + imagePath: string, + videoPath: string, + ffmpeg = resolveFfmpeg(), + run: (executable: string, args: string[]) => Promise = runFfmpeg, +): Promise { + if (!ffmpeg) throw new Error("The bundled FFmpeg encoder is unavailable."); + const temporary = videoPath.replace(/\.mp4$/i, ".encoding.mp4"); + const base = [ + "-hide_banner", + "-loglevel", + "error", + "-y", + "-loop", + "1", + "-framerate", + "24", + "-i", + imagePath, + "-frames:v", + String(VIDEO_FRAMES), + "-vf", + "pad=ceil(iw/2)*2:ceil(ih/2)*2", + "-pix_fmt", + "yuv420p", + "-movflags", + "+faststart", + "-an", + ]; + try { + try { + await run(ffmpeg, [...base, "-c:v", "libopenh264", "-b:v", "8M", temporary]); + } catch (error) { + if (!/Unknown encoder|Encoder .* not found/i.test(String(error))) throw error; + // LGPL builds on some platforms omit OpenH264. The original image stays lossless. + await fs.rm(temporary, { force: true }); + await run(ffmpeg, [...base, "-c:v", "mpeg4", "-q:v", "2", temporary]); + } + const stat = await fs.stat(temporary); + if (!stat.isFile() || stat.size < 1_000) throw new Error("Encoder returned an empty clip."); + await fs.rename(temporary, videoPath); + } finally { + await fs.rm(temporary, { force: true }); + } +} diff --git a/electron/ipc/nativeBridge.ts b/electron/ipc/nativeBridge.ts index 6e58d94f1..8557ef8c9 100644 --- a/electron/ipc/nativeBridge.ts +++ b/electron/ipc/nativeBridge.ts @@ -483,6 +483,7 @@ export function registerNativeBridgeHandlers(context: NativeBridgeContext) { request.payload.projectId, request.payload.path, request.payload.label, + request.payload.durationSec, ), ); case "document.removeAsset": @@ -552,6 +553,15 @@ export function registerNativeBridgeHandlers(context: NativeBridgeContext) { ), ); } + case "image.generateScene": + return createSuccessResponse( + requestId, + await aiEditionService.generateImageScene( + request.payload.projectId, + request.payload.sessionId, + request.payload.prompt, + ), + ); case "chat.undoLastBatch": return createSuccessResponse( requestId, diff --git a/electron/native-bridge/services/aiEditionService.lazyLlmConfig.test.ts b/electron/native-bridge/services/aiEditionService.lazyLlmConfig.test.ts index 66727f7b1..62a8594b0 100644 --- a/electron/native-bridge/services/aiEditionService.lazyLlmConfig.test.ts +++ b/electron/native-bridge/services/aiEditionService.lazyLlmConfig.test.ts @@ -4,6 +4,9 @@ import { AiEditionService, type AiEditionServiceOptions } from "./aiEditionServi const runtime = vi.hoisted(() => ({ status: vi.fn(async () => ({ available: true, connected: true, plan: "plus" })), + generateImage: vi.fn(async (_prompt: string): Promise => { + throw new Error("Codex subscription unavailable"); + }), logout: vi.fn(async () => undefined), models: vi.fn(async () => [ { @@ -237,6 +240,24 @@ describe("AiEditionService — LLM store resolution is deferred", () => { expect(await service.llmSubscriptionLogin("openai-oauth")).toMatchObject({ success: false }); expect(runtime.status).not.toHaveBeenCalled(); }); + + it("routes image generation to the subscription without changing the active chat provider", async () => { + const getProject = vi.fn(async () => ({})); + const llmConfig = vi.fn(() => { + throw new Error("Image generation must not read the active chat config."); + }); + const service = new AiEditionService({ + llmConfig, + documents: { getProject }, + selectSession: () => ({ id: "s", projectId: "p", messages: [] }), + } as unknown as AiEditionServiceOptions); + await expect(service.generateImageScene("p", "s", "a sphere")).rejects.toThrow( + "Codex subscription unavailable", + ); + expect(getProject).toHaveBeenCalledExactlyOnceWith("p"); + expect(runtime.generateImage).toHaveBeenCalledExactlyOnceWith("a sphere"); + expect(llmConfig).not.toHaveBeenCalled(); + }); it("does not build the store while the service is constructed", () => { const { builds } = serviceWithCountingFactory(); expect(builds()).toBe(0); diff --git a/electron/native-bridge/services/aiEditionService.ts b/electron/native-bridge/services/aiEditionService.ts index 30a882cf4..bab3f6213 100644 --- a/electron/native-bridge/services/aiEditionService.ts +++ b/electron/native-bridge/services/aiEditionService.ts @@ -12,6 +12,7 @@ import type { AiEditionChatSession, AiEditionChatSessionSummary, AiEditionDocumentResult, + AiEditionGeneratedSceneResult, AiEditionLlmConfig, AiEditionLlmDisconnectResult, AiEditionLlmModelOption, @@ -25,9 +26,11 @@ import { translateCaptionSegments, } from "../../ai-edition/caption-translate"; import type { ChatEventSink } from "../../ai-edition/chat-service"; +import { appendGeneratedScene } from "../../ai-edition/chat-service"; import { getClaudeCli } from "../../ai-edition/claude/cli"; import { getCodexAppServer } from "../../ai-edition/codex/app-server"; import type { DocumentService } from "../../ai-edition/document-service"; +import { encodeStillScene, saveGeneratedImage } from "../../ai-edition/generated-scene"; import type { LlmConfigStore, LlmCredential } from "../../ai-edition/llm-config-store"; import { listAnthropicModels, @@ -162,12 +165,43 @@ export class AiEditionService { } } - async addAsset(projectId: string, path: string, label?: string): Promise { - const document = await this.options.documents.addAsset(projectId, { path, label }); + async addAsset( + projectId: string, + path: string, + label?: string, + durationSec?: number, + ): Promise { + const document = await this.options.documents.addAsset(projectId, { + path, + label, + durationSec, + }); const assetId = document.project.primaryAssetId ?? document.assets.at(-1)?.id ?? ""; return { assetId, document }; } + async generateImageScene( + projectId: string, + sessionId: string, + prompt: string, + ): Promise { + if (!this.options.selectSession(projectId, sessionId)) + throw new Error("Chat session unavailable."); + await this.options.documents.getProject(projectId); + const image = await (await getCodexAppServer()).generateImage(prompt); + const { app } = await import("electron"); + const paths = await saveGeneratedImage(app.getPath("userData"), projectId, image); + try { + await encodeStillScene(paths.imagePath, paths.videoPath); + } catch (error) { + throw new Error( + `The original image was saved at ${paths.imagePath}, but its video clip could not be created: ${error instanceof Error ? error.message : String(error)}`, + ); + } + const messages = appendGeneratedScene(projectId, sessionId, prompt.trim(), paths.imagePath); + return { ...paths, ...messages }; + } + async removeAsset(projectId: string, assetId: string): Promise { const document = await this.options.documents.removeAsset(projectId, assetId); return { assetId, document }; diff --git a/src/components/ai-edition/ChatMessage.test.tsx b/src/components/ai-edition/ChatMessage.test.tsx index 52fc88c07..70df26ef3 100644 --- a/src/components/ai-edition/ChatMessage.test.tsx +++ b/src/components/ai-edition/ChatMessage.test.tsx @@ -60,6 +60,22 @@ describe("chat message rendering", () => { ); }); + it("previews only the explicit main-process generated image field", () => { + render( + , + ); + expect(screen.getByRole("img", { name: "chat.generatedImagePreview" })).toHaveAttribute( + "src", + "file:///C:/Studio/generated-scenes/proj_1/source.png", + ); + }); + it("reconciles partial Markdown while streaming and only exposes actions on completion", () => { const { container, rerender } = render( , diff --git a/src/components/ai-edition/ChatMessage.tsx b/src/components/ai-edition/ChatMessage.tsx index 8885c8ec9..5ec6369f1 100644 --- a/src/components/ai-edition/ChatMessage.tsx +++ b/src/components/ai-edition/ChatMessage.tsx @@ -3,6 +3,7 @@ import { memo } from "react"; import Markdown, { type Components } from "react-markdown"; import remarkGfm from "remark-gfm"; import { toast } from "sonner"; +import { toFileUrl } from "@/components/video-editor/projectPersistence"; import { useI18n, useScopedT } from "@/contexts/I18nContext"; import type { AiEditionToolCallSummary } from "@/native/contracts"; import styles from "./NewEditorShell.module.css"; @@ -16,6 +17,7 @@ export interface ChatDisplayMessage { checkpointId?: string | null; thinking?: string; interrupted?: boolean; + generatedImagePath?: string; } // Model output is untrusted: no raw HTML, remote images, local-file links or custom protocols. @@ -66,6 +68,7 @@ export const ChatMessage = memo(function ChatMessage({ const time = validDate ? date.toLocaleTimeString(locale, { hour: "numeric", minute: "2-digit" }) : message.time; + const content = message.generatedImagePath ? t("chat.imageSceneCreated") : message.content; return (
{message.thinking} ) : null} - {message.content ? ( + {content ? (
{user ? ( - message.content + content ) : ( - {message.content} + {content} )}
) : null} + {message.generatedImagePath ? ( + {t("chat.generatedImagePreview")} + ) : null} {message.toolCalls?.length ? (
    {message.toolCalls.map((call, i) => ( @@ -144,7 +155,7 @@ export const ChatMessage = memo(function ChatMessage({ title={t("chat.copyMessage")} aria-label={t("chat.copyMessage")} onClick={() => - void navigator.clipboard.writeText(message.content).then( + void navigator.clipboard.writeText(content).then( () => toast.success(t("chat.copiedToClipboard")), () => toast.error(t("chat.copyFailed")), ) diff --git a/src/components/ai-edition/LeftPanel.providerRefresh.test.tsx b/src/components/ai-edition/LeftPanel.providerRefresh.test.tsx index 5b34801f1..5842010ae 100644 --- a/src/components/ai-edition/LeftPanel.providerRefresh.test.tsx +++ b/src/components/ai-edition/LeftPanel.providerRefresh.test.tsx @@ -43,6 +43,7 @@ const chatSelectSession = vi.fn(); const chatCreateSession = vi.fn(); const chatRewind = vi.fn(); const chatRun = vi.fn<() => Promise>(); +const generateImageScene = vi.fn(); const modelList = vi.fn<() => Promise>(); const saveModel = vi.fn(async (_config: AiEditionLlmConfig) => ({ success: true })); let chatEvent: ((event: AiEditionChatEvent) => void) | undefined; @@ -55,6 +56,7 @@ vi.mock("@/native/client", () => ({ chatSelectSession: (...args: unknown[]) => chatSelectSession(...args), chatCreateSession: (...args: unknown[]) => chatCreateSession(...args), chatRun: () => chatRun(), + generateImageScene: (...args: unknown[]) => generateImageScene(...args), chatRewind: (...args: unknown[]) => chatRewind(...args), chatBudget: () => Promise.resolve(null), llmListProviderModels: () => modelList(), @@ -93,6 +95,7 @@ beforeEach(() => { chatCreateSession.mockReset(); chatRewind.mockReset().mockResolvedValue({ success: false, error: "test refused" }); chatRun.mockReset(); + generateImageScene.mockReset(); modelList.mockReset().mockResolvedValue({ models: [] }); saveModel.mockClear(); useProjectStore.setState({ projectId: null, document: null }); @@ -199,6 +202,103 @@ describe("ChatStripPanel streaming", () => { expect(screen.queryByText("AFTER FINISH")).toBeNull(); }); + it("uses the connected Codex subscription for images while Claude stays the chat provider", async () => { + llmGetSnapshot.mockResolvedValue({ + config: { provider: "claude-local", model: "opus[1m]" }, + connectedProviders: ["claude-local", "codex-subscription"], + availableProviders: [], + credentialSummary: [], + }); + useProjectStore.setState({ projectId: session.projectId }); + chatListSessions.mockResolvedValue([session]); + chatSelectSession.mockResolvedValue({ ...session, messages: [] }); + generateImageScene.mockResolvedValue({ + imagePath: "C:/studio/source.png", + videoPath: "C:/studio/scene.mp4", + userMessage: { + id: "generated-user", + role: "user", + content: "A violet sphere", + createdAt: "2026-08-27T12:00:00Z", + }, + assistantMessage: { + id: "generated-assistant", + role: "assistant", + content: "Generated", + createdAt: "2026-08-27T12:00:01Z", + }, + }); + const originalAddAsset = useProjectStore.getState().addAsset; + const addAsset = vi.fn(async () => null); + useProjectStore.setState({ addAsset }); + const confirm = vi.spyOn(window, "confirm").mockReturnValue(false); + try { + render( + + + , + ); + await waitFor(() => expect(screen.getByRole("textbox")).toBeEnabled()); + fireEvent.change(screen.getByRole("textbox"), { target: { value: "A violet sphere" } }); + expect(screen.getByRole("button", { name: "chat.imageSceneButton" })).toBeEnabled(); + fireEvent.click(screen.getByRole("button", { name: "chat.imageSceneButton" })); + expect(generateImageScene).not.toHaveBeenCalled(); + confirm.mockReturnValue(true); + fireEvent.click(screen.getByRole("button", { name: "chat.imageSceneButton" })); + await waitFor(() => + expect(generateImageScene).toHaveBeenCalledWith( + session.projectId, + session.id, + "A violet sphere", + ), + ); + await waitFor(() => + expect(addAsset).toHaveBeenCalledWith( + "C:/studio/scene.mp4", + "chat.imageSceneAssetLabel", + 5, + ), + ); + expect(screen.getByRole("img", { name: "chat.generatedImagePreview" })).toHaveAttribute( + "src", + "file:///C:/studio/source.png", + ); + expect(saveModel).not.toHaveBeenCalled(); + } finally { + confirm.mockRestore(); + useProjectStore.setState({ addAsset: originalAddAsset }); + } + }); + + it("allows an image-only prompt when Codex is connected but no chat provider is active", async () => { + llmGetSnapshot.mockResolvedValue({ + config: null, + connectedProviders: ["codex-subscription"], + availableProviders: [], + credentialSummary: [], + }); + useProjectStore.setState({ projectId: session.projectId }); + render( + + + , + ); + const composer = await screen.findByPlaceholderText("chat.imageScenePromptPlaceholder"); + expect(composer).toBeEnabled(); + fireEvent.change(composer, { target: { value: "A violet sphere" } }); + expect(screen.getByRole("button", { name: "chat.imageSceneButton" })).toBeEnabled(); + expect(screen.getByRole("button", { name: "chat.send" })).toBeDisabled(); + fireEvent.keyDown(composer, { key: "Enter" }); + expect(generateImageScene).not.toHaveBeenCalled(); + }); + + it("keeps image generation unavailable without a connected Codex subscription", async () => { + await ready(); + fireEvent.change(screen.getByRole("textbox"), { target: { value: "A violet sphere" } }); + expect(screen.getByRole("button", { name: "chat.imageSceneButton" })).toBeDisabled(); + expect(generateImageScene).not.toHaveBeenCalled(); + }); + it("keeps an interrupted partial reply, clearly marked incomplete", async () => { let finish!: (result: AiEditionChatResult) => void; chatRun.mockReturnValue( diff --git a/src/components/ai-edition/LeftPanel.tsx b/src/components/ai-edition/LeftPanel.tsx index 3aecdac8f..947595ac2 100644 --- a/src/components/ai-edition/LeftPanel.tsx +++ b/src/components/ai-edition/LeftPanel.tsx @@ -1,4 +1,4 @@ -import { ArrowLeft, Check, Film, MessageSquare, Plus, Search, X } from "lucide-react"; +import { ArrowLeft, Check, Film, ImagePlus, MessageSquare, Plus, Search, X } from "lucide-react"; import { useCallback, useEffect, useRef, useState } from "react"; import { createPortal } from "react-dom"; import { toast } from "sonner"; @@ -689,6 +689,9 @@ function ChatStripPanel() { const [activeModelCatalog, setActiveModelCatalog] = useState([]); // unknown ≠ none; see chatAvailability.ts. const canChat = canSendChat(llmConfig, connectedProviders); + // Image generation uses the separate, app-owned Codex subscription even when + // conversation turns use Claude or another provider. + const canGenerateImage = connectedProviders?.includes("codex-subscription") === true; const [modelPopoverOpen, setModelPopoverOpen] = useState(false); const modelButtonRef = useRef(null); const [modelPopoverRect, setModelPopoverRect] = useState<{ @@ -820,6 +823,7 @@ function ChatStripPanel() { time: m.createdAt, toolCalls: m.toolCalls, checkpointId: m.checkpointId ?? null, + generatedImagePath: m.generatedImagePath, })), ); } else { @@ -986,6 +990,100 @@ function ChatStripPanel() { } }; + const generateImageScene = async () => { + const prompt = input.trim(); + if (!projectId || !prompt || busy || activeRunRef.current) return; + if (!canGenerateImage) { + toast.error(t("chat.imageSceneRequiresCodex")); + return; + } + if (!window.confirm(t("chat.imageSceneConfirm"))) return; + const run = { projectId, sessionId: activeSessionId, text: "", thinking: "" }; + activeRunRef.current = run; + const optimisticUserId = `local_${Date.now()}_image`; + setInput(""); + setBusy(true); + setStreamText(t("chat.imageSceneWorking")); + followLatestRef.current = true; + setMessages((previous) => [ + ...previous, + { + id: optimisticUserId, + role: "user", + content: prompt, + time: new Date().toISOString(), + }, + ]); + try { + let sessionId = activeSessionId; + if (!sessionId) { + const created = await nativeBridgeClient.aiEdition.chatCreateSession(projectId); + sessionId = created.id; + if (activeRunRef.current === run && projectIdRef.current === projectId) { + activeSessionIdRef.current = sessionId; + setSessions((previous) => [...previous, created]); + setActiveSessionId(sessionId); + } + } + run.sessionId = sessionId; + const generated = await nativeBridgeClient.aiEdition.generateImageScene( + projectId, + sessionId, + prompt, + ); + if (activeRunRef.current === run && projectIdRef.current === projectId) { + setMessages((previous) => [ + ...previous.map((message) => + message.id === optimisticUserId + ? { ...message, id: generated.userMessage.id } + : message, + ), + { + id: generated.assistantMessage.id, + role: "assistant", + content: t("chat.imageSceneCreated"), + time: generated.assistantMessage.createdAt, + generatedImagePath: generated.imagePath, + }, + ]); + void refreshSessions(projectId); + } + try { + if (useProjectStore.getState().projectId === projectId) { + await useProjectStore + .getState() + .addAsset(generated.videoPath, t("chat.imageSceneAssetLabel"), 5); + } else { + await nativeBridgeClient.aiEdition.addAsset( + projectId, + generated.videoPath, + t("chat.imageSceneAssetLabel"), + 5, + ); + } + toast.success(t("chat.imageSceneReady")); + } catch (error) { + toast.error(t("chat.imageSceneImportFailed"), { + description: `${error instanceof Error ? error.message : String(error)} (${generated.videoPath})`, + }); + } + } catch (error) { + if (activeRunRef.current === run && projectIdRef.current === projectId) { + setMessages((previous) => previous.filter((message) => message.id !== optimisticUserId)); + setInput(prompt); + } + toast.error(t("chat.imageSceneFailed"), { + description: error instanceof Error ? error.message : String(error), + }); + } finally { + if (activeRunRef.current === run) { + activeRunRef.current = null; + setBusy(false); + setStreamText(""); + } + } + }; + // Auto-send a prompt handed over by another part of the UI (e.g. the // timeline's Auto-enhance → "Smart zooms + cuts with AI"). Routes through // the normal send() so sessions/checkpoints/rewind all keep working; the @@ -1044,6 +1142,7 @@ function ChatStripPanel() { time: m.createdAt, toolCalls: m.toolCalls, checkpointId: m.checkpointId ?? null, + generatedImagePath: m.generatedImagePath, })), ); setInput(result.prompt); @@ -1570,7 +1669,7 @@ function ChatStripPanel() { followLatestRef.current = el.scrollHeight - el.scrollTop - el.clientHeight < 80; }} > - {!canChat && messages.length === 0 ? ( + {!canChat && !canGenerateImage && messages.length === 0 ? ( ) : messages.length === 0 ? (

    - {t("chat.emptyState")} + {t(!canChat ? "chat.imageScenePromptPlaceholder" : "chat.emptyState")}

    ) : ( <> @@ -1630,15 +1729,19 @@ function ChatStripPanel() {