Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,12 @@ jobs:
- name: Install ffmpeg (homebrew-core) and verify headers/libs
run: |
brew update
# macos-14 images can retain an orphaned openssl@1.1 symlink.
# `brew unlink` removes zero links there, so remove this one stale
# symlink only when it still points at 1.1 before installing ffmpeg.
if [[ -L /opt/homebrew/bin/openssl && "$(readlink /opt/homebrew/bin/openssl)" == *openssl@1.1* ]]; then
rm /opt/homebrew/bin/openssl
fi
brew install ffmpeg
ls /opt/homebrew/opt/ffmpeg/include | head -5
ls /opt/homebrew/opt/ffmpeg/lib | head -5
Expand Down
15 changes: 15 additions & 0 deletions FORK-CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,21 @@ see [FORK.md](FORK.md).
For the dated review of newer upstream changes, see
[FORK-UPSTREAM-REVIEW.md](FORK-UPSTREAM-REVIEW.md).

## Unreleased — ChatGPT subscription image scenes

- Adds an explicit image-scene action to the AI session whenever ChatGPT
subscription (Codex) is connected, even if conversation turns use Claude or
another provider. The user confirms each generation because it consumes
subscription limits. No API key is passed to the CLI, and the active chat
provider/model is not changed.
- Copies the generated original image into the app-owned project media folder,
previews it in the conversation, and creates a five-second MP4 still clip for
the existing video-only Media panel. The original is kept beside the clip so
later scene changes can start from the full-quality image.
- Keeps the Codex run read-only with shell, web, apps and editor tools disabled.
The workflow does not yet animate images or integrate Blender, Hyperframe or
the PurplePrint Design Coach. It is not part of the published `.11` build.

## 1.10.0-subscription.11 — 2026-09-28 (Windows prerelease)

- Adopts the PurplePrint GraphOS family's neutral surfaces, restrained violet
Expand Down
21 changes: 21 additions & 0 deletions electron/ai-edition/chat-service.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import path from "node:path";
import { beforeEach, describe, expect, it } from "vitest";
import { type AxcutDocument, createEmptyDocument } from "../../src/lib/ai-edition/schema";
import {
appendGeneratedScene,
createSession,
deleteSession,
listSessions,
Expand Down Expand Up @@ -45,6 +46,26 @@ describe("chat-service sessions", () => {
expect(got2?.messages).toEqual([]);
});

it("records a generated image in its own session", () => {
const project = "proj_generated_scene";
const session = createSession(project);
const messages = appendGeneratedScene(
project,
session.id,
"A violet sphere",
"/tmp/source.png",
);
expect(selectSession(project, session.id)?.messages).toEqual([
messages.userMessage,
messages.assistantMessage,
]);
expect(messages.assistantMessage.generatedImagePath).toBe("/tmp/source.png");
expect(listSessions(project)[0].messageCount).toBe(2);
expect(() => appendGeneratedScene(project, "other-session", "x", "/tmp/x.png")).toThrow(
"unavailable",
);
});

it("selectSession returns null for an unknown sessionId", () => {
expect(selectSession(projectId, "nope")).toBeNull();
});
Expand Down
28 changes: 28 additions & 0 deletions electron/ai-edition/chat-service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,34 @@ export function selectSession(projectId: string, sessionId: string): ChatSession
};
}

/** Record a generated artifact in the current in-memory conversation. */
export function appendGeneratedScene(
projectId: string,
sessionId: string,
prompt: string,
imagePath: string,
): { userMessage: AiEditionChatMessage; assistantMessage: AiEditionChatMessage } {
const session = sessionsByProject.get(projectId)?.get(sessionId);
if (!session) throw new Error("Chat session unavailable.");
const now = new Date().toISOString();
const userMessage: AiEditionChatMessage = {
id: randomUUID(),
role: "user",
content: prompt,
createdAt: now,
};
const assistantMessage: AiEditionChatMessage = {
id: randomUUID(),
role: "assistant",
content:
"Generated a five-second still-image scene. The original image is saved beside the video clip.",
createdAt: new Date().toISOString(),
generatedImagePath: imagePath,
};
session.messages.push(userMessage, assistantMessage);
return { userMessage, assistantMessage };
}

export function renameSession(
projectId: string,
sessionId: string,
Expand Down
45 changes: 45 additions & 0 deletions electron/ai-edition/codex/app-server.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -279,6 +279,51 @@ describe("Codex app-server subscription transport", () => {
expect(f.sent.some((packet) => packet.method === "turn/interrupt")).toBe(false);
});

it("generates one image through the subscription capability without editor tools", async () => {
const f = fixture((packet, emit) => {
if (packet.method === "modelProvider/capabilities/read") {
emit({ id: packet.id, result: { imageGeneration: true } });
return true;
}
if (packet.method === "turn/start") {
emit({ id: packet.id, result: { turn: { id: "image-turn" } } });
emit({
method: "item/completed",
params: {
threadId: "thread-1",
item: { type: "imageGeneration", status: "completed", savedPath: "C:/image.png" },
},
});
emit({
method: "turn/completed",
params: { threadId: "thread-1", turn: { id: "image-turn", status: "completed" } },
});
return true;
}
return false;
});
expect(await f.runtime.generateImage("A violet sphere")).toEqual({
savedPath: "C:/image.png",
});
expect(f.sent.find((packet) => packet.method === "thread/start")?.params).toMatchObject({
sandbox: "read-only",
dynamicTools: [],
});
expect(f.sent.find((packet) => packet.method === "turn/start")?.params).toMatchObject({
input: [{ type: "text", text: "$imagegen A violet sphere", text_elements: [] }],
});
});

it("does not start an image turn when the subscription capability is absent", async () => {
const f = fixture((packet, emit) => {
if (packet.method !== "modelProvider/capabilities/read") return false;
emit({ id: packet.id, result: { imageGeneration: false } });
return true;
});
await expect(f.runtime.generateImage("A violet sphere")).rejects.toThrow("unavailable");
expect(f.sent.some((packet) => packet.method === "turn/start")).toBe(false);
});

it("interrupts timed-out turns and releases their thread", async () => {
const f = fixture((packet, emit) => {
if (packet.method !== "turn/start") return false;
Expand Down
39 changes: 38 additions & 1 deletion electron/ai-edition/codex/app-server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ export interface CodexRun {
input: string;
tools?: CodexTool[];
onText?: (text: string) => void;
onImageGeneration?: (item: ObjectValue) => void;
timeoutMs?: number;
signal?: AbortSignal;
}
interface PendingRequest {
Expand All @@ -40,6 +42,7 @@ interface PendingRequest {
interface ActiveTurn {
tools: Map<string, CodexTool>;
onText?: (text: string) => void;
onImageGeneration?: (item: ObjectValue) => void;
text: string;
turnId?: string;
toolCalls: number;
Expand Down Expand Up @@ -260,6 +263,9 @@ export class CodexAppServer {
if (payload.method === "item/agentMessage/delta" && typeof params.delta === "string") {
turn.text += params.delta;
turn.onText?.(params.delta);
} else if (payload.method === "item/completed") {
const item = object(params.item);
if (item.type === "imageGeneration") turn.onImageGeneration?.(item);
} else if (payload.method === "turn/started") {
turn.turnId = String(object(params.turn).id ?? "");
} else if (payload.method === "turn/completed") {
Expand Down Expand Up @@ -438,6 +444,7 @@ export class CodexAppServer {
const turn: ActiveTurn = {
tools: new Map((input.tools ?? []).map((tool) => [tool.name, tool])),
onText: input.onText,
onImageGeneration: input.onImageGeneration,
text: "",
toolCalls: 0,
resolve,
Expand All @@ -452,7 +459,7 @@ export class CodexAppServer {
}
timer = setTimeout(
() => reject(new Error("Codex AI request timed out. Please retry.")),
this.turnTimeoutMs,
input.timeoutMs ?? this.turnTimeoutMs,
);
void this.request("turn/start", {
threadId,
Expand All @@ -477,6 +484,36 @@ export class CodexAppServer {
});
}
}

/** Built-in Codex image generation, using only the app-owned ChatGPT subscription profile. */
async generateImage(prompt: string): Promise<{ savedPath?: string; result?: string }> {
const trimmed = prompt.trim();
if (!trimmed || trimmed.length > 4_000)
throw new Error("Image prompt must be 1–4,000 characters.");
const status = await this.status();
if (!status.connected)
throw new Error(status.error ?? "Connect your ChatGPT account in AI settings first.");
const capabilities = await this.request("modelProvider/capabilities/read");
if (capabilities.imageGeneration !== true)
throw new Error("Image generation is unavailable for this ChatGPT subscription.");
const images: ObjectValue[] = [];
await this.run({
instructions:
"Create exactly one still image for a video scene. Use only built-in image generation. " +
"Do not use shell, code, web, files, apps, or editor tools. Do not edit the project.",
input: `$imagegen ${trimmed}`,
tools: [],
timeoutMs: 360_000,
onImageGeneration: (item) => images.push(item),
});
if (images.length !== 1 || images[0].status !== "completed")
throw new Error("Codex did not return one completed image.");
const image = images[0];
return {
...(typeof image.savedPath === "string" ? { savedPath: image.savedPath } : {}),
...(typeof image.result === "string" ? { result: image.result } : {}),
};
}
}

let singleton: Promise<CodexAppServer> | undefined;
Expand Down
12 changes: 12 additions & 0 deletions electron/ai-edition/document-service.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,18 @@ describe("DocumentService", () => {
expect(updated.project.primaryAssetId).toBe(asset?.id);
});

it("stores a verified duration supplied for a generated clip", async () => {
const doc = await service.createProject("P");
const updated = await service.addAsset(doc.project.id, {
path: "/tmp/scene.mp4",
durationSec: 5,
});
expect(updated.assets[0]?.durationSec).toBe(5);
await expect(
service.addAsset(doc.project.id, { path: "/tmp/bad.mp4", durationSec: -1 }),
).rejects.toThrow("duration must be positive");
});

it("resolves relative paths against the cwd", async () => {
const doc = await service.createProject("P");
const updated = await service.addAsset(doc.project.id, { path: "recording.webm" });
Expand Down
7 changes: 7 additions & 0 deletions electron/ai-edition/document-service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ export interface ProjectSummary {
export interface AddAssetInput {
path: string;
label?: string;
durationSec?: number;
}

export class DocumentNotFoundError extends Error {
Expand Down Expand Up @@ -289,6 +290,11 @@ export class DocumentService {
projectId,
);
}
if (
input.durationSec !== undefined &&
(!Number.isFinite(input.durationSec) || input.durationSec <= 0)
)
throw new ProjectFileError("Asset duration must be positive and finite.", projectId);
const absolutePath = path.isAbsolute(input.path) ? input.path : path.resolve(input.path);
// P3.1 — capture the file size at import. Non-fatal: a stat failure
// (network drive, permissions) just leaves sizeBytes undefined.
Expand All @@ -303,6 +309,7 @@ export class DocumentService {
kind: "video",
label: input.label?.trim() || path.basename(absolutePath),
originalPath: absolutePath,
...(input.durationSec !== undefined ? { durationSec: input.durationSec } : {}),
sizeBytes,
cameraTrack: null,
};
Expand Down
71 changes: 71 additions & 0 deletions electron/ai-edition/generated-scene.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
import { encodeStillScene, saveGeneratedImage } from "./generated-scene";

const roots: string[] = [];
afterEach(async () => {
for (const root of roots.splice(0)) {
if (path.dirname(root) !== os.tmpdir() || !path.basename(root).startsWith("studio-scene-test-"))
throw new Error("Unsafe test cleanup path");
await fs.rm(root, { recursive: true, force: true });
}
});

async function temporaryRoot(): Promise<string> {
const root = await fs.mkdtemp(path.join(os.tmpdir(), "studio-scene-test-"));
roots.push(root);
return root;
}

function pngBytes(): Buffer {
const bytes = Buffer.alloc(256);
Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]).copy(bytes);
return bytes;
}

describe("generated still scene assets", () => {
it("copies a Codex image to app-owned project storage and retains the source", async () => {
const root = await temporaryRoot();
const source = path.join(root, "codex-output.png");
await fs.writeFile(source, pngBytes());
const saved = await saveGeneratedImage(root, "proj_123", { savedPath: source });
expect(saved.imagePath).toContain(path.join("generated-scenes", "proj_123"));
expect(saved.videoPath).toBe(saved.imagePath.replace(/\.png$/, ".mp4"));
expect(await fs.readFile(saved.imagePath)).toEqual(pngBytes());
expect(await fs.readFile(source)).toEqual(pngBytes());
});

it("accepts an inline image but rejects traversal and non-image data", async () => {
const root = await temporaryRoot();
const inline = `data:image/png;base64,${pngBytes().toString("base64")}`;
expect(await saveGeneratedImage(root, "proj_123", { result: inline })).toMatchObject({
imagePath: expect.stringMatching(/\.png$/),
});
await expect(saveGeneratedImage(root, "../other", { result: inline })).rejects.toThrow(
"Invalid project id",
);
await expect(
saveGeneratedImage(root, "proj_123", { result: Buffer.alloc(256).toString("base64") }),
).rejects.toThrow("unsupported or invalid");
});

it("encodes a five-second clip and falls back when OpenH264 is missing", async () => {
const root = await temporaryRoot();
const imagePath = path.join(root, "source.png");
const videoPath = path.join(root, "scene.mp4");
await fs.writeFile(imagePath, pngBytes());
const run = vi.fn(async (_exe: string, args: string[]) => {
if (args.includes("libopenh264")) throw new Error("Unknown encoder 'libopenh264'");
await fs.writeFile(args.at(-1)!, Buffer.alloc(2_000));
});
await encodeStillScene(imagePath, videoPath, "ffmpeg", run);
expect(run).toHaveBeenCalledTimes(2);
expect(run.mock.calls[0][1]).toContain("libopenh264");
expect(run.mock.calls[1][1]).toContain("mpeg4");
expect(run.mock.calls[1][1]).toContain("120");
expect((await fs.stat(videoPath)).size).toBe(2_000);
expect(await fs.readFile(imagePath)).toEqual(pngBytes());
});
});
Loading
Loading