Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 52 additions & 4 deletions apps/desktop/src/main/services/chat/agentChatService.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42858,6 +42858,49 @@ it("fails a cleanly ended OpenCode event stream and clears active child sessions
expect(doneEvent.event.modelId).toBe("droid/custom:claude-sonnet-5-thinking-32000");
});

it("sends Droid screenshots as attachment paths over worker IPC", async () => {
const events: AgentChatEventEnvelope[] = [];
const { service } = createService({
onEvent: (event: AgentChatEventEnvelope) => events.push(event),
});
const imagePath = path.join(tmpRoot, "droid-shot.png");
fs.writeFileSync(imagePath, Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]));

const session = await service.createSession({
laneId: "lane-1",
provider: "droid",
model: "custom:claude-sonnet-5-thinking-32000",
modelId: "droid/custom:claude-sonnet-5-thinking-32000",
});

await service.sendMessage({
sessionId: session.id,
text: "Look at this screenshot.",
attachments: [{ path: imagePath, type: "image" }],
}, { awaitDispatch: true });

await waitForEvent(
events,
(event): event is AgentChatEventEnvelope & {
event: Extract<AgentChatEventEnvelope["event"], { type: "done" }>;
} => event.event.type === "done" && event.sessionId === session.id,
);

const sentImages = mockState.droidPromptCalls[0]?.images as Array<{
path?: string;
data?: string;
mimeType?: string;
rootPath?: string;
}> | undefined;
expect(sentImages).toHaveLength(1);
expect(sentImages?.[0]?.data).toBeUndefined();
expect(sentImages?.[0]?.mimeType).toBe("image/png");
expect(sentImages?.[0]?.rootPath).toBe(tmpRoot);
expect(path.basename(sentImages?.[0]?.path ?? "")).toBe("droid-shot.png");
expect(String(mockState.droidPromptCalls[0]?.promptText ?? "")).toContain("Look at this screenshot.");
expect(String(mockState.droidPromptCalls[0]?.promptText ?? "")).not.toMatch(/iVBORw0KGgo/u);
});

it("uses Droid spec mode for ADE plan mode", async () => {
const events: AgentChatEventEnvelope[] = [];
const { service } = createService({
Expand Down Expand Up @@ -44542,8 +44585,11 @@ describe("orchestrator-lead provider-native tool denial", () => {
expect(mockState.droidAcquireCalls.at(-1)?.settings).toMatchObject({
disabledToolCategories: ["edit", "execute"],
});
expect(mockState.droidPromptCalls.at(-1)?.settings).toMatchObject({
disabledToolCategories: ["edit", "execute"],
// awaitDispatch returns at onDispatched, which is before sendPrompt.
await vi.waitFor(() => {
expect(mockState.droidPromptCalls.at(-1)?.settings).toMatchObject({
disabledToolCategories: ["edit", "execute"],
});
});

const worker = await service.createSession({
Expand All @@ -44554,8 +44600,10 @@ describe("orchestrator-lead provider-native tool denial", () => {
...workerArgs(created),
});
await service.sendMessage({ sessionId: worker.id, text: "Do the work." }, { awaitDispatch: true });
expect(mockState.droidPromptCalls.at(-1)?.settings)
.not.toHaveProperty("disabledToolCategories");
await vi.waitFor(() => {
expect(mockState.droidPromptCalls.at(-1)?.settings)
.not.toHaveProperty("disabledToolCategories");
});
} finally {
await orchestrationService.dispose();
}
Expand Down
16 changes: 7 additions & 9 deletions apps/desktop/src/main/services/chat/agentChatService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36705,7 +36705,7 @@ export function createAgentChatService(args: {
],
});

const buildPiWorkerPrompt = async (
const buildPathOnlyWorkerPrompt = async (
promptText: string,
resolvedAttachments: ResolvedAgentChatFileRef[],
): Promise<{ promptText: string; images: Array<{ path: string; mimeType: string; rootPath: string }> }> => {
Expand All @@ -36720,6 +36720,8 @@ export function createAgentChatService(args: {
images: pathImagesFromResolved(resolvedAttachments),
};
};
const buildPiWorkerPrompt = buildPathOnlyWorkerPrompt;
const buildDroidWorkerPrompt = buildPathOnlyWorkerPrompt;

const mapChatDecisionToDroidPermission = (
decision: AgentChatApprovalDecision | undefined,
Expand Down Expand Up @@ -39566,14 +39568,10 @@ export function createAgentChatService(args: {
"## User Request",
composed,
].join("\n");
const promptBlocks = await buildAgentPromptBlocks(sdkInput, args.resolvedAttachments);
const sdkPromptText = promptBlocks
.filter((block): block is { type: "text"; text: string } => block.type === "text")
.map((block) => block.text)
.join("\n\n");
const images = promptBlocks
.filter((block): block is { type: "image"; data: string; mimeType: string } => block.type === "image")
.map((block) => ({ data: block.data, mimeType: block.mimeType }));
const { promptText: sdkPromptText, images } = await buildDroidWorkerPrompt(
sdkInput,
args.resolvedAttachments,
);
const result = await runtime.sdk.sendPrompt({
promptText: sdkPromptText,
...(images.length ? { images } : {}),
Expand Down
52 changes: 52 additions & 0 deletions apps/desktop/src/main/services/chat/droidSdkPool.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,10 @@ class FakeSdkChild extends EventEmitter {
killed = false;
disposeCount = 0;
initPayloads: unknown[] = [];
sent: unknown[] = [];

send(message: { type?: string; requestId?: string; payload?: unknown }): boolean {
this.sent.push(message);
if (message.type === "init" && message.requestId) {
this.initPayloads.push(message.payload);
queueMicrotask(() => {
Expand All @@ -37,6 +39,16 @@ class FakeSdkChild extends EventEmitter {
});
});
}
if (message.type === "send" && message.requestId) {
queueMicrotask(() => {
this.emit("message", {
type: "response",
requestId: message.requestId,
ok: true,
result: {},
});
});
}
if (message.type === "dispose") {
this.disposeCount += 1;
}
Expand Down Expand Up @@ -134,6 +146,46 @@ describe("Droid SDK pool", () => {
expect(child.disposeCount).toBe(1);
});

it("sends screenshot paths over worker IPC instead of inline bytes", async () => {
const child = new FakeSdkChild();
forkMock.mockReturnValue(child);
const poolKey = `test-image-paths:${Date.now()}:${Math.random()}`;
const acquired = await acquireDroidSdkConnection({
poolKey,
droidPath: "/usr/local/bin/droid",
workspacePath: path.join(os.tmpdir(), "ade-workspace"),
sessionId: "session-1",
settings: {
modelId: "droid-model",
autonomyLevel: "medium",
interactionMode: "auto",
},
});

await acquired.pooled.sendPrompt({
promptText: "compare these screens",
images: [
{ path: "/repo/.ade/attachments/a.png", mimeType: "image/png", rootPath: "/repo" },
{ path: "/repo/.ade/attachments/b.png", mimeType: "image/png", rootPath: "/repo" },
],
settings: { modelId: "droid-model" },
});

const sendReq = child.sent.find((message) => (
message
&& typeof message === "object"
&& "type" in message
&& message.type === "send"
)) as { payload?: { images?: Array<{ path?: string; data?: string }> } } | undefined;
expect(sendReq?.payload?.images).toEqual([
{ path: "/repo/.ade/attachments/a.png", mimeType: "image/png", rootPath: "/repo" },
{ path: "/repo/.ade/attachments/b.png", mimeType: "image/png", rootPath: "/repo" },
]);
expect(sendReq?.payload?.images?.some((image) => image.data)).toBeFalsy();

releaseDroidSdkConnection(poolKey, acquired.generation);
});

it("rejects initialization instead of throwing when the worker IPC channel closes", async () => {
forkMock.mockReturnValue(new ExitingBeforeInitChild());
const poolKey = `test-exit:${Date.now()}:${Math.random()}`;
Expand Down
15 changes: 11 additions & 4 deletions apps/desktop/src/main/services/chat/droidSdkProtocol.ts
Original file line number Diff line number Diff line change
Expand Up @@ -120,10 +120,17 @@ export type DroidSdkWorkerInit = {
allowedMcpServerNames?: string[];
};

export type DroidSdkUserImage = {
data: string;
mimeType: string;
};
/**
* Worker-IPC image reference. Prefer `path` — never put multi-megabyte
* screenshot bytes on this object. The worker materializes `{ data, mimeType }`
* for `@factory/droid-sdk` locally. `data` remains for tests and tiny inline
* cases. Droid's stream API has no remote-URL image form, so `url` is not part
* of this union. Path images include `rootPath` so the worker re-opens through
* the attachment sandbox.
*/
export type DroidSdkUserImage =
| { path: string; mimeType: string; rootPath: string }
| { data: string; mimeType: string };

export type DroidSdkSendPrompt = {
promptText: string;
Expand Down
19 changes: 13 additions & 6 deletions apps/desktop/src/main/services/chat/droidSdkWorker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import { droidDisabledToolIdsForCategories, droidInteractionModeValue, droidMcpT
import { loadDroidSdk } from "../ai/droidSdkLoader";
import { summarizeDroidAskUser } from "./droidSdkAskUser";
import { ensureDroidSpawnsAreWindowless } from "./droidSdkWindowsHide";
import { materializeWorkerImages } from "./workerAttachmentImages";

// Must run before the SDK spawns `droid`; see droidSdkWindowsHide.ts.
ensureDroidSpawnsAreWindowless();
Expand Down Expand Up @@ -408,13 +409,19 @@ async function sendPrompt(payload: DroidSdkWorkerRequest & { type: "send" }): Pr
let tokenUsage: unknown = null;
let firstError: unknown = null;
try {
const images = payload.payload.images?.map((image) => ({
type: "base64" as const,
data: image.data,
mediaType: image.mimeType as DroidSdkTypes.Base64ImageSource["mediaType"],
}));
const materialized = await materializeWorkerImages(payload.payload.images, { label: "Droid SDK" });
const images = materialized.map((image) => {
if (!("data" in image)) {
throw new Error("Droid SDK image URLs are not supported.");
}
return {
type: "base64" as const,
data: image.data,
mediaType: image.mimeType as DroidSdkTypes.Base64ImageSource["mediaType"],
};
});
for await (const event of session.stream(payload.payload.promptText, {
...(images?.length ? { images } : {}),
...(images.length ? { images } : {}),
abortSignal: controller.signal,
})) {
if ((event as { type?: string }).type === "token_usage_update") tokenUsage = event;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@ export type WorkerMaterializedImage =
/**
* Path-only worker-IPC images. Never inline bytes — stuffing screenshot
* base64 through `child.send` JSON can fill the pipe and stall the turn.
* Remote URLs are a Cursor-only send shape and stay at that call site.
* Remote URLs are a Cursor-only send shape and stay at that call site;
* Pi and Droid turn image URLs into prompt text instead.
*/
export function workerPathImagesFromAttachments(
attachments: readonly WorkerPathImageSource[],
Expand Down
Loading
Loading