diff --git a/apps/server/src/orchestration-v2/CommandPolicy.test.ts b/apps/server/src/orchestration-v2/CommandPolicy.test.ts index 14a803b1d1d60..c18f7136eb643 100644 --- a/apps/server/src/orchestration-v2/CommandPolicy.test.ts +++ b/apps/server/src/orchestration-v2/CommandPolicy.test.ts @@ -234,6 +234,23 @@ it("does not steer a Grok follow-up by interrupting its running turn", () => { } }); +// regression: #37326 a steered Grok follow-up waits for the turn to end instead of interjecting +it("steers a Grok follow-up into its running turn without restarting it", () => { + const grok = makeGrokAcpAdapterFlavor( + { makeRuntime: () => Effect.never } as unknown as Parameters< + typeof makeGrokAcpAdapterFlavor + >[0], + "/home/ada", + ).capabilities; + const mode = CommandPolicy.resolveMessageDispatchIntent( + dispatchProjection(grok), + { type: "start_immediately" }, + "steer", + ); + assert.strictEqual(mode.type, "steer_active"); + assert.isTrue(grok.turns.supportsActiveSteering); +}); + it.each(["/compact", " /LOGOUT "])("queues a steer sent while %s runs", (text) => { const projection = dispatchProjection(baseCapabilities); const maintenance = { diff --git a/packages/provider-acp/src/server/adapter.ts b/packages/provider-acp/src/server/adapter.ts index b2a28c68c6c08..d642cfe200cbd 100644 --- a/packages/provider-acp/src/server/adapter.ts +++ b/packages/provider-acp/src/server/adapter.ts @@ -501,14 +501,16 @@ export interface AcpAdapterV2Flavor { /** * Native steering: `send` queues a message into the running turn and returns the * agent's id for it, which `injectedSteerId` reads from the update that delivers it. + * No id means the running turn reads the message once the agent accepts it. */ readonly steering?: { readonly send: ( runtime: AcpSessionRuntime.AcpSessionRuntime["Service"], sessionId: string, text: string, - ) => Effect.Effect; - readonly injectedSteerId: ( + images: ReadonlyArray, + ) => Effect.Effect; + readonly injectedSteerId?: ( notification: EffectAcpSchema.SessionNotification, ) => string | undefined; }; @@ -6638,7 +6640,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( context.nativePromptId ??= nativePromptId; } } - const injectedSteerId = flavor.steering?.injectedSteerId(rawNotification); + const injectedSteerId = flavor.steering?.injectedSteerId?.(rawNotification); if (injectedSteerId !== undefined) { yield* Deferred.succeed(steerInjection(injectedSteerId), undefined); } @@ -8321,6 +8323,38 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( } }); + const imageAttachmentBlocks = Effect.fnUntraced(function* ( + attachments: ReadonlyArray, + ) { + const blocks: Array = []; + for (const attachment of attachments.filter(isProviderNativeImageAttachment)) { + const path = host.resolveAttachmentPath(attachment as ChatAttachment); + if (path === null) { + return yield* new ProviderAdapter.ProviderAdapterProtocolError({ + driver, + detail: `Invalid attachment id '${attachment.id}'`, + }); + } + const bytes = yield* fileSystem.readFile(path).pipe( + Effect.mapError( + (cause) => + new ProviderAdapter.ProviderAdapterProtocolError({ + driver, + detail: `Failed to read attachment '${attachment.id}'`, + reason: "attachment_unreadable", + payload: cause, + }), + ), + ); + blocks.push({ + type: "image", + data: Buffer.from(bytes).toString("base64"), + mimeType: attachment.mimeType, + }); + } + return blocks; + }); + const resolvePromptParts = Effect.fnUntraced(function* ( turnInput: ProviderAdapter.ProviderAdapterV2TurnInput, sessionId: string, @@ -8349,41 +8383,18 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( resolveExtractedTextPath: host.resolveExtractedTextPath, }); const isNativeSlashCommand = userText.trimStart().startsWith("/"); - const imageAttachments = turnInput.message.attachments.filter( - isProviderNativeImageAttachment, - ); - if (imageAttachments.length > 0 && !supportsImagePrompts) { + if ( + !supportsImagePrompts && + turnInput.message.attachments.some(isProviderNativeImageAttachment) + ) { return yield* new ProviderAdapter.ProviderAdapterProtocolError({ driver, detail: "ACP driver did not negotiate image prompt support", }); } - const attachmentBlocks: Array = []; - for (const attachment of flavor.buildAttachmentPrompt ? [] : imageAttachments) { - const path = host.resolveAttachmentPath(attachment as ChatAttachment); - if (path === null) { - return yield* new ProviderAdapter.ProviderAdapterProtocolError({ - driver, - detail: `Invalid attachment id '${attachment.id}'`, - }); - } - const bytes = yield* fileSystem.readFile(path).pipe( - Effect.mapError( - (cause) => - new ProviderAdapter.ProviderAdapterProtocolError({ - driver, - detail: `Failed to read attachment '${attachment.id}'`, - reason: "attachment_unreadable", - payload: cause, - }), - ), - ); - attachmentBlocks.push({ - type: "image", - data: Buffer.from(bytes).toString("base64"), - mimeType: attachment.mimeType, - }); - } + const attachmentBlocks = flavor.buildAttachmentPrompt + ? [] + : yield* imageAttachmentBlocks(turnInput.message.attachments); if (flavor.buildAttachmentPrompt) { const setup = yield* Ref.get(activeSessionSetup); const modelOption = findModelConfigOption(yield* runtime.getConfigOptions); @@ -9262,8 +9273,12 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( steerTurn: Effect.fn("AcpAdapterV2.steerTurn")( function* (turnInput) { const steering = flavor.steering; - // The agent takes text only, so a steer with attachments waits for the next turn. - if (steering === undefined || turnInput.message.attachments.length > 0) { + // A text-only steer leaves attachments to the orchestrator's restart. + if ( + steering === undefined || + (turnInput.message.attachments.length > 0 && + flavor.capabilities.turns.activeSteeringAcceptsAttachments === false) + ) { return yield* new ProviderAdapter.ProviderAdapterSteerRunUnsupportedError({ driver, providerThreadId: turnInput.providerThread.id, @@ -9287,11 +9302,19 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( Effect.timeoutOption(startHold.bound), ); } + const text = yield* providerMessageTextWithAttachmentPaths({ + text: turnInput.message.text, + attachments: turnInput.message.attachments, + resolveAttachmentPath: host.resolveAttachmentPath, + resolveExtractedTextPath: host.resolveExtractedTextPath, + }); const steerId = yield* steering.send( runtime, context.nativeThreadId, - turnInput.message.text, + text, + yield* imageAttachmentBlocks(turnInput.message.attachments), ); + if (steerId === undefined) return; const injected = steerInjection(steerId); // The agent reads a steer at its next step. One that lands after the last // step is dropped, and the turn settles right after; the orchestrator then diff --git a/packages/provider-grok/src/server/adapter.test.ts b/packages/provider-grok/src/server/adapter.test.ts index 74ac3e1ca441d..d4a8819e8de14 100644 --- a/packages/provider-grok/src/server/adapter.test.ts +++ b/packages/provider-grok/src/server/adapter.test.ts @@ -31,7 +31,7 @@ import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderS import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; import * as TestProviderHost from "@t3tools/provider-testing/TestProviderHost"; import { GrokSettings } from "../settings.ts"; -import { makeGrokAdapterV2 } from "./adapter.ts"; +import { makeGrokAcpAdapterFlavor, makeGrokAdapterV2 } from "./adapter.ts"; const layerTest = Layer.mergeAll( NodeServices.layer, @@ -263,3 +263,37 @@ describe("Grok reasoning effort", () => { }), ); }); + +it.effect("a steer reaches Grok as an interjection that carries its images", () => + Effect.gen(function* () { + const requests: Array<{ readonly method: string; readonly params: unknown }> = []; + let reply: unknown = { result: { status: "queued" } }; + const runtime = { + request: (method: string, params: unknown) => { + requests.push({ method, params }); + return Effect.succeed(reply); + }, + } as unknown as AcpSessionRuntime.AcpSessionRuntime["Service"]; + const steering = makeGrokAcpAdapterFlavor( + { makeRuntime: () => Effect.never } as unknown as Parameters< + typeof makeGrokAcpAdapterFlavor + >[0], + "/home/ada", + ).steering!; + const image = { type: "image", data: "aGk=", mimeType: "image/png" } as const; + + assert.isUndefined(yield* steering.send(runtime, "session-1", "Look at this", [image])); + yield* steering.send(runtime, "session-1", "Text only", []); + assert.deepEqual(requests, [ + { + method: "_x.ai/interject", + params: { sessionId: "session-1", text: "Look at this", content: [image] }, + }, + { method: "_x.ai/interject", params: { sessionId: "session-1", text: "Text only" } }, + ]); + + reply = { result: { status: "rejected" } }; + const refused = yield* steering.send(runtime, "session-1", "Late", []).pipe(Effect.flip); + assert.strictEqual(refused._tag, "AcpRequestError"); + }), +); diff --git a/packages/provider-grok/src/server/adapter.ts b/packages/provider-grok/src/server/adapter.ts index 0e8d9b6e04714..db5a136697c10 100644 --- a/packages/provider-grok/src/server/adapter.ts +++ b/packages/provider-grok/src/server/adapter.ts @@ -117,10 +117,46 @@ const GrokProviderCapabilitiesV2 = { ...AcpProviderCapabilitiesV2.checkpointing, providerCanReadConversationSnapshot: true, }, - // Cancelling a Grok prompt kills its running foreground tool. - turns: { ...AcpProviderCapabilitiesV2.turns, supportsSteeringByInterruptRestart: false }, + turns: { + ...AcpProviderCapabilitiesV2.turns, + // `_x.ai/interject` lets the running tool finish; the model reads the message at its next step. + supportsActiveSteering: true, + activeSteeringInterruptsTools: false, + // Cancelling a Grok prompt kills its running foreground tool. + supportsSteeringByInterruptRestart: false, + }, } satisfies OrchestrationV2ProviderCapabilities; +const XAI_INTERJECT_METHOD = "_x.ai/interject"; +const isXAiInterjectQueued = Schema.is( + Schema.Struct({ result: Schema.Struct({ status: Schema.Literal("queued") }) }), +); + +/** A queued interjection joins the running prompt, so its acceptance is the delivery. */ +const grokSteering: NonNullable = { + send: (runtime, sessionId, text, images) => + runtime + .request(XAI_INTERJECT_METHOD, { + sessionId, + text, + ...(images.length === 0 ? {} : { content: images }), + }) + .pipe( + Effect.flatMap((result) => + isXAiInterjectQueued(result) + ? Effect.undefined + : Effect.fail( + new EffectAcpErrors.AcpRequestError({ + code: -32603, + errorMessage: "Grok did not queue the interjection.", + method: XAI_INTERJECT_METHOD, + data: result, + }), + ), + ), + ), +}; + export interface GrokAdapterV2Options { readonly instanceId: Parameters[0]["instanceId"]; readonly settings: GrokSettings; @@ -330,6 +366,7 @@ export function makeGrokAcpAdapterFlavor( // Grok accepts image blocks despite advertising promptCapabilities.image:false. supportsImagePrompts: true, supportsCompaction: true, + steering: grokSteering, // Grok expands only a leading `/name`, while the `$` picker inserts `$name` from the // `grok inspect` catalog. Ask that catalog again so other `$` text is left alone. rewriteUserText: (text, turnInput) => {