diff --git a/apps/server/src/orchestration-v2/ContextHandoffDelivery.ts b/apps/server/src/orchestration-v2/ContextHandoffDelivery.ts index 5211da2faf169..72471eb8cf944 100644 --- a/apps/server/src/orchestration-v2/ContextHandoffDelivery.ts +++ b/apps/server/src/orchestration-v2/ContextHandoffDelivery.ts @@ -37,7 +37,7 @@ export const deliverContextHandoffs = Effect.fn("orchestrationV2.deliverContextH handoff.delivery.status === "pending", ); if (pending.length === 0 || (input.deferInline && input.inject === undefined)) - return { context: "", delivered: Effect.void, unsent: Effect.void }; + return { context: "", delivered: Effect.void, unsent: Effect.void, inlineHandoffs: [] }; const budget = typeof input.budget === "number" ? input.budget : yield* input.budget; let coverage = pending .map( @@ -76,7 +76,8 @@ export const deliverContextHandoffs = Effect.fn("orchestrationV2.deliverContextH budget, }); if (historyCost(selected.messages, selected.context) > budget) { - if (input.deferInline) return { context: "", delivered: Effect.void, unsent: Effect.void }; + if (input.deferInline) + return { context: "", delivered: Effect.void, unsent: Effect.void, inlineHandoffs: [] }; return yield* new ContextHandoffBudgetError(); } const omittedItemIds = new Set(selected.omittedItemIds); @@ -127,7 +128,7 @@ export const deliverContextHandoffs = Effect.fn("orchestrationV2.deliverContextH }); if (injected) { yield* persist("injected"); - return { context: "", delivered: Effect.void, unsent: Effect.void }; + return { context: "", delivered: Effect.void, unsent: Effect.void, inlineHandoffs: [] }; } } else { yield* persist("pending"); @@ -135,12 +136,13 @@ export const deliverContextHandoffs = Effect.fn("orchestrationV2.deliverContextH if (input.deferInline) { // Compaction cannot carry inline transcripts; retain for the next turn if native injection is unsupported. yield* Effect.forEach(pending, input.persist, { discard: true }); - return { context: "", delivered: Effect.void, unsent: Effect.void }; + return { context: "", delivered: Effect.void, unsent: Effect.void, inlineHandoffs: [] }; } return { context: renderHistory(selected.messages, selected.context), delivered: persist("inline"), unsent: Effect.forEach(pending, input.persist, { discard: true }), + inlineHandoffs: pending, }; }, ); diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 4f1d2d671060e..995ff63b25747 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -28,6 +28,7 @@ import { CommandId, type GuestMessageId, isCompactionCommandMessage, + isDeliveredProviderTurn, isOrchestrationV2WorkActive, isProviderNativeSubagentThread, MessageId, @@ -1599,7 +1600,9 @@ function lastDeliveredRunForProviderThread( isHandoffSourceRun(run) && run.providerThreadId === providerThreadId && (run.status === "completed" || - projection.providerTurns.some((turn) => turn.runAttemptId === run.activeAttemptId)), + projection.providerTurns.some( + (turn) => turn.runAttemptId === run.activeAttemptId && isDeliveredProviderTurn(turn), + )), ); } diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index e47e88eda09a0..8e35271ce5aa9 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -16,6 +16,7 @@ import { type OrchestrationV2ThreadProjection, type OrchestrationV2TurnItem, isCompactionCommandMessage, + isDeliveredProviderTurn, parseGoalCommand, RunId, RunAttemptId, @@ -32,6 +33,7 @@ import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as Predicate from "effect/Predicate"; import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; import * as ServerConfig from "../config.ts"; import { @@ -173,7 +175,9 @@ const nativeSessionMissesKeptTurns = ( ): boolean => { const nativeId = providerThread.nativeThreadRef?.nativeId; if (nativeId === undefined) return false; - const acceptedAttemptIds = new Set(projection.providerTurns.map((turn) => turn.runAttemptId)); + const acceptedAttemptIds = new Set( + projection.providerTurns.filter(isDeliveredProviderTurn).map((turn) => turn.runAttemptId), + ); const accepted = projection.attempts.filter((attempt) => acceptedAttemptIds.has(attempt.id)); const runsById = new Map(projection.runs.map((source) => [source.id, source])); const ordinalOf = (attempt: OrchestrationV2RunAttempt) => @@ -1087,7 +1091,12 @@ export const layer: Layer.Layer< ...settledHandoffs.flatMap((handoff) => handoff.delivery?.omittedItemIds ?? []), ]); const deliveredAttemptIds = new Set( - projection.providerTurns.map((turn) => turn.runAttemptId), + projection.providerTurns.filter(isDeliveredProviderTurn).map((turn) => turn.runAttemptId), + ); + const droppedAttemptIds = new Set( + projection.providerTurns + .filter((turn) => !isDeliveredProviderTurn(turn)) + .map((turn) => turn.runAttemptId), ); const acceptedAttempts = projection.attempts.filter( (source) => @@ -1159,7 +1168,9 @@ export const layer: Layer.Layer< (source) => source.ordinal < run.ordinal && source.providerThreadId === providerThread.id && - (source.status === "failed" || source.status === "interrupted") && + (source.status === "failed" || + source.status === "interrupted" || + (source.status === "cancelled" && droppedAttemptIds.has(source.activeAttemptId))) && !deliveredAttemptIds.has(source.activeAttemptId), ); const missedRunIds = new Set(missedRuns.map((source) => source.id)); @@ -1536,7 +1547,9 @@ export const layer: Layer.Layer< const rejectedHistory = yield* Effect.gen(function* () { if (session.driver !== "codex") return false; const acceptedAttemptIds = new Set( - projection.providerTurns.map((turn) => turn.runAttemptId), + projection.providerTurns + .filter(isDeliveredProviderTurn) + .map((turn) => turn.runAttemptId), ); const previous = projection.runs.findLast( (source) => @@ -1605,7 +1618,9 @@ export const layer: Layer.Layer< // Dispatch prepares no import handoff for a session import: its history is in the native session. const importedNativeSession = projection.thread.historyOrigin === "v1_import" && - !projection.providerTurns.some((turn) => turn.providerThreadId === providerThread.id) && + !projection.providerTurns.some( + (turn) => turn.providerThreadId === providerThread.id && isDeliveredProviderTurn(turn), + ) && !projection.contextHandoffs.some( (handoff) => handoff.toProviderThreadId === providerThread.id && @@ -2025,6 +2040,10 @@ export const layer: Layer.Layer< ], }); }); + let inlineDeliveredHandoffs: ReadonlyArray = []; + let promptDroppedOnTerminal = false; + const inlineDeliveryLock = yield* Semaphore.make(1); + const retryHandoffIds = new Set(retryHandoff.handoffs.map((handoff) => handoff.id)); const startWithHandoffs = Effect.fn("ProviderTurnStartService.startWithHandoffs")( function* (turnInput: Parameters[0], compact = false) { const delivery = yield* deliverContextHandoffs({ @@ -2051,6 +2070,15 @@ export const layer: Layer.Layer< sideChatHandoff === undefined || compact || goalCommand ? "" : sideChatHandoff.summaryText; + // Record inline handoffs before startTurn so an immediate dropped terminal can restore them. + inlineDeliveredHandoffs = [ + ...delivery.inlineHandoffs.map((handoff) => + retryHandoffIds.has(handoff.id) + ? { ...handoff, status: "superseded" as const } + : handoff, + ), + ...(sideChatContext === "" || sideChatHandoff === undefined ? [] : [sideChatHandoff]), + ]; const context = [delivery.context, sideChatContext, restartNote] .filter((part) => part !== "") .join("\n\n"); @@ -2101,31 +2129,39 @@ export const layer: Layer.Layer< : Effect.void, ), ); - // After provider acceptance, failed delivery bookkeeping must not stop live ingestion. - const nativeThreadId = runningProviderThread.nativeThreadRef?.nativeId; - yield* Effect.forEach( - [ - delivery.delivered, - ...(sideChatContext === "" || sideChatHandoff === undefined || nativeThreadId == null - ? [] - : [ - persistHandoff({ - ...sideChatHandoff, - toProviderThreadId: runningProviderThread.id, - delivery: { nativeThreadId, status: "inline" as const, itemIds: [] }, - }), - ]), - ], - (record) => - record.pipe( - Effect.catchCause(() => - Effect.logWarning("Failed to record accepted context handoff delivery", { - runId: run.id, - deliveryStatus: "pending", - }), - ), - ), - { discard: true }, + // Serialize acceptance writes against dropped-terminal restoration so neither overwrites the other. + yield* inlineDeliveryLock.withPermits(1)( + Effect.gen(function* () { + if (promptDroppedOnTerminal) return; + // After provider acceptance, failed delivery bookkeeping must not stop live ingestion. + const nativeThreadId = runningProviderThread.nativeThreadRef?.nativeId; + yield* Effect.forEach( + [ + delivery.delivered, + ...(sideChatContext === "" || + sideChatHandoff === undefined || + nativeThreadId == null + ? [] + : [ + persistHandoff({ + ...sideChatHandoff, + toProviderThreadId: runningProviderThread.id, + delivery: { nativeThreadId, status: "inline" as const, itemIds: [] }, + }), + ]), + ], + (record) => + record.pipe( + Effect.catchCause(() => + Effect.logWarning("Failed to record accepted context handoff delivery", { + runId: run.id, + deliveryStatus: "pending", + }), + ), + ), + { discard: true }, + ); + }), ); }, Effect.mapError((cause) => @@ -2186,6 +2222,13 @@ export const layer: Layer.Layer< hasUnpairedRunInterruptRequest: runControls.hasUnpairedRunInterruptRequest, loadRunInterruptReason: runControls.loadRunInterruptReason, hasWorkspaceMoved: runControls.hasWorkspaceMoved, + loadDroppedInlineHandoffs: () => + inlineDeliveryLock.withPermits(1)( + Effect.sync(() => { + promptDroppedOnTerminal = true; + return inlineDeliveredHandoffs; + }), + ), loadUnsettledWork: runControls.loadUnsettledWork, message: { messageId: message.id, diff --git a/apps/server/src/orchestration-v2/RestartBackgroundNote.ts b/apps/server/src/orchestration-v2/RestartBackgroundNote.ts index ceaf2fd5de54b..58918bdef612d 100644 --- a/apps/server/src/orchestration-v2/RestartBackgroundNote.ts +++ b/apps/server/src/orchestration-v2/RestartBackgroundNote.ts @@ -1,14 +1,15 @@ -import type { - NodeId, - OrchestrationV2ConversationMessage, - OrchestrationV2PendingBackgroundTask, - OrchestrationV2ProviderThread, - OrchestrationV2ProviderTurn, - OrchestrationV2RestartCancelledBackgroundWork, - OrchestrationV2Run, - OrchestrationV2RunAttempt, - OrchestrationV2Subagent, - OrchestrationV2TurnItem, +import { + isDeliveredProviderTurn, + type NodeId, + type OrchestrationV2ConversationMessage, + type OrchestrationV2PendingBackgroundTask, + type OrchestrationV2ProviderThread, + type OrchestrationV2ProviderTurn, + type OrchestrationV2RestartCancelledBackgroundWork, + type OrchestrationV2Run, + type OrchestrationV2RunAttempt, + type OrchestrationV2Subagent, + type OrchestrationV2TurnItem, } from "@t3tools/contracts"; import { subagentHoldsCompletion } from "@t3tools/provider-core/server/subagentProjection"; import { backgroundWorkHoldsCompletion } from "@t3tools/shared/orchestrationV2PendingBackgroundWork"; @@ -151,7 +152,7 @@ export function restartCancelledBackgroundWorkNote( type ProviderTurnState = Pick< OrchestrationV2ProviderTurn, - "runAttemptId" | "providerThreadId" | "status" + "runAttemptId" | "providerThreadId" | "status" | "promptDropped" >; export function isRestartNoteSource( @@ -307,12 +308,14 @@ export function delegatedTaskResultRun( /** The provider never accepted this continuation's prompt, so starting it again repeats nothing. */ export function continuationFailedBeforePrompt( run: Pick, - providerTurns: ReadonlyArray>, + providerTurns: ReadonlyArray>, ): boolean { return ( run.status === "failed" && run.restartContinuationOfRunId !== undefined && - !providerTurns.some((turn) => turn.runAttemptId === run.activeAttemptId) + !providerTurns.some( + (turn) => turn.runAttemptId === run.activeAttemptId && isDeliveredProviderTurn(turn), + ) ); } @@ -323,7 +326,9 @@ export function continuationStartRetryDelayMs( run: OrchestrationV2Run, projection: { readonly runs: ReadonlyArray; - readonly providerTurns: ReadonlyArray>; + readonly providerTurns: ReadonlyArray< + Pick + >; readonly turnItems: Parameters[1]; }, ): number | undefined { diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index a23a3fdf87bff..1d13620b75623 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -8,6 +8,7 @@ import { NodeId, type OrchestrationV2AppThread, type OrchestrationV2CheckpointScope, + type OrchestrationV2ContextHandoff, type OrchestrationV2DomainEvent, type OrchestrationV2ExecutionNode, type OrchestrationV2PendingBackgroundTask, @@ -3702,6 +3703,49 @@ it.effect.each([ }), ); +it.effect("restores dropped inline handoffs when ingestion fails before the terminal", () => + Effect.gen(function* () { + const key = "dropped-turn-ingest-error"; + const ids = backgroundScenarioIds(key); + const handoff = { id: `handoff:${key}` } as unknown as OrchestrationV2ContextHandoff; + const { finalEvents } = yield* captureRootRunTermination({ + key, + shouldFinalizeRun: () => Effect.succeed(true), + loadDroppedInlineHandoffs: () => Effect.succeed([handoff]), + events: () => + Stream.make({ + type: "provider_turn.updated" as const, + driver, + threadId: ids.threadId, + providerTurn: { + id: ids.rootProviderTurnId, + providerThreadId: ids.providerThreadId, + runAttemptId: ids.attemptId, + ordinal: 1, + status: "failed", + promptDropped: true, + } as OrchestrationV2ProviderTurn, + }), + ingestNormalized: ({ event }) => + event.type === "provider_turn.updated" + ? Effect.fail( + new ProviderEventIngestor.ProviderEventPublishError({ + providerSessionId: ProviderSessionId.make(`session:${key}`), + eventCount: 1, + cause: new Error("SQLITE_FULL: database or disk is full"), + }), + ) + : Effect.succeed([]), + }); + assert.deepEqual( + finalEvents.flatMap((event) => + event.type === "context-handoff.updated" ? [event.payload.id] : [], + ), + [handoff.id], + ); + }), +); + it.effect("does not interrupt a provider when ingestion fails before the root turn is known", () => Effect.gen(function* () { const interrupts = yield* Ref.make(0); @@ -3989,6 +4033,7 @@ function captureRootRunTermination(input: { readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect; readonly hasWorkspaceMoved?: () => Effect.Effect; readonly loadRunInterruptReason?: () => Effect.Effect; + readonly loadDroppedInlineHandoffs?: RunExecutionService.RunExecutionServiceV2StartRootRunInput["loadDroppedInlineHandoffs"]; readonly hasPendingBackgroundWorkForThread?: ProviderAdapter.ProviderAdapterV2SessionRuntime["hasPendingBackgroundWorkForThread"]; readonly seedOpenSubagent?: boolean; readonly events?: ( @@ -4235,6 +4280,9 @@ function captureRootRunTermination(input: { ...(input.hasWorkspaceMoved === undefined ? {} : { hasWorkspaceMoved: input.hasWorkspaceMoved }), + ...(input.loadDroppedInlineHandoffs === undefined + ? {} + : { loadDroppedInlineHandoffs: input.loadDroppedInlineHandoffs }), message: { messageId: MessageId.make(`message:${input.key}`), text: input.messageText ?? "interrupt projection", diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 1bc83f639d5ae..34cc31f98fa09 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -9,6 +9,7 @@ import { type NodeId, type OrchestrationV2AppThread, type OrchestrationV2CheckpointScope, + type OrchestrationV2ContextHandoff, type OrchestrationV2DomainEvent, type OrchestrationV2ExecutionNode, type OrchestrationV2ProviderFailure, @@ -658,6 +659,11 @@ export interface RunExecutionServiceV2StartRootRunInput { readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect; readonly loadRunInterruptReason?: () => Effect.Effect; readonly hasWorkspaceMoved?: () => Effect.Effect; + /** Restores inline context handoffs if the provider drops this turn's prompt before persisting it. */ + readonly loadDroppedInlineHandoffs?: () => Effect.Effect< + ReadonlyArray, + never + >; /** * Stored rows of a thread's unsettled work. Work a dead runtime left is ended from these, since * Stop may have settled it after the snapshots this run holds. @@ -736,6 +742,7 @@ export const layer: Layer.Layer< readonly abandonedProviderTurn?: OrchestrationV2ProviderTurn; // Stored root tool calls still open when the provider runtime was lost; nothing else ends them. readonly lostToolCalls?: ReadonlyArray; + readonly restoredHandoffs?: ReadonlyArray; readonly terminal: ProviderTerminalEvent; readonly completedAt?: DateTime.Utc; readonly failureItemPersisted: boolean; @@ -752,24 +759,35 @@ export const layer: Layer.Layer< status: input.terminal.status, completedAt, }; + const allocateEventId = () => idAllocator.allocate.event({ threadId: input.run.threadId }); + const buildRestoredHandoffEvents = () => + input.terminal.promptDropped === true && + input.restoredHandoffs !== undefined && + input.restoredHandoffs.length > 0 + ? Effect.forEach(input.restoredHandoffs, (handoff) => + Effect.map(allocateEventId(), (id) => ({ + id, + type: "context-handoff.updated" as const, + threadId: input.run.threadId, + runId: input.run.id, + providerInstanceId: input.run.providerInstanceId, + occurredAt: completedAt, + payload: { ...handoff, updatedAt: completedAt }, + })), + ) + : Effect.succeed([]); const shouldFinalizeRun = input.shouldFinalizeRun === undefined ? true : yield* input.shouldFinalizeRun(); if (!shouldFinalizeRun) { - if (input.terminal.status === "interrupted") { - const hasUnpairedRequest = - input.hasUnpairedRunInterruptRequest === undefined - ? false - : yield* input.hasUnpairedRunInterruptRequest(); - if (hasUnpairedRequest) { - const reason = - input.loadRunInterruptReason === undefined - ? undefined - : yield* input.loadRunInterruptReason(); - yield* eventSink.writeWithEffects({ - effects: [], - events: [ + const restoredHandoffEvents = yield* buildRestoredHandoffEvents(); + const unpairedInterruptEvents = + input.terminal.status === "interrupted" && + (input.hasUnpairedRunInterruptRequest === undefined + ? false + : yield* input.hasUnpairedRunInterruptRequest()) + ? [ { - id: yield* idAllocator.allocate.event({ threadId: input.run.threadId }), + id: yield* allocateEventId(), type: "turn-item.updated" as const, threadId: input.run.threadId, runId: input.run.id, @@ -781,18 +799,25 @@ export const layer: Layer.Layer< run: input.run, rootNode: input.rootNode, providerThread: input.providerThread, - reason, + reason: + input.loadRunInterruptReason === undefined + ? undefined + : yield* input.loadRunInterruptReason(), completedAt, }), }, - ], - }); - yield* input.refreshAfterTurn; - } + ] + : []; + const earlyEvents = [...unpairedInterruptEvents, ...restoredHandoffEvents]; + if (earlyEvents.length > 0) { + yield* eventSink.writeWithEffects({ + effects: [], + events: earlyEvents, + }); + yield* input.refreshAfterTurn; } return false; } - const allocateEventId = () => idAllocator.allocate.event({ threadId: input.run.threadId }); const reason = input.terminal.status === "interrupted" ? ((input.loadRunInterruptReason === undefined @@ -839,6 +864,7 @@ export const layer: Layer.Layer< status: finalProviderThreadStatus(input.terminal.threadDisposition), updatedAt: completedAt, }; + const restoredHandoffEvents = yield* buildRestoredHandoffEvents(); const runEventId = yield* allocateEventId(); const nodeEventId = yield* allocateEventId(); const providerThreadEventId = yield* allocateEventId(); @@ -1005,6 +1031,7 @@ export const layer: Layer.Layer< occurredAt: completedAt, payload: finalizedProviderThread, }, + ...restoredHandoffEvents, ], } satisfies Parameters[0]; if (input.writeIfRunCurrent !== undefined) { @@ -1031,6 +1058,13 @@ export const layer: Layer.Layer< effects: finalization.effects, }); if (!result.committed) { + if (restoredHandoffEvents.length > 0) { + yield* eventSink.writeWithEffects({ + effects: [], + events: restoredHandoffEvents, + }); + yield* input.refreshAfterTurn; + } return false; } } else { @@ -1275,44 +1309,55 @@ export const layer: Layer.Layer< } const providerThread = yield* Ref.get(latestProviderThread); const openSubagents = yield* Ref.get(openRunOwnedSubagents); - const writeTerminal = (settledTerminal: ProviderTerminalEvent) => - writeFinalRunEvents({ - run: input.run, - rootNode: input.rootNode, - checkpointScope: input.checkpointScope, - providerThread, - attempt: input.attempt, - ...(input.shouldFinalizeRun === undefined - ? {} - : { - shouldFinalizeRun: input.shouldFinalizeRun, - // Stop may commit between the ownership read and this - // terminal write. Gate the events and checkpoint together. - writeIfRunCurrent: { - activeAttemptId: input.attempt.id, - expectedStatus: "running" as const, - unlessInterruptRequested: settledTerminal !== terminal, - }, - }), - ...(input.hasUnpairedRunInterruptRequest === undefined - ? {} - : { - hasUnpairedRunInterruptRequest: input.hasUnpairedRunInterruptRequest, - }), - ...(input.loadRunInterruptReason === undefined - ? {} - : { loadRunInterruptReason: input.loadRunInterruptReason }), - openRunOwnedSubagents: openSubagents, - openRunOwnedSubagentsTerminal: terminal, - ...(abandonedProviderTurn === undefined ? {} : { abandonedProviderTurn }), - terminal: settledTerminal, - failureItemPersisted: settledTerminal.status === "failed", - refreshAfterTurn, - }).pipe( - Effect.mapError( - (cause) => new RunExecutionIngestError({ runId: input.run.id, cause }), + const writeTerminal = Effect.fnUntraced( + function* (settledTerminal: ProviderTerminalEvent) { + const restoredHandoffs = + settledTerminal.promptDropped === true && + input.loadDroppedInlineHandoffs !== undefined + ? yield* input.loadDroppedInlineHandoffs() + : undefined; + return yield* writeFinalRunEvents({ + run: input.run, + rootNode: input.rootNode, + checkpointScope: input.checkpointScope, + providerThread, + attempt: input.attempt, + ...(input.shouldFinalizeRun === undefined + ? {} + : { + shouldFinalizeRun: input.shouldFinalizeRun, + // Stop may commit between the ownership read and this + // terminal write. Gate the events and checkpoint together. + writeIfRunCurrent: { + activeAttemptId: input.attempt.id, + expectedStatus: "running" as const, + unlessInterruptRequested: settledTerminal !== terminal, + }, + }), + ...(input.hasUnpairedRunInterruptRequest === undefined + ? {} + : { + hasUnpairedRunInterruptRequest: input.hasUnpairedRunInterruptRequest, + }), + ...(input.loadRunInterruptReason === undefined + ? {} + : { loadRunInterruptReason: input.loadRunInterruptReason }), + openRunOwnedSubagents: openSubagents, + openRunOwnedSubagentsTerminal: terminal, + ...(abandonedProviderTurn === undefined ? {} : { abandonedProviderTurn }), + ...(restoredHandoffs === undefined ? {} : { restoredHandoffs }), + terminal: settledTerminal, + failureItemPersisted: settledTerminal.status === "failed", + refreshAfterTurn, + }); + }, + (effect) => + effect.pipe( + Effect.mapError( + (cause) => new RunExecutionIngestError({ runId: input.run.id, cause }), + ), ), - ); + ); // A workspace handoff detaches the session and stops the turn that asked // for it: the run completes, while its native subagents still settle // as interrupted. The completed write refuses in the same transaction @@ -2116,6 +2161,13 @@ export const layer: Layer.Layer< : null; const latestItemOrdinal = yield* Ref.get(latestTurnItemOrdinal); const openSubagents = yield* Ref.get(openRunOwnedSubagents); + const droppedPrompt = + (yield* Ref.get(terminalEvent))?.promptDropped === true || + rootProviderTurn?.promptDropped === true; + const recoveredHandoffs = + droppedPrompt && input.loadDroppedInlineHandoffs !== undefined + ? yield* input.loadDroppedInlineHandoffs() + : undefined; const lostToolCalls = runtimeLost ? yield* loadLostRootToolCalls.pipe( Effect.catchCauseIf( @@ -2141,18 +2193,24 @@ export const layer: Layer.Layer< }, openRunOwnedSubagents: openSubagents, ...(abandonedProviderTurn === null ? {} : { abandonedProviderTurn }), + ...(recoveredHandoffs === undefined + ? {} + : { restoredHandoffs: recoveredHandoffs }), lostToolCalls, completedAt, - terminal: makeFailedTerminalEvent( - makeProviderFailure({ - cause: squashed, - message: - knownProviderFailureMessage(squashed) ?? - userFacingDatabaseErrorMessage(squashed), - class: "unknown", - }), - latestItemOrdinal + 1, - ), + terminal: { + ...makeFailedTerminalEvent( + makeProviderFailure({ + cause: squashed, + message: + knownProviderFailureMessage(squashed) ?? + userFacingDatabaseErrorMessage(squashed), + class: "unknown", + }), + latestItemOrdinal + 1, + ), + ...(droppedPrompt ? { promptDropped: true as const } : {}), + }, failureItemPersisted: false, refreshAfterTurn, }).pipe( diff --git a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts index ee048c2100592..ddf9259622599 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts @@ -118,6 +118,7 @@ function makeTestAdapter(input: { readonly failure?: ReturnType; readonly rejectedNativeThreads?: Ref.Ref>; readonly failedRunOrdinals?: ReadonlySet; + readonly promptDroppedRunOrdinals?: ReadonlySet; readonly brokenRunOrdinals?: ReadonlySet; readonly interruptedRunOrdinals?: ReadonlySet; readonly cancelledRunOrdinals?: ReadonlySet; @@ -410,6 +411,9 @@ function makeTestAdapter(input: { status: terminalStatus, startedAt: eventTime, completedAt: eventTime, + ...(input.promptDroppedRunOrdinals?.has(turnInput.runOrdinal) + ? { promptDropped: true as const } + : {}), }, }, { @@ -462,6 +466,9 @@ function makeTestAdapter(input: { threadDisposition: input.brokenRunOrdinals?.has(turnInput.runOrdinal) ? "broken" : "reusable", + ...(input.promptDroppedRunOrdinals?.has(turnInput.runOrdinal) + ? { promptDropped: true as const } + : {}), }, ]; const reportedUsage = input.tokenUsageByRunOrdinal?.[turnInput.runOrdinal]; @@ -6400,4 +6407,166 @@ describe("orchestration v2 provider switching", () => { ); }), ); + + it.effect.each([ + "same-provider-retry", + "cancelled-dropped", + "switch-away-and-return", + "persisted-failure", + ] as const)("handles an inline context handoff after a failed returning turn (%s)", (variant) => + Effect.gen(function* () { + const cwd = yield* checkpointWorkspace(`dropped-handoff-${variant}`); + const capturedTurns = yield* Ref.make>([]); + const dropped = variant !== "persisted-failure"; + const cancelled = variant === "cancelled-dropped"; + const layerRegistry = ProviderAdapterRegistry.layerFromAdapters([ + makeTestAdapter({ + instanceId: CODEX_MODEL_SELECTION.instanceId, + driver: CODEX_DRIVER, + capabilities: CodexProviderCapabilitiesV2, + modelSelection: CODEX_MODEL_SELECTION, + responseByRunOrdinal: { + 1: "Morning Codex summary.", + 3: "", + 4: "Recovered Codex answer.", + 5: "Recovered Codex after round-trip.", + }, + capturedTurns, + ...(cancelled + ? { cancelledRunOrdinals: new Set([3]) } + : { failedRunOrdinals: new Set([3]) }), + ...(dropped ? { promptDroppedRunOrdinals: new Set([3]) } : {}), + }), + makeTestAdapter({ + instanceId: CLAUDE_MODEL_SELECTION.instanceId, + driver: CLAUDE_DRIVER, + capabilities: ClaudeProviderCapabilitiesV2, + modelSelection: CLAUDE_MODEL_SELECTION, + responseByRunOrdinal: { + 2: "Afternoon Claude shipped b63 live draft.", + 4: "Extra Claude turn after dropped handoff.", + }, + capturedTurns, + }), + ]); + + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const testThreadId = ThreadId.make(`thread:dropped-handoff:${variant}`); + yield* orchestrator.dispatch({ + type: "thread.create", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make(`command:dropped-handoff:${variant}:create`), + threadId: testThreadId, + projectId, + title: `Dropped handoff ${variant}`, + modelSelection: CODEX_MODEL_SELECTION, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: cwd, + }); + let step = 0; + const send = (text: string, modelSelection: ModelSelection) => + Effect.gen(function* () { + step += 1; + yield* orchestrator.dispatch({ + type: "message.dispatch", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make(`command:dropped-handoff:${variant}:msg-${step}`), + threadId: testThreadId, + messageId: MessageId.make(`message:dropped-handoff:${variant}:${step}`), + text, + attachments: [], + modelSelection, + dispatchMode: { type: "start_immediately" }, + }); + yield* worker.drain(); + yield* waitForIdle(testThreadId); + yield* worker.drain(); + return yield* orchestrator.getThreadProjection(testThreadId); + }); + + // Run 1: establish native session on Codex. + yield* send("Morning task", CODEX_MODEL_SELECTION); + // Run 2: switch to Claude and do real work. + yield* send("Build live draft b63", CLAUDE_MODEL_SELECTION); + // Run 3: switch back to Codex (delta handoff covering run 2); fails. + const afterRun3 = yield* send("Resume after failure", CODEX_MODEL_SELECTION); + const run3 = afterRun3.runs.find((run) => run.ordinal === 3)!; + assert.equal(run3.status, cancelled ? "cancelled" : "failed"); + const run3Handoff = afterRun3.contextHandoffs.find( + (handoff) => handoff.targetRunId === run3.id, + ); + assert.isDefined(run3Handoff); + assert.deepEqual(run3Handoff?.coveredRunOrdinals, { from: 2, to: 2 }); + + if (dropped) { + assert.isUndefined( + run3Handoff?.delivery, + "dropping the failed prompt must clear inline handoff delivery", + ); + const run3Turn = afterRun3.providerTurns.find( + (turn) => turn.runAttemptId === run3.activeAttemptId, + ); + assert.equal(run3Turn?.promptDropped, true); + } else { + assert.equal(run3Handoff?.delivery?.status, "inline"); + } + + if (variant === "same-provider-retry" || variant === "cancelled-dropped") { + // Run 4: stay on Codex; must re-deliver run 2's handoff. + const afterRun4 = yield* send("Resume", CODEX_MODEL_SELECTION); + const turns = yield* Ref.get(capturedTurns); + const run4Turn = turns.at(-1)!; + assert.include(run4Turn.text, "Afternoon Claude shipped b63 live draft."); + assert.include(run4Turn.text, "Resume after failure"); + assert.include(run4Turn.text, "User message:\nResume"); + const updatedHandoff = afterRun4.contextHandoffs.find( + (handoff) => handoff.id === run3Handoff!.id, + ); + assert.equal(updatedHandoff?.delivery?.status, "inline"); + } else if (variant === "switch-away-and-return") { + // Run 4: bounce back to Claude, then Run 5: return to Codex. + yield* send("Continue on Claude", CLAUDE_MODEL_SELECTION); + const afterRun5 = yield* send("Back to Codex", CODEX_MODEL_SELECTION); + const run5 = afterRun5.runs.find((run) => run.ordinal === 5)!; + const run5Handoff = afterRun5.contextHandoffs.find( + (handoff) => handoff.targetRunId === run5.id, + ); + assert.isDefined(run5Handoff); + assert.deepEqual(run5Handoff?.coveredRunOrdinals, { from: 2, to: 4 }); + const run5Turn = (yield* Ref.get(capturedTurns)).at(-1)!; + assert.include(run5Turn.text, "Afternoon Claude shipped b63 live draft."); + assert.include(run5Turn.text, "Extra Claude turn after dropped handoff."); + } else { + // Control: run 3 persisted its prompt before failing, so run 4 must not re-send run 2. + yield* send("Resume", CODEX_MODEL_SELECTION); + const run4Turn = (yield* Ref.get(capturedTurns)).at(-1)!; + assert.notInclude(run4Turn.text, "Afternoon Claude shipped b63 live draft."); + } + }).pipe( + Effect.provide( + ProviderReplayHarness.layerWithRegistry( + { + name: `dropped-handoff-${variant}`, + runtimePolicyOverride: { + cwd, + approvalPolicy: "never", + sandboxPolicy: { + type: "readOnly", + access: { type: "fullAccess" }, + networkAccess: false, + }, + }, + }, + layerRegistry, + ), + ), + ); + }), + ); }); diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index 2d5bc546d1800..8b05c12e17871 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -1334,9 +1334,17 @@ export const OrchestrationV2ProviderTurn = Schema.Struct({ turnTokenUsage: Schema.optional(TurnTokenUsage), /** The model the provider reported running this turn's root agent, which can differ from the requested one. */ model: Schema.optional(TrimmedNonEmptyString), + /** True when the provider dropped this turn's request before persisting it to native history. */ + promptDropped: Schema.optional(Schema.Literal(true)), }); export type OrchestrationV2ProviderTurn = typeof OrchestrationV2ProviderTurn.Type; +export function isDeliveredProviderTurn( + turn: Pick, +): boolean { + return turn.promptDropped !== true; +} + /** Codex goals can span multiple native turns within one attempt. */ export function latestProviderTurnForAttempt< Turn extends Pick, diff --git a/packages/provider-core/src/server/ProviderAdapter.ts b/packages/provider-core/src/server/ProviderAdapter.ts index 18ebf74baa1c7..c0bf5156acb97 100644 --- a/packages/provider-core/src/server/ProviderAdapter.ts +++ b/packages/provider-core/src/server/ProviderAdapter.ts @@ -202,6 +202,8 @@ export const ProviderAdapterV2Event = Schema.Union([ failure: Schema.Null, reason: Schema.optional(Schema.String), threadDisposition: Schema.Literals(["reusable", "broken"]), + /** The provider dropped this turn's request before persisting it to native history. */ + promptDropped: Schema.optional(Schema.Literal(true)), retainedRuntimeRequestIds: Schema.optional(Schema.Array(RuntimeRequestId)), }), Schema.Struct({ @@ -216,6 +218,8 @@ export const ProviderAdapterV2Event = Schema.Union([ retry: Schema.optional(OrchestrationV2ProviderRetry), retryStartedAt: Schema.optional(Schema.DateTimeUtc), threadDisposition: Schema.Literals(["reusable", "broken"]), + /** The provider dropped this turn's request before persisting it to native history. */ + promptDropped: Schema.optional(Schema.Literal(true)), /** Child turns outlive root failure and report their own terminals; otherwise open children settle. */ childTurnsContinue: Schema.optional(Schema.Literal(true)), retainedRuntimeRequestIds: Schema.optional(Schema.Array(RuntimeRequestId)), diff --git a/packages/provider-cursor/src/server/adapter.test.ts b/packages/provider-cursor/src/server/adapter.test.ts index 5c0a1940d671c..dfa1ac4865269 100644 --- a/packages/provider-cursor/src/server/adapter.test.ts +++ b/packages/provider-cursor/src/server/adapter.test.ts @@ -756,6 +756,149 @@ describe("CursorAdapterV2", () => { ), ); + it.effect("marks failed turns as promptDropped only when Cursor did not checkpoint the run", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const workspace = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "cursor-v2-dropped-", + }); + const checkpointedRuns = new Set(); + let sendCalls = 0; + const instanceId = ProviderInstanceId.make("cursor"); + const threadId = ThreadId.make("cursor-dropped-thread"); + const modelSelection = { instanceId, model: "gemini-4-neon" }; + const runtimePolicy = ProviderAdapter.ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: workspace, + }); + const adapter = yield* makeCursorAdapterV2({ + instanceId, + environment: { HOME: workspace }, + }).pipe( + Effect.provideService(CursorAgentSdk.CursorAgentSdkRunner, { + assertComplete: Effect.void, + open: () => + Effect.succeed({ + agentId: "native-cursor-dropped", + listMessages: Effect.succeed([]), + close: Effect.void, + runCheckpointed: (runId) => + Effect.succeed(runId !== undefined && checkpointedRuns.has(runId)), + send: () => + Effect.sync(() => { + sendCalls += 1; + const runId = `native-cursor-run-${sendCalls}`; + return { + agentId: "native-cursor-dropped", + runId, + cancel: Effect.void, + wait: Effect.succeed({ + id: runId, + status: "error" as const, + error: { + message: + 'AI Model Not Found Invalid parameters for registry model: "gemini-4-neon"', + }, + }), + }; + }), + }), + textGenerate: () => Effect.die("unused"), + }), + Effect.provide(TestProviderHost.layer({ cwd: workspace })), + ); + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("cursor-dropped-session"), + modelSelection, + runtimePolicy, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + const appThread = { + id: threadId, + projectId: ProjectId.make("cursor-dropped-project"), + createdBy: "user" as const, + creationSource: "web" as const, + title: "Cursor dropped prompt", + providerInstanceId: instanceId, + modelSelection, + runtimeMode: "full-access" as const, + interactionMode: "default" as const, + branch: null, + worktreePath: null, + activeProviderThreadId: providerThread.id, + lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: threadId }, + forkedFrom: null, + createdAt: now, + updatedAt: now, + archivedAt: null, + settledOverride: null, + settledAt: null, + lastVisitedAt: null, + deletedAt: null, + }; + const runFailed = (ordinal: number) => + Effect.gen(function* () { + yield* runtime.startTurn({ + threadId, + providerThread, + modelSelection, + runtimePolicy, + runId: RunId.make(`cursor-dropped-run-${ordinal}`), + runOrdinal: ordinal, + providerTurnOrdinal: ordinal, + attemptId: RunAttemptId.make(`cursor-dropped-attempt-${ordinal}`), + rootNodeId: NodeId.make(`cursor-dropped-root-${ordinal}`), + appThread, + message: { + messageId: MessageId.make(`cursor-dropped-message-${ordinal}`), + createdBy: "user", + creationSource: "web", + text: "Resume", + attachments: [], + }, + }); + const events = yield* runtime.events.pipe( + Stream.takeUntil((event) => event.type === "turn.terminal"), + Stream.runCollect, + ); + const terminal = events.find((event) => event.type === "turn.terminal"); + const turn = events.findLast( + (event) => + event.type === "provider_turn.updated" && event.providerTurn.completedAt !== null, + ); + return { + terminal: terminal?.type === "turn.terminal" ? terminal : undefined, + turn: turn?.type === "provider_turn.updated" ? turn.providerTurn : undefined, + }; + }); + + // Uncheckpointed failure: Cursor never persisted the prompt. + const uncheckpointed = yield* runFailed(1); + assert.equal(uncheckpointed.terminal?.status, "failed"); + assert.equal(uncheckpointed.terminal?.promptDropped, true); + assert.equal(uncheckpointed.turn?.promptDropped, true); + + // Checkpointed failure: Cursor kept the run's prompt in its store. + checkpointedRuns.add("native-cursor-run-2"); + const checkpointed = yield* runFailed(2); + assert.equal(checkpointed.terminal?.status, "failed"); + assert.isUndefined(checkpointed.terminal?.promptDropped); + assert.isUndefined(checkpointed.turn?.promptDropped); + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll(NodeServices.layer, IdAllocator.layer, McpProviderSessions.layer), + ), + ), + ); + it.effect("sends T3 instructions to each Cursor conversation that lacks them", () => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; diff --git a/packages/provider-cursor/src/server/adapter.ts b/packages/provider-cursor/src/server/adapter.ts index 7aadadaf9ddd4..b4dd1033c2a9f 100644 --- a/packages/provider-cursor/src/server/adapter.ts +++ b/packages/provider-cursor/src/server/adapter.ts @@ -2510,6 +2510,7 @@ export const makeCursorAdapterV2 = Effect.fn("makeCursorAdapterV2")(function* ( readonly status: OrchestrationV2ProviderTurn["status"]; readonly completedAt: DateTime.Utc | null; readonly tokenUsage?: OrchestrationV2ProviderTurn["tokenUsage"]; + readonly promptDropped?: true; }): OrchestrationV2ProviderTurn => ({ id: input.context.providerTurnId, providerThreadId: input.context.input.providerThread.id, @@ -2525,6 +2526,7 @@ export const makeCursorAdapterV2 = Effect.fn("makeCursorAdapterV2")(function* ( startedAt: input.context.startedAt, completedAt: input.completedAt, ...(input.tokenUsage === undefined ? {} : { tokenUsage: input.tokenUsage }), + ...(input.promptDropped === true ? { promptDropped: true as const } : {}), ...(input.completedAt === null || input.context.usage === undefined ? {} : { @@ -2590,6 +2592,15 @@ export const makeCursorAdapterV2 = Effect.fn("makeCursorAdapterV2")(function* ( }).pipe(Effect.as(undefined)), ), ); + // Cursor only keeps a run in conversation history once it saves its first checkpoint. + const promptDropped = + (input.status === "failed" || input.status === "cancelled") && + !input.context.compacted && + input.context.prompt !== undefined && + !(yield* ( + input.context.session.runCheckpointed?.(input.context.run.runId) ?? + Effect.succeed(false) + ).pipe(Effect.orElseSucceed(() => false))); yield* emitProviderEvent({ type: "provider_turn.updated", driver: CursorAgentSdk.CURSOR_PROVIDER, @@ -2597,6 +2608,7 @@ export const makeCursorAdapterV2 = Effect.fn("makeCursorAdapterV2")(function* ( context: input.context, status: input.status, completedAt, + ...(promptDropped ? { promptDropped: true as const } : {}), ...(contextUsage === undefined ? {} : { @@ -2633,6 +2645,7 @@ export const makeCursorAdapterV2 = Effect.fn("makeCursorAdapterV2")(function* ( status: input.status, failure: input.failure ?? makeProviderFailure({ class: "provider_error" }), threadDisposition, + ...(promptDropped ? { promptDropped: true as const } : {}), } : { type: "turn.terminal", @@ -2643,6 +2656,7 @@ export const makeCursorAdapterV2 = Effect.fn("makeCursorAdapterV2")(function* ( status: input.status, failure: null, threadDisposition, + ...(promptDropped ? { promptDropped: true as const } : {}), }, ); if (input.context.runtimeInstructions !== undefined) { diff --git a/packages/shared/src/orchestrationV2ManualContinuation.test.ts b/packages/shared/src/orchestrationV2ManualContinuation.test.ts index 137bf6cd8f205..e7670749f6c5a 100644 --- a/packages/shared/src/orchestrationV2ManualContinuation.test.ts +++ b/packages/shared/src/orchestrationV2ManualContinuation.test.ts @@ -108,4 +108,16 @@ describe("unsentManualContinuationSource", () => { unsentManualContinuationSource(run("c", "a" as RunId), projection([a, b], [], [])), ).toBeUndefined(); }); + + it("treats a turn with promptDropped as unsent", () => { + const source = run("src"); + const a1 = attempt("a1", "src", 1); + const dropped = { ...turn("a1"), promptDropped: true as const }; + expect( + unsentManualContinuationSource( + run("r2", "src" as RunId), + projection([source], [a1], [dropped]), + ), + ).toBe(source); + }); }); diff --git a/packages/shared/src/orchestrationV2ManualContinuation.ts b/packages/shared/src/orchestrationV2ManualContinuation.ts index 3fb6c8d5a1866..ad64b63483e14 100644 --- a/packages/shared/src/orchestrationV2ManualContinuation.ts +++ b/packages/shared/src/orchestrationV2ManualContinuation.ts @@ -1,4 +1,8 @@ -import type { OrchestrationV2Run, OrchestrationV2ThreadProjection } from "@t3tools/contracts"; +import { + isDeliveredProviderTurn, + type OrchestrationV2Run, + type OrchestrationV2ThreadProjection, +} from "@t3tools/contracts"; export function unsentManualContinuationSource( run: Pick | undefined, @@ -7,7 +11,9 @@ export function unsentManualContinuationSource( let sourceId = run?.manualContinuationOfRunId; if (sourceId === undefined) return undefined; const runs = new Map(projection.runs.map((source) => [source.id, source])); - const acceptedAttempts = new Set(projection.providerTurns.map((turn) => turn.runAttemptId)); + const acceptedAttempts = new Set( + projection.providerTurns.filter(isDeliveredProviderTurn).map((turn) => turn.runAttemptId), + ); const latestAttempt = ( runId: OrchestrationV2Run["id"], matches: (attempt: (typeof projection.attempts)[number]) => boolean,