Skip to content
Merged
10 changes: 6 additions & 4 deletions apps/server/src/orchestration-v2/ContextHandoffDelivery.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ export const deliverContextHandoffs = Effect.fn("orchestrationV2.deliverContextH
handoff.delivery.status === "pending",
);
if (pending.length === 0 || (input.deferInline && input.inject === undefined))
return { context: "", delivered: Effect.void, unsent: Effect.void };
return { context: "", delivered: Effect.void, unsent: Effect.void, inlineHandoffs: [] };
const budget = typeof input.budget === "number" ? input.budget : yield* input.budget;
let coverage = pending
.map(
Expand Down Expand Up @@ -76,7 +76,8 @@ export const deliverContextHandoffs = Effect.fn("orchestrationV2.deliverContextH
budget,
});
if (historyCost(selected.messages, selected.context) > budget) {
if (input.deferInline) return { context: "", delivered: Effect.void, unsent: Effect.void };
if (input.deferInline)
return { context: "", delivered: Effect.void, unsent: Effect.void, inlineHandoffs: [] };
return yield* new ContextHandoffBudgetError();
}
const omittedItemIds = new Set(selected.omittedItemIds);
Expand Down Expand Up @@ -127,20 +128,21 @@ export const deliverContextHandoffs = Effect.fn("orchestrationV2.deliverContextH
});
if (injected) {
yield* persist("injected");
return { context: "", delivered: Effect.void, unsent: Effect.void };
return { context: "", delivered: Effect.void, unsent: Effect.void, inlineHandoffs: [] };
}
} else {
yield* persist("pending");
}
if (input.deferInline) {
// Compaction cannot carry inline transcripts; retain for the next turn if native injection is unsupported.
yield* Effect.forEach(pending, input.persist, { discard: true });
return { context: "", delivered: Effect.void, unsent: Effect.void };
return { context: "", delivered: Effect.void, unsent: Effect.void, inlineHandoffs: [] };
}
return {
context: renderHistory(selected.messages, selected.context),
delivered: persist("inline"),
unsent: Effect.forEach(pending, input.persist, { discard: true }),
inlineHandoffs: pending,
};
},
);
Expand Down
5 changes: 4 additions & 1 deletion apps/server/src/orchestration-v2/Orchestrator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import {
CommandId,
type GuestMessageId,
isCompactionCommandMessage,
isDeliveredProviderTurn,
isOrchestrationV2WorkActive,
isProviderNativeSubagentThread,
MessageId,
Expand Down Expand Up @@ -1599,7 +1600,9 @@ function lastDeliveredRunForProviderThread(
isHandoffSourceRun(run) &&
run.providerThreadId === providerThreadId &&
(run.status === "completed" ||
projection.providerTurns.some((turn) => turn.runAttemptId === run.activeAttemptId)),
projection.providerTurns.some(
(turn) => turn.runAttemptId === run.activeAttemptId && isDeliveredProviderTurn(turn),
)),
);
}

Expand Down
103 changes: 73 additions & 30 deletions apps/server/src/orchestration-v2/ProviderTurnStartService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import {
type OrchestrationV2ThreadProjection,
type OrchestrationV2TurnItem,
isCompactionCommandMessage,
isDeliveredProviderTurn,
parseGoalCommand,
RunId,
RunAttemptId,
Expand All @@ -32,6 +33,7 @@ import * as Option from "effect/Option";
import * as Path from "effect/Path";
import * as Predicate from "effect/Predicate";
import * as Schema from "effect/Schema";
import * as Semaphore from "effect/Semaphore";

import * as ServerConfig from "../config.ts";
import {
Expand Down Expand Up @@ -173,7 +175,9 @@ const nativeSessionMissesKeptTurns = (
): boolean => {
const nativeId = providerThread.nativeThreadRef?.nativeId;
if (nativeId === undefined) return false;
const acceptedAttemptIds = new Set(projection.providerTurns.map((turn) => turn.runAttemptId));
const acceptedAttemptIds = new Set(
projection.providerTurns.filter(isDeliveredProviderTurn).map((turn) => turn.runAttemptId),
);
const accepted = projection.attempts.filter((attempt) => acceptedAttemptIds.has(attempt.id));
const runsById = new Map(projection.runs.map((source) => [source.id, source]));
const ordinalOf = (attempt: OrchestrationV2RunAttempt) =>
Expand Down Expand Up @@ -1087,7 +1091,12 @@ export const layer: Layer.Layer<
...settledHandoffs.flatMap((handoff) => handoff.delivery?.omittedItemIds ?? []),
]);
const deliveredAttemptIds = new Set(
projection.providerTurns.map((turn) => turn.runAttemptId),
projection.providerTurns.filter(isDeliveredProviderTurn).map((turn) => turn.runAttemptId),
);
const droppedAttemptIds = new Set(
projection.providerTurns
.filter((turn) => !isDeliveredProviderTurn(turn))
.map((turn) => turn.runAttemptId),
);
const acceptedAttempts = projection.attempts.filter(
(source) =>
Expand Down Expand Up @@ -1159,7 +1168,9 @@ export const layer: Layer.Layer<
(source) =>
source.ordinal < run.ordinal &&
source.providerThreadId === providerThread.id &&
(source.status === "failed" || source.status === "interrupted") &&
(source.status === "failed" ||
source.status === "interrupted" ||
(source.status === "cancelled" && droppedAttemptIds.has(source.activeAttemptId))) &&
!deliveredAttemptIds.has(source.activeAttemptId),
);
const missedRunIds = new Set(missedRuns.map((source) => source.id));
Expand Down Expand Up @@ -1536,7 +1547,9 @@ export const layer: Layer.Layer<
const rejectedHistory = yield* Effect.gen(function* () {
if (session.driver !== "codex") return false;
const acceptedAttemptIds = new Set(
projection.providerTurns.map((turn) => turn.runAttemptId),
projection.providerTurns
.filter(isDeliveredProviderTurn)
.map((turn) => turn.runAttemptId),
);
const previous = projection.runs.findLast(
(source) =>
Expand Down Expand Up @@ -1605,7 +1618,9 @@ export const layer: Layer.Layer<
// Dispatch prepares no import handoff for a session import: its history is in the native session.
const importedNativeSession =
projection.thread.historyOrigin === "v1_import" &&
!projection.providerTurns.some((turn) => turn.providerThreadId === providerThread.id) &&
!projection.providerTurns.some(
(turn) => turn.providerThreadId === providerThread.id && isDeliveredProviderTurn(turn),
) &&
!projection.contextHandoffs.some(
(handoff) =>
handoff.toProviderThreadId === providerThread.id &&
Expand Down Expand Up @@ -2025,6 +2040,10 @@ export const layer: Layer.Layer<
],
});
});
let inlineDeliveredHandoffs: ReadonlyArray<OrchestrationV2ContextHandoff> = [];
let promptDroppedOnTerminal = false;
const inlineDeliveryLock = yield* Semaphore.make(1);
const retryHandoffIds = new Set(retryHandoff.handoffs.map((handoff) => handoff.id));
const startWithHandoffs = Effect.fn("ProviderTurnStartService.startWithHandoffs")(
function* (turnInput: Parameters<typeof session.startTurn>[0], compact = false) {
const delivery = yield* deliverContextHandoffs({
Expand All @@ -2051,6 +2070,15 @@ export const layer: Layer.Layer<
sideChatHandoff === undefined || compact || goalCommand
? ""
: sideChatHandoff.summaryText;
// Record inline handoffs before startTurn so an immediate dropped terminal can restore them.
inlineDeliveredHandoffs = [
...delivery.inlineHandoffs.map((handoff) =>
retryHandoffIds.has(handoff.id)
? { ...handoff, status: "superseded" as const }
: handoff,
),
...(sideChatContext === "" || sideChatHandoff === undefined ? [] : [sideChatHandoff]),
];
const context = [delivery.context, sideChatContext, restartNote]
.filter((part) => part !== "")
.join("\n\n");
Expand Down Expand Up @@ -2101,31 +2129,39 @@ export const layer: Layer.Layer<
: Effect.void,
),
);
// After provider acceptance, failed delivery bookkeeping must not stop live ingestion.
const nativeThreadId = runningProviderThread.nativeThreadRef?.nativeId;
yield* Effect.forEach(
[
delivery.delivered,
...(sideChatContext === "" || sideChatHandoff === undefined || nativeThreadId == null
? []
: [
persistHandoff({
...sideChatHandoff,
toProviderThreadId: runningProviderThread.id,
delivery: { nativeThreadId, status: "inline" as const, itemIds: [] },
}),
]),
],
(record) =>
record.pipe(
Effect.catchCause(() =>
Effect.logWarning("Failed to record accepted context handoff delivery", {
runId: run.id,
deliveryStatus: "pending",
}),
),
),
{ discard: true },
// Serialize acceptance writes against dropped-terminal restoration so neither overwrites the other.
yield* inlineDeliveryLock.withPermits(1)(
Effect.gen(function* () {
if (promptDroppedOnTerminal) return;
// After provider acceptance, failed delivery bookkeeping must not stop live ingestion.
const nativeThreadId = runningProviderThread.nativeThreadRef?.nativeId;
yield* Effect.forEach(
[
delivery.delivered,
...(sideChatContext === "" ||
sideChatHandoff === undefined ||
nativeThreadId == null
? []
: [
persistHandoff({
...sideChatHandoff,
toProviderThreadId: runningProviderThread.id,
delivery: { nativeThreadId, status: "inline" as const, itemIds: [] },
}),
]),
],
(record) =>
record.pipe(
Effect.catchCause(() =>
Effect.logWarning("Failed to record accepted context handoff delivery", {
runId: run.id,
deliveryStatus: "pending",
}),
),
),
{ discard: true },
);
}),
);
},
Effect.mapError((cause) =>
Expand Down Expand Up @@ -2186,6 +2222,13 @@ export const layer: Layer.Layer<
hasUnpairedRunInterruptRequest: runControls.hasUnpairedRunInterruptRequest,
loadRunInterruptReason: runControls.loadRunInterruptReason,
hasWorkspaceMoved: runControls.hasWorkspaceMoved,
loadDroppedInlineHandoffs: () =>
inlineDeliveryLock.withPermits(1)(
Effect.sync(() => {
promptDroppedOnTerminal = true;
return inlineDeliveredHandoffs;
}),
),
loadUnsettledWork: runControls.loadUnsettledWork,
message: {
messageId: message.id,
Expand Down
35 changes: 20 additions & 15 deletions apps/server/src/orchestration-v2/RestartBackgroundNote.ts
Original file line number Diff line number Diff line change
@@ -1,14 +1,15 @@
import type {
NodeId,
OrchestrationV2ConversationMessage,
OrchestrationV2PendingBackgroundTask,
OrchestrationV2ProviderThread,
OrchestrationV2ProviderTurn,
OrchestrationV2RestartCancelledBackgroundWork,
OrchestrationV2Run,
OrchestrationV2RunAttempt,
OrchestrationV2Subagent,
OrchestrationV2TurnItem,
import {
isDeliveredProviderTurn,
type NodeId,
type OrchestrationV2ConversationMessage,
type OrchestrationV2PendingBackgroundTask,
type OrchestrationV2ProviderThread,
type OrchestrationV2ProviderTurn,
type OrchestrationV2RestartCancelledBackgroundWork,
type OrchestrationV2Run,
type OrchestrationV2RunAttempt,
type OrchestrationV2Subagent,
type OrchestrationV2TurnItem,
} from "@t3tools/contracts";
import { subagentHoldsCompletion } from "@t3tools/provider-core/server/subagentProjection";
import { backgroundWorkHoldsCompletion } from "@t3tools/shared/orchestrationV2PendingBackgroundWork";
Expand Down Expand Up @@ -151,7 +152,7 @@ export function restartCancelledBackgroundWorkNote(

type ProviderTurnState = Pick<
OrchestrationV2ProviderTurn,
"runAttemptId" | "providerThreadId" | "status"
"runAttemptId" | "providerThreadId" | "status" | "promptDropped"
>;

export function isRestartNoteSource(
Expand Down Expand Up @@ -307,12 +308,14 @@ export function delegatedTaskResultRun(
/** The provider never accepted this continuation's prompt, so starting it again repeats nothing. */
export function continuationFailedBeforePrompt(
run: Pick<OrchestrationV2Run, "status" | "activeAttemptId" | "restartContinuationOfRunId">,
providerTurns: ReadonlyArray<Pick<ProviderTurnState, "runAttemptId">>,
providerTurns: ReadonlyArray<Pick<ProviderTurnState, "runAttemptId" | "promptDropped">>,
): boolean {
return (
run.status === "failed" &&
run.restartContinuationOfRunId !== undefined &&
!providerTurns.some((turn) => turn.runAttemptId === run.activeAttemptId)
!providerTurns.some(
(turn) => turn.runAttemptId === run.activeAttemptId && isDeliveredProviderTurn(turn),
)
);
}

Expand All @@ -323,7 +326,9 @@ export function continuationStartRetryDelayMs(
run: OrchestrationV2Run,
projection: {
readonly runs: ReadonlyArray<OrchestrationV2Run>;
readonly providerTurns: ReadonlyArray<Pick<ProviderTurnState, "runAttemptId">>;
readonly providerTurns: ReadonlyArray<
Pick<ProviderTurnState, "runAttemptId" | "promptDropped">
>;
readonly turnItems: Parameters<typeof latestRootProviderFailure>[1];
},
): number | undefined {
Expand Down
48 changes: 48 additions & 0 deletions apps/server/src/orchestration-v2/RunExecutionService.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import {
NodeId,
type OrchestrationV2AppThread,
type OrchestrationV2CheckpointScope,
type OrchestrationV2ContextHandoff,
type OrchestrationV2DomainEvent,
type OrchestrationV2ExecutionNode,
type OrchestrationV2PendingBackgroundTask,
Expand Down Expand Up @@ -3702,6 +3703,49 @@ it.effect.each([
}),
);

it.effect("restores dropped inline handoffs when ingestion fails before the terminal", () =>
Effect.gen(function* () {
const key = "dropped-turn-ingest-error";
const ids = backgroundScenarioIds(key);
const handoff = { id: `handoff:${key}` } as unknown as OrchestrationV2ContextHandoff;
const { finalEvents } = yield* captureRootRunTermination({
key,
shouldFinalizeRun: () => Effect.succeed(true),
loadDroppedInlineHandoffs: () => Effect.succeed([handoff]),
events: () =>
Stream.make({
type: "provider_turn.updated" as const,
driver,
threadId: ids.threadId,
providerTurn: {
id: ids.rootProviderTurnId,
providerThreadId: ids.providerThreadId,
runAttemptId: ids.attemptId,
ordinal: 1,
status: "failed",
promptDropped: true,
} as OrchestrationV2ProviderTurn,
}),
ingestNormalized: ({ event }) =>
event.type === "provider_turn.updated"
? Effect.fail(
new ProviderEventIngestor.ProviderEventPublishError({
providerSessionId: ProviderSessionId.make(`session:${key}`),
eventCount: 1,
cause: new Error("SQLITE_FULL: database or disk is full"),
}),
)
: Effect.succeed([]),
});
assert.deepEqual(
finalEvents.flatMap((event) =>
event.type === "context-handoff.updated" ? [event.payload.id] : [],
),
[handoff.id],
);
}),
);

it.effect("does not interrupt a provider when ingestion fails before the root turn is known", () =>
Effect.gen(function* () {
const interrupts = yield* Ref.make(0);
Expand Down Expand Up @@ -3989,6 +4033,7 @@ function captureRootRunTermination(input: {
readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect<boolean, never>;
readonly hasWorkspaceMoved?: () => Effect.Effect<boolean, never>;
readonly loadRunInterruptReason?: () => Effect.Effect<string | undefined, never>;
readonly loadDroppedInlineHandoffs?: RunExecutionService.RunExecutionServiceV2StartRootRunInput["loadDroppedInlineHandoffs"];
readonly hasPendingBackgroundWorkForThread?: ProviderAdapter.ProviderAdapterV2SessionRuntime["hasPendingBackgroundWorkForThread"];
readonly seedOpenSubagent?: boolean;
readonly events?: (
Expand Down Expand Up @@ -4235,6 +4280,9 @@ function captureRootRunTermination(input: {
...(input.hasWorkspaceMoved === undefined
? {}
: { hasWorkspaceMoved: input.hasWorkspaceMoved }),
...(input.loadDroppedInlineHandoffs === undefined
? {}
: { loadDroppedInlineHandoffs: input.loadDroppedInlineHandoffs }),
message: {
messageId: MessageId.make(`message:${input.key}`),
text: input.messageText ?? "interrupt projection",
Expand Down
Loading
Loading