Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 38 additions & 16 deletions apps/server/src/orchestration-v2/Orchestrator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13201,7 +13201,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
const markStoppedRun = Effect.fnUntraced(function* (input: {
readonly command: Extract<
OrchestrationV2ServerCommand,
{ readonly type: "run.interrupt" | "thread.stop" }
{ readonly type: "run.interrupt" | "thread.stop" | "thread.background-work.settle" }
>;
readonly events: Ref.Ref<Array<OrchestrationV2DomainEvent>>;
readonly thread: OrchestrationV2AppThread;
Expand Down Expand Up @@ -13250,6 +13250,36 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
});
});

const stopDelegatedCompletionCohorts = Effect.fnUntraced(function* (input: {
readonly command: Extract<
OrchestrationV2ServerCommand,
{ readonly type: "run.interrupt" | "thread.background-work.settle" }
>;
readonly events: Ref.Ref<Array<OrchestrationV2DomainEvent>>;
readonly run: OrchestrationV2Run;
readonly now: DateTime.Utc;
}) {
const { command, events, run, now } = input;
const projection = yield* getProjectionWithPendingEvents(command.threadId, events);
const delegatedCompletion = projection.messages.find(
(candidate) => candidate.id === run.userMessageId,
)?.delegatedCompletion;
const completionCohortRunId = delegatedCompletion?.parentRunId ?? run.id;
for (const parentRunId of new Set([completionCohortRunId, run.id])) {
yield* disposeDelegatedCompletionCohort({
command,
events,
projection: yield* getProjectionWithPendingEvents(command.threadId, events),
parentRunId,
disposition: "stopped",
now,
...(parentRunId !== run.id && delegatedCompletion !== undefined
? { interruptedWakeTaskIds: delegatedCompletion.taskIds }
: {}),
});
}
});

const holdStoppedThread = Effect.fn("Orchestrator.holdStoppedThread")(function* (input: {
readonly command: Extract<
OrchestrationV2ServerCommand,
Expand Down Expand Up @@ -13552,6 +13582,12 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
return;
const now = yield* DateTime.now;
if (stopped.providerTurn !== undefined && stoppedRun.status === "running") {
// Only Force stop settles a running run no Stop reached; it records that Stop itself.
if (!(yield* stopReachedRun(command, command.threadId, stoppedRun.id))) {
const { thread } = yield* getProjectionWithPendingEvents(command.threadId, events);
yield* markStoppedRun({ command, events, thread, run: stoppedRun, now });
yield* stopDelegatedCompletionCohorts({ command, events, run: stoppedRun, now });
}
const output = yield* projectionStore
.getThreadRecords(command.threadId, ["messages"], {
messageRunIds: [stoppedRun.id],
Expand Down Expand Up @@ -13792,21 +13828,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
cohortRunIds: [completionCohortRunId],
now,
})
: Effect.gen(function* () {
for (const parentRunId of new Set([completionCohortRunId, run.id])) {
yield* disposeDelegatedCompletionCohort({
command,
events,
projection: yield* getProjectionWithPendingEvents(command.threadId, events),
parentRunId,
disposition: "stopped",
now,
...(parentRunId !== run.id && completionMessage?.delegatedCompletion !== undefined
? { interruptedWakeTaskIds: completionMessage.delegatedCompletion.taskIds }
: {}),
});
}
});
: stopDelegatedCompletionCohorts({ command, events, run, now });

const emitEvent = emit(events, command);
const interruptRequestItem: OrchestrationV2TurnItem = {
Expand Down
13 changes: 12 additions & 1 deletion apps/server/src/orchestration-v2/ThreadManagementService.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -913,6 +913,17 @@ it.effect.each([
},
teardown: ["detach"],
},
{
name: "settles a running run whose provider turn already completed in a shared process",
others: ["thread:thread-management:force-stop:other"],
killed: false,
strandedRun: true,
expected: {
outcome: "shared",
sharedWithThreadIds: ["thread:thread-management:force-stop:other"],
},
teardown: ["detach"],
},
{
name: "detaches when a sibling starts a turn just before the session is released",
others: [] as ReadonlyArray<string>,
Expand Down Expand Up @@ -977,7 +988,7 @@ it.effect.each([
id: "provider-turn:force-stop",
providerThreadId,
runAttemptId: "attempt",
status: "settledRun" in row ? "completed" : "running",
status: "settledRun" in row || "strandedRun" in row ? "completed" : "running",
},
],
} as unknown as OrchestrationV2ThreadProjection),
Expand Down
3 changes: 2 additions & 1 deletion apps/server/src/orchestration-v2/ThreadManagementService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1439,7 +1439,8 @@ const make = Effect.gen(function* () {
const active = isActiveRun(run);
const providerTurn = records.providerTurns.findLast(
(turn) =>
turn.runAttemptId === run.activeAttemptId && (!active || turn.status === "running"),
turn.runAttemptId === run.activeAttemptId &&
(!active || run.status === "running" || turn.status === "running"),
);
const commandId = CommandId.make(
`server:force-stop:${run.id}:${yield* Clock.currentTimeMillis}`,
Expand Down
65 changes: 65 additions & 0 deletions apps/server/src/orchestration-v2/ThreadStop.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1839,6 +1839,71 @@ it.effect.each(["Stop", "Force stop"] as const)(
}).pipe(Effect.provide(layerTest)),
);

it.effect("Force stop alone stops a stranded run's delegated cohort and later agent watches", () =>
Effect.gen(function* () {
const orchestrator = yield* Orchestrator.OrchestratorV2;
const projections = yield* ProjectionStore.ProjectionStoreV2;
const threadId = ThreadId.make("thread:force-stop-stranded");
yield* createWatchingThread(threadId, 8);
yield* send(threadId, "parent", "start_immediately");
yield* delegate(threadId, "stranded child");
const now = yield* DateTime.now;
const started = yield* orchestrator.getThreadProjection(threadId);
const run = started.runs[0]!;
const attempt = started.attempts[0]!;
const apply = (event: Omit<OrchestrationV2DomainEvent, "id" | "threadId">) =>
projections.apply({
...event,
id: EventId.make(`event:force-stop-stranded:${event.type}`),
threadId,
} as OrchestrationV2DomainEvent);
yield* apply({
type: "run.updated",
runId: run.id,
occurredAt: now,
payload: {
...run,
status: "running",
startedAt: now,
delegatedCompletion: { disposition: "open", nextGeneration: 1, delivery: null },
},
});
yield* apply({
type: "run-attempt.updated",
occurredAt: now,
payload: { ...attempt, status: "completed", startedAt: now, completedAt: now },
});
yield* apply({
type: "provider-turn.updated",
occurredAt: now,
payload: {
id: ProviderTurnId.make("provider-turn:force-stop-stranded"),
providerThreadId: run.providerThreadId!,
nodeId: run.rootNodeId!,
runAttemptId: attempt.id,
nativeTurnRef: null,
ordinal: 1,
status: "completed",
startedAt: now,
completedAt: now,
},
});

const threads = yield* ThreadManagementService.ThreadManagementService;
yield* threads.forceStopThread({ threadId, runId: run.id });
const stopped = yield* orchestrator.getThreadProjection(threadId);
assert.deepEqual(
stopped.runs.map((candidate) => [
candidate.status,
candidate.delegatedCompletion?.disposition,
]),
[["interrupted", "stopped"]],
);
assert.equal(stopped.subagents[0]?.completionDelivery?.state, "disposed");
assert.isTrue(Exit.isFailure(yield* Effect.exit(watch(threadId, 9))));
}).pipe(Effect.provide(layerTest)),
);

// A finished delegated child whose native subagent the provider never reported ending,
// with no open turn item, and a nested native subagent on that agent's own thread.
it.effect(
Expand Down
Loading