From ca73b2b95fb9ef800beb6f79446cff478ce1f9d0 Mon Sep 17 00:00:00 2001 From: Victor Vogel Date: Sat, 21 Nov 2026 14:25:10 +0000 Subject: [PATCH 1/6] fix(opencode): dedupe redirected subagent rows and project steers --- .../Adapters/OpenCode2AdapterV2.test.ts | 1698 ++++++++++++++--- .../src/orchestration-v2/ProjectionStore.ts | 96 + .../ProviderEventIngestor.test.ts | 680 +++++++ .../orchestration-v2/ProviderEventIngestor.ts | 179 +- .../ProviderTurnControlService.test.ts | 2 + .../RunExecutionService.test.ts | 1161 +++++++++++ .../orchestration-v2/RunExecutionService.ts | 85 +- .../src/server/ProviderAdapter.ts | 8 + .../src/server/subagentProjection.ts | 4 +- .../src/server/v2/adapter.ts | 553 +++++- 10 files changed, 4088 insertions(+), 378 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts index 88563aeb43492..d77015f2c39e0 100644 --- a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts @@ -1751,16 +1751,21 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { returnedRunning(tool.id), ]; }; - const childReport = (inboxID: string, text: string, description = "Sleep") => + const childReport = ( + inboxID: string, + text: string, + description = "Sleep", + state: "completed" | "failed" | "cancelled" = "completed", + ) => event("session.inbox.enqueued", { inboxID, sessionID: SESSION, item: { type: "synthetic", payload: { - text: `\n${text}\n`, + text: `\n${text}\n`, description, - metadata: { source: "subagent", childID: CHILD, agent: "General", state: "completed" }, + metadata: { source: "subagent", childID: CHILD, agent: "General", state }, }, delivery: "steer", }, @@ -1933,223 +1938,1467 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { thread, ); yield* runtime.startTurn(withLineage(thread)); - yield* watch.until( - () => watch.calls.get("Sleep")?.status === "failed" && watch.state.roster?.length === 0, - ); - assert.includeMembers( - childUpdates.map((event) => event.providerThread.status), - ["active", "idle"], - ); - assert.isFalse( - childUpdates.some((event) => event.sessionScoped === true), - "reattached child status must route through the parent run", - ); - assert.deepEqual(watch.state.roster, []); - assert.isFalse(yield* runtime.hasPendingBackgroundWorkForThread!(thread)); + yield* watch.until( + () => watch.calls.get("Sleep")?.status === "failed" && watch.state.roster?.length === 0, + ); + assert.includeMembers( + childUpdates.map((event) => event.providerThread.status), + ["active", "idle"], + ); + assert.isFalse( + childUpdates.some((event) => event.sessionScoped === true), + "reattached child status must route through the parent run", + ); + assert.deepEqual(watch.state.roster, []); + assert.isFalse(yield* runtime.hasPendingBackgroundWorkForThread!(thread)); + }), + ); + + // regression: #21979 a failed background subagent pinned its thread as busy + it.effect("settles the background call a foreground call joined to a failed child", () => + Effect.gen(function* () { + const again = { sessionID: SESSION, assistantMessageID: "msg_assistant", id: "call-again" }; + const { runtime, thread } = yield* resumed([ + ...launchedAndRunning, + // A foreground call joins the running background child. It is only + // marked background once its own tool call returns, so the child can + // fail while this call is still the foreground one. + event("session.tool.input.started", { ...again, name: "subagent" }), + event("session.tool.called", { + ...again, + name: "subagent", + input: { description: "Again", prompt: "also this", sessionID: CHILD }, + executed: false, + }), + event("session.tool.progress", { + ...again, + metadata: { sessionID: CHILD, status: "running" }, + }), + out("session.update", { sessionID: CHILD, permissions: "" }), + reply("session.update", null), + // The child fails while the joined call is still the foreground one. + event("session.execution.failed", { + sessionID: CHILD, + error: { type: "provider.quota", message: "Rate limit exceeded." }, + }), + event("session.execution.succeeded", { sessionID: SESSION }), + ]); + const watch = yield* watchCalls(runtime, thread); + yield* runtime.startTurn(withLineage(thread)); + yield* watch.until( + () => watch.calls.get("Sleep")?.status === "failed" && watch.state.roster?.length === 0, + ); + assert.deepEqual(Object.fromEntries(watch.calls), { + Sleep: { status: "failed", result: "Rate limit exceeded." }, + }); + assert.deepEqual(watch.state.roster, []); + assert.isFalse(yield* runtime.hasPendingBackgroundWorkForThread!(thread)); + }), + ); + + it.effect("settles every call that joined a running subagent with its one report", () => + Effect.gen(function* () { + const { runtime, thread } = yield* resumed([ + ...launchedAndRunning, + ...callAgain({ background: true }), + event("session.execution.succeeded", { sessionID: SESSION }), + event("session.execution.succeeded", { sessionID: CHILD }), + childReport("msg_report", "CHILD_OK"), + ]); + const watch = yield* watchCalls(runtime, thread); + yield* runtime.startTurn(withLineage(thread)); + yield* watch.until( + () => watch.calls.get("Sleep")?.status === "completed" && watch.state.roster?.length === 0, + ); + assert.deepEqual(Object.fromEntries(watch.calls), { + Sleep: { status: "completed", result: "CHILD_OK" }, + }); + }), + ); + + it.effect("keeps a call to a finished subagent for its own run's report", () => + Effect.gen(function* () { + const { runtime, thread } = yield* resumed([ + ...launchedAndRunning, + event("session.execution.succeeded", { sessionID: CHILD }), + ...callAgain({ background: true }), + childReport("msg_report_1", "FIRST_OK"), + event("session.execution.started", { sessionID: CHILD }), + event("session.execution.succeeded", { sessionID: CHILD }), + event("session.execution.succeeded", { sessionID: SESSION }), + childReport("msg_report_2", "SECOND_OK"), + ]); + const watch = yield* watchCalls(runtime, thread); + yield* runtime.startTurn(withLineage(thread)); + yield* watch.until(() => watch.childTurns.has(`${CHILD}:turn:2`)); + assert.deepEqual(watch.callsAtChildTurnStart.get(`${CHILD}:turn:2`), { + Sleep: { status: "running", result: null }, + }); + yield* watch.until( + () => watch.calls.get("Sleep")?.status === "completed" && watch.state.roster?.length === 0, + ); + assert.deepEqual(Object.fromEntries(watch.calls), { + Sleep: { status: "completed", result: "SECOND_OK" }, + }); + }), + ); + + const childSaid = (text: string, run = 1) => + event("session.text.ended", { + sessionID: CHILD, + assistantMessageID: `msg_child_${run}`, + ordinal: 0, + text, + }); + it.effect.each([ + { + // regression: #26319 the report settled the older run and left the resumed call running + name: "an earlier run's report never arrived", + replay: [ + ...launchedAndRunning, + ...callAgain({ background: true }), + childSaid("FIRST_OK"), + event("session.execution.succeeded", { sessionID: CHILD }), + ...callAgain({ background: true }, { id: "call-later", writesRules: false }), + event("session.execution.started", { sessionID: CHILD }), + childSaid("SECOND_OK", 2), + event("session.execution.succeeded", { sessionID: CHILD }), + event("session.execution.succeeded", { sessionID: SESSION }), + childReport("msg_report_2", "SECOND_OK", "Again"), + ], + expected: { + [CHILD]: { status: "completed", result: "SECOND_OK" }, + }, + }, + { + name: "both runs' reports arrive after the runs ended", + replay: [ + ...launchedAndRunning, + event("session.execution.succeeded", { sessionID: CHILD }), + ...callAgain({ background: true, description: "Sleep" }), + event("session.execution.started", { sessionID: CHILD }), + event("session.execution.succeeded", { sessionID: CHILD }), + childReport("msg_report_1", "FIRST_OK"), + childReport("msg_report_2", "SECOND_OK"), + event("session.execution.succeeded", { sessionID: SESSION }), + ], + expected: { + [CHILD]: { status: "completed", result: "SECOND_OK" }, + }, + }, + { + name: "two runs with the same answer report late", + replay: [ + ...launchedAndRunning, + childSaid("DONE"), + event("session.execution.succeeded", { sessionID: CHILD }), + ...callAgain({ background: true }), + event("session.execution.started", { sessionID: CHILD }), + childSaid("DONE", 2), + event("session.execution.succeeded", { sessionID: CHILD }), + childReport("msg_report_1", "DONE"), + ...callAgain({ background: true }, { id: "call-later", writesRules: false }), + event("session.execution.started", { sessionID: CHILD }), + childReport("msg_report_2", "DONE", "Again"), + childSaid("THIRD_OK", 3), + event("session.execution.succeeded", { sessionID: CHILD }), + event("session.execution.succeeded", { sessionID: SESSION }), + childReport("msg_report_3", "THIRD_OK", "Again"), + ], + expected: { + [CHILD]: { status: "completed", result: "THIRD_OK" }, + }, + }, + { + name: "the call it joined already returned", + replay: [ + ...backgroundLaunch(CHILD), + event("session.execution.started", { sessionID: CHILD }), + ...callAgain({ background: true }), + event("session.execution.succeeded", { sessionID: CHILD }), + event( + "session.tool.success", + { + sessionID: SESSION, + assistantMessageID: "msg_assistant", + id: "call-background", + content: [{ type: "text", text: "FIRST_OK" }], + metadata: { sessionID: CHILD, status: "completed" }, + executed: true, + }, + undefined, + 2, + ), + event("session.execution.succeeded", { sessionID: SESSION }), + childReport("msg_report", "CHILD_OK", "Again"), + ], + expected: { + [CHILD]: { status: "completed", result: "CHILD_OK" }, + }, + }, + { + name: "a resumed foreground execution completes before an earlier run's report arrives", + replay: [ + ...launchedAndRunning, + event("session.execution.succeeded", { sessionID: CHILD }), + event("session.tool.input.started", { + sessionID: SESSION, + assistantMessageID: "msg_assistant", + id: "call-fg-resume", + name: "subagent", + }), + event("session.tool.called", { + sessionID: SESSION, + assistantMessageID: "msg_assistant", + id: "call-fg-resume", + name: "subagent", + input: { description: "Follow up", prompt: "second run", sessionID: CHILD }, + executed: false, + }), + event("session.tool.progress", { + sessionID: SESSION, + assistantMessageID: "msg_assistant", + id: "call-fg-resume", + metadata: { sessionID: CHILD, status: "running" }, + }), + out("session.update", { sessionID: CHILD, permissions: "" }), + reply("session.update", null), + event("session.execution.started", { sessionID: CHILD }), + childSaid("SECOND_OK", 2), + event("session.execution.succeeded", { sessionID: CHILD }), + event( + "session.tool.success", + { + sessionID: SESSION, + assistantMessageID: "msg_assistant", + id: "call-fg-resume", + content: [{ type: "text", text: "SECOND_OK" }], + metadata: { sessionID: CHILD, status: "completed" }, + executed: true, + }, + undefined, + 2, + ), + event("session.execution.succeeded", { sessionID: SESSION }), + childReport("msg_report_1", "FIRST_OK"), + ], + expected: { + [CHILD]: { status: "completed", result: "SECOND_OK" }, + }, + }, + { + name: "a resumed foreground execution completes before an earlier run settles as cancelled", + replay: [ + ...launchedAndRunning, + event("session.execution.succeeded", { sessionID: CHILD }), + event("session.tool.input.started", { + sessionID: SESSION, + assistantMessageID: "msg_assistant", + id: "call-fg-resume", + name: "subagent", + }), + event("session.tool.called", { + sessionID: SESSION, + assistantMessageID: "msg_assistant", + id: "call-fg-resume", + name: "subagent", + input: { description: "Follow up", prompt: "second run", sessionID: CHILD }, + executed: false, + }), + event("session.tool.progress", { + sessionID: SESSION, + assistantMessageID: "msg_assistant", + id: "call-fg-resume", + metadata: { sessionID: CHILD, status: "running" }, + }), + out("session.update", { sessionID: CHILD, permissions: "" }), + reply("session.update", null), + event("session.execution.started", { sessionID: CHILD }), + childSaid("SECOND_OK", 2), + event("session.execution.succeeded", { sessionID: CHILD }), + event( + "session.tool.success", + { + sessionID: SESSION, + assistantMessageID: "msg_assistant", + id: "call-fg-resume", + content: [{ type: "text", text: "SECOND_OK" }], + metadata: { sessionID: CHILD, status: "completed" }, + executed: true, + }, + undefined, + 2, + ), + event("session.execution.succeeded", { sessionID: SESSION }), + childReport("msg_report_1", "CANCELLED_OLD", "Sleep", "cancelled"), + ], + expected: { + [CHILD]: { status: "completed", result: "SECOND_OK" }, + }, + }, + { + name: "a newer foreground execution is interrupted before an earlier run's report succeeds", + replay: [ + ...launchedAndRunning, + event("session.execution.succeeded", { sessionID: CHILD }), + event("session.tool.input.started", { + sessionID: SESSION, + assistantMessageID: "msg_assistant", + id: "call-fg-interrupted", + name: "subagent", + }), + event("session.tool.called", { + sessionID: SESSION, + assistantMessageID: "msg_assistant", + id: "call-fg-interrupted", + name: "subagent", + input: { description: "Follow up", prompt: "second run", sessionID: CHILD }, + executed: false, + }), + event("session.tool.progress", { + sessionID: SESSION, + assistantMessageID: "msg_assistant", + id: "call-fg-interrupted", + metadata: { sessionID: CHILD, status: "running" }, + }), + out("session.update", { sessionID: CHILD, permissions: "" }), + reply("session.update", null), + event("session.execution.started", { sessionID: CHILD }), + event("session.execution.interrupted", { sessionID: CHILD, reason: "user" }), + event("session.execution.succeeded", { sessionID: SESSION }), + childReport("msg_report_1", "FIRST_OK"), + ], + expected: { + [CHILD]: { status: "interrupted", result: null }, + }, + }, + ])("settles a resumed subagent's calls when $name", ({ replay, expected }) => + Effect.gen(function* () { + const { runtime, thread } = yield* resumed(replay); + const watch = yield* watchCalls(runtime, thread); + yield* runtime.startTurn(withLineage(thread)); + yield* watch.until( + () => + watch.state.roster?.length === 0 && + Object.entries(expected).every( + ([id, want]) => watch.callsById.get(id)?.status === want.status, + ), + ); + assert.deepEqual(Object.fromEntries(watch.callsById), expected); + }), + ); + + it.effect.each([1, 5])( + "dedupes %i redirect(s) onto one subagent row, preserves title and model, and surfaces steered prompts", + (steers) => + Effect.gen(function* () { + const steerEntries: Array = []; + for (let index = 1; index <= steers; index += 1) { + steerEntries.push( + ...callAgain( + { + background: true, + description: `Redirect #${index}`, + prompt: `Redirect prompt #${index}`, + }, + { id: `call-steer-${index}`, writesRules: index === 1 }, + ), + event("session.inbox.enqueued", { + inboxID: `msg_steer_${index}`, + sessionID: CHILD, + item: { + type: "user", + payload: { text: `Redirect prompt #${index}`, files: [] }, + delivery: "steer", + }, + }), + ); + } + const { runtime, thread } = yield* resumed([ + ...backgroundLaunch(CHILD), + event("session.inbox.enqueued", { + inboxID: "msg_child_initial", + sessionID: CHILD, + item: { + type: "user", + payload: { text: "You are a subagent.\nsleep", files: [] }, + delivery: "steer", + }, + }), + returnedRunning("call-background"), + event("session.execution.started", { sessionID: CHILD }), + event("session.inbox.delivered", { sessionID: CHILD, inboxID: "msg_child_initial" }), + ...steerEntries, + event("session.inbox.enqueued", { + inboxID: "msg_steer_cancelled", + sessionID: CHILD, + item: { + type: "user", + payload: { text: "Cancelled steer", files: [] }, + delivery: "steer", + }, + }), + event("session.inbox.cancelled", { sessionID: CHILD, inboxID: "msg_steer_cancelled" }), + event("session.execution.succeeded", { sessionID: SESSION }), + ...Array.from({ length: steers }, (_, index) => + event("session.inbox.delivered", { + sessionID: CHILD, + inboxID: `msg_steer_${index + 1}`, + }), + ), + childSaid("SECOND"), + event("session.execution.succeeded", { sessionID: CHILD }), + childReport("msg_report", "SECOND"), + ]); + const subagentIds = new Set(); + const subagentItemIds = new Set(); + const subagentNodeIds = new Set(); + const childCreates: Array = []; + const childUserPrompts: Array<{ readonly text: string; readonly inputIntent: string }> = []; + let latestSubagent: + | Extract< + ProviderAdapter.ProviderAdapterV2Event, + { type: "subagent.updated" } + >["subagent"] + | undefined; + let maxRosterSubagents = 0; + let lastRoster: ReadonlyArray = []; + const done = yield* Deferred.make(); + yield* runtime.events.pipe( + Stream.tap((e) => + Effect.gen(function* () { + if (e.type === "app_thread.created") childCreates.push(e.appThread); + if (e.type === "node.updated" && e.node.kind === "subagent") { + subagentNodeIds.add(e.node.id); + } + if (e.type === "subagent.updated") { + subagentIds.add(e.subagent.id); + latestSubagent = e.subagent; + } + if (e.type === "turn_item.updated" && e.turnItem.type === "subagent") { + subagentItemIds.add(e.turnItem.id); + } + if ( + e.type === "turn_item.updated" && + e.turnItem.type === "user_message" && + e.turnItem.threadId === childCreates[0]?.id + ) { + childUserPrompts.push({ + text: e.turnItem.text, + inputIntent: e.turnItem.inputIntent, + }); + } + if (e.type === "provider_thread.updated" && e.providerThread.id === thread.id) { + const tasks = e.providerThread.pendingBackgroundTasks ?? []; + const count = tasks.filter((task) => task.kind === "subagent").length; + if (count > maxRosterSubagents) maxRosterSubagents = count; + lastRoster = tasks; + if (latestSubagent?.status === "completed" && tasks.length === 0) { + yield* Deferred.succeed(done, undefined); + } + } + }), + ), + Stream.runDrain, + Effect.forkScoped, + ); + yield* runtime.startTurn(withLineage(thread)); + yield* Deferred.await(done); + assert.equal(subagentIds.size, 1); + assert.equal(subagentItemIds.size, 1); + assert.equal(subagentNodeIds.size, 1); + assert.lengthOf(childCreates, 1); + assert.equal(childCreates[0]?.title, "Sleep"); + assert.equal(latestSubagent?.title, "Sleep"); + assert.equal(latestSubagent?.model, "opencode/big-pickle"); + assert.equal(latestSubagent?.status, "completed"); + assert.equal(latestSubagent?.result, "SECOND"); + assert.equal(maxRosterSubagents, 1); + assert.deepEqual(lastRoster, []); + assert.deepEqual(childUserPrompts, [ + { text: "sleep", inputIntent: "turn_start" }, + ...Array.from({ length: steers }, (_, index) => ({ + text: `Redirect prompt #${index + 1}`, + inputIntent: "steer", + })), + ]); + }), + ); + + it.effect( + "keeps separate subagent sessions distinct while deduping redirects to one of them", + () => + Effect.gen(function* () { + const CHILD_2 = "ses_secondchild00000000000000"; + const secondTool = { + sessionID: SESSION, + assistantMessageID: "msg_assistant", + id: "call-second", + }; + const { runtime, thread } = yield* resumed([ + ...launchedAndRunning, + event("session.tool.input.started", { ...secondTool, name: "subagent" }), + event("session.tool.called", { + ...secondTool, + name: "subagent", + input: { description: "Lint", prompt: "run lint", background: true }, + executed: false, + }), + event("session.created", { ...childCreated(CHILD_2), title: "Lint" }), + event("session.tool.progress", { + ...secondTool, + metadata: { sessionID: CHILD_2, status: "running" }, + }), + event("session.tool.success", { + ...secondTool, + content: [{ type: "text", text: "The subagent is working in the background." }], + metadata: { sessionID: CHILD_2, status: "running" }, + executed: true, + }), + event("session.execution.started", { sessionID: CHILD_2 }), + ...callAgain({ background: true, description: "Redirect sleep", prompt: "redirect" }), + event("session.execution.succeeded", { sessionID: SESSION }), + event("session.execution.succeeded", { sessionID: CHILD }), + childReport("msg_report_1", "CHILD_1_OK"), + event("session.execution.succeeded", { sessionID: CHILD_2 }), + event("session.inbox.enqueued", { + inboxID: "msg_report_2", + sessionID: SESSION, + item: { + type: "synthetic", + payload: { + text: `\nCHILD_2_OK\n`, + description: "Lint", + metadata: { + source: "subagent", + childID: CHILD_2, + agent: "General", + state: "completed", + }, + }, + delivery: "steer", + }, + }), + ]); + const byId = new Map< + string, + Extract["subagent"] + >(); + const done = yield* Deferred.make(); + yield* runtime.events.pipe( + Stream.tap((e) => + Effect.gen(function* () { + if (e.type === "subagent.updated") byId.set(e.subagent.id, e.subagent); + if ( + byId.size >= 2 && + [...byId.values()].every((entry) => entry.status === "completed") + ) { + yield* Deferred.succeed(done, undefined); + } + }), + ), + Stream.runDrain, + Effect.forkScoped, + ); + yield* runtime.startTurn(withLineage(thread)); + yield* Deferred.await(done); + assert.equal(byId.size, 2); + const summaries = [...byId.values()] + .map((entry) => ({ + title: entry.title, + model: entry.model, + status: entry.status, + result: entry.result, + })) + .toSorted((a, b) => (a.title ?? "").localeCompare(b.title ?? "")); + assert.deepEqual(summaries, [ + { + title: "Lint", + model: "opencode/big-pickle", + status: "completed", + result: "CHILD_2_OK", + }, + { + title: "Sleep", + model: "opencode/big-pickle", + status: "completed", + result: "CHILD_1_OK", + }, + ]); + }), + ); + + it.effect( + "reuses the original subagent row and model when resuming a completed child in a later run", + () => + Effect.gen(function* () { + const { runtime, thread } = yield* resumed([ + ...launchedAndRunning, + event("session.execution.succeeded", { sessionID: CHILD }), + childReport("msg_report_1", "FIRST_OK"), + event("session.execution.succeeded", { sessionID: SESSION }), + out("session.prompt", { sessionID: SESSION, text: "" }), + promptAccepted, + event("session.execution.started", { sessionID: SESSION }), + ...callAgain( + { background: true, description: "Follow up", prompt: "also check Y" }, + { id: "call-resume" }, + ), + event("session.execution.started", { sessionID: CHILD }), + event("session.execution.succeeded", { sessionID: CHILD }), + event("session.execution.succeeded", { sessionID: SESSION }), + childReport("msg_report_2", "SECOND_OK", "Follow up"), + ]); + const run2Id = RunId.make("run:opencode2-adapter:2"); + const subagentIds = new Set(); + const childCreates: Array = []; + const childUserPrompts: Array = []; + const reopenOrder: Array = []; + const run1Done = yield* Deferred.make(); + const run2Done = yield* Deferred.make(); + let run2Started = false; + let latestSubagent: + | Extract< + ProviderAdapter.ProviderAdapterV2Event, + { type: "subagent.updated" } + >["subagent"] + | undefined; + yield* runtime.events.pipe( + Stream.tap((e) => + Effect.gen(function* () { + if (e.type === "app_thread.created") childCreates.push(e.appThread); + if (e.type === "subagent.updated") { + subagentIds.add(e.subagent.id); + latestSubagent = e.subagent; + if (run2Started && e.subagent.runId === run2Id && e.subagent.status === "running") { + reopenOrder.push("subagent.running"); + } + if (e.subagent.runId === run2Id && e.subagent.status === "completed") { + yield* Deferred.succeed(run2Done, undefined); + } + } + if ( + run2Started && + e.type === "provider_thread.updated" && + e.providerThread.appThreadId === childCreates[0]?.id && + e.providerThread.status === "active" + ) { + reopenOrder.push("child.active"); + } + if ( + e.type === "turn_item.updated" && + e.turnItem.type === "user_message" && + e.turnItem.threadId === childCreates[0]?.id + ) { + childUserPrompts.push(e.turnItem.text); + } + if (e.type === "turn.terminal" && e.runOrdinal === 1) { + yield* Deferred.succeed(run1Done, undefined); + } + }), + ), + Stream.runDrain, + Effect.forkScoped, + ); + yield* runtime.startTurn(withLineage(thread)); + yield* Deferred.await(run1Done); + run2Started = true; + yield* runtime.startTurn({ + ...withLineage(thread), + runId: run2Id, + runOrdinal: 2, + providerTurnOrdinal: 2, + attemptId: RunAttemptId.make("attempt:opencode2-adapter:2"), + }); + yield* Deferred.await(run2Done); + assert.equal(subagentIds.size, 1); + assert.lengthOf(childCreates, 1); + assert.equal(childCreates[0]?.title, "Sleep"); + assert.equal(latestSubagent?.title, "Sleep"); + assert.equal(latestSubagent?.model, "opencode/big-pickle"); + assert.equal(latestSubagent?.runId, run2Id); + assert.equal(latestSubagent?.result, "SECOND_OK"); + assert.deepEqual(childUserPrompts, ["sleep", "also check Y"]); + assert.equal(reopenOrder[0], "subagent.running"); + assert.include(reopenOrder, "child.active"); + }), + ); + + it.effect( + "keeps a running background subagent active and routes its requests when a joined steer call fails", + () => + Effect.gen(function* () { + const badSteer = { + sessionID: SESSION, + assistantMessageID: "msg_assistant", + id: "call-bad-steer", + }; + const pendingSteer = { + sessionID: SESSION, + assistantMessageID: "msg_assistant", + id: "call-unattached-steer", + }; + const form = { ...colorForm, sessionID: CHILD }; + const { runtime, thread } = yield* resumed([ + ...launchedAndRunning, + event("session.tool.input.started", { ...badSteer, name: "subagent" }), + event("session.tool.called", { + ...badSteer, + name: "subagent", + input: { description: "Bad steer", prompt: "redirect", sessionID: CHILD }, + executed: false, + }), + event("session.tool.progress", { + ...badSteer, + metadata: { sessionID: CHILD, status: "running" }, + }), + out("session.update", { sessionID: CHILD, permissions: "" }), + reply("session.update", null), + // A second background steer that has not attached yet must not duplicate the roster entry. + event("session.tool.input.started", { ...pendingSteer, name: "subagent" }), + event("session.tool.called", { + ...pendingSteer, + name: "subagent", + input: { + description: "Pending steer", + prompt: "wait", + background: true, + sessionID: CHILD, + }, + executed: false, + }), + event( + "session.tool.failed", + { + ...badSteer, + error: { type: "aborted", message: "cancelled steer" }, + metadata: { sessionID: CHILD }, + executed: false, + }, + undefined, + 2, + ), + // Emit the child's form before pendingSteer attaches, proving settleCall restored + // child.subagent.call to the surviving background call. + event("form.created", { form }), + event("session.tool.progress", { + ...pendingSteer, + metadata: { sessionID: CHILD, status: "running" }, + }), + returnedRunning(pendingSteer.id), + event("session.execution.succeeded", { sessionID: SESSION }), + event("session.execution.succeeded", { sessionID: CHILD }), + childReport("msg_report", "STILL_OK"), + ]); + const statuses: Array = []; + const done = yield* Deferred.make(); + let finalResult: string | null = null; + let maxRosterSubagents = 0; + let seenForm: + | { readonly nativeId: string | undefined; readonly runId: RunId | null } + | undefined; + yield* runtime.events.pipe( + Stream.tap((e) => + Effect.gen(function* () { + if (e.type === "turn_item.updated" && e.turnItem.type === "user_input_request") { + seenForm = { + nativeId: e.turnItem.nativeItemRef?.nativeId ?? undefined, + runId: e.turnItem.runId, + }; + } + if (e.type === "provider_thread.updated" && e.providerThread.id === thread.id) { + const count = (e.providerThread.pendingBackgroundTasks ?? []).filter( + (task) => task.kind === "subagent", + ).length; + if (count > maxRosterSubagents) maxRosterSubagents = count; + } + if (e.type === "subagent.updated") { + statuses.push(e.subagent.status); + finalResult = e.subagent.result; + if (e.subagent.status === "completed") yield* Deferred.succeed(done, undefined); + } + }), + ), + Stream.runDrain, + Effect.forkScoped, + ); + yield* runtime.startTurn(withLineage(thread)); + yield* Deferred.await(done); + assert.isFalse(statuses.includes("interrupted")); + assert.isFalse(statuses.includes("failed")); + assert.equal(maxRosterSubagents, 1); + assert.deepEqual(seenForm, { nativeId: form.id, runId: withLineage(thread).runId }); + assert.equal(finalResult, "STILL_OK"); + }), + ); + + it.effect.each([false, true] as const)( + "preserves existing row identity, child thread, and turn IDs when resuming an uncached child (preRestored=%s)", + (preRestored) => + Effect.gen(function* () { + const now = yield* DateTime.now; + const childThreadId = ThreadId.make("thread:opencode2-adapter:child-cold-resume"); + const childProviderThread = { + ...providerThread(now), + id: ProviderThreadId.make("provider-thread:opencode2-adapter:child-cold-resume"), + appThreadId: childThreadId, + nativeThreadRef: { + driver: OPENCODE_PROVIDER, + nativeId: CHILD, + strength: "strong" as const, + }, + }; + const { runtime, thread } = yield* resumed([ + ...(preRestored + ? [ + out("session.get", { sessionID: CHILD }), + replyData("session.get", sessionInfo({ id: CHILD })), + out("permission.list", { sessionID: CHILD }), + replyData("permission.list", []), + out("session.form.list", { sessionID: CHILD }), + replyData("session.form.list", []), + out("session.update", { sessionID: CHILD, permissions: "" }), + reply("session.update", null), + ] + : []), + out("session.prompt", { sessionID: SESSION, text: "" }), + promptAccepted, + event("session.execution.started", { sessionID: SESSION }), + ...callAgain( + { background: true, description: "Resume after restart", prompt: "keep going" }, + { id: "call-restart-resume" }, + ), + event("session.execution.started", { sessionID: CHILD }), + event("session.step.started", { + sessionID: CHILD, + agent: "general", + model: { id: "big-pickle", providerID: "opencode", variant: "default" }, + assistantMessageID: "msg_child_step", + snapshot: "snap", + started: 1, + }), + event("session.execution.succeeded", { sessionID: CHILD }), + event("session.execution.succeeded", { sessionID: SESSION }), + childReport("msg_report", "RESUMED_OK", "Resume after restart"), + ]); + if (preRestored) { + yield* runtime.resumeThread({ + providerThread: childProviderThread, + threadId: childThreadId, + modelSelection: bigPickle, + runtimePolicy: policy(), + }); + } + const creates: Array< + Extract + > = []; + const subagentNodeEvents: Array< + Extract + > = []; + const subagentEvents: Array< + Extract + > = []; + const subagentItemEvents: Array< + Extract + > = []; + const rosterDescriptions: Array = []; + let childNativeTurnId: string | undefined; + const done = yield* Deferred.make(); + yield* runtime.events.pipe( + Stream.tap((e) => + Effect.gen(function* () { + if (e.type === "app_thread.created") creates.push(e); + if (e.type === "node.updated" && e.node.kind === "subagent") { + subagentNodeEvents.push(e); + } + if (e.type === "subagent.updated") { + subagentEvents.push(e); + if (e.subagent.status === "completed") yield* Deferred.succeed(done, undefined); + } + if (e.type === "turn_item.updated" && e.turnItem.type === "subagent") { + subagentItemEvents.push(e); + } + if (e.type === "provider_thread.updated" && e.providerThread.id === thread.id) { + for (const task of e.providerThread.pendingBackgroundTasks ?? []) { + if (task.kind === "subagent") rosterDescriptions.push(task.description); + } + } + if ( + e.type === "provider_turn.updated" && + e.threadId === creates[0]?.appThread.id && + childNativeTurnId === undefined + ) { + childNativeTurnId = e.providerTurn.nativeTurnRef?.nativeId ?? undefined; + } + }), + ), + Stream.runDrain, + Effect.forkScoped, + ); + yield* runtime.startTurn(withLineage(thread)); + yield* Deferred.await(done); + assert.lengthOf(creates, 1); + assert.isTrue(creates[0]?.createOnly); + assert.isTrue(subagentNodeEvents.every((e) => e.preserveIdentity === true)); + assert.isTrue(subagentEvents.every((e) => e.preserveIdentity === true)); + assert.isTrue(subagentItemEvents.every((e) => e.preserveIdentity === true)); + assert.isTrue(rosterDescriptions.every((description) => description === undefined)); + assert.equal(subagentEvents.at(-1)?.subagent.nativeTaskRef?.nativeId, CHILD); + assert.equal(subagentEvents.at(-1)?.subagent.model, "opencode/big-pickle"); + assert.equal(subagentEvents.at(-1)?.subagent.result, "RESUMED_OK"); + assert.isDefined(childNativeTurnId); + assert.notEqual(childNativeTurnId, `${CHILD}:turn:1`); + }), + ); + + it.effect( + "promotes the next queued prompt when an idle child's leading prompt is cancelled", + () => + Effect.gen(function* () { + const { runtime, thread } = yield* resumed([ + ...backgroundLaunch(CHILD), + event("session.inbox.enqueued", { + inboxID: "msg_child_first", + sessionID: CHILD, + item: { + type: "user", + payload: { text: "sleep", files: [] }, + delivery: "steer", + }, + }), + event("session.inbox.enqueued", { + inboxID: "msg_child_second", + sessionID: CHILD, + item: { + type: "user", + payload: { text: "Promoted prompt", files: [] }, + delivery: "steer", + }, + }), + event("session.inbox.cancelled", { sessionID: CHILD, inboxID: "msg_child_first" }), + returnedRunning("call-background"), + event("session.execution.started", { sessionID: CHILD }), + event("session.inbox.delivered", { sessionID: CHILD, inboxID: "msg_child_second" }), + childSaid("PROMOTED_OK"), + event("session.execution.succeeded", { sessionID: CHILD }), + event("session.execution.succeeded", { sessionID: SESSION }), + childReport("msg_report", "PROMOTED_OK"), + ]); + let childThreadId: ThreadId | undefined; + const prompts: Array<{ readonly text: string; readonly inputIntent: string }> = []; + const done = yield* Deferred.make(); + yield* runtime.events.pipe( + Stream.tap((e) => + Effect.gen(function* () { + if (e.type === "app_thread.created") childThreadId = e.appThread.id; + if ( + e.type === "turn_item.updated" && + e.turnItem.type === "user_message" && + e.turnItem.threadId === childThreadId + ) { + prompts.push({ text: e.turnItem.text, inputIntent: e.turnItem.inputIntent }); + } + if (e.type === "subagent.updated" && e.subagent.status === "completed") { + yield* Deferred.succeed(done, undefined); + } + }), + ), + Stream.runDrain, + Effect.forkScoped, + ); + yield* runtime.startTurn(withLineage(thread)); + yield* Deferred.await(done); + assert.deepEqual(prompts, [{ text: "Promoted prompt", inputIntent: "turn_start" }]); + }), + ); + + it.effect.each([true, false] as const)( + "recovers a child's delivered steer from history backfill on reconnect (sawInitialEnqueue=%s)", + (sawInitialEnqueue) => + Effect.gen(function* () { + const { runtime, thread } = yield* resumed([ + ...backgroundLaunch(CHILD), + ...(sawInitialEnqueue + ? [ + event("session.inbox.enqueued", { + inboxID: "msg_child_initial", + sessionID: CHILD, + item: { + type: "user", + payload: { text: "sleep", files: [] }, + delivery: "steer", + }, + }), + ] + : []), + returnedRunning("call-background"), + event("session.execution.started", { sessionID: CHILD }), + ...(sawInitialEnqueue + ? [event("session.inbox.delivered", { sessionID: CHILD, inboxID: "msg_child_initial" })] + : []), + ...callAgain({ background: true, description: "Pivot", prompt: "Pivot to B" }), + event("session.inbox.enqueued", { + inboxID: "msg_steer_gap", + sessionID: CHILD, + item: { + type: "user", + payload: { text: "Pivot to B", files: [] }, + delivery: "steer", + }, + }), + event("session.execution.succeeded", { sessionID: SESSION }), + { type: "runtime_exit", status: "success" }, + out("event.subscribe"), + out("session.active"), + replyData("session.active", { [CHILD]: { type: "running" } }), + out("message.list", { sessionID: CHILD, order: "desc", limit: "50" }), + reply("message.list", { + data: [ + { + id: "msg_child_reply_gap", + time: { created: 3, streamed: 3, completed: 3 }, + type: "assistant", + agent: "general", + model: { id: "big-pickle", providerID: "opencode", variant: "default" }, + content: [{ type: "text", text: "Switched to B" }], + finish: "stop", + }, + { + id: "msg_steer_gap", + time: { created: 2 }, + type: "user", + text: "Pivot to B", + }, + { + id: "msg_child_initial", + time: { created: 1 }, + type: "user", + text: "sleep", + }, + ], + cursor: {}, + }), + out("permission.list", { sessionID: CHILD }), + replyData("permission.list", []), + out("session.form.list", { sessionID: CHILD }), + replyData("session.form.list", []), + event("session.execution.succeeded", { sessionID: CHILD }), + childReport("msg_report", "Switched to B"), + ]); + let childThreadId: ThreadId | undefined; + const childTimeline: Array = []; + const done = yield* Deferred.make(); + yield* runtime.events.pipe( + Stream.tap((e) => + Effect.gen(function* () { + if (e.type === "app_thread.created") childThreadId = e.appThread.id; + if (e.type === "turn_item.updated" && e.turnItem.threadId === childThreadId) { + if (e.turnItem.type === "user_message") { + childTimeline.push(`${e.turnItem.inputIntent}:${e.turnItem.text}`); + } else if (e.turnItem.type === "assistant_message") { + childTimeline.push(`assistant:${e.turnItem.text}`); + } + } + if (e.type === "subagent.updated" && e.subagent.status === "completed") { + yield* Deferred.succeed(done, undefined); + } + }), + ), + Stream.runDrain, + Effect.forkScoped, + ); + yield* runtime.startTurn(withLineage(thread)); + yield* Deferred.await(done); + assert.deepEqual(childTimeline, [ + "turn_start:sleep", + "steer:Pivot to B", + "assistant:Switched to B", + ]); + }), + ); + + it.effect( + "moves a still-running subagent row to a later parent run when redirected before its report arrives", + () => + Effect.gen(function* () { + const { runtime, thread } = yield* resumed([ + ...launchedAndRunning, + event("session.execution.succeeded", { sessionID: SESSION }), + out("session.update", { sessionID: CHILD, permissions: "" }), + reply("session.update", null), + out("session.prompt", { sessionID: SESSION, text: "" }), + promptAccepted, + event("session.execution.started", { sessionID: SESSION }), + ...callAgain( + { background: true, description: "Redirect in run 2", prompt: "pivot in run 2" }, + { id: "call-run2-redirect", writesRules: false }, + ), + event("session.execution.succeeded", { sessionID: SESSION }), + event("session.execution.succeeded", { sessionID: CHILD }), + childReport("msg_report", "MOVED_OK"), + ]); + const run2Id = RunId.make("run:opencode2-adapter:2"); + const run1Done = yield* Deferred.make(); + const run2Done = yield* Deferred.make(); + const subagentIds = new Set(); + const seenRun2Statuses: Array = []; + let finalSubagent: + | Extract< + ProviderAdapter.ProviderAdapterV2Event, + { type: "subagent.updated" } + >["subagent"] + | undefined; + yield* runtime.events.pipe( + Stream.tap((e) => + Effect.gen(function* () { + if (e.type === "subagent.updated") { + subagentIds.add(e.subagent.id); + finalSubagent = e.subagent; + if (e.subagent.runId === run2Id) { + seenRun2Statuses.push(e.subagent.status); + if (e.subagent.status === "completed") + yield* Deferred.succeed(run2Done, undefined); + } + } + if (e.type === "turn.terminal" && e.runOrdinal === 1) { + yield* Deferred.succeed(run1Done, undefined); + } + }), + ), + Stream.runDrain, + Effect.forkScoped, + ); + yield* runtime.startTurn(withLineage(thread)); + yield* Deferred.await(run1Done); + yield* runtime.startTurn({ + ...withLineage(thread), + runId: run2Id, + runOrdinal: 2, + providerTurnOrdinal: 2, + attemptId: RunAttemptId.make("attempt:opencode2-adapter:2"), + }); + yield* Deferred.await(run2Done); + assert.equal(subagentIds.size, 1); + assert.deepEqual(seenRun2Statuses, ["running", "completed"]); + assert.equal(finalSubagent?.runId, run2Id); + assert.equal(finalSubagent?.title, "Sleep"); + assert.equal(finalSubagent?.result, "MOVED_OK"); + }), + ); + + it.effect( + "dedupes parent history subagent tool parts on reconnect without emitting generic tool calls", + () => + Effect.gen(function* () { + const { runtime, thread } = yield* resumed([ + ...launchedAndRunning, + { type: "runtime_exit", status: "success" }, + out("event.subscribe"), + out("session.active"), + replyData("session.active", { + [SESSION]: { type: "running" }, + [CHILD]: { type: "running" }, + }), + out("message.list", { sessionID: SESSION, order: "desc", limit: "50" }), + reply("message.list", { + data: [ + { + id: "msg_parent_gap", + time: { created: 2, streamed: 2, completed: 2 }, + type: "assistant", + agent: "build", + model: { id: "big-pickle", providerID: "opencode", variant: "default" }, + content: [ + { + type: "tool", + id: "call-background", + name: "subagent", + executed: true, + state: { + status: "completed", + input: { description: "Sleep", prompt: "sleep", background: true }, + content: [ + { type: "text", text: "The subagent is working in the background." }, + ], + metadata: { sessionID: CHILD, status: "running" }, + }, + time: { created: 2, ran: 2, completed: 2 }, + }, + { + type: "tool", + id: "call-gap-redirect", + name: "subagent", + executed: true, + state: { + status: "completed", + input: { + description: "Redirect in gap", + prompt: "redirect", + background: true, + sessionID: CHILD, + }, + content: [ + { type: "text", text: "The subagent is working in the background." }, + ], + metadata: { sessionID: CHILD, status: "running" }, + }, + time: { created: 2, ran: 2, completed: 2 }, + }, + ], + finish: "stop", + }, + { + id: `msg_t3_turn_${SESSION}:attempt:opencode2-adapter`, + time: { created: 1 }, + type: "user", + text: "hi", + }, + ], + cursor: {}, + }), + out("session.update", { sessionID: CHILD, permissions: "" }), + reply("session.update", null), + out("permission.list", { sessionID: SESSION }), + replyData("permission.list", []), + out("session.form.list", { sessionID: SESSION }), + replyData("session.form.list", []), + out("permission.list", { sessionID: CHILD }), + replyData("permission.list", []), + out("session.form.list", { sessionID: CHILD }), + replyData("session.form.list", []), + event("session.execution.succeeded", { sessionID: SESSION }), + event("session.execution.succeeded", { sessionID: CHILD }), + childReport("msg_report", "RECONNECTED_OK"), + ]); + const subagentItemIds = new Set(); + const genericSubagentTools: Array = []; + const done = yield* Deferred.make(); + yield* runtime.events.pipe( + Stream.tap((e) => + Effect.gen(function* () { + if (e.type === "turn_item.updated") { + if (e.turnItem.type === "subagent") subagentItemIds.add(e.turnItem.id); + if (e.turnItem.type === "dynamic_tool" && e.turnItem.toolName === "subagent") { + genericSubagentTools.push(e.turnItem.id); + } + } + if (e.type === "subagent.updated" && e.subagent.status === "completed") { + yield* Deferred.succeed(done, undefined); + } + }), + ), + Stream.runDrain, + Effect.forkScoped, + ); + yield* runtime.startTurn(withLineage(thread)); + yield* Deferred.await(done); + assert.deepEqual(genericSubagentTools, []); + assert.equal(subagentItemIds.size, 1); + }), + ); + + it.effect( + "replays a history-only redirect to a previously settled child on reconnect without replaying settled calls", + () => + Effect.gen(function* () { + const { runtime, thread } = yield* resumed([ + ...backgroundLaunch(CHILD), + event( + "session.tool.success", + { + sessionID: SESSION, + assistantMessageID: "msg_assistant", + id: "call-background", + content: [{ type: "text", text: "FIRST_OK" }], + metadata: { sessionID: CHILD, status: "completed" }, + executed: true, + }, + undefined, + 2, + ), + { type: "runtime_exit", status: "success" }, + out("event.subscribe"), + out("session.active"), + replyData("session.active", { [SESSION]: { type: "running" } }), + out("message.list", { sessionID: SESSION, order: "desc", limit: "50" }), + reply("message.list", { + data: [ + { + id: "msg_parent_gap", + time: { created: 2, streamed: 2, completed: 2 }, + type: "assistant", + agent: "build", + model: { id: "big-pickle", providerID: "opencode", variant: "default" }, + content: [ + { + type: "tool", + id: "call-background", + name: "subagent", + executed: true, + state: { + status: "completed", + input: { description: "Sleep", prompt: "sleep" }, + content: [{ type: "text", text: "FIRST_OK" }], + metadata: { sessionID: CHILD, status: "completed" }, + }, + time: { created: 2, ran: 2, completed: 2 }, + }, + { + type: "tool", + id: "call-gap-redirect", + name: "subagent", + executed: true, + state: { + status: "completed", + input: { + description: "Redirect after settle", + prompt: "redirect", + sessionID: CHILD, + }, + content: [{ type: "text", text: "SECOND_GAP_OK" }], + metadata: { sessionID: CHILD, status: "completed" }, + }, + time: { created: 3, ran: 3, completed: 3 }, + }, + ], + finish: "stop", + }, + { + id: `msg_t3_turn_${SESSION}:attempt:opencode2-adapter`, + time: { created: 1 }, + type: "user", + text: "hi", + }, + ], + cursor: {}, + }), + out("session.update", { sessionID: CHILD, permissions: "" }), + reply("session.update", null), + out("permission.list", { sessionID: SESSION }), + replyData("permission.list", []), + out("session.form.list", { sessionID: SESSION }), + replyData("session.form.list", []), + event("session.execution.succeeded", { sessionID: SESSION }), + ]); + const subagentIds = new Set(); + const results: Array = []; + let finalSubagent: + | Extract< + ProviderAdapter.ProviderAdapterV2Event, + { type: "subagent.updated" } + >["subagent"] + | undefined; + const done = yield* Deferred.make(); + yield* runtime.events.pipe( + Stream.tap((e) => + Effect.gen(function* () { + if (e.type === "subagent.updated") { + subagentIds.add(e.subagent.id); + finalSubagent = e.subagent; + if (e.subagent.status === "completed") results.push(e.subagent.result); + } + if (e.type === "turn.terminal") yield* Deferred.succeed(done, undefined); + }), + ), + Stream.runDrain, + Effect.forkScoped, + ); + yield* runtime.startTurn(withLineage(thread)); + yield* Deferred.await(done); + assert.equal(subagentIds.size, 1); + assert.deepEqual(results, ["FIRST_OK", "SECOND_GAP_OK"]); + assert.equal(finalSubagent?.title, "Sleep"); + assert.equal(finalSubagent?.result, "SECOND_GAP_OK"); }), ); - // regression: #21979 a failed background subagent pinned its thread as busy - it.effect("settles the background call a foreground call joined to a failed child", () => - Effect.gen(function* () { - const again = { sessionID: SESSION, assistantMessageID: "msg_assistant", id: "call-again" }; - const { runtime, thread } = yield* resumed([ - ...launchedAndRunning, - // A foreground call joins the running background child. It is only - // marked background once its own tool call returns, so the child can - // fail while this call is still the foreground one. - event("session.tool.input.started", { ...again, name: "subagent" }), - event("session.tool.called", { - ...again, - name: "subagent", - input: { description: "Again", prompt: "also this", sessionID: CHILD }, - executed: false, - }), - event("session.tool.progress", { - ...again, - metadata: { sessionID: CHILD, status: "running" }, - }), - out("session.update", { sessionID: CHILD, permissions: "" }), - reply("session.update", null), - // The child fails while the joined call is still the foreground one. - event("session.execution.failed", { - sessionID: CHILD, - error: { type: "provider.quota", message: "Rate limit exceeded." }, - }), - event("session.execution.succeeded", { sessionID: SESSION }), - ]); - const watch = yield* watchCalls(runtime, thread); - yield* runtime.startTurn(withLineage(thread)); - yield* watch.until( - () => - watch.calls.get("Sleep")?.status === "failed" && - watch.calls.get("Again")?.status === "failed" && - watch.state.roster?.length === 0, - ); - assert.deepEqual(watch.state.roster, []); - assert.isFalse(yield* runtime.hasPendingBackgroundWorkForThread!(thread)); - }), - ); - - it.effect("settles every call that joined a running subagent with its one report", () => - Effect.gen(function* () { - const { runtime, thread } = yield* resumed([ - ...launchedAndRunning, - ...callAgain({ background: true }), - event("session.execution.succeeded", { sessionID: SESSION }), - event("session.execution.succeeded", { sessionID: CHILD }), - childReport("msg_report", "CHILD_OK"), - ]); - const watch = yield* watchCalls(runtime, thread); - yield* runtime.startTurn(withLineage(thread)); - yield* watch.until( - () => - watch.calls.get("Again")?.status === "completed" && - watch.calls.get("Sleep")?.status === "completed" && - watch.state.roster?.length === 0, - ); - assert.deepEqual(Object.fromEntries(watch.calls), { - Sleep: { status: "completed", result: "CHILD_OK" }, - Again: { status: "completed", result: "CHILD_OK" }, - }); - }), - ); - - it.effect("keeps a call to a finished subagent for its own run's report", () => - Effect.gen(function* () { - const { runtime, thread } = yield* resumed([ - ...launchedAndRunning, - event("session.execution.succeeded", { sessionID: CHILD }), - ...callAgain({ background: true }), - childReport("msg_report_1", "FIRST_OK"), - event("session.execution.started", { sessionID: CHILD }), - event("session.execution.succeeded", { sessionID: CHILD }), - event("session.execution.succeeded", { sessionID: SESSION }), - childReport("msg_report_2", "SECOND_OK"), - ]); - const watch = yield* watchCalls(runtime, thread); - yield* runtime.startTurn(withLineage(thread)); - yield* watch.until(() => watch.childTurns.has(`${CHILD}:turn:2`)); - assert.deepEqual(watch.callsAtChildTurnStart.get(`${CHILD}:turn:2`), { - Sleep: { status: "completed", result: "FIRST_OK" }, - Again: { status: "running", result: null }, - }); - yield* watch.until(() => watch.calls.get("Again")?.status === "completed"); - assert.deepEqual(watch.calls.get("Again"), { status: "completed", result: "SECOND_OK" }); - }), - ); - - const childSaid = (text: string, run = 1) => - event("session.text.ended", { - sessionID: CHILD, - assistantMessageID: `msg_child_${run}`, - ordinal: 0, - text, - }); - it.effect.each([ - { - // regression: #26319 the report settled the older run and left the resumed call running - name: "an earlier run's report never arrived", - replay: [ - ...launchedAndRunning, - ...callAgain({ background: true }), - childSaid("FIRST_OK"), - event("session.execution.succeeded", { sessionID: CHILD }), - ...callAgain({ background: true }, { id: "call-later", writesRules: false }), - event("session.execution.started", { sessionID: CHILD }), - childSaid("SECOND_OK", 2), - event("session.execution.succeeded", { sessionID: CHILD }), - event("session.execution.succeeded", { sessionID: SESSION }), - childReport("msg_report_2", "SECOND_OK", "Again"), - ], - expected: { - "call-background": { status: "completed", result: "FIRST_OK" }, - "call-again": { status: "completed", result: "FIRST_OK" }, - "call-later": { status: "completed", result: "SECOND_OK" }, - }, - }, - { - name: "both runs' reports arrive after the runs ended", - replay: [ - ...launchedAndRunning, - event("session.execution.succeeded", { sessionID: CHILD }), - ...callAgain({ background: true, description: "Sleep" }), - event("session.execution.started", { sessionID: CHILD }), - event("session.execution.succeeded", { sessionID: CHILD }), - childReport("msg_report_1", "FIRST_OK"), - childReport("msg_report_2", "SECOND_OK"), - event("session.execution.succeeded", { sessionID: SESSION }), - ], - expected: { - "call-background": { status: "completed", result: "FIRST_OK" }, - "call-again": { status: "completed", result: "SECOND_OK" }, - }, - }, - { - name: "two runs with the same answer report late", - replay: [ - ...launchedAndRunning, - childSaid("DONE"), - event("session.execution.succeeded", { sessionID: CHILD }), - ...callAgain({ background: true }), - event("session.execution.started", { sessionID: CHILD }), - childSaid("DONE", 2), - event("session.execution.succeeded", { sessionID: CHILD }), - childReport("msg_report_1", "DONE"), - ...callAgain({ background: true }, { id: "call-later", writesRules: false }), - event("session.execution.started", { sessionID: CHILD }), - childReport("msg_report_2", "DONE", "Again"), - childSaid("THIRD_OK", 3), - event("session.execution.succeeded", { sessionID: CHILD }), - event("session.execution.succeeded", { sessionID: SESSION }), - childReport("msg_report_3", "THIRD_OK", "Again"), - ], - expected: { - "call-background": { status: "completed", result: "DONE" }, - "call-again": { status: "completed", result: "DONE" }, - "call-later": { status: "completed", result: "THIRD_OK" }, - }, - }, - { - name: "the call it joined already returned", - replay: [ - ...backgroundLaunch(CHILD), - event("session.execution.started", { sessionID: CHILD }), - ...callAgain({ background: true }), - event("session.execution.succeeded", { sessionID: CHILD }), - event( - "session.tool.success", - { + it.effect( + "recovers subagent prompt and title from history backfill when disconnected after input.started", + () => + Effect.gen(function* () { + const { runtime, thread } = yield* resumed([ + out("session.prompt", { sessionID: SESSION, text: "" }), + promptAccepted, + event("session.execution.started", { sessionID: SESSION }), + event("session.tool.input.started", { sessionID: SESSION, assistantMessageID: "msg_assistant", - id: "call-background", - content: [{ type: "text", text: "FIRST_OK" }], - metadata: { sessionID: CHILD, status: "completed" }, - executed: true, - }, - undefined, - 2, - ), - event("session.execution.succeeded", { sessionID: SESSION }), - childReport("msg_report", "CHILD_OK", "Again"), - ], - expected: { - "call-background": { status: "completed", result: "FIRST_OK" }, - "call-again": { status: "completed", result: "CHILD_OK" }, - }, - }, - ])("settles a resumed subagent's calls when $name", ({ replay, expected }) => - Effect.gen(function* () { - const { runtime, thread } = yield* resumed(replay); - const watch = yield* watchCalls(runtime, thread); - yield* runtime.startTurn(withLineage(thread)); - yield* watch.until( - () => - watch.state.roster?.length === 0 && - Object.keys(expected).every((id) => watch.callsById.get(id)?.status === "completed"), - ); - assert.deepEqual(Object.fromEntries(watch.callsById), expected); - }), + id: "call-partial", + name: "subagent", + }), + { type: "runtime_exit", status: "success" }, + out("event.subscribe"), + out("session.active"), + replyData("session.active", { [SESSION]: { type: "running" } }), + out("message.list", { sessionID: SESSION, order: "desc", limit: "50" }), + reply("message.list", { + data: [ + { + id: "msg_assistant", + time: { created: 2, streamed: 2, completed: 2 }, + type: "assistant", + agent: "build", + model: { id: "big-pickle", providerID: "opencode", variant: "default" }, + content: [ + { + type: "tool", + id: "call-partial", + name: "subagent", + executed: true, + state: { + status: "completed", + input: { description: "Inspect logs", prompt: "check stderr" }, + content: [{ type: "text", text: "LOGS_OK" }], + metadata: { sessionID: CHILD, status: "completed" }, + }, + time: { created: 2, ran: 2, completed: 2 }, + }, + ], + finish: "stop", + }, + { + id: `msg_t3_turn_${SESSION}:attempt:opencode2-adapter`, + time: { created: 1 }, + type: "user", + text: "hi", + }, + ], + cursor: {}, + }), + out("session.update", { sessionID: CHILD, permissions: "" }), + reply("session.update", null), + out("permission.list", { sessionID: SESSION }), + replyData("permission.list", []), + out("session.form.list", { sessionID: SESSION }), + replyData("session.form.list", []), + event("session.execution.succeeded", { sessionID: SESSION }), + ]); + let finalSubagent: + | Extract< + ProviderAdapter.ProviderAdapterV2Event, + { type: "subagent.updated" } + >["subagent"] + | undefined; + let childTitle: string | undefined; + const done = yield* Deferred.make(); + yield* runtime.events.pipe( + Stream.tap((e) => + Effect.gen(function* () { + if (e.type === "app_thread.created") childTitle = e.appThread.title; + if (e.type === "subagent.updated") finalSubagent = e.subagent; + if (e.type === "turn.terminal") yield* Deferred.succeed(done, undefined); + }), + ), + Stream.runDrain, + Effect.forkScoped, + ); + yield* runtime.startTurn(withLineage(thread)); + yield* Deferred.await(done); + assert.equal(childTitle, "Inspect logs"); + assert.equal(finalSubagent?.title, "Inspect logs"); + assert.equal(finalSubagent?.prompt, "check stderr"); + assert.equal(finalSubagent?.result, "LOGS_OK"); + }), ); const MIDDLE = "ses_middle0000000000000000000"; @@ -4266,8 +5515,21 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { event("form.created", { form }), event("session.execution.succeeded", { sessionID: SESSION }), ]); - const requested = yield* requestOf(runtime).pipe(Effect.forkScoped); + let request: + | Extract< + ProviderAdapter.ProviderAdapterV2Event, + { type: "runtime_request.updated" } + >["runtimeRequest"] + | undefined; + // Single consumer: runtime.events is queue-backed, so two fibers compete for items. const ended = yield* runtime.events.pipe( + Stream.tap((event) => + Effect.sync(() => { + if (event.type === "runtime_request.updated" && request === undefined) { + request = event.runtimeRequest; + } + }), + ), Stream.filter( (event) => event.type === "turn.terminal" || @@ -4278,9 +5540,9 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { Effect.forkScoped, ); yield* runtime.startTurn(withLineage(thread)); - const request = yield* Fiber.join(requested); + const terminals = yield* Fiber.join(ended); assert.equal(request?.nativeRequestRef?.nativeId, form.id); - for (const terminal of yield* Fiber.join(ended)) { + for (const terminal of terminals) { assert.deepEqual( "retainedRuntimeRequestIds" in terminal ? terminal.retainedRuntimeRequestIds : undefined, [request!.id], diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index ba91ba94d2520..1acbcdf3da8e2 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -591,6 +591,26 @@ export interface ProjectionStoreV2Shape { readonly threadId: ThreadId; readonly itemId: TurnItemId; }) => Effect.Effect; + /** Preserve an existing turn's ordinal or pick the next free ordinal for a runless provider turn. */ + readonly resolveProviderTurnOrdinal: (input: { + readonly threadId: ThreadId; + readonly providerTurnId: ProviderTurnId; + readonly providerThreadId: ProviderThreadId; + readonly ordinal: number; + }) => Effect.Effect; + /** Resolve a persisted subagent row and turn item by ID, child thread, or provider thread. */ + readonly findSubagentIdentity: (input: { + readonly threadId: ThreadId; + readonly subagentId: NodeId; + readonly childThreadId: ThreadId | null; + readonly providerThreadId: ProviderThreadId | null; + }) => Effect.Effect< + { + readonly subagent: OrchestrationV2Subagent; + readonly turnItem: Extract | null; + } | null, + ProjectionStoreV2Error + >; readonly getMessagesById: ( threadId: ThreadId, messageIds: ReadonlyArray, @@ -6801,6 +6821,80 @@ export const layer: Layer.Layer = Effect.mapError(controlReadError(threadId)), ); + const resolveProviderTurnOrdinal: ProjectionStoreV2Shape["resolveProviderTurnOrdinal"] = + Effect.fn("ProjectionStore.resolveProviderTurnOrdinal")(function* (input) { + const existingRows = yield* sql<{ ordinal: number }>` + SELECT ordinal + FROM orchestration_v2_projection_provider_turns + WHERE provider_turn_id = ${input.providerTurnId} + LIMIT 1 + `.pipe(Effect.mapError(controlReadError(input.threadId))); + if (existingRows[0] !== undefined) return existingRows[0].ordinal; + const occupantRows = yield* sql<{ provider_turn_id: string }>` + SELECT provider_turn_id + FROM orchestration_v2_projection_provider_turns + WHERE provider_thread_id = ${input.providerThreadId} AND ordinal = ${input.ordinal} + LIMIT 1 + `.pipe(Effect.mapError(controlReadError(input.threadId))); + if (occupantRows[0] === undefined) return input.ordinal; + const maxRows = yield* sql<{ max_ordinal: number | null }>` + SELECT MAX(ordinal) AS max_ordinal + FROM orchestration_v2_projection_provider_turns + WHERE provider_thread_id = ${input.providerThreadId} + `.pipe(Effect.mapError(controlReadError(input.threadId))); + return (maxRows[0]?.max_ordinal ?? input.ordinal) + 1; + }); + + const findSubagentIdentity: ProjectionStoreV2Shape["findSubagentIdentity"] = Effect.fn( + "ProjectionStore.findSubagentIdentity", + )(function* (input) { + let rows = yield* sql<{ payload_json: string }>` + SELECT payload_json + FROM orchestration_v2_projection_subagents + WHERE subagent_id = ${input.subagentId} AND thread_id = ${input.threadId} + LIMIT 1 + `.pipe(Effect.mapError(controlReadError(input.threadId))); + if (rows[0] === undefined && input.childThreadId !== null) { + rows = yield* sql<{ payload_json: string }>` + SELECT payload_json + FROM orchestration_v2_projection_subagents + WHERE child_thread_id = ${input.childThreadId} AND thread_id = ${input.threadId} + ORDER BY rowid ASC + LIMIT 1 + `.pipe(Effect.mapError(controlReadError(input.threadId))); + } + if (rows[0] === undefined && input.providerThreadId !== null) { + rows = yield* sql<{ payload_json: string }>` + SELECT payload_json + FROM orchestration_v2_projection_subagents + WHERE provider_thread_id = ${input.providerThreadId} AND thread_id = ${input.threadId} + ORDER BY rowid ASC + LIMIT 1 + `.pipe(Effect.mapError(controlReadError(input.threadId))); + } + if (rows[0] === undefined) return null; + const subagent = yield* decodeSubagentPayload(rows[0].payload_json).pipe( + Effect.mapError(controlReadError(input.threadId)), + ); + const itemRows = yield* sql<{ payload_json: string }>` + SELECT payload_json + FROM orchestration_v2_projection_turn_items + WHERE node_id = ${subagent.id} AND thread_id = ${input.threadId} AND type = 'subagent' + ORDER BY ordinal ASC + LIMIT 1 + `.pipe(Effect.mapError(controlReadError(input.threadId))); + const decodedItem = + itemRows[0] === undefined + ? null + : yield* decodeTurnItemPayload(itemRows[0].payload_json).pipe( + Effect.mapError(controlReadError(input.threadId)), + ); + return { + subagent, + turnItem: decodedItem?.type === "subagent" ? decodedItem : null, + }; + }); + // A run fork shows its source's history through the fork run without copying it, and every // run fork below it shows that history too. Returns the last source run live forks show, with // and without `leavingForkId`. @@ -9871,6 +9965,8 @@ export const layer: Layer.Layer = hasPendingDelegatedCompletion: readOnReader(hasPendingDelegatedCompletion), getNextTurnItemOrdinal: readOnReader(getNextTurnItemOrdinal), getTurnItem: readOnReader(getTurnItem), + resolveProviderTurnOrdinal: readOnReader(resolveProviderTurnOrdinal), + findSubagentIdentity: readOnReader(findSubagentIdentity), getMessagesById: readOnReader(getMessagesById), getThreadRecords: (threadId, fields, filter) => getThreadRecords(threadId, fields, filter).pipe(NodeSqliteClient.readOnly), diff --git a/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts b/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts index 938ad7e81eec5..a23748a69c21d 100644 --- a/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts +++ b/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts @@ -2462,6 +2462,686 @@ layer("ProviderEventIngestorV2", (it) => { ), ), ); + + it.effect.each(["session-keyed", "legacy-call-keyed", "legacy-duplicate-rows"] as const)( + "preserves subagent row identity and prior child turns across a cold restart resume (%s)", + (initialKeying) => + Effect.gen(function* () { + const now = yield* DateTime.now; + const t2 = DateTime.add(now, { seconds: 5 }); + const t3 = DateTime.add(now, { seconds: 10 }); + const eventSink = yield* EventSink.EventSinkV2; + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const threadEvent = yield* threadCreatedEvent(now); + const parentThread = threadEvent.payload as OrchestrationV2AppThread; + const parentThreadId = parentThread.id; + const parentProviderThreadId = parentThread.activeProviderThreadId!; + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId: parentThreadId, + }); + yield* eventSink.write({ events: [threadEvent] }); + + const driver = ProviderDriverKind.make("opencode"); + const instanceId = ProviderInstanceId.make("opencode"); + const childSessionId = `ses_cold_resume_child_${initialKeying}`; + const initialNativeItemId = + initialKeying === "session-keyed" ? childSessionId : "call-launch-1"; + const initialNodeId = idAllocator.derive.nodeFromProviderItem({ + driver, + nativeItemId: initialNativeItemId, + }); + const initialItemId = idAllocator.derive.turnItemFromProviderItem({ + driver, + nativeItemId: initialNativeItemId, + }); + const sessionNodeId = idAllocator.derive.nodeFromProviderItem({ + driver, + nativeItemId: childSessionId, + }); + const sessionItemId = idAllocator.derive.turnItemFromProviderItem({ + driver, + nativeItemId: childSessionId, + }); + const childThreadId = idAllocator.derive.subagentThread({ + driver, + nativeThreadId: childSessionId, + providerInstanceId: instanceId, + }); + const childProviderThreadId = idAllocator.derive.providerThread({ + driver, + nativeThreadId: childSessionId, + providerInstanceId: instanceId, + }); + const run1Id = RunId.make("run:cold-resume:1"); + const run2Id = RunId.make("run:cold-resume:2"); + const parentTurn1Id = ProviderTurnId.make("provider-turn:parent:1"); + const parentTurn2Id = ProviderTurnId.make("provider-turn:parent:2"); + const parentRoot1 = NodeId.make("node:parent:root:1"); + const parentRoot2 = NodeId.make("node:parent:root:2"); + const childTurn1Id = idAllocator.derive.providerTurn({ + driver, + nativeTurnId: `${childSessionId}:turn:1`, + }); + const childRoot1 = idAllocator.derive.nodeFromProviderItem({ + driver, + nativeItemId: `${childSessionId}:turn:1:root`, + }); + const childTurn2Id = idAllocator.derive.providerTurn({ + driver, + nativeTurnId: `${childSessionId}:resume:call-2:turn:1`, + }); + const childRoot2 = idAllocator.derive.nodeFromProviderItem({ + driver, + nativeItemId: `${childSessionId}:resume:call-2:turn:1:root`, + }); + const ingest = (runId: RunId, event: ProviderAdapter.ProviderAdapterV2Event) => + ingestor.ingestNormalized({ + providerSessionId, + providerInstanceId: instanceId, + threadId: parentThreadId, + runId, + event, + }); + const childAppThread = (title: string): OrchestrationV2AppThread => ({ + ...parentThread, + id: childThreadId, + title, + providerInstanceId: instanceId, + modelSelection: { instanceId, model: "opencode/big-pickle" }, + activeProviderThreadId: childProviderThreadId, + lineage: { + parentThreadId, + relationshipToParent: "subagent", + rootThreadId: parentThreadId, + }, + createdBy: "agent", + creationSource: "provider", + }); + + // Run 1: initial launch persists the subagent row and child turn 1 (ordinals 101, 102). + yield* ingest(run1Id, { + type: "app_thread.created", + driver, + appThread: childAppThread("Sleep"), + }); + yield* ingest(run1Id, { + type: "node.updated", + driver, + node: { + id: initialNodeId, + threadId: parentThreadId, + runId: run1Id, + parentNodeId: parentRoot1, + rootNodeId: parentRoot1, + kind: "subagent", + status: "completed", + countsForRun: false, + providerThreadId: childProviderThreadId, + providerTurnId: parentTurn1Id, + nativeItemRef: { driver, nativeId: initialNativeItemId, strength: "strong" }, + runtimeRequestId: null, + checkpointScopeId: null, + startedAt: now, + completedAt: now, + }, + }); + yield* ingest(run1Id, { + type: "subagent.updated", + driver, + subagent: { + id: initialNodeId, + threadId: parentThreadId, + runId: run1Id, + parentNodeId: parentRoot1, + origin: "provider_native", + createdBy: "agent", + driver, + providerInstanceId: instanceId, + providerThreadId: childProviderThreadId, + childThreadId, + nativeTaskRef: { driver, nativeId: initialNativeItemId, strength: "strong" }, + prompt: "sleep 180", + title: "Sleep", + model: "opencode/big-pickle", + status: "completed", + result: "FIRST_OK", + startedAt: now, + completedAt: now, + updatedAt: now, + }, + }); + yield* ingest(run1Id, { + type: "turn_item.updated", + driver, + turnItem: { + id: initialItemId, + threadId: parentThreadId, + runId: run1Id, + nodeId: initialNodeId, + providerThreadId: parentProviderThreadId, + providerTurnId: parentTurn1Id, + nativeItemRef: { driver, nativeId: initialNativeItemId, strength: "strong" }, + parentItemId: null, + ordinal: 1, + status: "completed", + title: "Sleep", + startedAt: now, + completedAt: now, + updatedAt: now, + type: "subagent", + subagentId: initialNodeId, + origin: "provider_native", + driver, + providerInstanceId: instanceId, + childThreadId, + prompt: "sleep 180", + result: "FIRST_OK", + }, + }); + if (initialKeying === "legacy-duplicate-rows") { + const duplicateNodeId = idAllocator.derive.nodeFromProviderItem({ + driver, + nativeItemId: "call-legacy-dup-2", + }); + const duplicateItemId = idAllocator.derive.turnItemFromProviderItem({ + driver, + nativeItemId: "call-legacy-dup-2", + }); + yield* ingest(run1Id, { + type: "node.updated", + driver, + node: { + id: duplicateNodeId, + threadId: parentThreadId, + runId: run1Id, + parentNodeId: parentRoot1, + rootNodeId: parentRoot1, + kind: "subagent", + status: "completed", + countsForRun: false, + providerThreadId: childProviderThreadId, + providerTurnId: parentTurn1Id, + nativeItemRef: { driver, nativeId: "call-legacy-dup-2", strength: "strong" }, + runtimeRequestId: null, + checkpointScopeId: null, + startedAt: t2, + completedAt: t2, + }, + }); + yield* ingest(run1Id, { + type: "subagent.updated", + driver, + subagent: { + id: duplicateNodeId, + threadId: parentThreadId, + runId: run1Id, + parentNodeId: parentRoot1, + origin: "provider_native", + createdBy: "agent", + driver, + providerInstanceId: instanceId, + providerThreadId: childProviderThreadId, + childThreadId, + nativeTaskRef: { driver, nativeId: "call-legacy-dup-2", strength: "strong" }, + prompt: "old redirect", + title: "Old redirect", + model: null, + status: "completed", + result: "FIRST_OK", + startedAt: t2, + completedAt: t2, + updatedAt: t2, + }, + }); + yield* ingest(run1Id, { + type: "turn_item.updated", + driver, + turnItem: { + id: duplicateItemId, + threadId: parentThreadId, + runId: run1Id, + nodeId: duplicateNodeId, + providerThreadId: parentProviderThreadId, + providerTurnId: parentTurn1Id, + nativeItemRef: { driver, nativeId: "call-legacy-dup-2", strength: "strong" }, + parentItemId: null, + ordinal: 2, + status: "completed", + title: "Old redirect", + startedAt: t2, + completedAt: t2, + updatedAt: t2, + type: "subagent", + subagentId: duplicateNodeId, + origin: "provider_native", + driver, + providerInstanceId: instanceId, + childThreadId, + prompt: "old redirect", + result: "FIRST_OK", + }, + }); + } + yield* ingest(run1Id, { + type: "provider_turn.updated", + driver, + threadId: childThreadId, + providerTurn: { + id: childTurn1Id, + providerThreadId: childProviderThreadId, + nodeId: childRoot1, + runAttemptId: null, + nativeTurnRef: { driver, nativeId: `${childSessionId}:turn:1`, strength: "weak" }, + ordinal: 1, + status: "completed", + startedAt: now, + completedAt: now, + }, + }); + yield* ingest(run1Id, { + type: "turn_item.updated", + driver, + turnItem: { + id: TurnItemId.make("turn-item:z:child:prompt:1"), + threadId: childThreadId, + runId: null, + nodeId: childRoot1, + providerThreadId: childProviderThreadId, + providerTurnId: childTurn1Id, + nativeItemRef: null, + parentItemId: null, + ordinal: 101, + status: "completed", + title: null, + startedAt: now, + completedAt: now, + updatedAt: now, + createdBy: "agent", + creationSource: "provider", + type: "user_message", + messageId: MessageId.make("message:z:child:prompt:1"), + inputIntent: "turn_start", + text: "sleep 180", + attachments: [], + }, + }); + yield* ingest(run1Id, { + type: "turn_item.updated", + driver, + turnItem: { + id: TurnItemId.make("turn-item:z:child:reply:1"), + threadId: childThreadId, + runId: null, + nodeId: childRoot1, + providerThreadId: childProviderThreadId, + providerTurnId: childTurn1Id, + nativeItemRef: null, + parentItemId: null, + ordinal: 102, + status: "completed", + title: null, + startedAt: now, + completedAt: now, + updatedAt: now, + type: "assistant_message", + messageId: MessageId.make("message:z:child:reply:1"), + text: "FIRST_OK", + streaming: false, + }, + }); + + // Run 2: cold restart resumes the same child at t3 > t2 with session-keyed IDs, running then completed. + yield* ingest(run2Id, { + type: "app_thread.created", + driver, + createOnly: true, + appThread: childAppThread("Resume after restart"), + }); + for (const phase of ["running", "completed"] as const) { + yield* ingest(run2Id, { + type: "node.updated", + driver, + preserveIdentity: true, + node: { + id: sessionNodeId, + threadId: parentThreadId, + runId: run2Id, + parentNodeId: parentRoot2, + rootNodeId: parentRoot2, + kind: "subagent", + status: phase, + countsForRun: false, + providerThreadId: childProviderThreadId, + providerTurnId: parentTurn2Id, + nativeItemRef: { driver, nativeId: childSessionId, strength: "strong" }, + runtimeRequestId: null, + checkpointScopeId: null, + startedAt: t3, + completedAt: phase === "completed" ? t3 : null, + }, + }); + yield* ingest(run2Id, { + type: "subagent.updated", + driver, + preserveIdentity: true, + subagent: { + id: sessionNodeId, + threadId: parentThreadId, + runId: run2Id, + parentNodeId: parentRoot2, + origin: "provider_native", + createdBy: "agent", + driver, + providerInstanceId: instanceId, + providerThreadId: childProviderThreadId, + childThreadId, + nativeTaskRef: { driver, nativeId: childSessionId, strength: "strong" }, + prompt: "keep going", + title: "Resume after restart", + model: null, + status: phase, + result: phase === "completed" ? "SECOND_OK" : null, + startedAt: t3, + completedAt: phase === "completed" ? t3 : null, + updatedAt: t3, + }, + }); + yield* ingest(run2Id, { + type: "turn_item.updated", + driver, + preserveIdentity: true, + turnItem: { + id: sessionItemId, + threadId: parentThreadId, + runId: run2Id, + nodeId: sessionNodeId, + providerThreadId: parentProviderThreadId, + providerTurnId: parentTurn2Id, + nativeItemRef: { driver, nativeId: childSessionId, strength: "strong" }, + parentItemId: null, + ordinal: 1, + status: phase, + title: "Resume after restart", + startedAt: t3, + completedAt: phase === "completed" ? t3 : null, + updatedAt: t3, + type: "subagent", + subagentId: sessionNodeId, + origin: "provider_native", + driver, + providerInstanceId: instanceId, + childThreadId, + prompt: "keep going", + result: phase === "completed" ? "SECOND_OK" : null, + }, + }); + yield* ingest(run2Id, { + type: "provider_thread.updated", + driver, + preserveIdentity: true, + providerThread: { + id: childProviderThreadId, + driver, + providerInstanceId: instanceId, + providerSessionId, + appThreadId: childThreadId, + ownerNodeId: sessionNodeId, + nativeThreadRef: { driver, nativeId: childSessionId, strength: "strong" }, + nativeConversationHeadRef: null, + status: phase === "running" ? "active" : "idle", + firstRunOrdinal: null, + lastRunOrdinal: null, + handoffIds: [], + forkedFrom: null, + createdAt: t3, + updatedAt: t3, + }, + }); + } + yield* ingest(run2Id, { + type: "provider_turn.updated", + driver, + threadId: childThreadId, + providerTurn: { + id: childTurn2Id, + providerThreadId: childProviderThreadId, + nodeId: childRoot2, + runAttemptId: null, + nativeTurnRef: { + driver, + nativeId: `${childSessionId}:resume:call-2:turn:1`, + strength: "weak", + }, + ordinal: 1, + status: "running", + startedAt: t3, + completedAt: null, + }, + }); + yield* ingest(run2Id, { + type: "turn_item.updated", + driver, + turnItem: { + id: TurnItemId.make("turn-item:a:child:prompt:2"), + threadId: childThreadId, + runId: null, + nodeId: childRoot2, + providerThreadId: childProviderThreadId, + providerTurnId: childTurn2Id, + nativeItemRef: null, + parentItemId: null, + ordinal: 101, + status: "completed", + title: null, + startedAt: t3, + completedAt: t3, + updatedAt: t3, + createdBy: "agent", + creationSource: "provider", + type: "user_message", + messageId: MessageId.make("message:a:child:prompt:2"), + inputIntent: "turn_start", + text: "keep going", + attachments: [], + }, + }); + yield* ingest(run2Id, { + type: "turn_item.updated", + driver, + turnItem: { + id: TurnItemId.make("turn-item:a:child:reply:2"), + threadId: childThreadId, + runId: null, + nodeId: childRoot2, + providerThreadId: childProviderThreadId, + providerTurnId: childTurn2Id, + nativeItemRef: null, + parentItemId: null, + ordinal: 102, + status: "completed", + title: null, + startedAt: t3, + completedAt: t3, + updatedAt: t3, + type: "assistant_message", + messageId: MessageId.make("message:a:child:reply:2"), + text: "SECOND_OK", + streaming: false, + }, + }); + yield* ingest(run2Id, { + type: "provider_turn.updated", + driver, + threadId: childThreadId, + providerTurn: { + id: childTurn2Id, + providerThreadId: childProviderThreadId, + nodeId: childRoot2, + runAttemptId: null, + nativeTurnRef: { + driver, + nativeId: `${childSessionId}:resume:call-2:turn:1`, + strength: "weak", + }, + ordinal: 1, + status: "completed", + startedAt: t3, + completedAt: t3, + }, + }); + + const parentProjection = yield* projectionStore.getThreadProjection(parentThreadId); + const childProjection = yield* projectionStore.getThreadProjection(childThreadId); + const subagentNodes = parentProjection.nodes.filter((node) => node.kind === "subagent"); + const subagentItems = parentProjection.turnItems.filter((item) => item.type === "subagent"); + const expectedCount = initialKeying === "legacy-duplicate-rows" ? 2 : 1; + assert.lengthOf(subagentNodes, expectedCount); + assert.lengthOf(parentProjection.subagents, expectedCount); + assert.lengthOf(subagentItems, expectedCount); + const canonicalNode = subagentNodes.find((node) => node.id === initialNodeId); + const canonicalSubagent = parentProjection.subagents.find( + (subagent) => subagent.id === initialNodeId, + ); + const canonicalItem = subagentItems.find((item) => item.id === initialItemId); + assert.equal(canonicalNode?.runId, run2Id); + assert.equal(canonicalNode?.status, "completed"); + assert.equal(canonicalSubagent?.runId, run2Id); + assert.equal(canonicalSubagent?.title, "Sleep"); + assert.equal(canonicalSubagent?.prompt, "sleep 180"); + assert.equal(canonicalSubagent?.model, "opencode/big-pickle"); + assert.equal(canonicalSubagent?.result, "SECOND_OK"); + assert.equal(canonicalItem?.nodeId, initialNodeId); + assert.equal(canonicalItem?.subagentId, initialNodeId); + assert.equal(canonicalItem?.runId, run2Id); + assert.equal(canonicalItem?.title, "Sleep"); + assert.equal(canonicalItem?.prompt, "sleep 180"); + assert.equal(canonicalItem?.result, "SECOND_OK"); + if (initialKeying !== "legacy-duplicate-rows") { + assert.isFalse(subagentItems.some((item) => item.runId === run1Id)); + } + assert.equal(childProjection.thread.title, "Sleep"); + assert.equal(childProjection.thread.modelSelection.model, "opencode/big-pickle"); + assert.equal(childProjection.providerThreads[0]?.ownerNodeId, initialNodeId); + assert.deepEqual( + childProjection.providerTurns.map((turn) => ({ + id: turn.id, + ordinal: turn.ordinal, + status: turn.status, + })), + [ + { id: childTurn1Id, ordinal: 1, status: "completed" }, + { id: childTurn2Id, ordinal: 2, status: "completed" }, + ], + ); + assert.deepEqual( + childProjection.turnItems + .filter( + ( + item, + ): item is Extract< + OrchestrationV2TurnItem, + { type: "user_message" | "assistant_message" } + > => item.type === "user_message" || item.type === "assistant_message", + ) + .map((item) => item.text), + ["sleep 180", "FIRST_OK", "keep going", "SECOND_OK"], + ); + + if (initialKeying === "legacy-call-keyed") { + // Nested root -> child -> legacy grandchild: provider_thread.updated is ingested with + // root `threadId: parentThreadId`, yet must resolve the grandchild's ownerNodeId on childThreadId. + const grandchildSessionId = "ses_cold_resume_grandchild"; + const legacyGrandchildNodeId = idAllocator.derive.nodeFromProviderItem({ + driver, + nativeItemId: "call-legacy-grandchild", + }); + const sessionGrandchildNodeId = idAllocator.derive.nodeFromProviderItem({ + driver, + nativeItemId: grandchildSessionId, + }); + const grandchildThreadId = idAllocator.derive.subagentThread({ + driver, + nativeThreadId: grandchildSessionId, + providerInstanceId: instanceId, + }); + const grandchildProviderThreadId = idAllocator.derive.providerThread({ + driver, + nativeThreadId: grandchildSessionId, + providerInstanceId: instanceId, + }); + yield* ingest(run1Id, { + type: "app_thread.created", + driver, + appThread: { + ...childAppThread("Grandchild"), + id: grandchildThreadId, + activeProviderThreadId: grandchildProviderThreadId, + lineage: { + parentThreadId: childThreadId, + relationshipToParent: "subagent", + rootThreadId: parentThreadId, + }, + }, + }); + yield* ingest(run1Id, { + type: "subagent.updated", + driver, + subagent: { + id: legacyGrandchildNodeId, + threadId: childThreadId, + runId: null, + parentNodeId: childRoot1, + origin: "provider_native", + createdBy: "agent", + driver, + providerInstanceId: instanceId, + providerThreadId: grandchildProviderThreadId, + childThreadId: grandchildThreadId, + nativeTaskRef: { driver, nativeId: "call-legacy-grandchild", strength: "strong" }, + prompt: "nested work", + title: "Grandchild", + model: "opencode/big-pickle", + status: "completed", + result: "NESTED_OK", + startedAt: now, + completedAt: now, + updatedAt: now, + }, + }); + yield* ingest(run2Id, { + type: "provider_thread.updated", + driver, + preserveIdentity: true, + providerThread: { + id: grandchildProviderThreadId, + driver, + providerInstanceId: instanceId, + providerSessionId, + appThreadId: grandchildThreadId, + ownerNodeId: sessionGrandchildNodeId, + nativeThreadRef: { driver, nativeId: grandchildSessionId, strength: "strong" }, + nativeConversationHeadRef: null, + status: "active", + firstRunOrdinal: null, + lastRunOrdinal: null, + handoffIds: [], + forkedFrom: null, + createdAt: t2, + updatedAt: t2, + }, + }); + const grandchildProjection = + yield* projectionStore.getThreadProjection(grandchildThreadId); + assert.equal( + grandchildProjection.providerThreads[0]?.ownerNodeId, + legacyGrandchildNodeId, + ); + } + }), + ); }); const ingestCompletedProviderTurn = ( diff --git a/apps/server/src/orchestration-v2/ProviderEventIngestor.ts b/apps/server/src/orchestration-v2/ProviderEventIngestor.ts index e036bbe8994c1..f2859480a9b5a 100644 --- a/apps/server/src/orchestration-v2/ProviderEventIngestor.ts +++ b/apps/server/src/orchestration-v2/ProviderEventIngestor.ts @@ -505,11 +505,28 @@ export const layer: Layer.Layer< }), ]; case "provider_thread.updated": { - const threadId = input.event.providerThread.appThreadId ?? input.threadId; + let providerThread = input.event.providerThread; + const threadId = providerThread.appThreadId ?? input.threadId; + if (input.event.preserveIdentity === true && providerThread.ownerNodeId !== null) { + const ownerThreadId = + providerThread.appThreadId === null + ? input.threadId + : ((yield* projections.findThread(providerThread.appThreadId))?.lineage + .parentThreadId ?? input.threadId); + const existing = yield* projections.findSubagentIdentity({ + threadId: ownerThreadId, + subagentId: providerThread.ownerNodeId, + childThreadId: providerThread.appThreadId, + providerThreadId: providerThread.id, + }); + if (existing !== null && existing.subagent.id !== providerThread.ownerNodeId) { + providerThread = { ...providerThread, ownerNodeId: existing.subagent.id }; + } + } const updated = yield* makeDomainEvent(input, { type: "provider-thread.updated", threadId, - payload: input.event.providerThread, + payload: providerThread, }); return input.event.retainedNativeTurnIds === undefined ? [updated] @@ -517,19 +534,32 @@ export const layer: Layer.Layer< updated, ...(yield* rollBackRewoundRuns(input, { threadId, - providerThreadId: input.event.providerThread.id, + providerThreadId: providerThread.id, retainedNativeTurnIds: new Set(input.event.retainedNativeTurnIds), })), ]; } - case "provider_turn.updated": + case "provider_turn.updated": { + let providerTurn = input.event.providerTurn; + if (providerTurn.runAttemptId === null) { + // Runless child turns can restart from ordinal 1 after a restart; keep SQLite ordinals unique. + const ordinal = yield* projections.resolveProviderTurnOrdinal({ + threadId: input.event.threadId ?? input.threadId, + providerTurnId: providerTurn.id, + providerThreadId: providerTurn.providerThreadId, + ordinal: providerTurn.ordinal, + }); + if (ordinal !== providerTurn.ordinal) { + providerTurn = { ...providerTurn, ordinal }; + } + } return [ ...(["completed", "interrupted", "failed", "cancelled"].includes( - input.event.providerTurn.status, + providerTurn.status, ) ? yield* dismissNativeUserInputs( input, - input.event.providerTurn.id, + providerTurn.id, input.event.threadId, input.event.retainedRuntimeRequestIds, ) @@ -537,33 +567,72 @@ export const layer: Layer.Layer< yield* makeDomainEvent(input, { type: "provider-turn.updated", ...(input.event.threadId === undefined ? {} : { threadId: input.event.threadId }), - payload: input.event.providerTurn, - nodeId: input.event.providerTurn.nodeId, + payload: providerTurn, + nodeId: providerTurn.nodeId, }), ]; - case "node.updated": + } + case "node.updated": { + let node = input.event.node; + if (input.event.preserveIdentity === true && node.kind === "subagent") { + const existing = yield* projections.findSubagentIdentity({ + threadId: node.threadId, + subagentId: node.id, + childThreadId: null, + providerThreadId: node.providerThreadId, + }); + if (existing !== null && existing.subagent.id !== node.id) { + node = { ...node, id: existing.subagent.id }; + } + } // Adapters send an unchanged node again with each text delta. Compare with the // stored node, not the last one sent, because the server also writes nodes. - if (yield* projections.isNodeStored(input.event.node)) return []; + if (yield* projections.isNodeStored(node)) return []; return [ yield* makeDomainEvent(input, { type: "node.updated", - threadId: input.event.node.threadId, - payload: input.event.node, - runId: input.event.node.runId, - nodeId: input.event.node.id, + threadId: node.threadId, + payload: node, + runId: node.runId, + nodeId: node.id, }), ]; - case "subagent.updated": + } + case "subagent.updated": { + let subagent = input.event.subagent; + if (input.event.preserveIdentity === true) { + const existing = yield* projections.findSubagentIdentity({ + threadId: subagent.threadId, + subagentId: subagent.id, + childThreadId: subagent.childThreadId, + providerThreadId: subagent.providerThreadId, + }); + if (existing !== null) { + subagent = { + ...subagent, + id: existing.subagent.id, + title: existing.subagent.title ?? subagent.title, + prompt: existing.subagent.prompt || subagent.prompt, + model: subagent.model ?? existing.subagent.model, + ...((subagent.modelSelection ?? existing.subagent.modelSelection) === undefined + ? {} + : { + modelSelection: + subagent.modelSelection ?? existing.subagent.modelSelection, + }), + }; + } + } return [ yield* makeDomainEvent(input, { type: "subagent.updated", - threadId: input.event.subagent.threadId, - payload: input.event.subagent, - runId: input.event.subagent.runId, - nodeId: input.event.subagent.id, + threadId: subagent.threadId, + payload: subagent, + runId: subagent.runId, + nodeId: subagent.id, }), ]; + } case "message.updated": return [ yield* makeDomainEvent(input, { @@ -574,18 +643,37 @@ export const layer: Layer.Layer< nodeId: input.event.message.nodeId, }), ]; - case "turn_item.updated": + case "turn_item.updated": { + let turnItem = input.event.turnItem; + if (input.event.preserveIdentity === true && turnItem.type === "subagent") { + const existing = yield* projections.findSubagentIdentity({ + threadId: turnItem.threadId, + subagentId: turnItem.subagentId, + childThreadId: turnItem.childThreadId, + providerThreadId: null, + }); + if (existing !== null) { + turnItem = { + ...turnItem, + ...(existing.turnItem === null ? {} : { id: existing.turnItem.id }), + nodeId: existing.subagent.id, + subagentId: existing.subagent.id, + title: existing.turnItem?.title ?? existing.subagent.title ?? turnItem.title, + prompt: + existing.turnItem?.prompt || existing.subagent.prompt || turnItem.prompt, + }; + } + } return [ yield* makeDomainEvent(input, { type: "turn-item.updated", - threadId: input.event.turnItem.threadId, - payload: boundFileChangeForStorage( - stripUnservedToolOutputImageBytes(input.event.turnItem), - ), - runId: input.event.turnItem.runId, - nodeId: input.event.turnItem.nodeId, + threadId: turnItem.threadId, + payload: boundFileChangeForStorage(stripUnservedToolOutputImageBytes(turnItem)), + runId: turnItem.runId, + nodeId: turnItem.nodeId, }), ]; + } case "runtime_request.updated": return [ yield* makeDomainEvent(input, { @@ -837,22 +925,29 @@ export const layer: Layer.Layer< ), ), ), - // @effect-diagnostics-next-line flatMapConditionalToFilterOrFail:off - The predicate narrows the captured event for the subagent update. - Effect.flatMap((storedEvents) => - storedEvents.length === 0 || input.event.type !== "subagent.updated" - ? Effect.succeed(storedEvents) - : syncSubagentThreadModel(input, input.event.subagent).pipe( - Effect.map((synced) => [...storedEvents, ...synced]), - Effect.mapError( - (cause) => - new ProviderEventPublishError({ - providerSessionId: input.providerSessionId, - eventCount: 1, - cause, - }), - ), - ), - ), + Effect.flatMap((storedEvents) => { + if (storedEvents.length === 0 || input.event.type !== "subagent.updated") { + return Effect.succeed(storedEvents); + } + const persisted = storedEvents.find( + (stored) => stored.event.type === "subagent.updated", + ); + const subagent = + persisted?.event.type === "subagent.updated" + ? persisted.event.payload + : input.event.subagent; + return syncSubagentThreadModel(input, subagent).pipe( + Effect.map((synced) => [...storedEvents, ...synced]), + Effect.mapError( + (cause) => + new ProviderEventPublishError({ + providerSessionId: input.providerSessionId, + eventCount: 1, + cause, + }), + ), + ); + }), Effect.tap((storedEvents) => Effect.gen(function* () { if (storedEvents.length === 0 || input.event.type !== "provider_turn.updated") diff --git a/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts b/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts index 51c38efa8a41a..6902f9bb92b87 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts @@ -175,6 +175,8 @@ function layerProjectionStore( listAppOwnedChildThreadIds: () => Effect.die("unused listAppOwnedChildThreadIds"), getNextTurnItemOrdinal: () => Effect.die("unused ordinal read"), getTurnItem: () => Effect.die("unused turn item read"), + resolveProviderTurnOrdinal: () => Effect.die("unused resolveProviderTurnOrdinal"), + findSubagentIdentity: () => Effect.die("unused findSubagentIdentity"), getMessagesById: () => Effect.die("unused getMessagesById"), getThreadRecords: () => Effect.die("unused record read"), getRuntimeRequest: () => Effect.die("unused getRuntimeRequest"), diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index a23a3fdf87bff..3534afcf09e4d 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -47,10 +47,14 @@ import { CodexAppServerRequestError } from "effect-codex-app-server/errors"; import { SqlError, classifySqliteError } from "effect/sql/SqlError"; import { toPersistenceSqlError } from "../persistence/Errors.ts"; +import * as SqlitePersistence from "../persistence/Sqlite.ts"; +import * as ProviderInstanceRegistry from "../provider/ProviderInstanceRegistry.ts"; import * as ServerSettings from "../serverSettings.ts"; +import * as GitVcsDriver from "../vcs/GitVcsDriver.ts"; import * as CheckpointService from "./CheckpointService.ts"; import type { PendingOrchestrationEffectV2 } from "./EffectOutbox.ts"; import * as EventSink from "./EventSink.ts"; +import * as EventStore from "./EventStore.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; import * as ProjectionStore from "./ProjectionStore.ts"; @@ -5237,3 +5241,1160 @@ it.effect("drops a child's held text once the child is handed to another run", ( assert.deepEqual(yield* Ref.get(staleWrites), []); }), ); + +it.effect( + "terminalizes a cold-resumed legacy subagent in place when the resuming run stream fails", + () => { + const layerDb = SqlitePersistence.layerMemory; + const layerStores = Layer.merge(EventStore.layer, ProjectionStore.layer).pipe( + Layer.provide(layerDb), + ); + const layerSink = EventSink.layer.pipe(Layer.provide(Layer.mergeAll(layerStores, layerDb))); + const layerInstances = Layer.mock(ProviderInstanceRegistry.ProviderInstanceRegistry)({ + getInstance: () => Effect.undefined, + }); + const layerGit = Layer.mock(GitVcsDriver.GitVcsDriver)({}); + const layerIngestor = ProviderEventIngestor.layer.pipe( + Layer.provide( + Layer.mergeAll( + layerInstances, + layerStores, + layerSink, + IdAllocator.layer, + ThreadCommandExecutor.layer, + layerGit, + ), + ), + ); + const layerRunExecution = RunExecutionService.layer.pipe( + Layer.provide( + Layer.mergeAll( + McpAppModelContext.layerEmpty, + PreviewAgentActivity.layer, + Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }), + layerTurnItemPositionsTest, + layerSink, + IdAllocator.layer, + ThreadCommandExecutor.layer, + layerIngestor, + ServerSettings.layerTest(), + ), + ), + ); + const layerFull = Layer.mergeAll( + layerDb, + layerStores, + layerSink, + IdAllocator.layer, + ThreadCommandExecutor.layer, + layerIngestor, + layerRunExecution, + ); + + return Effect.gen(function* () { + const now = yield* DateTime.now; + const t2 = DateTime.add(now, { seconds: 5 }); + const eventSink = yield* EventSink.EventSinkV2; + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const runExecution = yield* RunExecutionService.RunExecutionServiceV2; + + const opencodeDriver = ProviderDriverKind.make("opencode"); + const instanceId = ProviderInstanceId.make("opencode"); + const key = "legacy-stream-fail"; + const ids = backgroundScenarioIds(key); + const appThread: OrchestrationV2AppThread = { + ...rootAppThread(ids.threadId), + createdBy: "user", + creationSource: "web", + projectId: yield* idAllocator.allocate.project({ fixtureName: key }), + title: "Legacy stream fail parent", + providerInstanceId: instanceId, + modelSelection: { instanceId, model: "opencode/big-pickle" }, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + branchPullRequest: null, + activeOrderKey: null, + activeProviderThreadId: ids.providerThreadId, + lineage: { + parentThreadId: null, + relationshipToParent: null, + rootThreadId: ids.threadId, + }, + forkedFrom: null, + createdAt: now, + updatedAt: now, + archivedAt: null, + settledOverride: null, + settledAt: null, + lastVisitedAt: null, + deletedAt: null, + }; + const providerSessionId = ProviderSessionId.make(`session:${key}`); + const childSessionId = "ses_legacy_stream_fail_child"; + const legacyNodeId = idAllocator.derive.nodeFromProviderItem({ + driver: opencodeDriver, + nativeItemId: "call-legacy-launch", + }); + const legacyItemId = idAllocator.derive.turnItemFromProviderItem({ + driver: opencodeDriver, + nativeItemId: "call-legacy-launch", + }); + const sessionNodeId = idAllocator.derive.nodeFromProviderItem({ + driver: opencodeDriver, + nativeItemId: childSessionId, + }); + const sessionItemId = idAllocator.derive.turnItemFromProviderItem({ + driver: opencodeDriver, + nativeItemId: childSessionId, + }); + const childThreadId = idAllocator.derive.subagentThread({ + driver: opencodeDriver, + nativeThreadId: childSessionId, + providerInstanceId: instanceId, + }); + const childProviderThreadId = idAllocator.derive.providerThread({ + driver: opencodeDriver, + nativeThreadId: childSessionId, + providerInstanceId: instanceId, + }); + const run1Id = RunId.make(`run:${key}:1`); + const parentTurn1Id = ProviderTurnId.make(`provider-turn:${key}:1`); + const parentRoot1 = NodeId.make(`node:${key}:root:1`); + + yield* eventSink.write({ + events: [ + { + id: yield* idAllocator.allocate.event({ threadId: ids.threadId }), + type: "thread.created", + threadId: ids.threadId, + occurredAt: now, + payload: appThread, + }, + ], + }); + + // Seed the pre-upgrade call-keyed subagent row from run 1. + const seedIngest = (event: ProviderAdapter.ProviderAdapterV2Event) => + ingestor.ingestNormalized({ + providerSessionId, + providerInstanceId: instanceId, + threadId: ids.threadId, + runId: run1Id, + event, + }); + const childThread = (title: string): OrchestrationV2AppThread => ({ + ...appThread, + id: childThreadId, + title, + activeProviderThreadId: childProviderThreadId, + lineage: { + parentThreadId: ids.threadId, + relationshipToParent: "subagent", + rootThreadId: ids.threadId, + }, + createdBy: "agent", + creationSource: "provider", + }); + yield* seedIngest({ + type: "app_thread.created", + driver: opencodeDriver, + appThread: childThread("Sleep"), + }); + yield* seedIngest({ + type: "node.updated", + driver: opencodeDriver, + node: { + id: legacyNodeId, + threadId: ids.threadId, + runId: run1Id, + parentNodeId: parentRoot1, + rootNodeId: parentRoot1, + kind: "subagent", + status: "completed", + countsForRun: false, + providerThreadId: childProviderThreadId, + providerTurnId: parentTurn1Id, + nativeItemRef: { + driver: opencodeDriver, + nativeId: "call-legacy-launch", + strength: "strong", + }, + runtimeRequestId: null, + checkpointScopeId: null, + startedAt: now, + completedAt: now, + }, + }); + yield* seedIngest({ + type: "subagent.updated", + driver: opencodeDriver, + subagent: { + id: legacyNodeId, + threadId: ids.threadId, + runId: run1Id, + parentNodeId: parentRoot1, + origin: "provider_native", + createdBy: "agent", + driver: opencodeDriver, + providerInstanceId: instanceId, + providerThreadId: childProviderThreadId, + childThreadId, + nativeTaskRef: { + driver: opencodeDriver, + nativeId: "call-legacy-launch", + strength: "strong", + }, + prompt: "sleep 180", + title: "Sleep", + model: "opencode/big-pickle", + status: "completed", + result: "FIRST_OK", + startedAt: now, + completedAt: now, + updatedAt: now, + }, + }); + yield* seedIngest({ + type: "turn_item.updated", + driver: opencodeDriver, + turnItem: { + id: legacyItemId, + threadId: ids.threadId, + runId: run1Id, + nodeId: legacyNodeId, + providerThreadId: ids.providerThreadId, + providerTurnId: parentTurn1Id, + nativeItemRef: { + driver: opencodeDriver, + nativeId: "call-legacy-launch", + strength: "strong", + }, + parentItemId: null, + ordinal: 1, + status: "completed", + title: "Sleep", + startedAt: now, + completedAt: now, + updatedAt: now, + type: "subagent", + subagentId: legacyNodeId, + origin: "provider_native", + driver: opencodeDriver, + providerInstanceId: instanceId, + childThreadId, + prompt: "sleep 180", + result: "FIRST_OK", + }, + }); + + const rootTurn: OrchestrationV2ProviderTurn = { + id: ids.rootProviderTurnId, + providerThreadId: ids.providerThreadId, + nodeId: ids.rootNodeId, + runAttemptId: ids.attemptId, + nativeTurnRef: null, + ordinal: 2, + status: "running", + startedAt: t2, + completedAt: null, + }; + const streamEvents: ReadonlyArray = [ + { + type: "provider_turn.updated", + driver: opencodeDriver, + providerTurn: rootTurn, + }, + { + type: "app_thread.created", + driver: opencodeDriver, + createOnly: true, + appThread: childThread("Resume after upgrade"), + }, + { + type: "node.updated", + driver: opencodeDriver, + preserveIdentity: true, + node: { + id: sessionNodeId, + threadId: ids.threadId, + runId: ids.runId, + parentNodeId: ids.rootNodeId, + rootNodeId: ids.rootNodeId, + kind: "subagent", + status: "running", + countsForRun: false, + providerThreadId: childProviderThreadId, + providerTurnId: ids.rootProviderTurnId, + nativeItemRef: { driver: opencodeDriver, nativeId: childSessionId, strength: "strong" }, + runtimeRequestId: null, + checkpointScopeId: null, + startedAt: t2, + completedAt: null, + }, + }, + { + type: "subagent.updated", + driver: opencodeDriver, + preserveIdentity: true, + subagent: { + id: sessionNodeId, + threadId: ids.threadId, + runId: ids.runId, + parentNodeId: ids.rootNodeId, + origin: "provider_native", + createdBy: "agent", + driver: opencodeDriver, + providerInstanceId: instanceId, + providerThreadId: childProviderThreadId, + childThreadId, + nativeTaskRef: { driver: opencodeDriver, nativeId: childSessionId, strength: "strong" }, + prompt: "keep going", + title: "Resume after upgrade", + model: null, + status: "running", + result: null, + startedAt: t2, + completedAt: null, + updatedAt: t2, + }, + }, + { + type: "turn_item.updated", + driver: opencodeDriver, + preserveIdentity: true, + turnItem: { + id: sessionItemId, + threadId: ids.threadId, + runId: ids.runId, + nodeId: sessionNodeId, + providerThreadId: ids.providerThreadId, + providerTurnId: ids.rootProviderTurnId, + nativeItemRef: { driver: opencodeDriver, nativeId: childSessionId, strength: "strong" }, + parentItemId: null, + ordinal: 1, + status: "running", + title: "Resume after upgrade", + startedAt: t2, + completedAt: null, + updatedAt: t2, + type: "subagent", + subagentId: sessionNodeId, + origin: "provider_native", + driver: opencodeDriver, + providerInstanceId: instanceId, + childThreadId, + prompt: "keep going", + result: null, + }, + }, + { + type: "provider_thread.updated", + driver: opencodeDriver, + preserveIdentity: true, + providerThread: { + id: childProviderThreadId, + driver: opencodeDriver, + providerInstanceId: instanceId, + providerSessionId, + appThreadId: childThreadId, + ownerNodeId: sessionNodeId, + nativeThreadRef: { + driver: opencodeDriver, + nativeId: childSessionId, + strength: "strong", + }, + nativeConversationHeadRef: null, + status: "active", + firstRunOrdinal: null, + lastRunOrdinal: null, + handoffIds: [], + forkedFrom: null, + createdAt: t2, + updatedAt: t2, + }, + }, + // Simulate session.step.started filling row.model and re-emitting an unchanged node. + { + type: "node.updated", + driver: opencodeDriver, + preserveIdentity: true, + node: { + id: sessionNodeId, + threadId: ids.threadId, + runId: ids.runId, + parentNodeId: ids.rootNodeId, + rootNodeId: ids.rootNodeId, + kind: "subagent", + status: "running", + countsForRun: false, + providerThreadId: childProviderThreadId, + providerTurnId: ids.rootProviderTurnId, + nativeItemRef: { driver: opencodeDriver, nativeId: childSessionId, strength: "strong" }, + runtimeRequestId: null, + checkpointScopeId: null, + startedAt: t2, + completedAt: null, + }, + }, + { + type: "subagent.updated", + driver: opencodeDriver, + preserveIdentity: true, + subagent: { + id: sessionNodeId, + threadId: ids.threadId, + runId: ids.runId, + parentNodeId: ids.rootNodeId, + origin: "provider_native", + createdBy: "agent", + driver: opencodeDriver, + providerInstanceId: instanceId, + providerThreadId: childProviderThreadId, + childThreadId, + nativeTaskRef: { driver: opencodeDriver, nativeId: childSessionId, strength: "strong" }, + prompt: "keep going", + title: "Resume after upgrade", + model: "opencode/big-pickle", + status: "running", + result: null, + startedAt: t2, + completedAt: null, + updatedAt: t2, + }, + }, + ]; + + const ingestionDone = yield* Deferred.make(); + const parentProviderThread: OrchestrationV2ProviderThread = { + id: ids.providerThreadId, + driver: opencodeDriver, + providerInstanceId: instanceId, + providerSessionId, + appThreadId: ids.threadId, + ownerNodeId: null, + nativeThreadRef: { driver: opencodeDriver, nativeId: "ses_parent", strength: "strong" }, + nativeConversationHeadRef: null, + status: "active", + firstRunOrdinal: 1, + lastRunOrdinal: 2, + handoffIds: [], + forkedFrom: null, + createdAt: now, + updatedAt: t2, + }; + const checkpointScopeId = CheckpointScopeId.make(`checkpoint-scope:${key}`); + const parentRun: OrchestrationV2Run = { + id: ids.runId, + threadId: ids.threadId, + ordinal: 2, + providerInstanceId: instanceId, + modelSelection: { instanceId, model: "opencode/big-pickle" }, + providerThreadId: ids.providerThreadId, + userMessageId: MessageId.make(`message:${key}:user`), + rootNodeId: ids.rootNodeId, + activeAttemptId: ids.attemptId, + status: "running", + requestedAt: t2, + startedAt: t2, + completedAt: null, + checkpointId: null, + contextHandoffId: null, + }; + const parentRootNode: OrchestrationV2ExecutionNode = { + id: ids.rootNodeId, + threadId: ids.threadId, + runId: ids.runId, + parentNodeId: null, + rootNodeId: ids.rootNodeId, + kind: "root_turn", + status: "running", + countsForRun: true, + providerThreadId: ids.providerThreadId, + providerTurnId: ids.rootProviderTurnId, + nativeItemRef: null, + runtimeRequestId: null, + checkpointScopeId, + startedAt: t2, + completedAt: null, + }; + const parentAttempt: OrchestrationV2RunAttempt = { + id: ids.attemptId, + runId: ids.runId, + attemptOrdinal: 1, + rootNodeId: ids.rootNodeId, + providerInstanceId: instanceId, + providerThreadId: ids.providerThreadId, + providerTurnId: ids.rootProviderTurnId, + reason: "initial", + status: "running", + startedAt: t2, + completedAt: null, + }; + const parentCheckpointScope: OrchestrationV2CheckpointScope = { + id: checkpointScopeId, + threadId: ids.threadId, + runId: ids.runId, + nodeId: ids.rootNodeId, + parentScopeId: null, + providerThreadId: ids.providerThreadId, + ordinalWithinParent: 1, + advancesAppRunCount: true, + cwd: process.cwd(), + kind: "root_run", + createdAt: t2, + }; + yield* eventSink.write({ + events: [ + { + id: yield* idAllocator.allocate.event({ threadId: ids.threadId }), + type: "run.created", + threadId: ids.threadId, + runId: ids.runId, + occurredAt: t2, + payload: parentRun, + }, + { + id: yield* idAllocator.allocate.event({ threadId: ids.threadId }), + type: "run-attempt.created", + threadId: ids.threadId, + runId: ids.runId, + occurredAt: t2, + payload: parentAttempt, + }, + { + id: yield* idAllocator.allocate.event({ threadId: ids.threadId }), + type: "node.updated", + threadId: ids.threadId, + runId: ids.runId, + nodeId: ids.rootNodeId, + occurredAt: t2, + payload: parentRootNode, + }, + ], + }); + + yield* runExecution.startRootRun({ + commandId: CommandId.make(`command:${key}`), + appThread, + providerSessionId, + session: { + events: Stream.empty, + subscribeEvents: Effect.succeed({ + events: Stream.fromIterable(streamEvents).pipe( + Stream.concat( + Stream.fail( + new ProviderAdapter.ProviderAdapterEventStreamError({ + driver: opencodeDriver, + providerSessionId, + cause: "provider stream disconnected", + }), + ), + ), + ), + close: Deferred.succeed(ingestionDone, undefined), + }), + startTurn: () => Effect.void, + interruptTurn: () => Effect.void, + } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime, + providerThread: parentProviderThread, + run: parentRun, + attempt: parentAttempt, + attemptId: ids.attemptId, + rootNode: parentRootNode, + checkpointScope: parentCheckpointScope, + providerTurnOrdinal: 2, + message: { + messageId: MessageId.make(`message:${key}:user`), + text: "Resume the legacy subagent", + attachments: [], + createdBy: "user", + creationSource: "web", + }, + modelSelection: { instanceId, model: "opencode/big-pickle" }, + runtimePolicy: { + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + approvalPolicy: "never", + sandboxPolicy: { + type: "readOnly", + access: { type: "fullAccess" }, + networkAccess: false, + }, + }, + }); + yield* Deferred.await(ingestionDone); + + const parentProjection = yield* projectionStore.getThreadProjection(ids.threadId); + const childProjection = yield* projectionStore.getThreadProjection(childThreadId); + const subagentNodes = parentProjection.nodes.filter((node) => node.kind === "subagent"); + const subagentItems = parentProjection.turnItems.filter((item) => item.type === "subagent"); + assert.lengthOf(subagentNodes, 1); + assert.lengthOf(parentProjection.subagents, 1); + assert.lengthOf(subagentItems, 1); + assert.equal(subagentNodes[0]?.id, legacyNodeId); + assert.equal(subagentNodes[0]?.status, "failed"); + assert.equal(parentProjection.subagents[0]?.id, legacyNodeId); + assert.equal(parentProjection.subagents[0]?.status, "failed"); + assert.equal(parentProjection.subagents[0]?.title, "Sleep"); + assert.equal(parentProjection.subagents[0]?.prompt, "sleep 180"); + assert.equal(parentProjection.subagents[0]?.model, "opencode/big-pickle"); + assert.equal(subagentItems[0]?.id, legacyItemId); + assert.equal(subagentItems[0]?.nodeId, legacyNodeId); + assert.equal(subagentItems[0]?.subagentId, legacyNodeId); + assert.equal(subagentItems[0]?.status, "failed"); + assert.equal(subagentItems[0]?.title, "Sleep"); + assert.equal(subagentItems[0]?.prompt, "sleep 180"); + assert.equal(childProjection.providerThreads[0]?.ownerNodeId, legacyNodeId); + }).pipe(Effect.provide(layerFull)); + }, +); + +it.effect( + "detaches a cold-resumed legacy subagent from the owning run on cross-run handoff", + () => { + const layerDb = SqlitePersistence.layerMemory; + const layerStores = Layer.merge(EventStore.layer, ProjectionStore.layer).pipe( + Layer.provide(layerDb), + ); + const layerSink = EventSink.layer.pipe(Layer.provide(Layer.mergeAll(layerStores, layerDb))); + const layerInstances = Layer.mock(ProviderInstanceRegistry.ProviderInstanceRegistry)({ + getInstance: () => Effect.undefined, + }); + const layerGit = Layer.mock(GitVcsDriver.GitVcsDriver)({}); + const layerIngestor = ProviderEventIngestor.layer.pipe( + Layer.provide( + Layer.mergeAll( + layerInstances, + layerStores, + layerSink, + IdAllocator.layer, + ThreadCommandExecutor.layer, + layerGit, + ), + ), + ); + const layerRunExecution = RunExecutionService.layer.pipe( + Layer.provide( + Layer.mergeAll( + McpAppModelContext.layerEmpty, + PreviewAgentActivity.layer, + Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }), + layerTurnItemPositionsTest, + layerSink, + IdAllocator.layer, + ThreadCommandExecutor.layer, + layerIngestor, + ServerSettings.layerTest(), + ), + ), + ); + const layerFull = Layer.mergeAll( + layerDb, + layerStores, + layerSink, + IdAllocator.layer, + ThreadCommandExecutor.layer, + layerIngestor, + layerRunExecution, + ); + return Effect.gen(function* () { + const key = "legacy-cross-run-handoff"; + const ids = backgroundScenarioIds(key); + const now = yield* DateTime.now; + const t2 = DateTime.add(now, { seconds: 5 }); + const t3 = DateTime.add(now, { seconds: 10 }); + const opencodeDriver = ProviderDriverKind.make("opencode"); + const instanceId = ProviderInstanceId.make("opencode"); + const eventSink = yield* EventSink.EventSinkV2; + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const runExecution = yield* RunExecutionService.RunExecutionServiceV2; + + const appThread: OrchestrationV2AppThread = { + ...rootAppThread(ids.threadId), + createdBy: "user", + creationSource: "web", + projectId: yield* idAllocator.allocate.project({ fixtureName: key }), + title: "Legacy handoff parent", + providerInstanceId: instanceId, + modelSelection: { instanceId, model: "opencode/big-pickle" }, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + branchPullRequest: null, + activeOrderKey: null, + activeProviderThreadId: ids.providerThreadId, + lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: ids.threadId }, + forkedFrom: null, + createdAt: now, + updatedAt: now, + archivedAt: null, + settledOverride: null, + settledAt: null, + lastVisitedAt: null, + deletedAt: null, + }; + yield* eventSink.write({ + events: [ + { + id: yield* idAllocator.allocate.event({ threadId: ids.threadId }), + type: "thread.created", + threadId: ids.threadId, + occurredAt: now, + payload: appThread, + }, + ], + }); + + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: instanceId, + threadId: ids.threadId, + }); + const childSessionId = "ses_legacy_handoff_child"; + const legacyNodeId = idAllocator.derive.nodeFromProviderItem({ + driver: opencodeDriver, + nativeItemId: "call-legacy-handoff", + }); + const legacyItemId = idAllocator.derive.turnItemFromProviderItem({ + driver: opencodeDriver, + nativeItemId: "call-legacy-handoff", + }); + const sessionNodeId = idAllocator.derive.nodeFromProviderItem({ + driver: opencodeDriver, + nativeItemId: childSessionId, + }); + const sessionItemId = idAllocator.derive.turnItemFromProviderItem({ + driver: opencodeDriver, + nativeItemId: childSessionId, + }); + const childThreadId = idAllocator.derive.subagentThread({ + driver: opencodeDriver, + nativeThreadId: childSessionId, + providerInstanceId: instanceId, + }); + const childProviderThreadId = idAllocator.derive.providerThread({ + driver: opencodeDriver, + nativeThreadId: childSessionId, + providerInstanceId: instanceId, + }); + const run1Id = RunId.make(`run:${key}:1`); + const run3Id = RunId.make(`run:${key}:3`); + const parentTurn1Id = ProviderTurnId.make(`provider-turn:${key}:1`); + const parentRoot1 = NodeId.make(`node:${key}:root:1`); + const childThread: OrchestrationV2AppThread = { + ...appThread, + id: childThreadId, + title: "Sleep", + activeProviderThreadId: childProviderThreadId, + lineage: { + parentThreadId: ids.threadId, + relationshipToParent: "subagent", + rootThreadId: ids.threadId, + }, + createdBy: "agent", + creationSource: "provider", + }; + const seedIngest = (event: ProviderAdapter.ProviderAdapterV2Event) => + ingestor.ingestNormalized({ + providerSessionId, + providerInstanceId: instanceId, + threadId: ids.threadId, + runId: run1Id, + event, + }); + yield* seedIngest({ + type: "app_thread.created", + driver: opencodeDriver, + appThread: childThread, + }); + yield* seedIngest({ + type: "node.updated", + driver: opencodeDriver, + node: { + id: legacyNodeId, + threadId: ids.threadId, + runId: run1Id, + parentNodeId: parentRoot1, + rootNodeId: parentRoot1, + kind: "subagent", + status: "completed", + countsForRun: false, + providerThreadId: childProviderThreadId, + providerTurnId: parentTurn1Id, + nativeItemRef: { + driver: opencodeDriver, + nativeId: "call-legacy-handoff", + strength: "strong", + }, + runtimeRequestId: null, + checkpointScopeId: null, + startedAt: now, + completedAt: now, + }, + }); + yield* seedIngest({ + type: "subagent.updated", + driver: opencodeDriver, + subagent: { + id: legacyNodeId, + threadId: ids.threadId, + runId: run1Id, + parentNodeId: parentRoot1, + origin: "provider_native", + createdBy: "agent", + driver: opencodeDriver, + providerInstanceId: instanceId, + providerThreadId: childProviderThreadId, + childThreadId, + nativeTaskRef: { + driver: opencodeDriver, + nativeId: "call-legacy-handoff", + strength: "strong", + }, + prompt: "sleep 180", + title: "Sleep", + model: "opencode/big-pickle", + status: "completed", + result: "FIRST_OK", + startedAt: now, + completedAt: now, + updatedAt: now, + }, + }); + yield* seedIngest({ + type: "turn_item.updated", + driver: opencodeDriver, + turnItem: { + id: legacyItemId, + threadId: ids.threadId, + runId: run1Id, + nodeId: legacyNodeId, + providerThreadId: ids.providerThreadId, + providerTurnId: parentTurn1Id, + nativeItemRef: { + driver: opencodeDriver, + nativeId: "call-legacy-handoff", + strength: "strong", + }, + parentItemId: null, + ordinal: 1, + status: "completed", + title: "Sleep", + startedAt: now, + completedAt: now, + updatedAt: now, + type: "subagent", + subagentId: legacyNodeId, + origin: "provider_native", + driver: opencodeDriver, + providerInstanceId: instanceId, + childThreadId, + prompt: "sleep 180", + result: "FIRST_OK", + }, + }); + + const eventQueue = yield* Queue.unbounded(); + const ingestionDone = yield* Deferred.make(); + const parentProviderThread: OrchestrationV2ProviderThread = { + id: ids.providerThreadId, + driver: opencodeDriver, + providerInstanceId: instanceId, + providerSessionId, + appThreadId: ids.threadId, + ownerNodeId: null, + nativeThreadRef: { driver: opencodeDriver, nativeId: "ses_parent", strength: "strong" }, + nativeConversationHeadRef: null, + status: "active", + firstRunOrdinal: 1, + lastRunOrdinal: 2, + handoffIds: [], + forkedFrom: null, + createdAt: now, + updatedAt: t2, + }; + const checkpointScopeId = CheckpointScopeId.make(`checkpoint-scope:${key}`); + const parentRun: OrchestrationV2Run = { + id: ids.runId, + threadId: ids.threadId, + ordinal: 2, + providerInstanceId: instanceId, + modelSelection: { instanceId, model: "opencode/big-pickle" }, + providerThreadId: ids.providerThreadId, + userMessageId: MessageId.make(`message:${key}:user`), + rootNodeId: ids.rootNodeId, + activeAttemptId: ids.attemptId, + status: "running", + requestedAt: t2, + startedAt: t2, + completedAt: null, + checkpointId: null, + contextHandoffId: null, + }; + const parentRootNode: OrchestrationV2ExecutionNode = { + id: ids.rootNodeId, + threadId: ids.threadId, + runId: ids.runId, + parentNodeId: null, + rootNodeId: ids.rootNodeId, + kind: "root_turn", + status: "running", + countsForRun: true, + providerThreadId: ids.providerThreadId, + providerTurnId: ids.rootProviderTurnId, + nativeItemRef: null, + runtimeRequestId: null, + checkpointScopeId, + startedAt: t2, + completedAt: null, + }; + const parentAttempt: OrchestrationV2RunAttempt = { + id: ids.attemptId, + runId: ids.runId, + attemptOrdinal: 1, + rootNodeId: ids.rootNodeId, + providerInstanceId: instanceId, + providerThreadId: ids.providerThreadId, + providerTurnId: ids.rootProviderTurnId, + reason: "initial", + status: "running", + startedAt: t2, + completedAt: null, + }; + const parentCheckpointScope: OrchestrationV2CheckpointScope = { + id: checkpointScopeId, + threadId: ids.threadId, + runId: ids.runId, + nodeId: ids.rootNodeId, + parentScopeId: null, + providerThreadId: ids.providerThreadId, + ordinalWithinParent: 1, + advancesAppRunCount: true, + cwd: process.cwd(), + kind: "root_run", + createdAt: t2, + }; + yield* eventSink.write({ + events: [ + { + id: yield* idAllocator.allocate.event({ threadId: ids.threadId }), + type: "run.created", + threadId: ids.threadId, + runId: ids.runId, + occurredAt: t2, + payload: parentRun, + }, + { + id: yield* idAllocator.allocate.event({ threadId: ids.threadId }), + type: "run-attempt.created", + threadId: ids.threadId, + runId: ids.runId, + occurredAt: t2, + payload: parentAttempt, + }, + { + id: yield* idAllocator.allocate.event({ threadId: ids.threadId }), + type: "node.updated", + threadId: ids.threadId, + runId: ids.runId, + nodeId: ids.rootNodeId, + occurredAt: t2, + payload: parentRootNode, + }, + ], + }); + + yield* runExecution.startRootRun({ + commandId: CommandId.make(`command:${key}`), + appThread, + providerSessionId, + session: { + events: Stream.empty, + subscribeEvents: Effect.succeed({ + events: Stream.fromQueue(eventQueue), + close: Deferred.succeed(ingestionDone, undefined), + }), + startTurn: () => Effect.void, + interruptTurn: () => Effect.void, + } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime, + providerThread: parentProviderThread, + run: parentRun, + attempt: parentAttempt, + attemptId: ids.attemptId, + rootNode: parentRootNode, + checkpointScope: parentCheckpointScope, + providerTurnOrdinal: 2, + message: { + messageId: MessageId.make(`message:${key}:user`), + text: "Resume then hand off", + attachments: [], + createdBy: "user", + creationSource: "web", + }, + modelSelection: { instanceId, model: "opencode/big-pickle" }, + runtimePolicy: { + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + approvalPolicy: "never", + sandboxPolicy: { + type: "readOnly", + access: { type: "fullAccess" }, + networkAccess: false, + }, + }, + }); + + yield* Queue.offerAll(eventQueue, [ + { + type: "provider_turn.updated", + driver: opencodeDriver, + providerTurn: { + id: ids.rootProviderTurnId, + providerThreadId: ids.providerThreadId, + nodeId: ids.rootNodeId, + runAttemptId: ids.attemptId, + nativeTurnRef: null, + ordinal: 2, + status: "running", + startedAt: t2, + completedAt: null, + }, + }, + { + type: "app_thread.created", + driver: opencodeDriver, + createOnly: true, + appThread: childThread, + }, + { + type: "node.updated", + driver: opencodeDriver, + preserveIdentity: true, + node: { + id: sessionNodeId, + threadId: ids.threadId, + runId: ids.runId, + parentNodeId: ids.rootNodeId, + rootNodeId: ids.rootNodeId, + kind: "subagent", + status: "running", + countsForRun: false, + providerThreadId: childProviderThreadId, + providerTurnId: ids.rootProviderTurnId, + nativeItemRef: { driver: opencodeDriver, nativeId: childSessionId, strength: "strong" }, + runtimeRequestId: null, + checkpointScopeId: null, + startedAt: t2, + completedAt: null, + }, + }, + { + type: "subagent.updated", + driver: opencodeDriver, + preserveIdentity: true, + subagent: { + id: sessionNodeId, + threadId: ids.threadId, + runId: ids.runId, + parentNodeId: ids.rootNodeId, + origin: "provider_native", + createdBy: "agent", + driver: opencodeDriver, + providerInstanceId: instanceId, + providerThreadId: childProviderThreadId, + childThreadId, + nativeTaskRef: { driver: opencodeDriver, nativeId: childSessionId, strength: "strong" }, + prompt: "keep going", + title: "Resume after upgrade", + model: null, + status: "running", + result: null, + startedAt: t2, + completedAt: null, + updatedAt: t2, + }, + }, + { + type: "turn_item.updated", + driver: opencodeDriver, + preserveIdentity: true, + turnItem: { + id: sessionItemId, + threadId: ids.threadId, + runId: ids.runId, + nodeId: sessionNodeId, + providerThreadId: ids.providerThreadId, + providerTurnId: ids.rootProviderTurnId, + nativeItemRef: { driver: opencodeDriver, nativeId: childSessionId, strength: "strong" }, + parentItemId: null, + ordinal: 1, + status: "running", + title: "Resume after upgrade", + startedAt: t2, + completedAt: null, + updatedAt: t2, + type: "subagent", + subagentId: sessionNodeId, + origin: "provider_native", + driver: opencodeDriver, + providerInstanceId: instanceId, + childThreadId, + prompt: "keep going", + result: null, + }, + }, + { + type: "turn.terminal", + driver: opencodeDriver, + providerThreadId: ids.providerThreadId, + providerTurnId: ids.rootProviderTurnId, + runOrdinal: 2, + status: "completed", + failure: null, + threadDisposition: "reusable", + }, + // A newer run claims the still-running legacy child via its session-keyed subagent ID. + { + type: "subagent.updated", + driver: opencodeDriver, + preserveIdentity: true, + subagent: { + id: sessionNodeId, + threadId: ids.threadId, + runId: run3Id, + parentNodeId: NodeId.make(`node:${key}:root:3`), + origin: "provider_native", + createdBy: "agent", + driver: opencodeDriver, + providerInstanceId: instanceId, + providerThreadId: childProviderThreadId, + childThreadId, + nativeTaskRef: { driver: opencodeDriver, nativeId: childSessionId, strength: "strong" }, + prompt: "redirect in run 3", + title: "Redirect in run 3", + model: "opencode/big-pickle", + status: "running", + result: null, + startedAt: t3, + completedAt: null, + updatedAt: t3, + }, + }, + ]); + + // Run 2 must detach the legacy row and close its subscription while Run 3 still owns the child. + yield* Deferred.await(ingestionDone); + const parentProjection = yield* projectionStore.getThreadProjection(ids.threadId); + assert.equal(parentProjection.subagents[0]?.id, legacyNodeId); + assert.equal(parentProjection.subagents[0]?.status, "running"); + }).pipe(Effect.provide(layerFull)); + }, +); diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 1bc83f639d5ae..e02d452defdb6 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -1682,21 +1682,41 @@ export const layer: Layer.Layer< ) { const moved = event.subagent; const detached = yield* Ref.modify(openRunOwnedSubagents, (current) => { - const subagentIds = new Set([moved.id]); + const subagentIds = new Set([moved.id]); for (const subagent of current.subagents.values()) { - if (subagent.threadId === moved.childThreadId) subagentIds.add(subagent.id); + if ( + subagent.childThreadId === moved.childThreadId || + subagent.threadId === moved.childThreadId + ) { + subagentIds.add(subagent.id); + } + } + for (const item of current.turnItems.values()) { + if (item.childThreadId === moved.childThreadId) { + subagentIds.add(item.subagentId); + if (item.nodeId !== null) subagentIds.add(item.nodeId); + } } const childTurnItems = new Map( [...current.childTurnItems].filter( ([, item]) => item.threadId !== moved.childThreadId, ), ); - const turnItems = new Map(current.turnItems); - turnItems.delete(moved.id); + const turnItems = new Map( + [...current.turnItems].filter( + ([subagentId, item]) => + !subagentIds.has(subagentId) && item.childThreadId !== moved.childThreadId, + ), + ); const detachedItems = new Set( [...current.childTurnItems.values(), ...current.turnItems.values()] .filter( - (item) => item.threadId === moved.childThreadId || item.nodeId === moved.id, + (item) => + item.threadId === moved.childThreadId || + (item.nodeId !== null && subagentIds.has(item.nodeId)) || + (item.type === "subagent" && + (item.childThreadId === moved.childThreadId || + subagentIds.has(item.subagentId))), ) .map((item) => item.id), ); @@ -1708,7 +1728,11 @@ export const layer: Layer.Layer< childTurnItems, nodes: new Map( [...current.nodes].filter( - ([id, node]) => id !== moved.id && node.threadId !== moved.childThreadId, + ([id, node]) => + !subagentIds.has(id) && + node.threadId !== moved.childThreadId && + (moved.providerThreadId === null || + node.providerThreadId !== moved.providerThreadId), ), ), linkedChildThreadIds: new Set( @@ -1805,7 +1829,7 @@ export const layer: Layer.Layer< if (isRootProviderThreadUpdate && rootTerminalAlreadySeen && storedEvents.length === 0) { yield* Ref.set(providerThreadOwnerLost, true); } - return storedEvents.length; + return storedEvents; }); // The timer only wakes ingestion. The flush runs as an ingestion step outside any provider // batch, so it stays ordered with routing and snapshots, and a write has committed when @@ -1841,6 +1865,29 @@ export const layer: Layer.Layer< idleFlushWake = undefined; return wake === undefined ? Effect.void : Fiber.interrupt(wake.fiber); }); + const trackedEventFromStored = ( + event: ProviderAdapter.ProviderAdapterV2Event, + storedEvents: Effect.Success>, + ): ProviderAdapter.ProviderAdapterV2Event => { + if (storedEvents.length === 0) return event; + if (event.type === "subagent.updated") { + const stored = storedEvents.find((item) => item.event.type === "subagent.updated"); + if (stored?.event.type === "subagent.updated") { + return { ...event, subagent: stored.event.payload }; + } + } else if (event.type === "node.updated") { + const stored = storedEvents.find((item) => item.event.type === "node.updated"); + if (stored?.event.type === "node.updated") { + return { ...event, node: stored.event.payload }; + } + } else if (event.type === "turn_item.updated") { + const stored = storedEvents.find((item) => item.event.type === "turn-item.updated"); + if (stored?.event.type === "turn-item.updated") { + return { ...event, turnItem: stored.event.payload }; + } + } + return event; + }; // Per-tick ingestion avoids adding a span for every idle wake. const flushIdleAssistantText = Effect.fnUntraced(function* (wokeAt: number) { // A tick means its timer slept until `wokeAt`, but the wall clock can still read up @@ -1857,8 +1904,8 @@ export const layer: Layer.Layer< if (!routeProviderEvent(event, routeIdentity, yield* Ref.get(eventRouting))[0]) continue; // A failed write fails ingestion like any provider write. - yield* ingestDelivered(event, event); - yield* trackChildLifecycle(event, true); + const stored = yield* ingestDelivered(event, event); + yield* trackChildLifecycle(trackedEventFromStored(event, stored), true); } yield* scheduleIdleFlush; }); @@ -1877,7 +1924,7 @@ export const layer: Layer.Layer< ) { yield* Ref.set(latestRootProviderTurn, event.providerTurn); } - let storedEventCount = 0; + let storedEvents: Effect.Success> = []; const deliveredEvent = assistantStreaming.filter( event, DateTime.toEpochMillis(yield* DateTime.now), @@ -1902,7 +1949,7 @@ export const layer: Layer.Layer< shouldStartProviderTurn().pipe(Effect.orElseSucceed(() => true)), ]), ); - storedEventCount = stored; + storedEvents = stored; if (!owned) { yield* Ref.get(latestProviderThread).pipe( Effect.flatMap((providerThread) => @@ -1923,7 +1970,7 @@ export const layer: Layer.Layer< ); } } else if (deliveredEvent) { - storedEventCount = yield* ingestDelivered(event, deliveredEvent); + storedEvents = yield* ingestDelivered(event, deliveredEvent); } else if ( event.type === "turn_item.updated" && (event.turnItem.type === "assistant_message" || event.turnItem.type === "reasoning") && @@ -1937,6 +1984,8 @@ export const layer: Layer.Layer< }); reservedHeldTurnItemIds.add(event.turnItem.id); } + const storedEventCount = storedEvents.length; + const trackedEvent = trackedEventFromStored(event, storedEvents); if (event.type === "provider_thread.updated") { if (event.providerThread.id === input.providerThread.id && storedEventCount > 0) { yield* Ref.set(latestProviderThread, event.providerThread); @@ -1971,7 +2020,17 @@ export const layer: Layer.Layer< pendingBackgroundProviderTurns.delete(event.providerTurnId); } } - yield* trackChildLifecycle(event, deliveredEvent !== null); + // A deduplicated preserveIdentity node.updated matched the stored node, so skip tracking its raw ID. + if ( + !( + event.type === "node.updated" && + event.preserveIdentity === true && + deliveredEvent !== null && + storedEventCount === 0 + ) + ) { + yield* trackChildLifecycle(trackedEvent, deliveredEvent !== null); + } yield* scheduleIdleFlush; }); const providerEventFiber = yield* eventSink diff --git a/packages/provider-core/src/server/ProviderAdapter.ts b/packages/provider-core/src/server/ProviderAdapter.ts index 18ebf74baa1c7..7c74583275a22 100644 --- a/packages/provider-core/src/server/ProviderAdapter.ts +++ b/packages/provider-core/src/server/ProviderAdapter.ts @@ -116,6 +116,8 @@ export const ProviderAdapterV2Event = Schema.Union([ pendingBackgroundWork: Schema.optional(Schema.Boolean), /** Persist idle background snapshots before a run subscriber exists. */ sessionScoped: Schema.optional(Schema.Boolean), + /** Reuse an existing persisted subagent owner node ID when cold-resuming an uncached child. */ + preserveIdentity: Schema.optional(Schema.Boolean), retainedNativeTurnIds: Schema.optional(Schema.Array(Schema.String)), }), Schema.Struct({ @@ -130,11 +132,15 @@ export const ProviderAdapterV2Event = Schema.Union([ type: Schema.Literal("node.updated"), driver: ProviderDriverKind, node: OrchestrationV2ExecutionNode, + /** Reuse an existing persisted subagent node ID when cold-resuming an uncached child. */ + preserveIdentity: Schema.optional(Schema.Boolean), }), Schema.Struct({ type: Schema.Literal("subagent.updated"), driver: ProviderDriverKind, subagent: OrchestrationV2Subagent, + /** Keep previously persisted title, prompt, and model when cold-resuming an uncached child. */ + preserveIdentity: Schema.optional(Schema.Boolean), }), Schema.Struct({ type: Schema.Literal("message.updated"), @@ -147,6 +153,8 @@ export const ProviderAdapterV2Event = Schema.Union([ turnItem: OrchestrationV2TurnItem, /** Persist recovered activity independently of a run subscription. */ sessionScoped: Schema.optional(Schema.Boolean), + /** Keep previously persisted title and prompt when cold-resuming an uncached child. */ + preserveIdentity: Schema.optional(Schema.Boolean), }), /** A steer the provider queued stays `pending` until the agent reads it; `cancelled` if dropped unread. */ Schema.Struct({ diff --git a/packages/provider-core/src/server/subagentProjection.ts b/packages/provider-core/src/server/subagentProjection.ts index d1638b8022029..fbbbdb3fd0b63 100644 --- a/packages/provider-core/src/server/subagentProjection.ts +++ b/packages/provider-core/src/server/subagentProjection.ts @@ -12,6 +12,7 @@ import type { OrchestrationV2Run, OrchestrationV2ThreadProjection, OrchestrationV2TurnItem, + OrchestrationV2UserMessageInputIntent, ProviderInstanceId, ProviderThreadId, ProviderTurnId, @@ -114,6 +115,7 @@ export function makeSubagentConversationArtifacts(input: { readonly providerTurnId: ProviderTurnId | null; readonly nativeItemRef: OrchestrationV2ProviderRef | null; readonly role: "user" | "assistant"; + readonly inputIntent?: OrchestrationV2UserMessageInputIntent; readonly text: string; readonly workflowScript?: boolean; readonly ordinal: number; @@ -169,7 +171,7 @@ export function makeSubagentConversationArtifacts(input: { type: "user_message", ...(input.senderThreadId === undefined ? {} : { senderThreadId: input.senderThreadId }), ...(input.workflowScript === undefined ? {} : { workflowScript: input.workflowScript }), - inputIntent: "turn_start", + inputIntent: input.inputIntent ?? "turn_start", attachments: [], } : { diff --git a/packages/provider-opencode/src/server/v2/adapter.ts b/packages/provider-opencode/src/server/v2/adapter.ts index 9e14f8c4ef36b..d30ade489ea2e 100644 --- a/packages/provider-opencode/src/server/v2/adapter.ts +++ b/packages/provider-opencode/src/server/v2/adapter.ts @@ -273,6 +273,7 @@ interface ActiveTurn { unsent: boolean; readonly steers: Set; readonly settledInbox: Set; + readonly settledCalls: Set; // Reconnects replay finished tools, so each nested html_render takes a page only once. readonly takenPages: Set; heldEnd: TurnTerminal | undefined; @@ -282,6 +283,24 @@ interface ActiveTurn { usedSubagents: boolean; } +interface SubagentRow { + readonly nativeId: string; + readonly state: ThreadState; + readonly preserveIdentity: boolean; + turn: ActiveTurn; + ordinal: number; + startedAt: DateTime.Utc; + prompt: string; + title: string | null; + model: string | null; + status: OrchestrationV2Subagent["status"]; + result: string | null; + resultRun: number; + resultStatus: OrchestrationV2Subagent["status"] | null; + completedAt: DateTime.Utc | null; + emitted: boolean; +} + interface SubagentCall { readonly toolId: string; readonly nativeId: string; @@ -294,14 +313,14 @@ interface SubagentCall { agent: string | undefined; model: string | null; background: boolean; + targetSessionId: string | undefined; + row: SubagentRow | undefined; /** The child's turn this call prompted or steered; unset until that turn starts. */ run: number | undefined; /** That turn's last text once it ended, which OpenCode sends as the run's report. */ answer: string | undefined; child: ThreadState | undefined; status: OrchestrationV2Subagent["status"]; - result: string | null; - completedAt: DateTime.Utc | null; } interface Wake { @@ -367,11 +386,16 @@ interface ThreadState { readonly calls: Map; subagent: | { + readonly row: SubagentRow; call: SubagentCall; readonly appThread: OrchestrationV2AppThread; + readonly coldResumeKey: string | undefined; turns: number; - prompt: string | undefined; + callPrompt: string | undefined; + queuedPrompt: { readonly inboxId: string; readonly text: string } | undefined; queued: string | undefined; + readonly steers: Map; + readonly deliveredSteers: Array<{ readonly inboxId: string; readonly text: string }>; } | undefined; readonly wakes: Array; @@ -959,14 +983,14 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr const rootOf = (state: ThreadState): ThreadState => { let current = state; - while (current.subagent !== undefined) current = current.subagent.call.state; + while (current.subagent !== undefined) current = current.subagent.row.state; return current; }; const sessionsOf = (thread: ThreadState): ReadonlyArray => [ thread, ...[...threads.values()].filter((state) => { - for (let above = state.subagent?.call.state; above; above = above.subagent?.call.state) { + for (let above = state.subagent?.row.state; above; above = above.subagent?.row.state) { if (above === thread) return true; } return false; @@ -981,7 +1005,7 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr * the ones that joined it while it ran. */ const callsAttachedTo = (child: ThreadState): ReadonlyArray => { - const parent = child.subagent?.call.state; + const parent = child.subagent?.row.state; if (parent === undefined) return []; return [...parent.calls.values()].filter((candidate) => candidate.child === child); }; @@ -990,8 +1014,11 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr const calls: Array = []; let current = threads.get(sessionId); while (current?.subagent !== undefined) { - calls.push(current.subagent.call); - current = current.subagent.call.state; + const attached = callsAttachedTo(current); + calls.push( + attached.findLast((call) => call.background) ?? attached.at(-1) ?? current.subagent.call, + ); + current = current.subagent.row.state; } return calls; }; @@ -1079,6 +1106,7 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr providerThread: state.providerThread, pendingBackgroundWork, ...(state.sessionScopedBackground === true ? { sessionScoped: true } : {}), + ...(state.subagent?.row.preserveIdentity === true ? { preserveIdentity: true } : {}), }); }); @@ -1355,6 +1383,36 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr }); }); + // Mid-turn steers delivered to a running child never start a new turn; project them in place. + const emitChildSteer = Effect.fnUntraced(function* ( + state: ThreadState, + turn: ActiveTurn, + inboxId: string, + text: string, + ) { + const subagent = state.subagent; + if (subagent === undefined || text.trim().length === 0) return; + turn.closingText = undefined; + const now = yield* DateTime.now; + const artifacts = makeSubagentConversationArtifacts({ + messageId: idAllocator.derive.messageFromProviderItem({ driver, nativeItemId: inboxId }), + senderThreadId: subagent.call.turn.input.threadId, + turnItemId: idAllocator.derive.turnItemFromProviderItem({ driver, nativeItemId: inboxId }), + threadId: turn.input.threadId, + rootNodeId: turn.input.rootNodeId, + providerThreadId: state.providerThread.id, + providerTurnId: turn.providerTurn.id, + nativeItemRef: ref(inboxId), + role: "user", + inputIntent: "steer", + text, + ordinal: ordinalOf(turn, inboxId), + now, + }); + yield* emit({ type: "message.updated", driver, message: artifacts.message }); + yield* emit({ type: "turn_item.updated", driver, turnItem: artifacts.turnItem }); + }); + // Todo updates share the provider event span. const emitTodo = Effect.fnUntraced(function* ( state: ThreadState, @@ -1679,6 +1737,7 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr unsent: false, steers: new Set(), settledInbox: new Set(), + settledCalls: new Set(), takenPages: new Set(), heldEnd: undefined, awaitingStart: options.awaitingStart, @@ -1686,21 +1745,48 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr usedSubagents: false, }); - const emitSubagent = Effect.fnUntraced(function* (call: SubagentCall) { + const makeSubagentRow = ( + call: SubagentCall, + nativeId: string, + preserveIdentity: boolean, + ): SubagentRow => ({ + nativeId, + state: call.state, + preserveIdentity, + turn: call.turn, + ordinal: call.ordinal, + startedAt: call.startedAt, + prompt: call.prompt, + title: call.title, + model: call.model, + status: call.status, + result: null, + resultRun: -1, + resultStatus: null, + completedAt: null, + emitted: false, + }); + + const emitSubagentRow = Effect.fnUntraced(function* ( + row: SubagentRow, + child: ThreadState | undefined, + ) { + row.emitted = true; const updatedAt = yield* DateTime.now; - const { turn, state } = call; + const { turn, state } = row; const nodeId = idAllocator.derive.nodeFromProviderItem({ driver, - nativeItemId: call.nativeId, + nativeItemId: row.nativeId, }); - const completedAt = isOrchestrationV2WorkActive(call.status) + const completedAt = isOrchestrationV2WorkActive(row.status) ? null - : (call.completedAt ?? updatedAt); - const childProviderThreadId = call.child?.providerThread.id ?? null; - const childThreadId = call.child?.subagent?.appThread.id ?? null; + : (row.completedAt ?? updatedAt); + const childProviderThreadId = child?.providerThread.id ?? null; + const childThreadId = child?.subagent?.appThread.id ?? null; yield* emit({ type: "node.updated", driver, + ...(row.preserveIdentity ? { preserveIdentity: true } : {}), node: { id: nodeId, threadId: turn.input.threadId, @@ -1708,20 +1794,21 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr parentNodeId: turn.input.rootNodeId, rootNodeId: turn.input.rootNodeId, kind: "subagent", - status: call.status, + status: row.status, countsForRun: false, providerThreadId: childProviderThreadId ?? state.providerThread.id, providerTurnId: turn.providerTurn.id, - nativeItemRef: ref(call.nativeId), + nativeItemRef: ref(row.nativeId), runtimeRequestId: null, checkpointScopeId: null, - startedAt: call.startedAt, + startedAt: row.startedAt, completedAt, }, }); yield* emit({ type: "subagent.updated", driver, + ...(row.preserveIdentity ? { preserveIdentity: true } : {}), subagent: { id: nodeId, threadId: turn.input.threadId, @@ -1733,13 +1820,13 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr providerInstanceId: instanceId, providerThreadId: childProviderThreadId, childThreadId, - nativeTaskRef: ref(call.nativeId), - prompt: call.prompt, - title: call.title, - model: call.model, - status: call.status, - result: call.result, - startedAt: call.startedAt, + nativeTaskRef: ref(row.nativeId), + prompt: row.prompt, + title: row.title, + model: row.model, + status: row.status, + result: row.result, + startedAt: row.startedAt, completedAt, updatedAt, }, @@ -1747,19 +1834,20 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr yield* emit({ type: "turn_item.updated", driver, + ...(row.preserveIdentity ? { preserveIdentity: true } : {}), turnItem: { - id: idAllocator.derive.turnItemFromProviderItem({ driver, nativeItemId: call.nativeId }), + id: idAllocator.derive.turnItemFromProviderItem({ driver, nativeItemId: row.nativeId }), threadId: turn.input.threadId, runId: turn.input.runId, nodeId, providerThreadId: state.providerThread.id, providerTurnId: turn.providerTurn.id, - nativeItemRef: ref(call.nativeId), + nativeItemRef: ref(row.nativeId), parentItemId: null, - ordinal: call.ordinal, - status: call.status, - title: call.title, - startedAt: call.startedAt, + ordinal: row.ordinal, + status: row.status, + title: row.title, + startedAt: row.startedAt, completedAt, updatedAt, type: "subagent", @@ -1768,25 +1856,34 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr driver, providerInstanceId: instanceId, childThreadId, - prompt: call.prompt, - result: call.result, + prompt: row.prompt, + result: row.result, }, }); - yield* syncRosterAncestors(call.state); + yield* syncRosterAncestors(row.state); }); const syncRoster = Effect.fnUntraced(function* (state: ThreadState) { - const roster: Array = - runningCalls(state) - .filter((call) => call.background) - .map((call) => ({ - taskId: call.nativeId, - kind: "subagent" as const, - ...(call.title?.trim() ? { description: call.title.trim() } : {}), - ...(call.child?.subagent === undefined - ? {} - : { childThreadId: call.child.subagent.appThread.id }), - })); + const subagentTasks = new Map< + string, + import("@t3tools/contracts").OrchestrationV2PendingBackgroundTask + >(); + for (const call of runningCalls(state)) { + if (!call.background || call.row === undefined) continue; + const { row } = call; + const taskId = row.nativeId; + if (subagentTasks.has(taskId)) continue; + const title = row.preserveIdentity ? undefined : row.title?.trim(); + subagentTasks.set(taskId, { + taskId, + kind: "subagent" as const, + ...(title ? { description: title } : {}), + ...(call.child?.subagent === undefined + ? {} + : { childThreadId: call.child.subagent.appThread.id }), + }); + } + const roster = [...subagentTasks.values()]; for (const [taskId, shell] of shells) { if (shell.running && sessionsOf(state).includes(shell.state)) { roster.push({ @@ -1796,7 +1893,7 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr ...(shell.awaited === true ? { awaited: true } : {}), ...(shell.state.subagent === undefined ? {} - : { parentTaskId: shell.state.subagent.call.nativeId }), + : { parentTaskId: shell.state.subagent.row.nativeId }), }); } } @@ -1841,7 +1938,7 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr let current: ThreadState | undefined = state; while (current !== undefined) { yield* syncRoster(current); - current = current.subagent?.call.state; + current = current.subagent?.row.state; } }); @@ -2111,9 +2208,79 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr const info = announced.get(childId); announced.delete(childId); const now = yield* DateTime.now; + const previous = threads.get(childId); + const existing = + previous?.subagent !== undefined && previous.subagent.row.state === call.state + ? previous.subagent + : undefined; + if (existing !== undefined) { + const child = previous!; + const { row } = existing; + const needsReopen = !isOrchestrationV2WorkActive(row.status); + // A call to a running child steers that run; OpenCode reports it once for all its calls. + if (child.active !== undefined) call.run = existing.turns; + const childModel = info?.model ?? child.model; + if (childModel !== undefined) call.model ??= `${childModel.providerID}/${childModel.id}`; + const nextTitle = call.title ?? info?.title ?? null; + const filledMeta = + (row.model === null && call.model !== null) || (row.title === null && nextTitle !== null); + row.model ??= call.model; + row.title ??= nextTitle; + call.row = row; + call.child = child; + existing.call = call; + if (child.active === undefined) existing.callPrompt ??= call.prompt; + child.sessionScopedBackground = false; + child.model = childModel; + const nextProviderThread = withReportedModel( + { ...child.providerThread, status: "active", updatedAt: now }, + child.model, + ); + const providerChanged = + child.providerThread.status !== "active" || + nextProviderThread.nativeMetadata !== child.providerThread.nativeMetadata; + child.providerThread = nextProviderThread; + child.directory = call.state.directory; + child.agent = info?.agent ?? call.agent ?? child.agent; + child.rules = info?.permissions ?? child.rules; + childOwners.set(childId, rootOf(call.state)); + const movedTurn = row.turn !== call.turn; + if (movedTurn) { + row.turn = call.turn; + row.ordinal = call.ordinal; + } + if (needsReopen) { + row.startedAt = call.startedAt; + row.status = "running"; + row.result = null; + row.resultRun = -1; + row.resultStatus = null; + row.completedAt = null; + yield* emitSubagentRow(row, child); + yield* emitProviderThread(child); + } else { + if (movedTurn || filledMeta || !row.emitted) yield* emitSubagentRow(row, child); + else yield* syncRosterAncestors(call.state); + if (providerChanged) yield* emitProviderThread(child); + } + // Reused child sessions may retain old mode rules; refresh policy as soon as the call identifies them. + if (info === undefined) { + yield* writeRules(child, rootOf(call.state).policy).pipe( + Effect.timeout(REQUEST_REPLY_TIMEOUT), + Effect.ignore({ log: true }), + ); + } + return; + } + const coldResume = info === undefined && previous?.subagent === undefined; + const rowNativeId = + previous?.subagent !== undefined && previous.subagent.row.state !== call.state + ? call.nativeId + : childId; + const row = (call.row ??= makeSubagentRow(call, rowNativeId, coldResume)); const nodeId = idAllocator.derive.nodeFromProviderItem({ driver, - nativeItemId: call.nativeId, + nativeItemId: row.nativeId, }); const childThreadId = idAllocator.derive.subagentThread({ driver, @@ -2125,8 +2292,14 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr nativeThreadId: childId, providerInstanceId: instanceId, }); - if (info?.model !== undefined) call.model = `${info.model.providerID}/${info.model.id}`; - call.title = call.title ?? info?.title ?? null; + const childModel = info?.model ?? previous?.model; + if (childModel !== undefined) { + const slug = `${childModel.providerID}/${childModel.id}`; + call.model = slug; + row.model = slug; + } + row.title ??= call.title ?? info?.title ?? null; + call.title = row.title; const parentThread = call.turn.input.appThread; const appThread = makeSubagentChildThread({ parentThread, @@ -2136,13 +2309,13 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr providerInstanceId: instanceId, modelSelection: { instanceId, - model: call.model ?? call.turn.input.modelSelection.model, + model: row.model ?? call.turn.input.modelSelection.model, }, title: subagentThreadTitle({ parentTitle: parentThread.title, - title: call.title, - prompt: call.prompt, - ordinal: call.ordinal, + title: row.title, + prompt: row.prompt, + ordinal: row.ordinal, }), now, createdBy: "agent", @@ -2165,21 +2338,24 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr createdAt: now, updatedAt: now, }; - const previous = threads.get(childId); - // A call to a running child steers that run; OpenCode reports it once for all its calls. if (previous?.active !== undefined) call.run = previous.subagent?.turns; const subagent = { + row, call, appThread, + coldResumeKey: coldResume ? call.nativeId : previous?.subagent?.coldResumeKey, turns: previous?.subagent?.turns ?? 0, - prompt: call.prompt, + callPrompt: call.prompt, + queuedPrompt: undefined, queued: undefined, + steers: new Map(), + deliveredSteers: [], }; const child = previous ?? newThreadState(childId, providerThread, call.state.directory, subagent); child.subagent = subagent; child.sessionScopedBackground = false; - child.model = info?.model ?? child.model; + child.model = childModel; child.providerThread = withReportedModel(providerThread, child.model); child.directory = call.state.directory; child.agent = info?.agent ?? call.agent ?? child.agent; @@ -2187,9 +2363,14 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr threads.set(childId, child); childOwners.set(childId, rootOf(call.state)); call.child = child; - yield* emit({ type: "app_thread.created", driver, appThread }); + yield* emit({ + type: "app_thread.created", + driver, + appThread, + ...(info === undefined ? { createOnly: true } : {}), + }); + yield* emitSubagentRow(row, child); yield* emitProviderThread(child); - yield* emitSubagent(call); // Reused child sessions may retain old mode rules; refresh policy as soon as the call identifies them. if (info === undefined) { yield* writeRules(child, rootOf(call.state).policy).pipe( @@ -2205,12 +2386,29 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr result?: string, ) => Effect.Effect = Effect.fnUntraced(function* (call, status, result) { if (!isOrchestrationV2WorkActive(call.status)) return; + const now = yield* DateTime.now; call.status = status; - call.completedAt = yield* DateTime.now; - if (result !== undefined) call.result = result; + call.turn.settledCalls.add(call.toolId); call.state.calls.delete(call.toolId); const child = call.child; - if (child !== undefined) { + const remaining = child === undefined ? [] : callsAttachedTo(child); + if (child?.subagent !== undefined && child.subagent.call === call && remaining.length > 0) { + child.subagent.call = remaining.at(-1)!; + } + const row = call.row; + if (row !== undefined) { + const callRun = call.run ?? child?.subagent?.turns ?? 0; + const supersededByLiveCall = remaining.some( + (other) => (other.run ?? child?.subagent?.turns ?? 0) >= callRun, + ); + if ((!supersededByLiveCall || status === "completed") && callRun >= row.resultRun) { + if (callRun > row.resultRun) row.result = null; + row.resultRun = callRun; + row.resultStatus = status; + if (result !== undefined) row.result = result; + } + } + if (child !== undefined && remaining.length === 0) { for (const nested of Array.from(child.calls.values())) { if (nested.background && status !== "failed") continue; yield* settleCall(nested, status === "completed" ? "interrupted" : status); @@ -2218,15 +2416,38 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr if (child.active !== undefined && status !== "completed") { yield* finishTurn(child, { status: "interrupted" }); } + if (child.active === undefined && child.subagent !== undefined) { + child.subagent.callPrompt = undefined; + child.subagent.queuedPrompt = undefined; + child.subagent.queued = undefined; + child.subagent.steers.clear(); + child.subagent.deliveredSteers.length = 0; + } child.sessionScopedBackground = false; child.providerThread = { ...child.providerThread, status: "idle", - updatedAt: call.completedAt, + updatedAt: now, }; yield* emitProviderThread(child); } - yield* emitSubagent(call); + if (row !== undefined && remaining.length === 0) { + row.status = row.resultStatus ?? status; + row.completedAt = now; + yield* emitSubagentRow(row, child); + } else if ( + row === undefined && + (call.targetSessionId === undefined || + (status !== "completed" && !threads.has(call.targetSessionId))) + ) { + const fallbackRow = makeSubagentRow(call, call.nativeId, false); + fallbackRow.status = status; + fallbackRow.completedAt = now; + if (result !== undefined) fallbackRow.result = result; + yield* emitSubagentRow(fallbackRow, undefined); + } else { + yield* syncRosterAncestors(call.state); + } }); const startChildTurn = Effect.fnUntraced(function* (child: ThreadState) { @@ -2235,7 +2456,10 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr subagent.turns += 1; for (const call of callsAttachedTo(child)) call.run ??= subagent.turns; const startedAt = yield* DateTime.now; - const nativeTurnId = `${child.sessionId}:turn:${subagent.turns}`; + const nativeTurnId = + subagent.coldResumeKey === undefined + ? `${child.sessionId}:turn:${subagent.turns}` + : `${child.sessionId}:resume:${subagent.coldResumeKey}:turn:${subagent.turns}`; const rootNodeId = idAllocator.derive.nodeFromProviderItem({ driver, nativeItemId: `${nativeTurnId}:root`, @@ -2290,8 +2514,9 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr }, }); yield* emitProviderTurn(child, turn, providerTurn); - const prompt = subagent.prompt; - subagent.prompt = undefined; + const prompt = subagent.queuedPrompt?.text ?? subagent.callPrompt; + subagent.queuedPrompt = undefined; + subagent.callPrompt = undefined; if (prompt !== undefined && prompt.length > 0) { const nativeId = `${nativeTurnId}:prompt`; const artifacts = makeSubagentConversationArtifacts({ @@ -2314,6 +2539,9 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr yield* emit({ type: "message.updated", driver, message: artifacts.message }); yield* emit({ type: "turn_item.updated", driver, turnItem: artifacts.turnItem }); } + for (const steer of subagent.deliveredSteers.splice(0)) { + yield* emitChildSteer(child, turn, steer.inboxId, steer.text); + } }); const offerWake = Effect.fnUntraced(function* (state: ThreadState, wake: Wake) { @@ -2423,19 +2651,21 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr }); // Subagent turns are runless; their calls own settlement. if (state.subagent !== undefined) { - const call = state.subagent.call; for (const attached of callsAttachedTo(state)) { if (attached.run === state.subagent.turns) attached.answer = turn.answer; } - if (!call.background && terminal.status !== "completed") { - yield* settleCall(call, terminal.status); - } - if (terminal.status === "failed") { + if (terminal.status !== "completed") { // Background calls settle when the parent reads a report; failed children have no report. // Fail every attached call so earlier background calls cannot leave phantom work, // regardless of which call joined last or is still foreground. for (const attached of callsAttachedTo(state)) { - yield* settleCall(attached, terminal.status, terminal.failure.message); + if (terminal.status === "failed" || !attached.background) { + yield* settleCall( + attached, + terminal.status, + terminal.status === "failed" ? terminal.failure.message : undefined, + ); + } } } return; @@ -2816,16 +3046,15 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr agent: undefined, model: null, background: false, + targetSessionId: undefined, + row: undefined, run: undefined, answer: undefined, child: undefined, status: "running", - result: null, - completedAt: null, }; turn.usedSubagents = true; state.calls.set(toolId, call); - yield* emitSubagent(call); }); const onTurnEvent = Effect.fnUntraced(function* ( @@ -2872,7 +3101,9 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr call.agent = stringField(input, "agent"); call.model = stringField(input, "model") ?? call.model; call.background = input["background"] === true; - return yield* emitSubagent(call); + call.targetSessionId = + stringField(input, "sessionID") ?? stringField(input, "session_id"); + return; } const tool = turn.tools.get(event.data.id); if (tool !== undefined) tool.input = event.data.input; @@ -2893,7 +3124,7 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr if (event.data.metadata?.["status"] === "running") { if (call.background) return; call.background = true; - return yield* emitSubagent(call); + return yield* syncRosterAncestors(call.state); } return yield* settleCall(call, "completed", subagentOutput(textOf(event.data.content))); } @@ -3245,14 +3476,20 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr written !== undefined && candidate.run < written ? [candidate.call] : [], ); const outcome = reportOutcome(stringField(payload.metadata, "state")); + const existingChild = threads.get(childId); + const labelRow = call?.row ?? existingChild?.subagent?.row; state.reports.set(inboxId, { inboxId, childId, text: payload.text, report: { kind: "subagent", - label: call?.title ?? stringField(payload.metadata, "description"), - childThreadId: call?.child?.subagent?.appThread.id, + label: + labelRow?.preserveIdentity === true + ? undefined + : (labelRow?.title ?? call?.title ?? stringField(payload.metadata, "description")), + childThreadId: + call?.child?.subagent?.appThread.id ?? existingChild?.subagent?.appThread.id, outcome, }, }); @@ -3502,6 +3739,12 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr state.providerThread = { ...providerThread, updatedAt: yield* DateTime.now }; yield* emitProviderThread(state); } + if (state.subagent !== undefined && state.subagent.row.model === null) { + state.subagent.row.model = `${event.data.model.providerID}/${event.data.model.id}`; + if (state.subagent.row.emitted) { + yield* emitSubagentRow(state.subagent.row, state); + } + } } if ( event.type === "session.inbox.enqueued" && @@ -3518,10 +3761,35 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr state.reports.delete(event.data.inboxID); const external = state.externalPrompts.get(event.data.inboxID); state.externalPrompts.delete(event.data.inboxID); + const childSteer = state.subagent?.steers.get(event.data.inboxID); + state.subagent?.steers.delete(event.data.inboxID); + if (event.type === "session.inbox.cancelled" && state.subagent !== undefined) { + if (state.subagent.queued === event.data.inboxID) { + state.subagent.queued = undefined; + } + if (state.subagent.queuedPrompt?.inboxId === event.data.inboxID) { + state.subagent.queuedPrompt = undefined; + if (state.active === undefined) { + const next = state.subagent.steers.entries().next(); + if (!next.done) { + const [nextId, nextText] = next.value; + state.subagent.steers.delete(nextId); + state.subagent.queuedPrompt = { inboxId: nextId, text: nextText }; + state.subagent.queued ??= nextId; + } + } + } + } const turn = state.active; if (turn !== undefined && turn.steers.delete(event.data.inboxID)) { turn.settledInbox.add(event.data.inboxID); if (event.type === "session.inbox.cancelled") yield* endIfSettled(state, turn); + } else if (event.type === "session.inbox.delivered" && childSteer !== undefined) { + if (turn !== undefined) { + yield* emitChildSteer(state, turn, event.data.inboxID, childSteer); + } else if (state.subagent !== undefined) { + state.subagent.deliveredSteers.push({ inboxId: event.data.inboxID, text: childSteer }); + } } else if ( event.type === "session.inbox.delivered" && external !== undefined && @@ -3537,22 +3805,31 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr } return; } - // No T3 turn existed while a wake held these, so its user items came from another client. - if ( - event.type === "session.inbox.enqueued" && - event.data.item.type === "user" && - (state.active === undefined || (held && state.subagent === undefined)) - ) { + if (event.type === "session.inbox.enqueued" && event.data.item.type === "user") { const text = event.data.item.payload.text; - if (state.subagent !== undefined) state.subagent.prompt ??= text; - // T3's own prompts can be admitted after a failed submit ended their turn. - else if (!isOwnPrompt(event.data.inboxID)) { - state.externalPrompts.set(event.data.inboxID, { - text, - leads: !held && state.reports.size === 0, - }); + if (state.subagent !== undefined) { + if (state.active === undefined && state.subagent.queuedPrompt === undefined) { + state.subagent.queuedPrompt = { + inboxId: event.data.inboxID, + text: state.subagent.callPrompt || text, + }; + state.subagent.callPrompt = undefined; + } else { + state.subagent.steers.set(event.data.inboxID, text); + } + return; + } + // No T3 turn existed while a wake held these, so its user items came from another client. + if (state.active === undefined || held) { + // T3's own prompts can be admitted after a failed submit ended their turn. + if (!isOwnPrompt(event.data.inboxID)) { + state.externalPrompts.set(event.data.inboxID, { + text, + leads: !held && state.reports.size === 0, + }); + } + return; } - return; } const started = event.type === "session.execution.started" || event.type === "unreadable.execution.started"; @@ -3603,13 +3880,22 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr if (turn === undefined) return undefined; const promptId = promptOf(turn.providerTurn); const { before } = turn; - if (promptId === undefined && before === undefined) return undefined; + const pendingSteers = + state.subagent !== undefined && state.subagent.steers.size > 0 + ? new Set(state.subagent.steers.keys()) + : undefined; + if (promptId === undefined && before === undefined && pendingSteers === undefined) { + return undefined; + } const start = before !== undefined ? before : promptId; + const oldestSteer = pendingSteers?.values().next().value; const read = yield* OpenCode2Client.paginate( { sessionID: Session.ID.make(sessionId), order: "desc" as const, limit: 50 }, client.message.list, ).pipe( - Stream.takeUntil((message) => message.id === start), + Stream.takeUntil((message) => + start !== undefined ? message.id === start : message.id === oldestSteer, + ), Stream.runCollect, Effect.catchTags({ SessionNotFoundError: () => Effect.succeed(null) }), ); @@ -3617,10 +3903,17 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr Effect.gen(function* () { if (state.active !== turn) return undefined; if (read === null) return { outcome: "missing" as const }; - // Without this turn's start in history, all retrieved turns are older. - const found = start === null || read.at(-1)?.id === start; - if (!found) return undefined; - const recent = before !== undefined && before !== null ? read.slice(0, -1) : read; + let recent: ReadonlyArray<(typeof read)[number]>; + if (start !== undefined) { + // Without this turn's start in history, all retrieved turns are older. + const found = start === null || read.at(-1)?.id === start; + if (!found) return undefined; + recent = before !== undefined && before !== null ? read.slice(0, -1) : read; + } else { + const anchorIndex = read.findLastIndex((message) => pendingSteers!.has(message.id)); + if (anchorIndex === -1) return undefined; + recent = read.slice(0, anchorIndex + 1); + } for (const message of recent) { if (message.type === "user" && turn.steers.delete(message.id)) { turn.settledInbox.add(message.id); @@ -3647,6 +3940,12 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr turn.retry = undefined; } for (const message of recent.toReversed()) { + if (message.type === "user" && state.subagent !== undefined && state.active === turn) { + const steer = state.subagent.steers.get(message.id) ?? message.text; + state.subagent.steers.delete(message.id); + yield* emitChildSteer(state, turn, message.id, steer); + continue; + } if (message.type !== "assistant" || state.active !== turn) continue; const ordinals = { text: 0, reasoning: 0 }; for (const [index, part] of message.content.entries()) { @@ -3668,6 +3967,49 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr } if (part.type !== "tool") continue; turn.closingText = undefined; + if (part.name === "subagent") { + if (turn.settledCalls.has(part.id)) continue; + const input = + typeof part.state.input === "object" && part.state.input !== null + ? (part.state.input as Record) + : {}; + const childId = + part.state.status === "streaming" + ? undefined + : stringField(part.state.metadata, "sessionID"); + let call = state.calls.get(part.id); + if (call === undefined) { + yield* startCall(state, turn, part.id); + call = state.calls.get(part.id); + } + if (call !== undefined) { + call.prompt ||= stringField(input, "prompt") ?? ""; + call.title ??= stringField(input, "description") ?? null; + call.agent ??= stringField(input, "agent"); + call.model ??= stringField(input, "model") ?? null; + if (input["background"] === true) call.background = true; + call.targetSessionId ??= + stringField(input, "sessionID") ?? stringField(input, "session_id"); + if (childId !== undefined) yield* attachChild(call, childId); + if (part.state.status === "completed") { + if (part.state.metadata?.["status"] === "running") { + if (!call.background) { + call.background = true; + yield* syncRosterAncestors(call.state); + } + } else { + yield* settleCall( + call, + "completed", + subagentOutput(textOf(part.state.content)), + ); + } + } else if (part.state.status === "error") { + yield* settleCall(call, "failed", part.state.error.message); + } + } + continue; + } if (!turn.tools.has(part.id)) { turn.tools.set(part.id, { name: part.name, input: {} }); turn.startedAt.set(part.id, yield* DateTime.now); @@ -4602,7 +4944,7 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr (caller) => caller.subagent !== undefined && busy.has(caller.sessionId) && - !isOrchestrationV2WorkActive(caller.subagent.call.status), + callsAttachedTo(caller).length === 0, ); // Child sessions are announced before call progress names them; use announcements to stop unnamed children. const announcedTo = (caller: ThreadState) => @@ -4995,9 +5337,12 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr Effect.timeout(REQUEST_REPLY_TIMEOUT), Effect.ignore({ log: true }), ); + const runningChildren = new Set(); for (const call of runningCalls(state)) { - if (call.child === undefined) continue; - yield* writeRules(call.child, turnInput.runtimePolicy).pipe( + if (call.child !== undefined) runningChildren.add(call.child); + } + for (const child of runningChildren) { + yield* writeRules(child, turnInput.runtimePolicy).pipe( Effect.timeout(REQUEST_REPLY_TIMEOUT), Effect.ignore({ log: true }), ); From 313b6d231a4a75961110a4a565010b23f9c365c1 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sat, 21 Nov 2026 14:42:01 +0000 Subject: [PATCH 2/6] test(opencode): trim subagent dedupe tests to the regressions they guard Drop edge-case adapter tests that duplicate coverage of the row model, run the redirect dedupe case once, and share the legacy-row layer setup between the two RunExecutionService cases. Co-Authored-By: Claude Opus 5.5 --- .../Adapters/OpenCode2AdapterV2.test.ts | 549 +----------------- .../RunExecutionService.test.ts | 144 ++--- 2 files changed, 52 insertions(+), 641 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts index d77015f2c39e0..8be8130c6b0b3 100644 --- a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts @@ -2288,7 +2288,7 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { }), ); - it.effect.each([1, 5])( + it.effect.each([2])( "dedupes %i redirect(s) onto one subagent row, preserves title and model, and surfaces steered prompts", (steers) => Effect.gen(function* () { @@ -2624,114 +2624,6 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { }), ); - it.effect( - "keeps a running background subagent active and routes its requests when a joined steer call fails", - () => - Effect.gen(function* () { - const badSteer = { - sessionID: SESSION, - assistantMessageID: "msg_assistant", - id: "call-bad-steer", - }; - const pendingSteer = { - sessionID: SESSION, - assistantMessageID: "msg_assistant", - id: "call-unattached-steer", - }; - const form = { ...colorForm, sessionID: CHILD }; - const { runtime, thread } = yield* resumed([ - ...launchedAndRunning, - event("session.tool.input.started", { ...badSteer, name: "subagent" }), - event("session.tool.called", { - ...badSteer, - name: "subagent", - input: { description: "Bad steer", prompt: "redirect", sessionID: CHILD }, - executed: false, - }), - event("session.tool.progress", { - ...badSteer, - metadata: { sessionID: CHILD, status: "running" }, - }), - out("session.update", { sessionID: CHILD, permissions: "" }), - reply("session.update", null), - // A second background steer that has not attached yet must not duplicate the roster entry. - event("session.tool.input.started", { ...pendingSteer, name: "subagent" }), - event("session.tool.called", { - ...pendingSteer, - name: "subagent", - input: { - description: "Pending steer", - prompt: "wait", - background: true, - sessionID: CHILD, - }, - executed: false, - }), - event( - "session.tool.failed", - { - ...badSteer, - error: { type: "aborted", message: "cancelled steer" }, - metadata: { sessionID: CHILD }, - executed: false, - }, - undefined, - 2, - ), - // Emit the child's form before pendingSteer attaches, proving settleCall restored - // child.subagent.call to the surviving background call. - event("form.created", { form }), - event("session.tool.progress", { - ...pendingSteer, - metadata: { sessionID: CHILD, status: "running" }, - }), - returnedRunning(pendingSteer.id), - event("session.execution.succeeded", { sessionID: SESSION }), - event("session.execution.succeeded", { sessionID: CHILD }), - childReport("msg_report", "STILL_OK"), - ]); - const statuses: Array = []; - const done = yield* Deferred.make(); - let finalResult: string | null = null; - let maxRosterSubagents = 0; - let seenForm: - | { readonly nativeId: string | undefined; readonly runId: RunId | null } - | undefined; - yield* runtime.events.pipe( - Stream.tap((e) => - Effect.gen(function* () { - if (e.type === "turn_item.updated" && e.turnItem.type === "user_input_request") { - seenForm = { - nativeId: e.turnItem.nativeItemRef?.nativeId ?? undefined, - runId: e.turnItem.runId, - }; - } - if (e.type === "provider_thread.updated" && e.providerThread.id === thread.id) { - const count = (e.providerThread.pendingBackgroundTasks ?? []).filter( - (task) => task.kind === "subagent", - ).length; - if (count > maxRosterSubagents) maxRosterSubagents = count; - } - if (e.type === "subagent.updated") { - statuses.push(e.subagent.status); - finalResult = e.subagent.result; - if (e.subagent.status === "completed") yield* Deferred.succeed(done, undefined); - } - }), - ), - Stream.runDrain, - Effect.forkScoped, - ); - yield* runtime.startTurn(withLineage(thread)); - yield* Deferred.await(done); - assert.isFalse(statuses.includes("interrupted")); - assert.isFalse(statuses.includes("failed")); - assert.equal(maxRosterSubagents, 1); - assert.deepEqual(seenForm, { nativeId: form.id, runId: withLineage(thread).runId }); - assert.equal(finalResult, "STILL_OK"); - }), - ); - it.effect.each([false, true] as const)( "preserves existing row identity, child thread, and turn IDs when resuming an uncached child (preRestored=%s)", (preRestored) => @@ -2851,241 +2743,6 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { }), ); - it.effect( - "promotes the next queued prompt when an idle child's leading prompt is cancelled", - () => - Effect.gen(function* () { - const { runtime, thread } = yield* resumed([ - ...backgroundLaunch(CHILD), - event("session.inbox.enqueued", { - inboxID: "msg_child_first", - sessionID: CHILD, - item: { - type: "user", - payload: { text: "sleep", files: [] }, - delivery: "steer", - }, - }), - event("session.inbox.enqueued", { - inboxID: "msg_child_second", - sessionID: CHILD, - item: { - type: "user", - payload: { text: "Promoted prompt", files: [] }, - delivery: "steer", - }, - }), - event("session.inbox.cancelled", { sessionID: CHILD, inboxID: "msg_child_first" }), - returnedRunning("call-background"), - event("session.execution.started", { sessionID: CHILD }), - event("session.inbox.delivered", { sessionID: CHILD, inboxID: "msg_child_second" }), - childSaid("PROMOTED_OK"), - event("session.execution.succeeded", { sessionID: CHILD }), - event("session.execution.succeeded", { sessionID: SESSION }), - childReport("msg_report", "PROMOTED_OK"), - ]); - let childThreadId: ThreadId | undefined; - const prompts: Array<{ readonly text: string; readonly inputIntent: string }> = []; - const done = yield* Deferred.make(); - yield* runtime.events.pipe( - Stream.tap((e) => - Effect.gen(function* () { - if (e.type === "app_thread.created") childThreadId = e.appThread.id; - if ( - e.type === "turn_item.updated" && - e.turnItem.type === "user_message" && - e.turnItem.threadId === childThreadId - ) { - prompts.push({ text: e.turnItem.text, inputIntent: e.turnItem.inputIntent }); - } - if (e.type === "subagent.updated" && e.subagent.status === "completed") { - yield* Deferred.succeed(done, undefined); - } - }), - ), - Stream.runDrain, - Effect.forkScoped, - ); - yield* runtime.startTurn(withLineage(thread)); - yield* Deferred.await(done); - assert.deepEqual(prompts, [{ text: "Promoted prompt", inputIntent: "turn_start" }]); - }), - ); - - it.effect.each([true, false] as const)( - "recovers a child's delivered steer from history backfill on reconnect (sawInitialEnqueue=%s)", - (sawInitialEnqueue) => - Effect.gen(function* () { - const { runtime, thread } = yield* resumed([ - ...backgroundLaunch(CHILD), - ...(sawInitialEnqueue - ? [ - event("session.inbox.enqueued", { - inboxID: "msg_child_initial", - sessionID: CHILD, - item: { - type: "user", - payload: { text: "sleep", files: [] }, - delivery: "steer", - }, - }), - ] - : []), - returnedRunning("call-background"), - event("session.execution.started", { sessionID: CHILD }), - ...(sawInitialEnqueue - ? [event("session.inbox.delivered", { sessionID: CHILD, inboxID: "msg_child_initial" })] - : []), - ...callAgain({ background: true, description: "Pivot", prompt: "Pivot to B" }), - event("session.inbox.enqueued", { - inboxID: "msg_steer_gap", - sessionID: CHILD, - item: { - type: "user", - payload: { text: "Pivot to B", files: [] }, - delivery: "steer", - }, - }), - event("session.execution.succeeded", { sessionID: SESSION }), - { type: "runtime_exit", status: "success" }, - out("event.subscribe"), - out("session.active"), - replyData("session.active", { [CHILD]: { type: "running" } }), - out("message.list", { sessionID: CHILD, order: "desc", limit: "50" }), - reply("message.list", { - data: [ - { - id: "msg_child_reply_gap", - time: { created: 3, streamed: 3, completed: 3 }, - type: "assistant", - agent: "general", - model: { id: "big-pickle", providerID: "opencode", variant: "default" }, - content: [{ type: "text", text: "Switched to B" }], - finish: "stop", - }, - { - id: "msg_steer_gap", - time: { created: 2 }, - type: "user", - text: "Pivot to B", - }, - { - id: "msg_child_initial", - time: { created: 1 }, - type: "user", - text: "sleep", - }, - ], - cursor: {}, - }), - out("permission.list", { sessionID: CHILD }), - replyData("permission.list", []), - out("session.form.list", { sessionID: CHILD }), - replyData("session.form.list", []), - event("session.execution.succeeded", { sessionID: CHILD }), - childReport("msg_report", "Switched to B"), - ]); - let childThreadId: ThreadId | undefined; - const childTimeline: Array = []; - const done = yield* Deferred.make(); - yield* runtime.events.pipe( - Stream.tap((e) => - Effect.gen(function* () { - if (e.type === "app_thread.created") childThreadId = e.appThread.id; - if (e.type === "turn_item.updated" && e.turnItem.threadId === childThreadId) { - if (e.turnItem.type === "user_message") { - childTimeline.push(`${e.turnItem.inputIntent}:${e.turnItem.text}`); - } else if (e.turnItem.type === "assistant_message") { - childTimeline.push(`assistant:${e.turnItem.text}`); - } - } - if (e.type === "subagent.updated" && e.subagent.status === "completed") { - yield* Deferred.succeed(done, undefined); - } - }), - ), - Stream.runDrain, - Effect.forkScoped, - ); - yield* runtime.startTurn(withLineage(thread)); - yield* Deferred.await(done); - assert.deepEqual(childTimeline, [ - "turn_start:sleep", - "steer:Pivot to B", - "assistant:Switched to B", - ]); - }), - ); - - it.effect( - "moves a still-running subagent row to a later parent run when redirected before its report arrives", - () => - Effect.gen(function* () { - const { runtime, thread } = yield* resumed([ - ...launchedAndRunning, - event("session.execution.succeeded", { sessionID: SESSION }), - out("session.update", { sessionID: CHILD, permissions: "" }), - reply("session.update", null), - out("session.prompt", { sessionID: SESSION, text: "" }), - promptAccepted, - event("session.execution.started", { sessionID: SESSION }), - ...callAgain( - { background: true, description: "Redirect in run 2", prompt: "pivot in run 2" }, - { id: "call-run2-redirect", writesRules: false }, - ), - event("session.execution.succeeded", { sessionID: SESSION }), - event("session.execution.succeeded", { sessionID: CHILD }), - childReport("msg_report", "MOVED_OK"), - ]); - const run2Id = RunId.make("run:opencode2-adapter:2"); - const run1Done = yield* Deferred.make(); - const run2Done = yield* Deferred.make(); - const subagentIds = new Set(); - const seenRun2Statuses: Array = []; - let finalSubagent: - | Extract< - ProviderAdapter.ProviderAdapterV2Event, - { type: "subagent.updated" } - >["subagent"] - | undefined; - yield* runtime.events.pipe( - Stream.tap((e) => - Effect.gen(function* () { - if (e.type === "subagent.updated") { - subagentIds.add(e.subagent.id); - finalSubagent = e.subagent; - if (e.subagent.runId === run2Id) { - seenRun2Statuses.push(e.subagent.status); - if (e.subagent.status === "completed") - yield* Deferred.succeed(run2Done, undefined); - } - } - if (e.type === "turn.terminal" && e.runOrdinal === 1) { - yield* Deferred.succeed(run1Done, undefined); - } - }), - ), - Stream.runDrain, - Effect.forkScoped, - ); - yield* runtime.startTurn(withLineage(thread)); - yield* Deferred.await(run1Done); - yield* runtime.startTurn({ - ...withLineage(thread), - runId: run2Id, - runOrdinal: 2, - providerTurnOrdinal: 2, - attemptId: RunAttemptId.make("attempt:opencode2-adapter:2"), - }); - yield* Deferred.await(run2Done); - assert.equal(subagentIds.size, 1); - assert.deepEqual(seenRun2Statuses, ["running", "completed"]); - assert.equal(finalSubagent?.runId, run2Id); - assert.equal(finalSubagent?.title, "Sleep"); - assert.equal(finalSubagent?.result, "MOVED_OK"); - }), - ); - it.effect( "dedupes parent history subagent tool parts on reconnect without emitting generic tool calls", () => @@ -3197,210 +2854,6 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { }), ); - it.effect( - "replays a history-only redirect to a previously settled child on reconnect without replaying settled calls", - () => - Effect.gen(function* () { - const { runtime, thread } = yield* resumed([ - ...backgroundLaunch(CHILD), - event( - "session.tool.success", - { - sessionID: SESSION, - assistantMessageID: "msg_assistant", - id: "call-background", - content: [{ type: "text", text: "FIRST_OK" }], - metadata: { sessionID: CHILD, status: "completed" }, - executed: true, - }, - undefined, - 2, - ), - { type: "runtime_exit", status: "success" }, - out("event.subscribe"), - out("session.active"), - replyData("session.active", { [SESSION]: { type: "running" } }), - out("message.list", { sessionID: SESSION, order: "desc", limit: "50" }), - reply("message.list", { - data: [ - { - id: "msg_parent_gap", - time: { created: 2, streamed: 2, completed: 2 }, - type: "assistant", - agent: "build", - model: { id: "big-pickle", providerID: "opencode", variant: "default" }, - content: [ - { - type: "tool", - id: "call-background", - name: "subagent", - executed: true, - state: { - status: "completed", - input: { description: "Sleep", prompt: "sleep" }, - content: [{ type: "text", text: "FIRST_OK" }], - metadata: { sessionID: CHILD, status: "completed" }, - }, - time: { created: 2, ran: 2, completed: 2 }, - }, - { - type: "tool", - id: "call-gap-redirect", - name: "subagent", - executed: true, - state: { - status: "completed", - input: { - description: "Redirect after settle", - prompt: "redirect", - sessionID: CHILD, - }, - content: [{ type: "text", text: "SECOND_GAP_OK" }], - metadata: { sessionID: CHILD, status: "completed" }, - }, - time: { created: 3, ran: 3, completed: 3 }, - }, - ], - finish: "stop", - }, - { - id: `msg_t3_turn_${SESSION}:attempt:opencode2-adapter`, - time: { created: 1 }, - type: "user", - text: "hi", - }, - ], - cursor: {}, - }), - out("session.update", { sessionID: CHILD, permissions: "" }), - reply("session.update", null), - out("permission.list", { sessionID: SESSION }), - replyData("permission.list", []), - out("session.form.list", { sessionID: SESSION }), - replyData("session.form.list", []), - event("session.execution.succeeded", { sessionID: SESSION }), - ]); - const subagentIds = new Set(); - const results: Array = []; - let finalSubagent: - | Extract< - ProviderAdapter.ProviderAdapterV2Event, - { type: "subagent.updated" } - >["subagent"] - | undefined; - const done = yield* Deferred.make(); - yield* runtime.events.pipe( - Stream.tap((e) => - Effect.gen(function* () { - if (e.type === "subagent.updated") { - subagentIds.add(e.subagent.id); - finalSubagent = e.subagent; - if (e.subagent.status === "completed") results.push(e.subagent.result); - } - if (e.type === "turn.terminal") yield* Deferred.succeed(done, undefined); - }), - ), - Stream.runDrain, - Effect.forkScoped, - ); - yield* runtime.startTurn(withLineage(thread)); - yield* Deferred.await(done); - assert.equal(subagentIds.size, 1); - assert.deepEqual(results, ["FIRST_OK", "SECOND_GAP_OK"]); - assert.equal(finalSubagent?.title, "Sleep"); - assert.equal(finalSubagent?.result, "SECOND_GAP_OK"); - }), - ); - - it.effect( - "recovers subagent prompt and title from history backfill when disconnected after input.started", - () => - Effect.gen(function* () { - const { runtime, thread } = yield* resumed([ - out("session.prompt", { sessionID: SESSION, text: "" }), - promptAccepted, - event("session.execution.started", { sessionID: SESSION }), - event("session.tool.input.started", { - sessionID: SESSION, - assistantMessageID: "msg_assistant", - id: "call-partial", - name: "subagent", - }), - { type: "runtime_exit", status: "success" }, - out("event.subscribe"), - out("session.active"), - replyData("session.active", { [SESSION]: { type: "running" } }), - out("message.list", { sessionID: SESSION, order: "desc", limit: "50" }), - reply("message.list", { - data: [ - { - id: "msg_assistant", - time: { created: 2, streamed: 2, completed: 2 }, - type: "assistant", - agent: "build", - model: { id: "big-pickle", providerID: "opencode", variant: "default" }, - content: [ - { - type: "tool", - id: "call-partial", - name: "subagent", - executed: true, - state: { - status: "completed", - input: { description: "Inspect logs", prompt: "check stderr" }, - content: [{ type: "text", text: "LOGS_OK" }], - metadata: { sessionID: CHILD, status: "completed" }, - }, - time: { created: 2, ran: 2, completed: 2 }, - }, - ], - finish: "stop", - }, - { - id: `msg_t3_turn_${SESSION}:attempt:opencode2-adapter`, - time: { created: 1 }, - type: "user", - text: "hi", - }, - ], - cursor: {}, - }), - out("session.update", { sessionID: CHILD, permissions: "" }), - reply("session.update", null), - out("permission.list", { sessionID: SESSION }), - replyData("permission.list", []), - out("session.form.list", { sessionID: SESSION }), - replyData("session.form.list", []), - event("session.execution.succeeded", { sessionID: SESSION }), - ]); - let finalSubagent: - | Extract< - ProviderAdapter.ProviderAdapterV2Event, - { type: "subagent.updated" } - >["subagent"] - | undefined; - let childTitle: string | undefined; - const done = yield* Deferred.make(); - yield* runtime.events.pipe( - Stream.tap((e) => - Effect.gen(function* () { - if (e.type === "app_thread.created") childTitle = e.appThread.title; - if (e.type === "subagent.updated") finalSubagent = e.subagent; - if (e.type === "turn.terminal") yield* Deferred.succeed(done, undefined); - }), - ), - Stream.runDrain, - Effect.forkScoped, - ); - yield* runtime.startTurn(withLineage(thread)); - yield* Deferred.await(done); - assert.equal(childTitle, "Inspect logs"); - assert.equal(finalSubagent?.title, "Inspect logs"); - assert.equal(finalSubagent?.prompt, "check stderr"); - assert.equal(finalSubagent?.result, "LOGS_OK"); - }), - ); - const MIDDLE = "ses_middle0000000000000000000"; const DEEP = "ses_grandchild00000000000000"; const toolOf = (session: string, id: string) => ({ diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index 3534afcf09e4d..f1443feee88a7 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -5242,55 +5242,58 @@ it.effect("drops a child's held text once the child is handed to another run", ( }), ); -it.effect( - "terminalizes a cold-resumed legacy subagent in place when the resuming run stream fails", - () => { - const layerDb = SqlitePersistence.layerMemory; - const layerStores = Layer.merge(EventStore.layer, ProjectionStore.layer).pipe( - Layer.provide(layerDb), - ); - const layerSink = EventSink.layer.pipe(Layer.provide(Layer.mergeAll(layerStores, layerDb))); - const layerInstances = Layer.mock(ProviderInstanceRegistry.ProviderInstanceRegistry)({ - getInstance: () => Effect.undefined, - }); - const layerGit = Layer.mock(GitVcsDriver.GitVcsDriver)({}); - const layerIngestor = ProviderEventIngestor.layer.pipe( - Layer.provide( - Layer.mergeAll( - layerInstances, - layerStores, - layerSink, - IdAllocator.layer, - ThreadCommandExecutor.layer, - layerGit, - ), +// Real SQLite projections and ingestion, so legacy subagent rows resolve the way they do in production. +const makeLegacySubagentLayer = () => { + const layerDb = SqlitePersistence.layerMemory; + const layerStores = Layer.merge(EventStore.layer, ProjectionStore.layer).pipe( + Layer.provide(layerDb), + ); + const layerSink = EventSink.layer.pipe(Layer.provide(Layer.mergeAll(layerStores, layerDb))); + const layerInstances = Layer.mock(ProviderInstanceRegistry.ProviderInstanceRegistry)({ + getInstance: () => Effect.undefined, + }); + const layerGit = Layer.mock(GitVcsDriver.GitVcsDriver)({}); + const layerIngestor = ProviderEventIngestor.layer.pipe( + Layer.provide( + Layer.mergeAll( + layerInstances, + layerStores, + layerSink, + IdAllocator.layer, + ThreadCommandExecutor.layer, + layerGit, ), - ); - const layerRunExecution = RunExecutionService.layer.pipe( - Layer.provide( - Layer.mergeAll( - McpAppModelContext.layerEmpty, - PreviewAgentActivity.layer, - Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }), - layerTurnItemPositionsTest, - layerSink, - IdAllocator.layer, - ThreadCommandExecutor.layer, - layerIngestor, - ServerSettings.layerTest(), - ), + ), + ); + const layerRunExecution = RunExecutionService.layer.pipe( + Layer.provide( + Layer.mergeAll( + McpAppModelContext.layerEmpty, + PreviewAgentActivity.layer, + Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }), + layerTurnItemPositionsTest, + layerSink, + IdAllocator.layer, + ThreadCommandExecutor.layer, + layerIngestor, + ServerSettings.layerTest(), ), - ); - const layerFull = Layer.mergeAll( - layerDb, - layerStores, - layerSink, - IdAllocator.layer, - ThreadCommandExecutor.layer, - layerIngestor, - layerRunExecution, - ); + ), + ); + return Layer.mergeAll( + layerDb, + layerStores, + layerSink, + IdAllocator.layer, + ThreadCommandExecutor.layer, + layerIngestor, + layerRunExecution, + ); +}; +it.effect( + "terminalizes a cold-resumed legacy subagent in place when the resuming run stream fails", + () => { return Effect.gen(function* () { const now = yield* DateTime.now; const t2 = DateTime.add(now, { seconds: 5 }); @@ -5850,58 +5853,13 @@ it.effect( assert.equal(subagentItems[0]?.title, "Sleep"); assert.equal(subagentItems[0]?.prompt, "sleep 180"); assert.equal(childProjection.providerThreads[0]?.ownerNodeId, legacyNodeId); - }).pipe(Effect.provide(layerFull)); + }).pipe(Effect.provide(makeLegacySubagentLayer())); }, ); it.effect( "detaches a cold-resumed legacy subagent from the owning run on cross-run handoff", () => { - const layerDb = SqlitePersistence.layerMemory; - const layerStores = Layer.merge(EventStore.layer, ProjectionStore.layer).pipe( - Layer.provide(layerDb), - ); - const layerSink = EventSink.layer.pipe(Layer.provide(Layer.mergeAll(layerStores, layerDb))); - const layerInstances = Layer.mock(ProviderInstanceRegistry.ProviderInstanceRegistry)({ - getInstance: () => Effect.undefined, - }); - const layerGit = Layer.mock(GitVcsDriver.GitVcsDriver)({}); - const layerIngestor = ProviderEventIngestor.layer.pipe( - Layer.provide( - Layer.mergeAll( - layerInstances, - layerStores, - layerSink, - IdAllocator.layer, - ThreadCommandExecutor.layer, - layerGit, - ), - ), - ); - const layerRunExecution = RunExecutionService.layer.pipe( - Layer.provide( - Layer.mergeAll( - McpAppModelContext.layerEmpty, - PreviewAgentActivity.layer, - Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }), - layerTurnItemPositionsTest, - layerSink, - IdAllocator.layer, - ThreadCommandExecutor.layer, - layerIngestor, - ServerSettings.layerTest(), - ), - ), - ); - const layerFull = Layer.mergeAll( - layerDb, - layerStores, - layerSink, - IdAllocator.layer, - ThreadCommandExecutor.layer, - layerIngestor, - layerRunExecution, - ); return Effect.gen(function* () { const key = "legacy-cross-run-handoff"; const ids = backgroundScenarioIds(key); @@ -6395,6 +6353,6 @@ it.effect( const parentProjection = yield* projectionStore.getThreadProjection(ids.threadId); assert.equal(parentProjection.subagents[0]?.id, legacyNodeId); assert.equal(parentProjection.subagents[0]?.status, "running"); - }).pipe(Effect.provide(layerFull)); + }).pipe(Effect.provide(makeLegacySubagentLayer())); }, ); From 0c17cb2bbb97b709b343233922164db1543cf0c8 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sat, 21 Nov 2026 14:55:56 +0000 Subject: [PATCH 3/6] fix(opencode): stream children found during reconnect and bound steer history reads A child that launched and started while the parent was disconnected is attached from the parent's history with no replayed start, so its output was dropped. Reconcile now starts a turn for such a child while OpenCode still lists it as active. A steer-anchored history read now stops after one page, because an unread steer is not in history yet. Co-Authored-By: Claude Opus 5.5 --- .../Adapters/OpenCode2AdapterV2.test.ts | 94 +++++++++++++++++++ .../src/server/v2/adapter.ts | 14 ++- 2 files changed, 107 insertions(+), 1 deletion(-) diff --git a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts index 8be8130c6b0b3..48e373c75393d 100644 --- a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts @@ -2854,6 +2854,100 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { }), ); + it.effect("streams a child that launched and started while the parent was disconnected", () => + Effect.gen(function* () { + const { runtime, thread } = yield* resumed([ + out("session.prompt", { sessionID: SESSION, text: "" }), + promptAccepted, + event("session.execution.started", { sessionID: SESSION }), + { type: "runtime_exit", status: "success" }, + out("event.subscribe"), + out("session.active"), + replyData("session.active", { + [SESSION]: { type: "running" }, + [CHILD]: { type: "running" }, + }), + out("message.list", { sessionID: SESSION, order: "desc", limit: "50" }), + reply("message.list", { + data: [ + { + id: "msg_parent_gap", + time: { created: 2, streamed: 2, completed: 2 }, + type: "assistant", + agent: "build", + model: { id: "big-pickle", providerID: "opencode", variant: "default" }, + content: [ + { + type: "tool", + id: "call-background", + name: "subagent", + executed: true, + state: { + status: "completed", + input: { description: "Sleep", prompt: "sleep", background: true }, + content: [{ type: "text", text: "The subagent is working in the background." }], + metadata: { sessionID: CHILD, status: "running" }, + }, + time: { created: 2, ran: 2, completed: 2 }, + }, + ], + finish: "stop", + }, + { + id: `msg_t3_turn_${SESSION}:attempt:opencode2-adapter`, + time: { created: 1 }, + type: "user", + text: "hi", + }, + ], + cursor: {}, + }), + out("session.update", { sessionID: CHILD, permissions: "" }), + reply("session.update", null), + out("permission.list", { sessionID: SESSION }), + replyData("permission.list", []), + out("session.form.list", { sessionID: SESSION }), + replyData("session.form.list", []), + out("permission.list", { sessionID: CHILD }), + replyData("permission.list", []), + out("session.form.list", { sessionID: CHILD }), + replyData("session.form.list", []), + event("session.text.ended", { + sessionID: CHILD, + assistantMessageID: "msg_child_gap", + ordinal: 0, + text: "GAP_OK", + }), + event("session.execution.succeeded", { sessionID: CHILD }), + event("session.execution.succeeded", { sessionID: SESSION }), + childReport("msg_report", "GAP_OK"), + ]); + const childTexts: Array = []; + const done = yield* Deferred.make(); + yield* runtime.events.pipe( + Stream.tap((e) => + Effect.gen(function* () { + if ( + e.type === "turn_item.updated" && + e.turnItem.type === "assistant_message" && + e.turnItem.threadId !== threadId + ) { + childTexts.push(e.turnItem.text); + } + if (e.type === "subagent.updated" && e.subagent.status === "completed") { + yield* Deferred.succeed(done, undefined); + } + }), + ), + Stream.runDrain, + Effect.forkScoped, + ); + yield* runtime.startTurn(withLineage(thread)); + yield* Deferred.await(done); + assert.include(childTexts, "GAP_OK"); + }), + ); + const MIDDLE = "ses_middle0000000000000000000"; const DEEP = "ses_grandchild00000000000000"; const toolOf = (session: string, id: string) => ({ diff --git a/packages/provider-opencode/src/server/v2/adapter.ts b/packages/provider-opencode/src/server/v2/adapter.ts index d30ade489ea2e..3e8a0e1352296 100644 --- a/packages/provider-opencode/src/server/v2/adapter.ts +++ b/packages/provider-opencode/src/server/v2/adapter.ts @@ -3889,13 +3889,16 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr } const start = before !== undefined ? before : promptId; const oldestSteer = pendingSteers?.values().next().value; - const read = yield* OpenCode2Client.paginate( + const history = OpenCode2Client.paginate( { sessionID: Session.ID.make(sessionId), order: "desc" as const, limit: 50 }, client.message.list, ).pipe( Stream.takeUntil((message) => start !== undefined ? message.id === start : message.id === oldestSteer, ), + ); + // An unread steer is not in history yet, so a steer anchor reads one page at most. + const read = yield* (start === undefined ? history.pipe(Stream.take(50)) : history).pipe( Stream.runCollect, Effect.catchTags({ SessionNotFoundError: () => Effect.succeed(null) }), ); @@ -4098,6 +4101,9 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr return; } const active = { ...(yield* client.session.active()) }; + const attachedBefore = new Set( + [...threads.values()].filter((state) => state.subagent !== undefined), + ); for (const sessionId of busy) if (!(sessionId in active)) busy.delete(sessionId); for (const state of threads.values()) { if (state.unsettled && !(state.sessionId in active)) state.unsettledStarted = true; @@ -4139,6 +4145,12 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr }), ); } + // A child the parent's backfill found running started in the gap; its start is not replayed. + for (const state of threads.values()) { + if (state.subagent === undefined || state.active !== undefined) continue; + if (attachedBefore.has(state) || !(state.sessionId in active)) continue; + yield* lock.withPermit(startChildTurn(state)); + } for (const state of threads.values()) { yield* lock .withPermit(recoverImages(state).pipe(Effect.timeout(INVENTORY_TIMEOUT))) From 8235863b8678caa2d744330e3a17807c62c8b37e Mon Sep 17 00:00:00 2001 From: Victor Vogel Date: Sat, 21 Nov 2026 15:56:11 +0000 Subject: [PATCH 4/6] fix(opencode): address review findings on subagent dedupe and steer replay --- .../Adapters/OpenCode2AdapterV2.test.ts | 544 +++++++++++++++--- .../Adapters/OpenCode2AdapterV2.testkit.ts | 8 + .../ProviderEventIngestor.test.ts | 41 +- .../orchestration-v2/ProviderEventIngestor.ts | 32 +- .../RunExecutionService.test.ts | 306 +++++++--- .../orchestration-v2/RunExecutionService.ts | 82 ++- .../src/server/v2/adapter.ts | 127 +++- 7 files changed, 909 insertions(+), 231 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts index 48e373c75393d..2c4c43ecdba8d 100644 --- a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts @@ -1755,7 +1755,7 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { inboxID: string, text: string, description = "Sleep", - state: "completed" | "failed" | "cancelled" = "completed", + state: "completed" | "error" | "cancelled" = "completed", ) => event("session.inbox.enqueued", { inboxID, @@ -2300,6 +2300,7 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { background: true, description: `Redirect #${index}`, prompt: `Redirect prompt #${index}`, + ...(index === 1 ? { model: "opencode/other-model" } : {}), }, { id: `call-steer-${index}`, writesRules: index === 1 }, ), @@ -2657,7 +2658,12 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { promptAccepted, event("session.execution.started", { sessionID: SESSION }), ...callAgain( - { background: true, description: "Resume after restart", prompt: "keep going" }, + { + background: true, + description: "Resume after restart", + prompt: "keep going", + model: "opencode/redirect-hint", + }, { id: "call-restart-resume" }, ), event("session.execution.started", { sessionID: CHILD }), @@ -2705,10 +2711,10 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { } if (e.type === "subagent.updated") { subagentEvents.push(e); - if (e.subagent.status === "completed") yield* Deferred.succeed(done, undefined); } if (e.type === "turn_item.updated" && e.turnItem.type === "subagent") { subagentItemEvents.push(e); + if (e.turnItem.status === "completed") yield* Deferred.succeed(done, undefined); } if (e.type === "provider_thread.updated" && e.providerThread.id === thread.id) { for (const task of e.providerThread.pendingBackgroundTasks ?? []) { @@ -2731,6 +2737,9 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { yield* Deferred.await(done); assert.lengthOf(creates, 1); assert.isTrue(creates[0]?.createOnly); + assert.isNotEmpty(subagentNodeEvents); + assert.isNotEmpty(subagentEvents); + assert.isNotEmpty(subagentItemEvents); assert.isTrue(subagentNodeEvents.every((e) => e.preserveIdentity === true)); assert.isTrue(subagentEvents.every((e) => e.preserveIdentity === true)); assert.isTrue(subagentItemEvents.every((e) => e.preserveIdentity === true)); @@ -2854,98 +2863,455 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { }), ); - it.effect("streams a child that launched and started while the parent was disconnected", () => - Effect.gen(function* () { - const { runtime, thread } = yield* resumed([ - out("session.prompt", { sessionID: SESSION, text: "" }), - promptAccepted, - event("session.execution.started", { sessionID: SESSION }), - { type: "runtime_exit", status: "success" }, - out("event.subscribe"), - out("session.active"), - replyData("session.active", { - [SESSION]: { type: "running" }, - [CHILD]: { type: "running" }, - }), - out("message.list", { sessionID: SESSION, order: "desc", limit: "50" }), - reply("message.list", { - data: [ - { - id: "msg_parent_gap", - time: { created: 2, streamed: 2, completed: 2 }, - type: "assistant", - agent: "build", - model: { id: "big-pickle", providerID: "opencode", variant: "default" }, - content: [ - { - type: "tool", - id: "call-background", + it.effect.each(["launched", "redirected"] as const)( + "streams a child that %s and started while the parent was disconnected", + (mode) => + Effect.gen(function* () { + const { runtime, thread } = yield* resumed([ + out("session.prompt", { sessionID: SESSION, text: "" }), + promptAccepted, + event("session.execution.started", { sessionID: SESSION }), + ...(mode === "redirected" + ? [ + event("session.created", { + session: { id: CHILD, title: "Sleep", parentID: SESSION }, + }), + event("session.tool.started", { + sessionID: SESSION, + assistantMessageID: "msg_parent_initial", + toolID: "call-initial", + name: "subagent", + input: { description: "Sleep", prompt: "first", background: true }, + }), + event("session.tool.progress", { + sessionID: SESSION, + assistantMessageID: "msg_parent_initial", + toolID: "call-initial", + name: "subagent", + metadata: { sessionID: CHILD, status: "running" }, + }), + event("session.execution.started", { sessionID: CHILD }), + event("session.execution.succeeded", { sessionID: CHILD }), + event("session.tool.completed", { + sessionID: SESSION, + assistantMessageID: "msg_parent_initial", + toolID: "call-initial", name: "subagent", - executed: true, - state: { - status: "completed", - input: { description: "Sleep", prompt: "sleep", background: true }, - content: [{ type: "text", text: "The subagent is working in the background." }], - metadata: { sessionID: CHILD, status: "running" }, + output: "The subagent is working in the background.", + metadata: { sessionID: CHILD, status: "running" }, + }), + childReport("msg_report_initial", "FIRST_OK"), + ] + : []), + { type: "runtime_exit", status: "success" }, + out("event.subscribe"), + out("session.active"), + replyData("session.active", { + [SESSION]: { type: "running" }, + [CHILD]: { type: "running" }, + }), + out("message.list", { sessionID: SESSION, order: "desc", limit: "50" }), + reply("message.list", { + data: [ + { + id: "msg_parent_gap", + time: { created: 2, streamed: 2, completed: 2 }, + type: "assistant", + agent: "build", + model: { id: "big-pickle", providerID: "opencode", variant: "default" }, + content: [ + { + type: "tool", + id: mode === "redirected" ? "call-redirect" : "call-background", + name: "subagent", + executed: true, + state: { + status: "completed", + input: + mode === "redirected" + ? { + sessionID: CHILD, + description: "Redirect", + prompt: "second", + background: true, + } + : { description: "Sleep", prompt: "sleep", background: true }, + content: [ + { type: "text", text: "The subagent is working in the background." }, + ], + metadata: { sessionID: CHILD, status: "running" }, + }, + time: { created: 2, ran: 2, completed: 2 }, }, - time: { created: 2, ran: 2, completed: 2 }, - }, - ], - finish: "stop", + ], + finish: "stop", + }, + { + id: `msg_t3_turn_${SESSION}:attempt:opencode2-adapter`, + time: { created: 1 }, + type: "user", + text: "hi", + }, + ], + cursor: {}, + }), + out("session.update", { sessionID: CHILD, permissions: "" }), + reply("session.update", null), + out("permission.list", { sessionID: SESSION }), + replyData("permission.list", []), + out("session.form.list", { sessionID: SESSION }), + replyData("session.form.list", []), + out("permission.list", { sessionID: CHILD }), + replyData("permission.list", []), + out("session.form.list", { sessionID: CHILD }), + replyData("session.form.list", []), + event("session.text.ended", { + sessionID: CHILD, + assistantMessageID: "msg_child_gap", + ordinal: 0, + text: "GAP_OK", + }), + event("session.execution.succeeded", { sessionID: CHILD }), + event("session.execution.succeeded", { sessionID: SESSION }), + childReport("msg_report", "GAP_OK"), + ]); + const childTexts: Array = []; + const done = yield* Deferred.make(); + yield* runtime.events.pipe( + Stream.tap((e) => + Effect.gen(function* () { + if ( + e.type === "turn_item.updated" && + e.turnItem.type === "assistant_message" && + e.turnItem.threadId !== threadId + ) { + childTexts.push(e.turnItem.text); + } + if ( + e.type === "subagent.updated" && + e.subagent.status === "completed" && + e.subagent.result === "GAP_OK" + ) { + yield* Deferred.succeed(done, undefined); + } + }), + ), + Stream.runDrain, + Effect.forkScoped, + ); + yield* runtime.startTurn(withLineage(thread)); + yield* Deferred.await(done); + assert.include(childTexts, "GAP_OK"); + }), + ); + + it.effect( + "discards a failed redirect's prompt override before the next redirect enqueues and skips launch prompt in steer backfill", + () => + Effect.gen(function* () { + const failedTool = { + sessionID: SESSION, + assistantMessageID: "msg_assistant", + id: "call-failed-redirect", + }; + const { runtime, thread } = yield* resumed([ + ...launchedAndRunning, + event("session.execution.succeeded", { sessionID: CHILD }), + // Child is idle while call-background still awaits its report; redirect fails before enqueueing. + event("session.tool.input.started", { ...failedTool, name: "subagent" }), + event("session.tool.called", { + ...failedTool, + name: "subagent", + input: { + description: "Bad redirect", + prompt: "stale failed prompt", + background: true, + sessionID: CHILD, }, - { - id: `msg_t3_turn_${SESSION}:attempt:opencode2-adapter`, - time: { created: 1 }, + executed: false, + }), + event("session.tool.progress", { + ...failedTool, + metadata: { sessionID: CHILD, status: "running" }, + }), + out("session.update", { sessionID: CHILD, permissions: "" }), + reply("session.update", null), + event("session.tool.failed", { + ...failedTool, + error: { type: "other", message: "boom" }, + executed: true, + }), + // A synthetic report enters the idle child's inbox before the real next redirect's user prompt. + event("session.inbox.enqueued", { + inboxID: "msg_child_prior_report", + sessionID: CHILD, + item: { + type: "synthetic", + payload: { text: "prior nested note" }, + delivery: "steer", + }, + }), + ...callAgain( + { background: true, description: "Good redirect", prompt: "real next prompt" }, + { id: "call-good-redirect", writesRules: false }, + ), + event("session.inbox.enqueued", { + inboxID: "msg_real_launch", + sessionID: CHILD, + item: { type: "user", - text: "hi", + payload: { text: "You are a subagent.\nreal next prompt", files: [] }, + delivery: "steer", }, - ], - cursor: {}, - }), - out("session.update", { sessionID: CHILD, permissions: "" }), - reply("session.update", null), - out("permission.list", { sessionID: SESSION }), - replyData("permission.list", []), - out("session.form.list", { sessionID: SESSION }), - replyData("session.form.list", []), - out("permission.list", { sessionID: CHILD }), - replyData("permission.list", []), - out("session.form.list", { sessionID: CHILD }), - replyData("session.form.list", []), - event("session.text.ended", { - sessionID: CHILD, - assistantMessageID: "msg_child_gap", - ordinal: 0, - text: "GAP_OK", - }), - event("session.execution.succeeded", { sessionID: CHILD }), - event("session.execution.succeeded", { sessionID: SESSION }), - childReport("msg_report", "GAP_OK"), - ]); - const childTexts: Array = []; - const done = yield* Deferred.make(); - yield* runtime.events.pipe( - Stream.tap((e) => - Effect.gen(function* () { - if ( - e.type === "turn_item.updated" && - e.turnItem.type === "assistant_message" && - e.turnItem.threadId !== threadId - ) { - childTexts.push(e.turnItem.text); - } - if (e.type === "subagent.updated" && e.subagent.status === "completed") { - yield* Deferred.succeed(done, undefined); - } }), - ), - Stream.runDrain, - Effect.forkScoped, - ); - yield* runtime.startTurn(withLineage(thread)); - yield* Deferred.await(done); - assert.include(childTexts, "GAP_OK"); - }), + event("session.execution.started", { sessionID: CHILD }), + event("session.inbox.delivered", { sessionID: CHILD, inboxID: "msg_real_launch" }), + // Disconnect and reconnect while the child is running; backfill excludes the prior report. + { type: "runtime_exit", status: "success" }, + out("event.subscribe"), + out("session.active"), + replyData("session.active", { + [SESSION]: { type: "running" }, + [CHILD]: { type: "running" }, + }), + out("message.list", { sessionID: CHILD, order: "desc", limit: "50" }), + reply("message.list", { + data: [ + { + id: "msg_mid_steer", + time: { created: 4 }, + type: "user", + text: "genuine mid-turn steer", + }, + { + id: "msg_real_launch", + time: { created: 3 }, + type: "user", + text: "You are a subagent.\nreal next prompt", + }, + { + id: "msg_child_prior_report", + time: { created: 2 }, + type: "synthetic", + text: "prior nested note", + }, + ], + cursor: {}, + }), + out("message.list", { sessionID: SESSION, order: "desc", limit: "50" }), + reply("message.list", { + data: [ + { + id: `msg_t3_turn_${SESSION}:attempt:opencode2-adapter`, + time: { created: 1 }, + type: "user", + text: "hi", + }, + ], + cursor: {}, + }), + out("permission.list", { sessionID: SESSION }), + replyData("permission.list", []), + out("session.form.list", { sessionID: SESSION }), + replyData("session.form.list", []), + out("permission.list", { sessionID: CHILD }), + replyData("permission.list", []), + out("session.form.list", { sessionID: CHILD }), + replyData("session.form.list", []), + event("session.execution.succeeded", { sessionID: CHILD }), + event("session.execution.succeeded", { sessionID: SESSION }), + childReport("msg_report_1", "FIRST_OK"), + childReport("msg_report_2", "SECOND_OK"), + ]); + const childUserPrompts: Array<{ readonly text: string; readonly inputIntent: string }> = []; + let childThreadId: string | undefined; + const done = yield* Deferred.make(); + yield* runtime.events.pipe( + Stream.tap((e) => + Effect.gen(function* () { + if (e.type === "app_thread.created") childThreadId = e.appThread.id; + if ( + e.type === "turn_item.updated" && + e.turnItem.type === "user_message" && + e.turnItem.threadId === childThreadId + ) { + childUserPrompts.push({ + text: e.turnItem.text, + inputIntent: e.turnItem.inputIntent, + }); + } + if (e.type === "subagent.updated" && e.subagent.status === "completed") { + yield* Deferred.succeed(done, undefined); + } + }), + ), + Stream.runDrain, + Effect.forkScoped, + ); + yield* runtime.startTurn(withLineage(thread)); + yield* Deferred.await(done); + assert.deepEqual(childUserPrompts, [ + { text: "sleep", inputIntent: "turn_start" }, + { text: "real next prompt", inputIntent: "turn_start" }, + { text: "genuine mid-turn steer", inputIntent: "steer" }, + ]); + }), + ); + + it.effect.each(["unread", "delivered-beyond-page"] as const)( + "bounds unread steer backfill and recovers a delivered steer older than 50 records (%s)", + (mode) => + Effect.gen(function* () { + const steerId = "msg_steer_anchor"; + const { runtime, thread } = yield* resumed([ + ...launchedAndRunning, + event("session.inbox.enqueued", { + inboxID: steerId, + sessionID: CHILD, + item: { + type: "user", + payload: { text: "late steer", files: [] }, + delivery: "steer", + }, + }), + { type: "runtime_exit", status: "success" }, + out("event.subscribe"), + out("session.active"), + replyData( + "session.active", + mode === "unread" + ? { [SESSION]: { type: "running" }, [CHILD]: { type: "running" } } + : { [SESSION]: { type: "running" } }, + ), + out("message.list", { sessionID: CHILD, order: "desc", limit: "50" }), + reply("message.list", { + data: + mode === "unread" + ? [{ id: "msg_older_1", time: { created: 1 }, type: "user", text: "old" }] + : [ + { + id: "msg_child_idle", + time: { created: 60 }, + type: "idle", + outcome: "succeeded", + }, + { + id: "msg_child_done", + time: { created: 59, streamed: 59, completed: 59 }, + type: "assistant", + agent: "general", + model: { id: "big-pickle", providerID: "opencode", variant: "default" }, + content: [{ type: "text", text: "RECOVERED_BEYOND_50" }], + finish: "stop", + }, + ], + cursor: { next: "cur_page_2" }, + }), + out("session.inbox.list", { sessionID: CHILD }), + replyData( + "session.inbox.list", + mode === "unread" + ? [ + { + id: steerId, + sessionID: CHILD, + time: { created: 2 }, + type: "user", + payload: { text: "late steer" }, + delivery: "steer", + }, + ] + : [], + ), + ...(mode === "delivered-beyond-page" + ? [ + out("session.message.get", { sessionID: CHILD, messageID: steerId }), + replyData("session.message.get", { + id: steerId, + time: { created: 5 }, + type: "user", + text: "late steer", + }), + out("message.list", { sessionID: CHILD, limit: "50", cursor: "cur_page_2" }), + reply("message.list", { + data: [{ id: steerId, time: { created: 5 }, type: "user", text: "late steer" }], + cursor: {}, + }), + ] + : []), + out("message.list", { sessionID: SESSION, order: "desc", limit: "50" }), + reply("message.list", { + data: [ + { + id: `msg_t3_turn_${SESSION}:attempt:opencode2-adapter`, + time: { created: 1 }, + type: "user", + text: "hi", + }, + ], + cursor: {}, + }), + out("permission.list", { sessionID: SESSION }), + replyData("permission.list", []), + out("session.form.list", { sessionID: SESSION }), + replyData("session.form.list", []), + ...(mode === "unread" + ? [ + out("permission.list", { sessionID: CHILD }), + replyData("permission.list", []), + out("session.form.list", { sessionID: CHILD }), + replyData("session.form.list", []), + event("session.inbox.delivered", { sessionID: CHILD, inboxID: steerId }), + childSaid("UNREAD_THEN_OK"), + event("session.execution.succeeded", { sessionID: CHILD }), + ] + : []), + event("session.execution.succeeded", { sessionID: SESSION }), + ...(mode === "unread" ? [childReport("msg_report", "UNREAD_THEN_OK")] : []), + ]); + const childTexts: Array = []; + const childSteers: Array = []; + let childTurnStatus: string | undefined; + let childThreadId: string | undefined; + const done = yield* Deferred.make(); + yield* runtime.events.pipe( + Stream.tap((e) => + Effect.gen(function* () { + if (e.type === "app_thread.created") childThreadId = e.appThread.id; + if ( + e.type === "provider_turn.updated" && + e.threadId === childThreadId && + e.providerTurn.completedAt !== null + ) { + childTurnStatus = e.providerTurn.status; + } + if (e.type === "turn_item.updated" && e.turnItem.threadId === childThreadId) { + if (e.turnItem.type === "assistant_message") childTexts.push(e.turnItem.text); + if (e.turnItem.type === "user_message" && e.turnItem.inputIntent === "steer") { + childSteers.push(e.turnItem.text); + } + } + if ( + mode === "unread" && + e.type === "subagent.updated" && + e.subagent.status === "completed" + ) { + yield* Deferred.succeed(done, undefined); + } + if (mode === "delivered-beyond-page" && e.type === "turn.terminal") { + yield* Deferred.succeed(done, undefined); + } + }), + ), + Stream.runDrain, + Effect.forkScoped, + ); + yield* runtime.startTurn(withLineage(thread)); + yield* Deferred.await(done); + assert.equal(childTurnStatus, "completed"); + assert.deepEqual(childSteers, ["late steer"]); + assert.include(childTexts, mode === "unread" ? "UNREAD_THEN_OK" : "RECOVERED_BEYOND_50"); + }), ); const MIDDLE = "ses_middle0000000000000000000"; diff --git a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.testkit.ts b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.testkit.ts index 000176e334696..ef3ac447e507d 100644 --- a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.testkit.ts +++ b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.testkit.ts @@ -124,12 +124,20 @@ const operationOf = ( return { type: "session.revert.commit", input }; } if (method === "DELETE" && rest === "/revert") return { type: "session.revert.clear", input }; + if (method === "GET" && rest === "/inbox") return { type: "session.inbox.list", input }; const inbox = /^\/inbox\/([^/]+)$/.exec(rest); if (method === "DELETE" && inbox !== null) { const inboxID = decodeURIComponent(inbox[1] ?? ""); return { type: "session.inbox.cancel", input: { ...input, inboxID } }; } if (method === "GET" && rest === "/message") return { type: "message.list", input }; + const message = /^\/message\/([^/]+)$/.exec(rest); + if (method === "GET" && message !== null) { + return { + type: "session.message.get", + input: { ...input, messageID: decodeURIComponent(message[1] ?? "") }, + }; + } if (method === "GET" && rest === "/permission") return { type: "permission.list", input }; if (method === "GET" && rest === "/form") return { type: "session.form.list", input }; const form = /^\/form\/([^/]+)(\/reply)?$/.exec(rest); diff --git a/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts b/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts index a23748a69c21d..83c96c015df67 100644 --- a/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts +++ b/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts @@ -2451,6 +2451,32 @@ layer("ProviderEventIngestorV2", (it) => { (yield* projectionStore.getThread(childThreadId)).modelSelection, reportedSelection, ); + + // A persistent model-sync failure must still return the committed subagent update. + const failingSyncSink = EventSink.EventSinkV2.of({ + ...eventSink, + write: (writeInput) => + writeInput.events.some((event) => event.type === "thread.model-selection-updated") + ? Effect.fail(constraintFailure) + : eventSink.write(writeInput), + }); + const failingSyncIngestor = yield* ProviderEventIngestor.ProviderEventIngestorV2.pipe( + Effect.provide(Layer.fresh(ProviderEventIngestor.layer)), + Effect.provideService(EventSink.EventSinkV2, failingSyncSink), + ); + const committedDespiteSyncFailure = yield* failingSyncIngestor.ingestNormalized({ + providerSessionId, + providerInstanceId: modelSelection.instanceId, + threadId: rootEvent.threadId, + event: { + ...subagentUpdated, + subagent: { ...subagentUpdated.subagent, model: "gpt-6.2-sol", status: "completed" }, + }, + }); + assert.deepEqual( + committedDespiteSyncFailure.map((stored) => stored.event.type), + ["subagent.updated"], + ); }).pipe( // Fresh: the shared test layer has already built the ingestor this replaces. Effect.provide( @@ -2606,6 +2632,11 @@ layer("ProviderEventIngestorV2", (it) => { prompt: "sleep 180", title: "Sleep", model: "opencode/big-pickle", + modelSelection: { + instanceId, + model: "opencode/big-pickle", + options: [{ id: "variant", value: "high" }], + }, status: "completed", result: "FIRST_OK", startedAt: now, @@ -2841,7 +2872,10 @@ layer("ProviderEventIngestorV2", (it) => { nativeTaskRef: { driver, nativeId: childSessionId, strength: "strong" }, prompt: "keep going", title: "Resume after restart", - model: null, + model: phase === "running" ? "opencode/redirect-hint" : null, + ...(phase === "running" + ? { modelSelection: { instanceId, model: "opencode/redirect-hint" } } + : {}), status: phase, result: phase === "completed" ? "SECOND_OK" : null, startedAt: t3, @@ -3012,6 +3046,11 @@ layer("ProviderEventIngestorV2", (it) => { assert.equal(canonicalSubagent?.title, "Sleep"); assert.equal(canonicalSubagent?.prompt, "sleep 180"); assert.equal(canonicalSubagent?.model, "opencode/big-pickle"); + assert.deepEqual(canonicalSubagent?.modelSelection, { + instanceId, + model: "opencode/big-pickle", + options: [{ id: "variant", value: "high" }], + }); assert.equal(canonicalSubagent?.result, "SECOND_OK"); assert.equal(canonicalItem?.nodeId, initialNodeId); assert.equal(canonicalItem?.subagentId, initialNodeId); diff --git a/apps/server/src/orchestration-v2/ProviderEventIngestor.ts b/apps/server/src/orchestration-v2/ProviderEventIngestor.ts index f2859480a9b5a..a22c03d7e57ae 100644 --- a/apps/server/src/orchestration-v2/ProviderEventIngestor.ts +++ b/apps/server/src/orchestration-v2/ProviderEventIngestor.ts @@ -18,6 +18,7 @@ import { type RuntimeRequestId, ThreadId, } from "@t3tools/contracts"; +import * as Cause from "effect/Cause"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; @@ -608,18 +609,21 @@ export const layer: Layer.Layer< providerThreadId: subagent.providerThreadId, }); if (existing !== null) { + const { modelSelection: incomingSelection, ...rest } = subagent; + const hasStoredModel = + existing.subagent.model !== null || + existing.subagent.modelSelection !== undefined; + const model = hasStoredModel ? existing.subagent.model : subagent.model; + const modelSelection = hasStoredModel + ? existing.subagent.modelSelection + : incomingSelection; subagent = { - ...subagent, + ...rest, id: existing.subagent.id, title: existing.subagent.title ?? subagent.title, prompt: existing.subagent.prompt || subagent.prompt, - model: subagent.model ?? existing.subagent.model, - ...((subagent.modelSelection ?? existing.subagent.modelSelection) === undefined - ? {} - : { - modelSelection: - subagent.modelSelection ?? existing.subagent.modelSelection, - }), + model, + ...(modelSelection === undefined ? {} : { modelSelection }), }; } } @@ -938,14 +942,16 @@ export const layer: Layer.Layer< : input.event.subagent; return syncSubagentThreadModel(input, subagent).pipe( Effect.map((synced) => [...storedEvents, ...synced]), - Effect.mapError( + Effect.catchCauseIf( + (cause) => !Cause.hasInterruptsOnly(cause), (cause) => - new ProviderEventPublishError({ - providerSessionId: input.providerSessionId, - eventCount: 1, + Effect.logWarning("Native subagent thread model sync failed", { + threadId: subagent.childThreadId ?? input.threadId, + subagentId: subagent.id, cause, - }), + }).pipe(Effect.as(storedEvents)), ), + Effect.orDie, ); }), Effect.tap((storedEvents) => diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index f1443feee88a7..372585a03dfe8 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -3634,40 +3634,97 @@ it.effect.each([ { ended: false, raisedByT3: true }, // A Stop that already settled the subagent keeps its result. { ended: false, stopped: true }, + // An already-committed subagent completion survives an ingestion failure on a live root run. + { ended: false, rootLive: true, settledStatus: "completed" as const }, + // Moving child C to another run keeps nested subagent G owned by this run for lost-work cleanup. + { ended: true, rootLive: true, handedOffWithNested: true }, ] as const)( "ends a subagent that outlived its root when the provider stream fails (%o)", - ({ ended, raisedByT3, stopped }) => + ({ ended, raisedByT3, stopped, rootLive, settledStatus, handedOffWithNested }) => Effect.gen(function* () { const providerInstanceId = ProviderInstanceId.make("codex"); + const grandchildId = NodeId.make("node:stream-error-outlived-subagent:grandchild"); + const grandchildThreadId = ThreadId.make("thread:stream-error-outlived-subagent:grandchild"); + const grandchildItemId = TurnItemId.make( + "turn-item:stream-error-outlived-subagent:grandchild", + ); const { finalEvents } = yield* captureRootRunTermination({ key: "stream-error-outlived-subagent", shouldFinalizeRun: () => Effect.succeed(true), - events: (ids) => - Stream.make( - { - type: "subagent.updated" as const, - driver, - subagent: makeRunOwnedSubagentFixture({ - ids, - providerInstanceId, - childThreadId: ids.childThreadId, + events: (ids) => { + const baseSubagent = makeRunOwnedSubagentFixture({ + ids, + providerInstanceId, + childThreadId: ids.childThreadId, + driver, + status: "running", + }); + const baseTurnItem = makeRunOwnedSubagentTurnItemFixture({ + ids, + providerInstanceId, + childThreadId: ids.childThreadId, + driver, + status: "running", + }); + const events: Array = [ + { type: "subagent.updated", driver, subagent: baseSubagent }, + { type: "turn_item.updated", driver, turnItem: baseTurnItem }, + ]; + if (handedOffWithNested === true) { + events.push( + { + type: "node.updated", driver, - status: "running", - }), - }, - { - type: "turn_item.updated" as const, - driver, - turnItem: makeRunOwnedSubagentTurnItemFixture({ - ids, - providerInstanceId, - childThreadId: ids.childThreadId, + node: makeRunOwnedSubagentNodeFixture({ ids, status: "running" }), + }, + { + type: "subagent.updated", driver, - status: "running", - }), - }, - rootTerminalEvent(ids, "completed"), - ).pipe( + subagent: { + ...baseSubagent, + id: grandchildId, + threadId: ids.childThreadId, + runId: null, + parentNodeId: baseSubagent.id, + childThreadId: grandchildThreadId, + }, + }, + { + type: "node.updated", + driver, + node: { + ...makeRunOwnedSubagentNodeFixture({ ids, status: "running" }), + id: grandchildId, + threadId: ids.childThreadId, + runId: null, + parentNodeId: baseSubagent.id, + rootNodeId: baseSubagent.id, + }, + }, + { + type: "turn_item.updated", + driver, + turnItem: { + ...baseTurnItem, + id: grandchildItemId, + threadId: ids.childThreadId, + runId: null, + nodeId: grandchildId, + subagentId: grandchildId, + childThreadId: grandchildThreadId, + }, + }, + { + type: "subagent.updated", + driver, + subagent: { ...baseSubagent, runId: RunId.make("run:other-owner") }, + }, + ); + } + if (rootLive !== true) { + events.push(rootTerminalEvent(ids, "completed")); + } + return Stream.fromIterable(events).pipe( Stream.concat( Stream.fail( new ProviderAdapter.ProviderAdapterEventStreamError({ @@ -3680,28 +3737,61 @@ it.effect.each([ }), ), ), - ), - ...(stopped === true + ); + }, + ...(stopped === true || settledStatus !== undefined ? { - settleStored: (work) => ({ - ...work, - subagents: work.subagents.map((row) => ({ ...row, status: "interrupted" })), - turnItems: work.turnItems.map((row) => ({ ...row, status: "interrupted" })), - }), + settleStored: (work) => { + const status = settledStatus ?? "interrupted"; + return { + ...work, + subagents: work.subagents.map((row) => ({ ...row, status })), + turnItems: work.turnItems.map((row) => ({ ...row, status })), + }; + }, } : {}), }); + const expected = + handedOffWithNested === true + ? [ + "run.updated:failed", + `subagent.updated:failed:${grandchildId}`, + `node.updated:failed:${grandchildId}`, + `turn-item.updated:failed:${grandchildItemId}`, + ] + : rootLive === true + ? ["run.updated:failed"] + : ended + ? ["run.updated:waiting", "subagent.updated:failed", "turn-item.updated:failed"] + : ["run.updated:waiting"]; assert.deepEqual( - finalEvents.flatMap((event) => - event.type === "run.updated" || - event.type === "subagent.updated" || - event.type === "turn-item.updated" - ? [`${event.type}:${event.payload.status}`] - : [], - ), - ended - ? ["run.updated:waiting", "subagent.updated:failed", "turn-item.updated:failed"] - : ["run.updated:waiting"], + finalEvents.flatMap((event) => { + if (event.type === "run.updated") return [`${event.type}:${event.payload.status}`]; + if (event.type === "subagent.updated") { + return [ + handedOffWithNested === true + ? `${event.type}:${event.payload.status}:${event.payload.id}` + : `${event.type}:${event.payload.status}`, + ]; + } + if ( + handedOffWithNested === true && + event.type === "node.updated" && + event.payload.kind === "subagent" + ) { + return [`${event.type}:${event.payload.status}:${event.payload.id}`]; + } + if (event.type === "turn-item.updated" && event.payload.type === "subagent") { + return [ + handedOffWithNested === true + ? `${event.type}:${event.payload.status}:${event.payload.id}` + : `${event.type}:${event.payload.status}`, + ]; + } + return []; + }), + expected, ); }), ); @@ -5620,7 +5710,7 @@ it.effect( updatedAt: t2, }, }, - // Simulate session.step.started filling row.model and re-emitting an unchanged node. + // Simulate a repeated running update (still model: null) re-emitting an unchanged node. { type: "node.updated", driver: opencodeDriver, @@ -5661,7 +5751,7 @@ it.effect( nativeTaskRef: { driver: opencodeDriver, nativeId: childSessionId, strength: "strong" }, prompt: "keep going", title: "Resume after upgrade", - model: "opencode/big-pickle", + model: null, status: "running", result: null, startedAt: t2, @@ -6210,6 +6300,84 @@ it.effect( }, }); + const run3ClaimSubagent = { + type: "subagent.updated", + driver: opencodeDriver, + preserveIdentity: true, + subagent: { + id: sessionNodeId, + threadId: ids.threadId, + runId: run3Id, + parentNodeId: NodeId.make(`node:${key}:root:3`), + origin: "provider_native", + createdBy: "agent", + driver: opencodeDriver, + providerInstanceId: instanceId, + providerThreadId: childProviderThreadId, + childThreadId, + nativeTaskRef: { driver: opencodeDriver, nativeId: childSessionId, strength: "strong" }, + prompt: "redirect in run 3", + title: "Redirect in run 3", + model: "opencode/big-pickle", + status: "running", + result: null, + startedAt: t3, + completedAt: null, + updatedAt: t3, + }, + } as const; + const run3ClaimNode = { + type: "node.updated", + driver: opencodeDriver, + preserveIdentity: true, + node: { + id: sessionNodeId, + threadId: ids.threadId, + runId: run3Id, + parentNodeId: NodeId.make(`node:${key}:root:3`), + rootNodeId: NodeId.make(`node:${key}:root:3`), + kind: "subagent", + status: "running", + countsForRun: false, + providerThreadId: childProviderThreadId, + providerTurnId: ProviderTurnId.make(`provider-turn:${key}:3`), + nativeItemRef: { driver: opencodeDriver, nativeId: childSessionId, strength: "strong" }, + runtimeRequestId: null, + checkpointScopeId: null, + startedAt: t3, + completedAt: null, + }, + } as const; + const run3ClaimTurnItem = { + type: "turn_item.updated", + driver: opencodeDriver, + preserveIdentity: true, + turnItem: { + id: sessionItemId, + threadId: ids.threadId, + runId: run3Id, + nodeId: sessionNodeId, + providerThreadId: ids.providerThreadId, + providerTurnId: ProviderTurnId.make(`provider-turn:${key}:3`), + nativeItemRef: { driver: opencodeDriver, nativeId: childSessionId, strength: "strong" }, + parentItemId: null, + ordinal: 1, + status: "running", + title: "Redirect in run 3", + startedAt: t3, + completedAt: null, + updatedAt: t3, + type: "subagent", + subagentId: sessionNodeId, + origin: "provider_native", + driver: opencodeDriver, + providerInstanceId: instanceId, + childThreadId, + prompt: "redirect in run 3", + result: null, + }, + } as const; + yield* Queue.offerAll(eventQueue, [ { type: "provider_turn.updated", @@ -6320,39 +6488,35 @@ it.effect( threadDisposition: "reusable", }, // A newer run claims the still-running legacy child via its session-keyed subagent ID. - { - type: "subagent.updated", - driver: opencodeDriver, - preserveIdentity: true, - subagent: { - id: sessionNodeId, - threadId: ids.threadId, - runId: run3Id, - parentNodeId: NodeId.make(`node:${key}:root:3`), - origin: "provider_native", - createdBy: "agent", - driver: opencodeDriver, - providerInstanceId: instanceId, - providerThreadId: childProviderThreadId, - childThreadId, - nativeTaskRef: { driver: opencodeDriver, nativeId: childSessionId, strength: "strong" }, - prompt: "redirect in run 3", - title: "Redirect in run 3", - model: "opencode/big-pickle", - status: "running", - result: null, - startedAt: t3, - completedAt: null, - updatedAt: t3, - }, - }, + run3ClaimSubagent, ]); - // Run 2 must detach the legacy row and close its subscription while Run 3 still owns the child. + // Run 2 detaches the legacy row and closes its subscription while Run 3 ingests the claim. yield* Deferred.await(ingestionDone); + for (const claimEvent of [run3ClaimNode, run3ClaimSubagent, run3ClaimTurnItem]) { + yield* ingestor.ingestNormalized({ + providerSessionId, + providerInstanceId: instanceId, + threadId: ids.threadId, + runId: run3Id, + event: claimEvent, + }); + } const parentProjection = yield* projectionStore.getThreadProjection(ids.threadId); + const subagentNodes = parentProjection.nodes.filter((node) => node.kind === "subagent"); + const subagentItems = parentProjection.turnItems.filter((item) => item.type === "subagent"); + assert.lengthOf(subagentNodes, 1); + assert.lengthOf(parentProjection.subagents, 1); + assert.lengthOf(subagentItems, 1); + assert.equal(subagentNodes[0]?.id, legacyNodeId); + assert.equal(subagentNodes[0]?.runId, run3Id); + assert.equal(subagentNodes[0]?.status, "running"); assert.equal(parentProjection.subagents[0]?.id, legacyNodeId); + assert.equal(parentProjection.subagents[0]?.runId, run3Id); assert.equal(parentProjection.subagents[0]?.status, "running"); + assert.equal(subagentItems[0]?.id, legacyItemId); + assert.equal(subagentItems[0]?.runId, run3Id); + assert.equal(subagentItems[0]?.status, "running"); }).pipe(Effect.provide(makeLegacySubagentLayer())); }, ); diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index e02d452defdb6..529ab54209b70 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -277,7 +277,8 @@ export const cascadeTerminalizeRunOwnedSubagents = Effect.fn( const node = input.open.nodes.get(key); if ( node !== undefined && - ((node.threadId === input.run.threadId && node.runId === input.run.id) || + (subagent !== undefined || + (node.threadId === input.run.threadId && node.runId === input.run.id) || childThreadIds.has(node.threadId)) && isOpenExecutionNodeStatus(node.status) ) { @@ -320,7 +321,12 @@ export const cascadeTerminalizeRunOwnedSubagents = Effect.fn( } } for (const turnItem of input.open.childTurnItems.values()) { - if (!childThreadIds.has(turnItem.threadId) || isSettledTurnItemStatus(turnItem.status)) { + const ownedByOpenSubagent = + turnItem.type === "subagent" && input.open.subagents.has(turnItem.subagentId); + if ( + (!childThreadIds.has(turnItem.threadId) && !ownedByOpenSubagent) || + isSettledTurnItemStatus(turnItem.status) + ) { continue; } events.push({ @@ -1373,8 +1379,8 @@ export const layer: Layer.Layer< ); // Subagents and tool calls that outlived the root run die with the provider runtime. Stop // can settle them meanwhile, so each write holds only while its stored row is unchanged. - const endWorkLostAfterRootRun = Effect.fn("RunExecutionService.endWorkLostAfterRootRun")( - function* (completedAt: DateTime.Utc) { + const endLostWork = Effect.fn("RunExecutionService.endLostWork")( + function* (completedAt: DateTime.Utc, options: { readonly includeToolCalls: boolean }) { const allocateEventId = () => idAllocator.allocate.event({ threadId: input.run.threadId }); const open = yield* Ref.get(openRunOwnedSubagents); @@ -1400,7 +1406,9 @@ export const layer: Layer.Layer< childTurnItems: new Map( [...stored.turnItems].filter( ([id, item]) => - open.childTurnItems.has(id) && item.threadId === subagent.childThreadId, + open.childTurnItems.has(id) && + (item.threadId === subagent.childThreadId || + (item.type === "subagent" && item.subagentId === subagent.id)), ), ), nodes: new Map( @@ -1419,20 +1427,22 @@ export const layer: Layer.Layer< }); yield* eventSink.writeIfSubagentUnchanged({ expected: subagent, events }); } - for (const toolCall of stored.turnItems.values()) { - if (!openRootToolCalls.has(toolCall.id)) continue; - yield* eventSink.writeIfTurnItemUnchanged({ - expected: toolCall, - events: yield* interruptLostToolCalls({ - run: input.run, - toolCalls: [toolCall], - completedAt, - allocateEventId, - }), - }); + if (options.includeToolCalls) { + for (const toolCall of stored.turnItems.values()) { + if (!openRootToolCalls.has(toolCall.id)) continue; + yield* eventSink.writeIfTurnItemUnchanged({ + expected: toolCall, + events: yield* interruptLostToolCalls({ + run: input.run, + toolCalls: [toolCall], + completedAt, + allocateEventId, + }), + }); + } + openRootToolCalls.clear(); } yield* Ref.set(openRunOwnedSubagents, emptyOpenRunOwnedSubagentProjection()); - openRootToolCalls.clear(); }, (effect) => effect.pipe( @@ -1684,10 +1694,7 @@ export const layer: Layer.Layer< const detached = yield* Ref.modify(openRunOwnedSubagents, (current) => { const subagentIds = new Set([moved.id]); for (const subagent of current.subagents.values()) { - if ( - subagent.childThreadId === moved.childThreadId || - subagent.threadId === moved.childThreadId - ) { + if (subagent.childThreadId === moved.childThreadId) { subagentIds.add(subagent.id); } } @@ -1697,9 +1704,19 @@ export const layer: Layer.Layer< if (item.nodeId !== null) subagentIds.add(item.nodeId); } } + // Nested subagents launched by the moved child keep their own child threads in this + // run's routing, so keep their row/node/item entries for lost-work cleanup. + const keptNestedSubagentIds = new Set(); + for (const subagent of current.subagents.values()) { + if (!subagentIds.has(subagent.id) && subagent.threadId === moved.childThreadId) { + keptNestedSubagentIds.add(subagent.id); + } + } const childTurnItems = new Map( [...current.childTurnItems].filter( - ([, item]) => item.threadId !== moved.childThreadId, + ([, item]) => + item.threadId !== moved.childThreadId || + (item.type === "subagent" && keptNestedSubagentIds.has(item.subagentId)), ), ); const turnItems = new Map( @@ -1729,10 +1746,11 @@ export const layer: Layer.Layer< nodes: new Map( [...current.nodes].filter( ([id, node]) => - !subagentIds.has(id) && - node.threadId !== moved.childThreadId && - (moved.providerThreadId === null || - node.providerThreadId !== moved.providerThreadId), + keptNestedSubagentIds.has(id) || + (!subagentIds.has(id) && + node.threadId !== moved.childThreadId && + (moved.providerThreadId === null || + node.providerThreadId !== moved.providerThreadId)), ), ), linkedChildThreadIds: new Set( @@ -2119,7 +2137,7 @@ export const layer: Layer.Layer< ); if (handedOff) { // Its tool calls, the handoff's own among them, died with that process. - yield* endWorkLostAfterRootRun(completedAt); + yield* endLostWork(completedAt, { includeToolCalls: true }); return; } } @@ -2134,7 +2152,7 @@ export const layer: Layer.Layer< isProviderEventStreamError(squashed.cause) && squashed.cause.raisedByT3 !== true; if (yield* Ref.get(rootRunFinalized)) { - if (runtimeLost) yield* endWorkLostAfterRootRun(completedAt); + if (runtimeLost) yield* endLostWork(completedAt, { includeToolCalls: true }); return; } const providerThread = yield* Ref.get(latestProviderThread); @@ -2188,7 +2206,8 @@ export const layer: Layer.Layer< ) : []; let nextRetryWarningAt = completedAt; - yield* writeFinalRunEvents({ + const reloadSubagents = input.loadUnsettledWork !== undefined; + const finalized = yield* writeFinalRunEvents({ run: input.run, rootNode: input.rootNode, checkpointScope: input.checkpointScope, @@ -2198,7 +2217,7 @@ export const layer: Layer.Layer< activeAttemptId: input.attempt.id, expectedStatus: "running", }, - openRunOwnedSubagents: openSubagents, + ...(reloadSubagents ? {} : { openRunOwnedSubagents: openSubagents }), ...(abandonedProviderTurn === null ? {} : { abandonedProviderTurn }), lostToolCalls, completedAt, @@ -2249,6 +2268,9 @@ export const layer: Layer.Layer< Effect.forkIn(scope), Effect.flatMap(Fiber.join), ); + if (finalized && reloadSubagents) { + yield* endLostWork(completedAt, { includeToolCalls: false }); + } }), ), Effect.ensuring(endPreviewActivity), diff --git a/packages/provider-opencode/src/server/v2/adapter.ts b/packages/provider-opencode/src/server/v2/adapter.ts index 3e8a0e1352296..82431e24b147e 100644 --- a/packages/provider-opencode/src/server/v2/adapter.ts +++ b/packages/provider-opencode/src/server/v2/adapter.ts @@ -259,6 +259,7 @@ interface ActiveTurn { readonly models: Set; lastStep: Tokens | undefined; before: string | null | undefined; + launchPromptId: string | undefined; compaction: { readonly nativeId: string; readonly startedAt: DateTime.Utc } | undefined; compactions: number; todoPlanId: PlanId | undefined; @@ -391,7 +392,7 @@ interface ThreadState { readonly appThread: OrchestrationV2AppThread; readonly coldResumeKey: string | undefined; turns: number; - callPrompt: string | undefined; + readonly pendingPrompts: Array<{ readonly call: SubagentCall; readonly text: string }>; queuedPrompt: { readonly inboxId: string; readonly text: string } | undefined; queued: string | undefined; readonly steers: Map; @@ -1729,6 +1730,7 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr models: new Set(), lastStep: undefined, before: undefined, + launchPromptId: undefined, compaction: undefined, compactions: 0, todoPlanId: undefined, @@ -1758,7 +1760,7 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr startedAt: call.startedAt, prompt: call.prompt, title: call.title, - model: call.model, + model: preserveIdentity ? null : call.model, status: call.status, result: null, resultRun: -1, @@ -2229,7 +2231,9 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr call.row = row; call.child = child; existing.call = call; - if (child.active === undefined) existing.callPrompt ??= call.prompt; + if (child.active === undefined && call.prompt.length > 0) { + existing.pendingPrompts.push({ call, text: call.prompt }); + } child.sessionScopedBackground = false; child.model = childModel; const nextProviderThread = withReportedModel( @@ -2345,7 +2349,7 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr appThread, coldResumeKey: coldResume ? call.nativeId : previous?.subagent?.coldResumeKey, turns: previous?.subagent?.turns ?? 0, - callPrompt: call.prompt, + pendingPrompts: call.prompt.length > 0 ? [{ call, text: call.prompt }] : [], queuedPrompt: undefined, queued: undefined, steers: new Map(), @@ -2392,8 +2396,15 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr call.state.calls.delete(call.toolId); const child = call.child; const remaining = child === undefined ? [] : callsAttachedTo(child); - if (child?.subagent !== undefined && child.subagent.call === call && remaining.length > 0) { - child.subagent.call = remaining.at(-1)!; + if (child?.subagent !== undefined) { + for (let i = child.subagent.pendingPrompts.length - 1; i >= 0; i -= 1) { + if (child.subagent.pendingPrompts[i]?.call === call) { + child.subagent.pendingPrompts.splice(i, 1); + } + } + if (child.subagent.call === call && remaining.length > 0) { + child.subagent.call = remaining.at(-1)!; + } } const row = call.row; if (row !== undefined) { @@ -2417,7 +2428,7 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr yield* finishTurn(child, { status: "interrupted" }); } if (child.active === undefined && child.subagent !== undefined) { - child.subagent.callPrompt = undefined; + child.subagent.pendingPrompts.length = 0; child.subagent.queuedPrompt = undefined; child.subagent.queued = undefined; child.subagent.steers.clear(); @@ -2514,9 +2525,10 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr }, }); yield* emitProviderTurn(child, turn, providerTurn); - const prompt = subagent.queuedPrompt?.text ?? subagent.callPrompt; + turn.launchPromptId = subagent.queuedPrompt?.inboxId; + const prompt = subagent.queuedPrompt?.text ?? subagent.pendingPrompts[0]?.text; subagent.queuedPrompt = undefined; - subagent.callPrompt = undefined; + subagent.pendingPrompts.length = 0; if (prompt !== undefined && prompt.length > 0) { const nativeId = `${nativeTurnId}:prompt`; const artifacts = makeSubagentConversationArtifacts({ @@ -3809,11 +3821,17 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr const text = event.data.item.payload.text; if (state.subagent !== undefined) { if (state.active === undefined && state.subagent.queuedPrompt === undefined) { + const matchIndex = state.subagent.pendingPrompts.findIndex( + (entry) => text === entry.text || text.includes(entry.text), + ); + const matched = + matchIndex === -1 + ? undefined + : state.subagent.pendingPrompts.splice(matchIndex, 1)[0]; state.subagent.queuedPrompt = { inboxId: event.data.inboxID, - text: state.subagent.callPrompt || text, + text: matched?.text ?? text, }; - state.subagent.callPrompt = undefined; } else { state.subagent.steers.set(event.data.inboxID, text); } @@ -3889,19 +3907,69 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr } const start = before !== undefined ? before : promptId; const oldestSteer = pendingSteers?.values().next().value; - const history = OpenCode2Client.paginate( - { sessionID: Session.ID.make(sessionId), order: "desc" as const, limit: 50 }, - client.message.list, - ).pipe( - Stream.takeUntil((message) => - start !== undefined ? message.id === start : message.id === oldestSteer, - ), - ); - // An unread steer is not in history yet, so a steer anchor reads one page at most. - const read = yield* (start === undefined ? history.pipe(Stream.take(50)) : history).pipe( - Stream.runCollect, - Effect.catchTags({ SessionNotFoundError: () => Effect.succeed(null) }), - ); + const read = yield* Effect.gen(function* () { + const sessionID = Session.ID.make(sessionId); + if (start !== undefined) { + return yield* OpenCode2Client.paginate( + { sessionID, order: "desc" as const, limit: 50 }, + client.message.list, + ).pipe( + Stream.takeUntil((message) => message.id === start), + Stream.runCollect, + Effect.catchTags({ SessionNotFoundError: () => Effect.succeed(null) }), + ); + } + const firstPage = yield* client.message + .list({ sessionID, order: "desc", limit: 50 }) + .pipe(Effect.catchTags({ SessionNotFoundError: () => Effect.succeed(null) })); + if (firstPage === null) return null; + if ( + firstPage.cursor.next === undefined || + firstPage.data.some((message) => message.id === oldestSteer) + ) { + return firstPage.data; + } + // Only scan past page 1 when a pending steer is proven to exist in message history. + const inbox = yield* client.session.inbox + .list({ sessionID }) + .pipe(Effect.catchTags({ SessionNotFoundError: () => Effect.succeed(null) })); + if (inbox === null) return null; + const unread = new Set(inbox.map((item) => item.id)); + const seen = new Set(firstPage.data.map((message) => message.id)); + let deliveredAnchor: string | undefined; + for (const steerId of pendingSteers!) { + if (seen.has(steerId)) break; + if (unread.has(steerId)) continue; + const probe = yield* client.session.message + .get({ sessionID, messageID: SessionMessage.ID.make(steerId) }) + .pipe( + Effect.as("found" as const), + Effect.catchTags({ + MessageNotFoundError: () => Effect.succeed("absent" as const), + SessionNotFoundError: () => Effect.succeed("missing" as const), + }), + ); + if (probe === "missing") return null; + if (probe === "found") { + deliveredAnchor = steerId; + break; + } + } + if (deliveredAnchor === undefined) return firstPage.data; + const rest = yield* OpenCode2Client.paginate( + { + sessionID, + limit: 50, + cursor: firstPage.cursor.next as string | undefined, + }, + client.message.list, + ).pipe( + Stream.takeUntil((message) => message.id === deliveredAnchor), + Stream.runCollect, + Effect.catchTags({ SessionNotFoundError: () => Effect.succeed(null) }), + ); + return rest === null ? null : [...firstPage.data, ...rest]; + }); return yield* lock.withPermit( Effect.gen(function* () { if (state.active !== turn) return undefined; @@ -3944,6 +4012,7 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr } for (const message of recent.toReversed()) { if (message.type === "user" && state.subagent !== undefined && state.active === turn) { + if (message.id === turn.launchPromptId) continue; const steer = state.subagent.steers.get(message.id) ?? message.text; state.subagent.steers.delete(message.id); yield* emitChildSteer(state, turn, message.id, steer); @@ -4101,8 +4170,10 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr return; } const active = { ...(yield* client.session.active()) }; - const attachedBefore = new Set( - [...threads.values()].filter((state) => state.subagent !== undefined), + const attachedBefore = new Map( + [...threads.values()].flatMap((state) => + state.subagent === undefined ? [] : [[state, state.subagent.call] as const], + ), ); for (const sessionId of busy) if (!(sessionId in active)) busy.delete(sessionId); for (const state of threads.values()) { @@ -4148,7 +4219,9 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr // A child the parent's backfill found running started in the gap; its start is not replayed. for (const state of threads.values()) { if (state.subagent === undefined || state.active !== undefined) continue; - if (attachedBefore.has(state) || !(state.sessionId in active)) continue; + if (attachedBefore.get(state) === state.subagent.call || !(state.sessionId in active)) { + continue; + } yield* lock.withPermit(startChildTurn(state)); } for (const state of threads.values()) { From 8ad8da5cf8f5e40fa48b2fba914c2a9d360267a8 Mon Sep 17 00:00:00 2001 From: Victor Vogel Date: Sat, 21 Nov 2026 17:12:03 +0000 Subject: [PATCH 5/6] fix(opencode): guard lost-work ownership, retry cleanup, and refine reconnect recovery --- .../Adapters/OpenCode2AdapterV2.test.ts | 215 +++++++--- .../src/orchestration-v2/EventSink.test.ts | 184 +++++++++ apps/server/src/orchestration-v2/EventSink.ts | 108 +++-- .../src/orchestration-v2/ProjectionStore.ts | 2 +- .../RunExecutionService.test.ts | 265 +++++++++++- .../orchestration-v2/RunExecutionService.ts | 386 ++++++++++-------- .../src/server/v2/adapter.ts | 61 ++- 7 files changed, 948 insertions(+), 273 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts index 2c4c43ecdba8d..d41247a9919c7 100644 --- a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts @@ -2322,7 +2322,7 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { sessionID: CHILD, item: { type: "user", - payload: { text: "You are a subagent.\nsleep", files: [] }, + payload: { text: "You are a subagent spawned by another session.\nsleep", files: [] }, delivery: "steer", }, }), @@ -2863,46 +2863,34 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { }), ); - it.effect.each(["launched", "redirected"] as const)( + it.effect.each(["launched", "redirected", "redirected-before-disconnect"] as const)( "streams a child that %s and started while the parent was disconnected", (mode) => Effect.gen(function* () { + const isRedirect = mode !== "launched"; const { runtime, thread } = yield* resumed([ - out("session.prompt", { sessionID: SESSION, text: "" }), - promptAccepted, - event("session.execution.started", { sessionID: SESSION }), - ...(mode === "redirected" + ...(isRedirect ? [ - event("session.created", { - session: { id: CHILD, title: "Sleep", parentID: SESSION }, - }), - event("session.tool.started", { - sessionID: SESSION, - assistantMessageID: "msg_parent_initial", - toolID: "call-initial", - name: "subagent", - input: { description: "Sleep", prompt: "first", background: true }, - }), - event("session.tool.progress", { - sessionID: SESSION, - assistantMessageID: "msg_parent_initial", - toolID: "call-initial", - name: "subagent", - metadata: { sessionID: CHILD, status: "running" }, - }), - event("session.execution.started", { sessionID: CHILD }), + ...launchedAndRunning, event("session.execution.succeeded", { sessionID: CHILD }), - event("session.tool.completed", { - sessionID: SESSION, - assistantMessageID: "msg_parent_initial", - toolID: "call-initial", - name: "subagent", - output: "The subagent is working in the background.", - metadata: { sessionID: CHILD, status: "running" }, - }), childReport("msg_report_initial", "FIRST_OK"), + ...(mode === "redirected-before-disconnect" + ? callAgain( + { + sessionID: CHILD, + description: "Redirect", + prompt: "second", + background: true, + }, + { id: "call-redirect", writesRules: true }, + ) + : []), ] - : []), + : [ + out("session.prompt", { sessionID: SESSION, text: "" }), + promptAccepted, + event("session.execution.started", { sessionID: SESSION }), + ]), { type: "runtime_exit", status: "success" }, out("event.subscribe"), out("session.active"), @@ -2922,20 +2910,19 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { content: [ { type: "tool", - id: mode === "redirected" ? "call-redirect" : "call-background", + id: isRedirect ? "call-redirect" : "call-background", name: "subagent", executed: true, state: { status: "completed", - input: - mode === "redirected" - ? { - sessionID: CHILD, - description: "Redirect", - prompt: "second", - background: true, - } - : { description: "Sleep", prompt: "sleep", background: true }, + input: isRedirect + ? { + sessionID: CHILD, + description: "Redirect", + prompt: "second", + background: true, + } + : { description: "Sleep", prompt: "sleep", background: true }, content: [ { type: "text", text: "The subagent is working in the background." }, ], @@ -2955,8 +2942,12 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { ], cursor: {}, }), - out("session.update", { sessionID: CHILD, permissions: "" }), - reply("session.update", null), + ...(mode === "redirected-before-disconnect" + ? [] + : [ + out("session.update", { sessionID: CHILD, permissions: "" }), + reply("session.update", null), + ]), out("permission.list", { sessionID: SESSION }), replyData("permission.list", []), out("session.form.list", { sessionID: SESSION }), @@ -3060,7 +3051,10 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { sessionID: CHILD, item: { type: "user", - payload: { text: "You are a subagent.\nreal next prompt", files: [] }, + payload: { + text: "You are a subagent spawned by another session.\nreal next prompt", + files: [], + }, delivery: "steer", }, }), @@ -3087,7 +3081,7 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { id: "msg_real_launch", time: { created: 3 }, type: "user", - text: "You are a subagent.\nreal next prompt", + text: "You are a subagent spawned by another session.\nreal next prompt", }, { id: "msg_child_prior_report", @@ -3158,6 +3152,131 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { }), ); + it.effect( + "matches full unwrapped child prompt instead of a prefix and preserves unknown wrappers", + () => + Effect.gen(function* () { + const { runtime, thread } = yield* resumed([ + ...launchedAndRunning, + event("session.execution.succeeded", { sessionID: CHILD }), + // Two pending redirects where the first prompt is a prefix of the second. + ...callAgain( + { background: true, description: "Prefix redirect", prompt: "fix" }, + { id: "call-prefix", writesRules: true }, + ), + ...callAgain( + { background: true, description: "Full redirect", prompt: "fix tests" }, + { id: "call-full", writesRules: false }, + ), + event("session.inbox.enqueued", { + inboxID: "msg_cancelled_launch", + sessionID: CHILD, + item: { + type: "user", + payload: { text: "cancelled placeholder", files: [] }, + delivery: "steer", + }, + }), + event("session.inbox.enqueued", { + inboxID: "msg_full_launch", + sessionID: CHILD, + item: { + type: "user", + payload: { + text: "You are a subagent spawned by another session.\nfix tests", + files: [], + }, + delivery: "steer", + }, + }), + // Cancelling the queued item promotes msg_full_launch from steers and unwraps it. + event("session.inbox.cancelled", { + sessionID: CHILD, + inboxID: "msg_cancelled_launch", + }), + event("session.execution.started", { sessionID: CHILD }), + event("session.inbox.delivered", { sessionID: CHILD, inboxID: "msg_full_launch" }), + event("session.execution.succeeded", { sessionID: CHILD }), + // Direct idle-enqueue with full match (without cancellation) and unknown wrapper fallback. + ...callAgain( + { background: true, description: "Direct wrapped", prompt: "fix lint" }, + { id: "call-direct-wrapped", writesRules: false }, + ), + event("session.inbox.enqueued", { + inboxID: "msg_direct_launch", + sessionID: CHILD, + item: { + type: "user", + payload: { + text: "You are a subagent spawned by another session.\nfix lint", + files: [], + }, + delivery: "steer", + }, + }), + event("session.execution.started", { sessionID: CHILD }), + event("session.inbox.delivered", { sessionID: CHILD, inboxID: "msg_direct_launch" }), + event("session.execution.succeeded", { sessionID: CHILD }), + // Unknown wrapper cannot match unambiguously: keep inbox text verbatim. + ...callAgain( + { background: true, description: "Unknown wrapper redirect", prompt: "fix" }, + { id: "call-unknown-wrapper", writesRules: false }, + ), + event("session.inbox.enqueued", { + inboxID: "msg_unknown_wrapper", + sessionID: CHILD, + item: { + type: "user", + payload: { text: "Custom wrapper.\nfix", files: [] }, + delivery: "steer", + }, + }), + event("session.execution.started", { sessionID: CHILD }), + event("session.inbox.delivered", { sessionID: CHILD, inboxID: "msg_unknown_wrapper" }), + event("session.execution.succeeded", { sessionID: CHILD }), + event("session.execution.succeeded", { sessionID: SESSION }), + childReport("msg_report_1", "ONE"), + childReport("msg_report_2", "TWO"), + childReport("msg_report_3", "THREE"), + childReport("msg_report_4", "FOUR"), + childReport("msg_report_5", "FIVE"), + ]); + const childUserPrompts: Array<{ readonly text: string; readonly inputIntent: string }> = []; + let childThreadId: string | undefined; + const done = yield* Deferred.make(); + yield* runtime.events.pipe( + Stream.tap((e) => + Effect.gen(function* () { + if (e.type === "app_thread.created") childThreadId = e.appThread.id; + if ( + e.type === "turn_item.updated" && + e.turnItem.type === "user_message" && + e.turnItem.threadId === childThreadId + ) { + childUserPrompts.push({ + text: e.turnItem.text, + inputIntent: e.turnItem.inputIntent, + }); + } + if (e.type === "subagent.updated" && e.subagent.status === "completed") { + yield* Deferred.succeed(done, undefined); + } + }), + ), + Stream.runDrain, + Effect.forkScoped, + ); + yield* runtime.startTurn(withLineage(thread)); + yield* Deferred.await(done); + assert.deepEqual(childUserPrompts, [ + { text: "sleep", inputIntent: "turn_start" }, + { text: "fix tests", inputIntent: "turn_start" }, + { text: "fix lint", inputIntent: "turn_start" }, + { text: "Custom wrapper.\nfix", inputIntent: "turn_start" }, + ]); + }), + ); + it.effect.each(["unread", "delivered-beyond-page"] as const)( "bounds unread steer backfill and recovers a delivered steer older than 50 records (%s)", (mode) => @@ -4023,7 +4142,7 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { Stream.takeUntil( (event) => event.type === "provider_turn.updated" && - event.providerTurn.nativeTurnRef?.nativeId?.startsWith(`${CHILD}:turn:`) === true, + event.providerTurn.nativeTurnRef?.nativeId === `${CHILD}:turn:2`, ), Stream.runCollect, Effect.forkScoped, diff --git a/apps/server/src/orchestration-v2/EventSink.test.ts b/apps/server/src/orchestration-v2/EventSink.test.ts index e89951ddfcc16..717f9ed6c01ab 100644 --- a/apps/server/src/orchestration-v2/EventSink.test.ts +++ b/apps/server/src/orchestration-v2/EventSink.test.ts @@ -2,11 +2,15 @@ import { assert, it } from "@effect/vitest"; import { CommandId, EventId, + NodeId, ProjectId, + ProviderDriverKind, ProviderInstanceId, RunId, ThreadId, type OrchestrationV2AppThread, + type OrchestrationV2ExecutionNode, + type OrchestrationV2Subagent, } from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; @@ -107,3 +111,183 @@ it.effect("rejects a planned fork whose source was deleted before it committed", assert.instanceOf(missing, ProjectionStore.ProjectionStoreThreadNotFoundError); }).pipe(Effect.provide(layer)), ); + +it.effect("rejects writeIfSubagentUnchanged when expectedNode changed to another run", () => + Effect.gen(function* () { + const sink = yield* EventSink.EventSinkV2; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make("subagent-node-guard"); + const run1Id = RunId.make("run:1"); + const run2Id = RunId.make("run:2"); + const rootNodeId = NodeId.make("node:root"); + const subagentId = NodeId.make("node:subagent"); + const providerInstanceId = ProviderInstanceId.make("opencode"); + const driver = ProviderDriverKind.make("opencode"); + const thread: OrchestrationV2AppThread = { + id: threadId, + projectId: ProjectId.make("project"), + title: "Parent", + createdBy: "user", + creationSource: "web", + providerInstanceId, + modelSelection: { instanceId: providerInstanceId, model: "opencode/big-pickle" }, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + activeProviderThreadId: null, + lineage: { rootThreadId: threadId, parentThreadId: null, relationshipToParent: null }, + forkedFrom: null, + createdAt: now, + updatedAt: now, + archivedAt: null, + deletedAt: null, + settledOverride: null, + settledAt: null, + snoozedUntil: null, + snoozedAt: null, + lastVisitedAt: null, + }; + const subagent: OrchestrationV2Subagent = { + id: subagentId, + threadId, + runId: run1Id, + parentNodeId: rootNodeId, + origin: "provider_native", + createdBy: "agent", + driver, + providerInstanceId, + providerThreadId: null, + childThreadId: null, + nativeTaskRef: null, + prompt: "work", + title: "Worker", + model: null, + status: "running", + result: null, + startedAt: now, + completedAt: null, + updatedAt: now, + }; + const node: OrchestrationV2ExecutionNode = { + id: subagentId, + threadId, + runId: run1Id, + parentNodeId: rootNodeId, + rootNodeId, + kind: "subagent", + status: "running", + countsForRun: false, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + runtimeRequestId: null, + checkpointScopeId: null, + startedAt: now, + completedAt: null, + }; + yield* sink.write({ + events: [ + { + id: EventId.make("subagent-guard:thread"), + type: "thread.created", + threadId, + occurredAt: now, + payload: thread, + }, + { + id: EventId.make("subagent-guard:subagent"), + type: "subagent.updated", + threadId, + runId: run1Id, + nodeId: subagentId, + driver, + providerInstanceId, + occurredAt: now, + payload: subagent, + }, + { + id: EventId.make("subagent-guard:node"), + type: "node.updated", + threadId, + runId: run1Id, + nodeId: subagentId, + providerInstanceId, + occurredAt: now, + payload: node, + }, + ], + }); + const claimedNode: OrchestrationV2ExecutionNode = { ...node, runId: run2Id }; + yield* sink.write({ + events: [ + { + id: EventId.make("subagent-guard:claim-node"), + type: "node.updated", + threadId, + runId: run2Id, + nodeId: subagentId, + providerInstanceId, + occurredAt: now, + payload: claimedNode, + }, + ], + }); + const result = yield* sink.writeIfSubagentUnchanged({ + expected: subagent, + expectedNode: node, + events: [ + { + id: EventId.make("subagent-guard:fail-subagent"), + type: "subagent.updated", + threadId, + runId: run1Id, + nodeId: subagentId, + driver, + providerInstanceId, + occurredAt: now, + payload: { ...subagent, status: "failed", completedAt: now, updatedAt: now }, + }, + { + id: EventId.make("subagent-guard:fail-node"), + type: "node.updated", + threadId, + runId: run1Id, + nodeId: subagentId, + providerInstanceId, + occurredAt: now, + payload: { ...node, status: "failed", completedAt: now }, + }, + ], + }); + assert.isFalse(result.committed); + const projection = yield* projections.getThreadProjection(threadId); + assert.equal(projection.subagents[0]?.status, "running"); + assert.equal(projection.nodes[0]?.runId, run2Id); + assert.equal(projection.nodes[0]?.status, "running"); + + const accepted = yield* sink.writeIfSubagentUnchanged({ + expected: subagent, + expectedNode: claimedNode, + events: [ + { + id: EventId.make("subagent-guard:fail-subagent-accepted"), + type: "subagent.updated", + threadId, + runId: run1Id, + nodeId: subagentId, + driver, + providerInstanceId, + occurredAt: now, + payload: { ...subagent, status: "failed", completedAt: now, updatedAt: now }, + }, + ], + }); + assert.isTrue(accepted.committed); + const afterAccept = yield* projections.getThreadProjection(threadId); + assert.equal(afterAccept.subagents[0]?.status, "failed"); + assert.equal(afterAccept.nodes[0]?.runId, run2Id); + assert.equal(afterAccept.nodes[0]?.status, "running"); + }).pipe(Effect.provide(layer)), +); diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index 3f6744e576edd..8327b8813ae66 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -1,5 +1,6 @@ import { CommandId, + type OrchestrationV2ExecutionNode, type OrchestrationV2Run, OrchestrationV2DomainEvent, type OrchestrationV2ProviderThread, @@ -193,9 +194,10 @@ export interface EventSinkV2Shape { }, EventSinkV2Error >; - /** Commits only if the stored subagent still equals expected. */ + /** Commits only if the stored subagent (and optional node) still equal expected. */ readonly writeIfSubagentUnchanged: (input: { readonly expected: OrchestrationV2Subagent; + readonly expectedNode?: OrchestrationV2ExecutionNode; readonly events: ReadonlyArray; }) => Effect.Effect< { @@ -681,21 +683,25 @@ const layerBase: Layer.Layer< ); }); - // Appends the events only if the stored payload still matches, in one transaction. - const writeIfStoredPayloadMatches = ( - readStoredPayload: Effect.Effect, E>, - expectedJson: string, + // Appends the events only if every stored payload still matches, in one transaction. + const writeIfStoredPayloadsMatch = ( + checks: ReadonlyArray<{ + readonly read: Effect.Effect, E>; + readonly expectedJson: string; + }>, events: ReadonlyArray, ) => commitThenPublish( projectedThreadIds(events), Effect.gen(function* () { - const rows = yield* readStoredPayload; - if (rows[0]?.payload_json !== expectedJson) { - return { - committed: false as const, - storedEvents: [] as ReadonlyArray, - }; + for (const check of checks) { + const rows = yield* check.read; + if (rows[0]?.payload_json !== check.expectedJson) { + return { + committed: false as const, + storedEvents: [] as ReadonlyArray, + }; + } } const normalized = yield* normalizeEvents(events); @@ -717,14 +723,18 @@ const layerBase: Layer.Layer< "orchestration_v2.provider_thread_id": input.expected.id, }); const expectedJson = yield* ProjectionStore.encodeProviderThreadPayload(input.expected); - return yield* writeIfStoredPayloadMatches( - sql<{ readonly payload_json: string }>` - SELECT payload_json - FROM orchestration_v2_projection_provider_threads - WHERE provider_thread_id = ${input.expected.id} - LIMIT 1 - `, - expectedJson, + return yield* writeIfStoredPayloadsMatch( + [ + { + read: sql<{ readonly payload_json: string }>` + SELECT payload_json + FROM orchestration_v2_projection_provider_threads + WHERE provider_thread_id = ${input.expected.id} + LIMIT 1 + `, + expectedJson, + }, + ], input.events, ); }); @@ -737,14 +747,18 @@ const layerBase: Layer.Layer< "orchestration_v2.turn_item_id": input.expected.id, }); const expectedJson = yield* ProjectionStore.encodeTurnItemPayload(input.expected); - return yield* writeIfStoredPayloadMatches( - sql<{ readonly payload_json: string }>` - SELECT payload_json - FROM orchestration_v2_projection_turn_items - WHERE turn_item_id = ${input.expected.id} - LIMIT 1 - `, - expectedJson, + return yield* writeIfStoredPayloadsMatch( + [ + { + read: sql<{ readonly payload_json: string }>` + SELECT payload_json + FROM orchestration_v2_projection_turn_items + WHERE turn_item_id = ${input.expected.id} + LIMIT 1 + `, + expectedJson, + }, + ], input.events, ); }); @@ -757,16 +771,36 @@ const layerBase: Layer.Layer< "orchestration_v2.subagent_id": input.expected.id, }); const expectedJson = yield* ProjectionStore.encodeSubagentPayload(input.expected); - return yield* writeIfStoredPayloadMatches( - sql<{ readonly payload_json: string }>` - SELECT payload_json - FROM orchestration_v2_projection_subagents - WHERE subagent_id = ${input.expected.id} - LIMIT 1 - `, - expectedJson, - input.events, - ); + const checks: Array<{ + readonly read: Effect.Effect< + ReadonlyArray<{ readonly payload_json: string }>, + SqlError.SqlError + >; + readonly expectedJson: string; + }> = [ + { + read: sql<{ readonly payload_json: string }>` + SELECT payload_json + FROM orchestration_v2_projection_subagents + WHERE subagent_id = ${input.expected.id} + LIMIT 1 + `, + expectedJson, + }, + ]; + if (input.expectedNode !== undefined) { + const expectedNode = input.expectedNode; + checks.push({ + read: sql<{ readonly payload_json: string }>` + SELECT payload_json + FROM orchestration_v2_projection_nodes + WHERE node_id = ${expectedNode.id} + LIMIT 1 + `, + expectedJson: yield* ProjectionStore.encodeNodePayload(expectedNode), + }); + } + return yield* writeIfStoredPayloadsMatch(checks, input.events); }); const existingCommandResult = diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 1acbcdf3da8e2..1ca913164e5ae 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -1384,7 +1384,7 @@ const encodeRunOutput = Schema.encodeEffect( const encodeRunAttemptPayload = Schema.encodeEffect( Schema.fromJsonString(OrchestrationV2RunAttemptJsonSchema), ); -const encodeNodePayload = Schema.encodeEffect( +export const encodeNodePayload = Schema.encodeEffect( Schema.fromJsonString(OrchestrationV2ExecutionNodeJsonSchema), ); export const encodeSubagentPayload = Schema.encodeEffect( diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index 372585a03dfe8..e3ac0c6f35930 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -3638,9 +3638,25 @@ it.effect.each([ { ended: false, rootLive: true, settledStatus: "completed" as const }, // Moving child C to another run keeps nested subagent G owned by this run for lost-work cleanup. { ended: true, rootLive: true, handedOffWithNested: true }, + // A legacy subagent claimed by a newer run under the same ID is not failed by the older run's cascade. + { ended: false, rootLive: true, claimedByOtherRun: true }, + // Even if only the shared node has been claimed by a newer run, the older run skips the subagent. + { ended: false, rootLive: true, nodeClaimedByOtherRun: true }, + // If the shared node is claimed between reload and conditional write, expectedNode rejects the write. + { ended: false, rootLive: true, nodeClaimedBeforeWrite: true }, ] as const)( "ends a subagent that outlived its root when the provider stream fails (%o)", - ({ ended, raisedByT3, stopped, rootLive, settledStatus, handedOffWithNested }) => + ({ + ended, + raisedByT3, + stopped, + rootLive, + settledStatus, + handedOffWithNested, + claimedByOtherRun, + nodeClaimedByOtherRun, + nodeClaimedBeforeWrite, + }) => Effect.gen(function* () { const providerInstanceId = ProviderInstanceId.make("codex"); const grandchildId = NodeId.make("node:stream-error-outlived-subagent:grandchild"); @@ -3670,6 +3686,17 @@ it.effect.each([ { type: "subagent.updated", driver, subagent: baseSubagent }, { type: "turn_item.updated", driver, turnItem: baseTurnItem }, ]; + if ( + claimedByOtherRun === true || + nodeClaimedByOtherRun === true || + nodeClaimedBeforeWrite === true + ) { + events.push({ + type: "node.updated", + driver, + node: makeRunOwnedSubagentNodeFixture({ ids, status: "running" }), + }); + } if (handedOffWithNested === true) { events.push( { @@ -3739,9 +3766,38 @@ it.effect.each([ ), ); }, - ...(stopped === true || settledStatus !== undefined + ...(nodeClaimedBeforeWrite === true + ? { + beforeSubagentWrite: (payload, stored) => + Effect.sync(() => { + const node = stored.nodes.get(payload.expected.id); + if (node !== undefined) { + stored.nodes.set(node.id, { ...node, runId: RunId.make("run:other-owner") }); + } + }), + } + : {}), + ...(stopped === true || + settledStatus !== undefined || + claimedByOtherRun === true || + nodeClaimedByOtherRun === true ? { settleStored: (work) => { + if (claimedByOtherRun === true) { + const otherRunId = RunId.make("run:other-owner"); + return { + subagents: work.subagents.map((row) => ({ ...row, runId: otherRunId })), + nodes: work.nodes.map((row) => ({ ...row, runId: otherRunId })), + turnItems: work.turnItems.map((row) => ({ ...row, runId: otherRunId })), + }; + } + if (nodeClaimedByOtherRun === true) { + const otherRunId = RunId.make("run:other-owner"); + return { + ...work, + nodes: work.nodes.map((row) => ({ ...row, runId: otherRunId })), + }; + } const status = settledStatus ?? "interrupted"; return { ...work, @@ -3776,7 +3832,10 @@ it.effect.each([ ]; } if ( - handedOffWithNested === true && + (handedOffWithNested === true || + claimedByOtherRun === true || + nodeClaimedByOtherRun === true || + nodeClaimedBeforeWrite === true) && event.type === "node.updated" && event.payload.kind === "subagent" ) { @@ -3796,6 +3855,144 @@ it.effect.each([ }), ); +it.effect( + "retries lost-subagent cleanup after root finalization without rewriting settled rows", + () => + Effect.gen(function* () { + const key = "retry-lost-subagent-cleanup"; + const providerInstanceId = ProviderInstanceId.make("codex"); + const subagent1Id = NodeId.make(`node:${key}:subagent:1`); + const subagent2Id = NodeId.make(`node:${key}:subagent:2`); + const subagent3Id = NodeId.make(`node:${key}:subagent:3`); + const subagent1ItemId = TurnItemId.make(`turn-item:${key}:subagent:1`); + const subagent2ItemId = TurnItemId.make(`turn-item:${key}:subagent:2`); + const subagent3ItemId = TurnItemId.make(`turn-item:${key}:subagent:3`); + const rootWrites = yield* Ref.make(0); + const subagentWriteAttempts = yield* Ref.make>([]); + const transientFailed = yield* Deferred.make(); + let failedSubagent2Once = false; + let subagent3Completed = false; + + const scenario = yield* captureRootRunTermination({ + key, + shouldFinalizeRun: () => Effect.succeed(true), + beforeTerminalWrite: () => + Ref.update(rootWrites, (count) => count + 1).pipe(Effect.as(true)), + beforeSubagentWrite: (payload) => + Effect.gen(function* () { + yield* Ref.update(subagentWriteAttempts, (current) => [ + ...current, + payload.expected.id, + ]); + if (payload.expected.id === subagent2Id && !failedSubagent2Once) { + failedSubagent2Once = true; + subagent3Completed = true; + yield* Deferred.succeed(transientFailed, undefined); + return yield* new EventSink.EventSinkWriteError({ + eventCount: payload.events.length, + cause: new Error("SQLITE_BUSY"), + }); + } + }), + settleStored: (work) => + subagent3Completed + ? { + ...work, + subagents: work.subagents.map((row) => + row.id === subagent3Id ? { ...row, status: "completed" as const } : row, + ), + turnItems: work.turnItems.map((row) => + row.id === subagent3ItemId ? { ...row, status: "completed" as const } : row, + ), + } + : work, + events: (ids) => { + const baseSubagent = makeRunOwnedSubagentFixture({ + ids, + providerInstanceId, + childThreadId: ids.childThreadId, + driver, + status: "running", + }); + const baseTurnItem = makeRunOwnedSubagentTurnItemFixture({ + ids, + providerInstanceId, + childThreadId: ids.childThreadId, + driver, + status: "running", + }); + const specs = [ + { id: subagent1Id, itemId: subagent1ItemId, child: ThreadId.make(`thread:${key}:c1`) }, + { id: subagent2Id, itemId: subagent2ItemId, child: ThreadId.make(`thread:${key}:c2`) }, + { id: subagent3Id, itemId: subagent3ItemId, child: ThreadId.make(`thread:${key}:c3`) }, + ] as const; + return Stream.fromIterable( + specs.flatMap( + ({ id, itemId, child }): ReadonlyArray => [ + { + type: "subagent.updated", + driver, + subagent: { ...baseSubagent, id, childThreadId: child }, + }, + { + type: "turn_item.updated", + driver, + turnItem: { + ...baseTurnItem, + id: itemId, + nodeId: id, + subagentId: id, + childThreadId: child, + }, + }, + ], + ), + ).pipe( + Stream.concat( + Stream.fail( + new ProviderAdapter.ProviderAdapterEventStreamError({ + driver, + providerSessionId: ProviderSessionId.make(`session:${key}`), + cause: "provider process exited", + }), + ), + ), + ); + }, + }).pipe(Effect.forkChild); + + yield* Deferred.await(transientFailed); + yield* TestClock.adjust("1 second"); + const { finalEvents } = yield* Fiber.join(scenario); + + assert.equal(yield* Ref.get(rootWrites), 1); + assert.deepEqual(yield* Ref.get(subagentWriteAttempts), [ + subagent1Id, + subagent2Id, + subagent2Id, + ]); + assert.deepEqual( + finalEvents.flatMap((event) => { + if (event.type === "run.updated") return [`${event.type}:${event.payload.status}`]; + if (event.type === "subagent.updated") { + return [`${event.type}:${event.payload.status}:${event.payload.id}`]; + } + if (event.type === "turn-item.updated" && event.payload.type === "subagent") { + return [`${event.type}:${event.payload.status}:${event.payload.id}`]; + } + return []; + }), + [ + "run.updated:failed", + `subagent.updated:failed:${subagent1Id}`, + `turn-item.updated:failed:${subagent1ItemId}`, + `subagent.updated:failed:${subagent2Id}`, + `turn-item.updated:failed:${subagent2ItemId}`, + ], + ); + }), +); + it.effect("does not interrupt a provider when ingestion fails before the root turn is known", () => Effect.gen(function* () { const interrupts = yield* Ref.make(0); @@ -4106,6 +4303,14 @@ function captureRootRunTermination(input: { readonly beforeTerminalWrite?: ( request: Parameters[0], ) => Effect.Effect; + readonly beforeSubagentWrite?: ( + request: Parameters[0], + stored: { + readonly subagents: Map; + readonly nodes: Map; + readonly turnItems: Map; + }, + ) => Effect.Effect; readonly afterStart?: ( closed: Deferred.Deferred, ) => Effect.Effect; @@ -4137,7 +4342,11 @@ function captureRootRunTermination(input: { nodes: [...stored.nodes.values()].filter((row) => row.threadId === threadId), turnItems: [...stored.turnItems.values()].filter((row) => row.threadId === threadId), }; - return input.settleStored?.(work) ?? work; + const settled = input.settleStored?.(work) ?? work; + for (const row of settled.subagents) stored.subagents.set(row.id, row); + for (const row of settled.nodes) stored.nodes.set(row.id, row); + for (const row of settled.turnItems) stored.turnItems.set(row.id, row); + return settled; }); const providerInstanceId = ProviderInstanceId.make("codex"); const runningSubagent = makeRunOwnedSubagentFixture({ @@ -4209,11 +4418,31 @@ function captureRootRunTermination(input: { }), // Committed only against the row that was read, as the real conditional writes are. writeIfSubagentUnchanged: (payload) => - stored.subagents.get(payload.expected.id) === payload.expected - ? captureFinalEvents(payload.events).pipe( - Effect.as({ committed: true, storedEvents: [] }), - ) - : Effect.succeed({ committed: false, storedEvents: [] }), + Effect.gen(function* () { + if (input.beforeSubagentWrite !== undefined) { + yield* input.beforeSubagentWrite(payload, stored); + } + if ( + stored.subagents.get(payload.expected.id) !== payload.expected || + (payload.expectedNode !== undefined && + stored.nodes.get(payload.expectedNode.id) !== payload.expectedNode) + ) { + return { committed: false, storedEvents: [] }; + } + for (const event of payload.events) { + if (event.type === "subagent.updated") { + stored.subagents.set(event.payload.id, event.payload); + } + if (event.type === "node.updated") { + stored.nodes.set(event.payload.id, event.payload); + } + if (event.type === "turn-item.updated") { + stored.turnItems.set(event.payload.id, event.payload); + } + } + yield* captureFinalEvents(payload.events); + return { committed: true, storedEvents: [] }; + }), writeIfTurnItemUnchanged: (payload) => stored.turnItems.get(payload.expected.id) === payload.expected ? captureFinalEvents(payload.events).pipe( @@ -6493,6 +6722,24 @@ it.effect( // Run 2 detaches the legacy row and closes its subscription while Run 3 ingests the claim. yield* Deferred.await(ingestionDone); + const beforeRun3Projection = yield* projectionStore.getThreadProjection(ids.threadId); + const beforeRun3Nodes = beforeRun3Projection.nodes.filter((node) => node.kind === "subagent"); + const beforeRun3Items = beforeRun3Projection.turnItems.filter( + (item) => item.type === "subagent", + ); + assert.lengthOf(beforeRun3Nodes, 1); + assert.lengthOf(beforeRun3Projection.subagents, 1); + assert.lengthOf(beforeRun3Items, 1); + assert.equal(beforeRun3Nodes[0]?.id, legacyNodeId); + assert.equal(beforeRun3Nodes[0]?.runId, ids.runId); + assert.equal(beforeRun3Nodes[0]?.status, "running"); + assert.equal(beforeRun3Projection.subagents[0]?.id, legacyNodeId); + assert.equal(beforeRun3Projection.subagents[0]?.runId, ids.runId); + assert.equal(beforeRun3Projection.subagents[0]?.status, "running"); + assert.equal(beforeRun3Items[0]?.id, legacyItemId); + assert.equal(beforeRun3Items[0]?.runId, ids.runId); + assert.equal(beforeRun3Items[0]?.status, "running"); + for (const claimEvent of [run3ClaimNode, run3ClaimSubagent, run3ClaimTurnItem]) { yield* ingestor.ingestNormalized({ providerSessionId, diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 529ab54209b70..d94816cd1147f 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -228,6 +228,13 @@ function withLinkedChildThreadId( return { ...current, linkedChildThreadIds }; } +function isOwnedByRun( + row: { readonly runId: OrchestrationV2Run["id"] | null; readonly threadId: ThreadId }, + run: OrchestrationV2Run, +): boolean { + return row.runId === run.id || (row.threadId !== run.threadId && row.runId === null); +} + export const cascadeTerminalizeRunOwnedSubagents = Effect.fn( "RunExecutionService.cascadeTerminalizeRunOwnedSubagents", )(function* (input: { @@ -244,8 +251,13 @@ export const cascadeTerminalizeRunOwnedSubagents = Effect.fn( // Prefer lifetime linkage over currently-open rows: subagent/turn-item // snapshots may terminalize before the linked child-thread node settles. const childThreadIds = new Set(input.open.linkedChildThreadIds); - for (const item of [...input.open.subagents.values(), ...input.open.turnItems.values()]) { - if (item.childThreadId !== null) { + for (const subagent of input.open.subagents.values()) { + if (isOwnedByRun(subagent, input.run) && subagent.childThreadId !== null) { + childThreadIds.add(subagent.childThreadId); + } + } + for (const item of input.open.turnItems.values()) { + if (item.runId === input.run.id && item.childThreadId !== null) { childThreadIds.add(item.childThreadId); } } @@ -256,18 +268,20 @@ export const cascadeTerminalizeRunOwnedSubagents = Effect.fn( ]); for (const key of keys) { const subagent = input.open.subagents.get(key); - if (subagent !== undefined && !isSettledSubagentStatus(subagent.status)) { + const ownedSubagent = + subagent !== undefined && isOwnedByRun(subagent, input.run) ? subagent : undefined; + if (ownedSubagent !== undefined && !isSettledSubagentStatus(ownedSubagent.status)) { events.push({ id: yield* input.allocateEventId(), type: "subagent.updated", - threadId: subagent.threadId, + threadId: ownedSubagent.threadId, runId: input.run.id, - nodeId: subagent.id, - driver: subagent.driver, - providerInstanceId: subagent.providerInstanceId, + nodeId: ownedSubagent.id, + driver: ownedSubagent.driver, + providerInstanceId: ownedSubagent.providerInstanceId, occurredAt: input.completedAt, payload: { - ...subagent, + ...ownedSubagent, status: input.status, completedAt: input.completedAt, updatedAt: input.completedAt, @@ -277,7 +291,8 @@ export const cascadeTerminalizeRunOwnedSubagents = Effect.fn( const node = input.open.nodes.get(key); if ( node !== undefined && - (subagent !== undefined || + isOwnedByRun(node, input.run) && + (ownedSubagent !== undefined || (node.threadId === input.run.threadId && node.runId === input.run.id) || childThreadIds.has(node.threadId)) && isOpenExecutionNodeStatus(node.status) @@ -321,9 +336,11 @@ export const cascadeTerminalizeRunOwnedSubagents = Effect.fn( } } for (const turnItem of input.open.childTurnItems.values()) { - const ownedByOpenSubagent = - turnItem.type === "subagent" && input.open.subagents.has(turnItem.subagentId); + const owner = + turnItem.type === "subagent" ? input.open.subagents.get(turnItem.subagentId) : undefined; + const ownedByOpenSubagent = owner !== undefined && isOwnedByRun(owner, input.run); if ( + (turnItem.runId !== null && turnItem.runId !== input.run.id) || (!childThreadIds.has(turnItem.threadId) && !ownedByOpenSubagent) || isSettledTurnItemStatus(turnItem.status) ) { @@ -1342,11 +1359,9 @@ export const layer: Layer.Layer< const nodes = new Map(); const turnItems = new Map(); if (input.loadUnsettledWork === undefined) return { subagents, nodes, turnItems }; - const trackedItemIds = new Set([ - ...openRootToolCalls.keys(), - ...open.childTurnItems.keys(), - ...[...open.turnItems.values()].map((item) => item.id), - ]); + const trackedSubagentItemIds = new Set( + [...open.turnItems.values()].map((item) => item.id), + ); for (const threadId of new Set(threadIds)) { const stored = yield* input.loadUnsettledWork( threadId, @@ -1354,18 +1369,41 @@ export const layer: Layer.Layer< .filter((record) => record.threadId === threadId) .map((record) => record.id), ); - for (const subagent of stored.subagents) { - if (open.subagents.has(subagent.id) && !isSettledSubagentStatus(subagent.status)) { - subagents.set(subagent.id, subagent); - } - } + const foreignOwnedNodeIds = new Set(); for (const node of stored.nodes) { + if (!isOwnedByRun(node, input.run)) { + foreignOwnedNodeIds.add(node.id); + continue; + } if (open.nodes.has(node.id) && isOpenExecutionNodeStatus(node.status)) { nodes.set(node.id, node); } } + for (const subagent of stored.subagents) { + if ( + open.subagents.has(subagent.id) && + !foreignOwnedNodeIds.has(subagent.id) && + isOwnedByRun(subagent, input.run) && + !isSettledSubagentStatus(subagent.status) + ) { + subagents.set(subagent.id, subagent); + } + } for (const item of stored.turnItems) { - if (trackedItemIds.has(item.id) && !isSettledTurnItemStatus(item.status)) { + if (isSettledTurnItemStatus(item.status)) continue; + const trackedRootToolCall = + openRootToolCalls.has(item.id) && + item.threadId === input.run.threadId && + (item.runId === input.run.id || + (item.runId !== null && + inheritedBackgroundTurnItemsById.get(item.id) === item.runId)); + const trackedSubagentItem = + trackedSubagentItemIds.has(item.id) && item.runId === input.run.id; + const trackedChildItem = + open.childTurnItems.has(item.id) && + item.threadId !== input.run.threadId && + (item.runId === null || item.runId === input.run.id); + if (trackedRootToolCall || trackedSubagentItem || trackedChildItem) { turnItems.set(item.id, item); } } @@ -1379,83 +1417,83 @@ export const layer: Layer.Layer< ); // Subagents and tool calls that outlived the root run die with the provider runtime. Stop // can settle them meanwhile, so each write holds only while its stored row is unchanged. - const endLostWork = Effect.fn("RunExecutionService.endLostWork")( - function* (completedAt: DateTime.Utc, options: { readonly includeToolCalls: boolean }) { - const allocateEventId = () => - idAllocator.allocate.event({ threadId: input.run.threadId }); - const open = yield* Ref.get(openRunOwnedSubagents); - const stored = yield* loadTrackedWork([ - input.run.threadId, - ...[ - ...open.subagents.values(), - ...open.nodes.values(), - ...open.childTurnItems.values(), - ].map((record) => record.threadId), - ]); - for (const subagent of stored.subagents.values()) { - const subagentItemId = open.turnItems.get(subagent.id)?.id; - const subagentItem = - subagentItemId === undefined ? undefined : stored.turnItems.get(subagentItemId); - const events = yield* cascadeTerminalizeRunOwnedSubagents({ - run: input.run, - open: { - subagents: new Map([[subagent.id, subagent]]), - turnItems: new Map( - subagentItem?.type === "subagent" ? [[subagent.id, subagentItem]] : [], - ), - childTurnItems: new Map( - [...stored.turnItems].filter( - ([id, item]) => - open.childTurnItems.has(id) && - (item.threadId === subagent.childThreadId || - (item.type === "subagent" && item.subagentId === subagent.id)), - ), - ), - nodes: new Map( - [...stored.nodes].filter( - ([id, node]) => - id === subagent.id || node.threadId === subagent.childThreadId, - ), + const endLostWorkOnce = Effect.fn("RunExecutionService.endLostWork")(function* ( + completedAt: DateTime.Utc, + options: { readonly includeToolCalls: boolean }, + ) { + const allocateEventId = () => + idAllocator.allocate.event({ threadId: input.run.threadId }); + const open = yield* Ref.get(openRunOwnedSubagents); + const hasSubagents = open.subagents.size > 0; + const hasToolCalls = options.includeToolCalls && openRootToolCalls.size > 0; + if (!hasSubagents && !hasToolCalls) { + yield* Ref.set(openRunOwnedSubagents, emptyOpenRunOwnedSubagentProjection()); + return; + } + const stored = yield* loadTrackedWork([ + input.run.threadId, + ...[ + ...open.subagents.values(), + ...open.nodes.values(), + ...open.childTurnItems.values(), + ].map((record) => record.threadId), + ]); + for (const subagent of stored.subagents.values()) { + const subagentItemId = open.turnItems.get(subagent.id)?.id; + const subagentItem = + subagentItemId === undefined ? undefined : stored.turnItems.get(subagentItemId); + const events = yield* cascadeTerminalizeRunOwnedSubagents({ + run: input.run, + open: { + subagents: new Map([[subagent.id, subagent]]), + turnItems: new Map( + subagentItem?.type === "subagent" ? [[subagent.id, subagentItem]] : [], + ), + childTurnItems: new Map( + [...stored.turnItems].filter( + ([id, item]) => + open.childTurnItems.has(id) && + (item.threadId === subagent.childThreadId || + (item.type === "subagent" && item.subagentId === subagent.id)), ), - linkedChildThreadIds: new Set( - subagent.childThreadId === null ? [] : [subagent.childThreadId], + ), + nodes: new Map( + [...stored.nodes].filter( + ([id, node]) => id === subagent.id || node.threadId === subagent.childThreadId, ), - }, - status: "failed", - completedAt, - allocateEventId, + ), + linkedChildThreadIds: new Set( + subagent.childThreadId === null ? [] : [subagent.childThreadId], + ), + }, + status: "failed", + completedAt, + allocateEventId, + }); + const expectedNode = stored.nodes.get(subagent.id); + yield* eventSink.writeIfSubagentUnchanged({ + expected: subagent, + ...(expectedNode === undefined ? {} : { expectedNode }), + events, + }); + } + if (options.includeToolCalls) { + for (const toolCall of stored.turnItems.values()) { + if (!openRootToolCalls.has(toolCall.id)) continue; + yield* eventSink.writeIfTurnItemUnchanged({ + expected: toolCall, + events: yield* interruptLostToolCalls({ + run: input.run, + toolCalls: [toolCall], + completedAt, + allocateEventId, + }), }); - yield* eventSink.writeIfSubagentUnchanged({ expected: subagent, events }); - } - if (options.includeToolCalls) { - for (const toolCall of stored.turnItems.values()) { - if (!openRootToolCalls.has(toolCall.id)) continue; - yield* eventSink.writeIfTurnItemUnchanged({ - expected: toolCall, - events: yield* interruptLostToolCalls({ - run: input.run, - toolCalls: [toolCall], - completedAt, - allocateEventId, - }), - }); - } - openRootToolCalls.clear(); } - yield* Ref.set(openRunOwnedSubagents, emptyOpenRunOwnedSubagentProjection()); - }, - (effect) => - effect.pipe( - Effect.catchCauseIf( - (cause) => !Cause.hasInterruptsOnly(cause), - (cause) => - Effect.logWarning( - "orchestration V2 failed to end work lost with the provider runtime", - { runId: input.run.id, cause }, - ), - ), - ), - ); + openRootToolCalls.clear(); + } + yield* Ref.set(openRunOwnedSubagents, emptyOpenRunOwnedSubagentProjection()); + }); // Runs for every provider event, including streamed text updates; avoid a per-event span. const trackChildLifecycle = Effect.fnUntraced(function* ( event: ProviderAdapter.ProviderAdapterV2Event, @@ -2101,6 +2139,47 @@ export const layer: Layer.Layer< const routing = yield* Ref.get(eventRouting); const rootProviderTurn = yield* Ref.get(latestRootProviderTurn); const squashed = Cause.squash(cause); + const retryRecoveryWrite = ( + effect: Effect.Effect, + warningMessage: string, + ) => { + let nextRetryWarningAt = completedAt; + return effect.pipe( + // Storage failures can be defects; retry both failure channels. + Effect.catchCause((writeCause) => + Effect.gen(function* () { + if (Cause.hasInterruptsOnly(writeCause)) { + return yield* Effect.failCause(writeCause); + } + const now = yield* DateTime.now; + if (DateTime.isGreaterThanOrEqualTo(now, nextRetryWarningAt)) { + yield* Effect.logWarning(warningMessage, { + runId: input.run.id, + attemptId: input.attempt.id, + cause: writeCause, + }); + // Persistent storage failures must not flood the same disk. + nextRetryWarningAt = DateTime.add(now, { minutes: 10 }); + } + return yield* Effect.fail(writeCause); + }), + ), + Effect.retry({ + schedule: Schedule.exponential("1 second").pipe( + Schedule.modifyDelay(({ duration }) => + Effect.succeed(Duration.min(duration, Duration.seconds(30))), + ), + ), + }), + ); + }; + const runInRecoveryScope = (effect: Effect.Effect) => + effect.pipe( + // Startup may cancel the consumer and fail its own terminal + // write. Only service shutdown or the atomic gate ends recovery. + Effect.forkIn(scope), + Effect.flatMap(Fiber.join), + ); // An exclusive session closes when its thread detaches, and no terminal // follows. The process is gone, so a handoff settles here with no interrupt. if ( @@ -2137,7 +2216,12 @@ export const layer: Layer.Layer< ); if (handedOff) { // Its tool calls, the handoff's own among them, died with that process. - yield* endLostWork(completedAt, { includeToolCalls: true }); + yield* runInRecoveryScope( + retryRecoveryWrite( + endLostWorkOnce(completedAt, { includeToolCalls: true }), + "orchestration V2 lost-work cleanup retry", + ), + ); return; } } @@ -2152,7 +2236,14 @@ export const layer: Layer.Layer< isProviderEventStreamError(squashed.cause) && squashed.cause.raisedByT3 !== true; if (yield* Ref.get(rootRunFinalized)) { - if (runtimeLost) yield* endLostWork(completedAt, { includeToolCalls: true }); + if (runtimeLost) { + yield* runInRecoveryScope( + retryRecoveryWrite( + endLostWorkOnce(completedAt, { includeToolCalls: true }), + "orchestration V2 lost-work cleanup retry", + ), + ); + } return; } const providerThread = yield* Ref.get(latestProviderThread); @@ -2205,72 +2296,47 @@ export const layer: Layer.Layer< ), ) : []; - let nextRetryWarningAt = completedAt; const reloadSubagents = input.loadUnsettledWork !== undefined; - const finalized = yield* writeFinalRunEvents({ - run: input.run, - rootNode: input.rootNode, - checkpointScope: input.checkpointScope, - providerThread, - attempt: input.attempt, - writeIfRunCurrent: { - activeAttemptId: input.attempt.id, - expectedStatus: "running", - }, - ...(reloadSubagents ? {} : { openRunOwnedSubagents: openSubagents }), - ...(abandonedProviderTurn === null ? {} : { abandonedProviderTurn }), - lostToolCalls, - completedAt, - terminal: makeFailedTerminalEvent( - makeProviderFailure({ - cause: squashed, - message: - knownProviderFailureMessage(squashed) ?? - userFacingDatabaseErrorMessage(squashed), - class: "unknown", - }), - latestItemOrdinal + 1, - ), - failureItemPersisted: false, - refreshAfterTurn, - }).pipe( - // Storage failures can be defects; retry both failure channels. - Effect.catchCause((writeCause) => - Effect.gen(function* () { - if (Cause.hasInterruptsOnly(writeCause)) { - return yield* Effect.failCause(writeCause); - } - const now = yield* DateTime.now; - if (DateTime.isGreaterThanOrEqualTo(now, nextRetryWarningAt)) { - yield* Effect.logWarning( - "orchestration V2 run failure finalization retry", - { - runId: input.run.id, - attemptId: input.attempt.id, - cause: writeCause, - }, - ); - // Persistent storage failures must not flood the same disk. - nextRetryWarningAt = DateTime.add(now, { minutes: 10 }); - } - return yield* Effect.fail(writeCause); - }), - ), - Effect.retry({ - schedule: Schedule.exponential("1 second").pipe( - Schedule.modifyDelay(({ duration }) => - Effect.succeed(Duration.min(duration, Duration.seconds(30))), - ), - ), + yield* runInRecoveryScope( + Effect.gen(function* () { + const finalized = yield* retryRecoveryWrite( + writeFinalRunEvents({ + run: input.run, + rootNode: input.rootNode, + checkpointScope: input.checkpointScope, + providerThread, + attempt: input.attempt, + writeIfRunCurrent: { + activeAttemptId: input.attempt.id, + expectedStatus: "running", + }, + ...(reloadSubagents ? {} : { openRunOwnedSubagents: openSubagents }), + ...(abandonedProviderTurn === null ? {} : { abandonedProviderTurn }), + lostToolCalls, + completedAt, + terminal: makeFailedTerminalEvent( + makeProviderFailure({ + cause: squashed, + message: + knownProviderFailureMessage(squashed) ?? + userFacingDatabaseErrorMessage(squashed), + class: "unknown", + }), + latestItemOrdinal + 1, + ), + failureItemPersisted: false, + refreshAfterTurn, + }), + "orchestration V2 run failure finalization retry", + ); + if (finalized && reloadSubagents) { + yield* retryRecoveryWrite( + endLostWorkOnce(completedAt, { includeToolCalls: false }), + "orchestration V2 lost-subagent cleanup retry", + ); + } }), - // Startup may cancel the consumer and fail its own terminal - // write. Only service shutdown or the atomic gate ends recovery. - Effect.forkIn(scope), - Effect.flatMap(Fiber.join), ); - if (finalized && reloadSubagents) { - yield* endLostWork(completedAt, { includeToolCalls: false }); - } }), ), Effect.ensuring(endPreviewActivity), diff --git a/packages/provider-opencode/src/server/v2/adapter.ts b/packages/provider-opencode/src/server/v2/adapter.ts index 82431e24b147e..330cbcfdfdbbc 100644 --- a/packages/provider-opencode/src/server/v2/adapter.ts +++ b/packages/provider-opencode/src/server/v2/adapter.ts @@ -105,6 +105,28 @@ import { openCodeToolTurnItem, } from "../toolItems.ts"; +const SUBAGENT_PROMPT_WRAPPER = "You are a subagent spawned by another session.\n"; + +function takeMatchingPendingPrompt( + pendingPrompts: Array<{ readonly call: SubagentCall; readonly text: string }>, + text: string, +): string { + const unwrapped = text.startsWith(SUBAGENT_PROMPT_WRAPPER) + ? text.slice(SUBAGENT_PROMPT_WRAPPER.length) + : undefined; + const matchedTexts = new Set(); + let matchIndex = -1; + for (let i = 0; i < pendingPrompts.length; i += 1) { + const candidate = pendingPrompts[i]!.text; + if (candidate === text || (unwrapped !== undefined && candidate === unwrapped)) { + matchedTexts.add(candidate); + if (matchIndex === -1) matchIndex = i; + } + } + if (matchedTexts.size !== 1) return text; + return pendingPrompts.splice(matchIndex, 1)[0]!.text; +} + const NativeGoal = Schema.NullOr( Schema.Struct({ objective: Schema.NonEmptyString, @@ -3786,7 +3808,10 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr if (!next.done) { const [nextId, nextText] = next.value; state.subagent.steers.delete(nextId); - state.subagent.queuedPrompt = { inboxId: nextId, text: nextText }; + state.subagent.queuedPrompt = { + inboxId: nextId, + text: takeMatchingPendingPrompt(state.subagent.pendingPrompts, nextText), + }; state.subagent.queued ??= nextId; } } @@ -3821,16 +3846,9 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr const text = event.data.item.payload.text; if (state.subagent !== undefined) { if (state.active === undefined && state.subagent.queuedPrompt === undefined) { - const matchIndex = state.subagent.pendingPrompts.findIndex( - (entry) => text === entry.text || text.includes(entry.text), - ); - const matched = - matchIndex === -1 - ? undefined - : state.subagent.pendingPrompts.splice(matchIndex, 1)[0]; state.subagent.queuedPrompt = { inboxId: event.data.inboxID, - text: matched?.text ?? text, + text: takeMatchingPendingPrompt(state.subagent.pendingPrompts, text), }; } else { state.subagent.steers.set(event.data.inboxID, text); @@ -4170,11 +4188,6 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr return; } const active = { ...(yield* client.session.active()) }; - const attachedBefore = new Map( - [...threads.values()].flatMap((state) => - state.subagent === undefined ? [] : [[state, state.subagent.call] as const], - ), - ); for (const sessionId of busy) if (!(sessionId in active)) busy.delete(sessionId); for (const state of threads.values()) { if (state.unsettled && !(state.sessionId in active)) state.unsettledStarted = true; @@ -4216,13 +4229,25 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr }), ); } - // A child the parent's backfill found running started in the gap; its start is not replayed. + // A locally idle child listed as active started in the gap while a call awaited execution. for (const state of threads.values()) { - if (state.subagent === undefined || state.active !== undefined) continue; - if (attachedBefore.get(state) === state.subagent.call || !(state.sessionId in active)) { + if ( + state.subagent === undefined || + state.active !== undefined || + state.unsettled || + !(state.sessionId in active) + ) { continue; } - yield* lock.withPermit(startChildTurn(state)); + yield* lock.withPermit( + Effect.gen(function* () { + if (state.subagent === undefined || state.active !== undefined || state.unsettled) { + return; + } + if (!callsAttachedTo(state).some((call) => call.run === undefined)) return; + yield* startChildTurn(state); + }), + ); } for (const state of threads.values()) { yield* lock From f8a21169ceb2cdd39764874247ed54b03f596c1f Mon Sep 17 00:00:00 2001 From: Victor Vogel Date: Sat, 21 Nov 2026 17:50:57 +0000 Subject: [PATCH 6/6] fix(server): isolate nested subagent cleanup, guard turn items, and retry conflicts --- .../src/orchestration-v2/EventSink.test.ts | 118 +++++++- apps/server/src/orchestration-v2/EventSink.ts | 19 +- .../RunExecutionService.test.ts | 269 +++++++++++++++--- .../orchestration-v2/RunExecutionService.ts | 189 ++++++++---- 4 files changed, 491 insertions(+), 104 deletions(-) diff --git a/apps/server/src/orchestration-v2/EventSink.test.ts b/apps/server/src/orchestration-v2/EventSink.test.ts index 717f9ed6c01ab..7cbd32c11a5f7 100644 --- a/apps/server/src/orchestration-v2/EventSink.test.ts +++ b/apps/server/src/orchestration-v2/EventSink.test.ts @@ -8,9 +8,11 @@ import { ProviderInstanceId, RunId, ThreadId, + TurnItemId, type OrchestrationV2AppThread, type OrchestrationV2ExecutionNode, type OrchestrationV2Subagent, + type OrchestrationV2TurnItem, } from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; @@ -112,16 +114,18 @@ it.effect("rejects a planned fork whose source was deleted before it committed", }).pipe(Effect.provide(layer)), ); -it.effect("rejects writeIfSubagentUnchanged when expectedNode changed to another run", () => +it.effect("guards writeIfSubagentUnchanged against concurrent node and turn-item updates", () => Effect.gen(function* () { const sink = yield* EventSink.EventSinkV2; const projections = yield* ProjectionStore.ProjectionStoreV2; const now = yield* DateTime.now; const threadId = ThreadId.make("subagent-node-guard"); + const childThreadId = ThreadId.make("subagent-node-guard:child"); const run1Id = RunId.make("run:1"); const run2Id = RunId.make("run:2"); const rootNodeId = NodeId.make("node:root"); const subagentId = NodeId.make("node:subagent"); + const childTurnItemId = TurnItemId.make("turn-item:child-command"); const providerInstanceId = ProviderInstanceId.make("opencode"); const driver = ProviderDriverKind.make("opencode"); const thread: OrchestrationV2AppThread = { @@ -149,6 +153,16 @@ it.effect("rejects writeIfSubagentUnchanged when expectedNode changed to another snoozedAt: null, lastVisitedAt: null, }; + const childThread: OrchestrationV2AppThread = { + ...thread, + id: childThreadId, + title: "Child", + lineage: { + rootThreadId: threadId, + parentThreadId: threadId, + relationshipToParent: "subagent", + }, + }; const subagent: OrchestrationV2Subagent = { id: subagentId, threadId, @@ -159,7 +173,7 @@ it.effect("rejects writeIfSubagentUnchanged when expectedNode changed to another driver, providerInstanceId, providerThreadId: null, - childThreadId: null, + childThreadId, nativeTaskRef: null, prompt: "work", title: "Worker", @@ -187,6 +201,24 @@ it.effect("rejects writeIfSubagentUnchanged when expectedNode changed to another startedAt: now, completedAt: null, }; + const childTurnItem: OrchestrationV2TurnItem = { + id: childTurnItemId, + threadId: childThreadId, + runId: null, + nodeId: null, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 1, + status: "running", + title: "Running command", + startedAt: now, + completedAt: null, + updatedAt: now, + type: "command_execution", + input: "echo ok", + }; yield* sink.write({ events: [ { @@ -196,6 +228,13 @@ it.effect("rejects writeIfSubagentUnchanged when expectedNode changed to another occurredAt: now, payload: thread, }, + { + id: EventId.make("subagent-guard:child-thread"), + type: "thread.created", + threadId: childThreadId, + occurredAt: now, + payload: childThread, + }, { id: EventId.make("subagent-guard:subagent"), type: "subagent.updated", @@ -217,6 +256,15 @@ it.effect("rejects writeIfSubagentUnchanged when expectedNode changed to another occurredAt: now, payload: node, }, + { + id: EventId.make("subagent-guard:child-item"), + type: "turn-item.updated", + threadId: childThreadId, + runId: run1Id, + providerInstanceId, + occurredAt: now, + payload: childTurnItem, + }, ], }); const claimedNode: OrchestrationV2ExecutionNode = { ...node, runId: run2Id }; @@ -234,9 +282,10 @@ it.effect("rejects writeIfSubagentUnchanged when expectedNode changed to another }, ], }); - const result = yield* sink.writeIfSubagentUnchanged({ + const nodeRejected = yield* sink.writeIfSubagentUnchanged({ expected: subagent, - expectedNode: node, + expectedNodes: [node], + expectedTurnItems: [childTurnItem], events: [ { id: EventId.make("subagent-guard:fail-subagent"), @@ -261,15 +310,69 @@ it.effect("rejects writeIfSubagentUnchanged when expectedNode changed to another }, ], }); - assert.isFalse(result.committed); + assert.isFalse(nodeRejected.committed); const projection = yield* projections.getThreadProjection(threadId); assert.equal(projection.subagents[0]?.status, "running"); assert.equal(projection.nodes[0]?.runId, run2Id); assert.equal(projection.nodes[0]?.status, "running"); + // A concurrent child turn-item completion also rejects the batch in the same transaction. + const completedChildItem: OrchestrationV2TurnItem = { + ...childTurnItem, + status: "completed", + title: "Finished command", + completedAt: now, + updatedAt: now, + }; + yield* sink.write({ + events: [ + { + id: EventId.make("subagent-guard:complete-child-item"), + type: "turn-item.updated", + threadId: childThreadId, + runId: run1Id, + providerInstanceId, + occurredAt: now, + payload: completedChildItem, + }, + ], + }); + const itemRejected = yield* sink.writeIfSubagentUnchanged({ + expected: subagent, + expectedNodes: [claimedNode], + expectedTurnItems: [childTurnItem], + events: [ + { + id: EventId.make("subagent-guard:fail-subagent-stale-item"), + type: "subagent.updated", + threadId, + runId: run1Id, + nodeId: subagentId, + driver, + providerInstanceId, + occurredAt: now, + payload: { ...subagent, status: "failed", completedAt: now, updatedAt: now }, + }, + { + id: EventId.make("subagent-guard:fail-child-item-stale"), + type: "turn-item.updated", + threadId: childThreadId, + runId: run1Id, + providerInstanceId, + occurredAt: now, + payload: { ...childTurnItem, status: "failed", completedAt: now, updatedAt: now }, + }, + ], + }); + assert.isFalse(itemRejected.committed); + const childBeforeAccept = yield* projections.getThreadProjection(childThreadId); + assert.equal(childBeforeAccept.turnItems[0]?.status, "completed"); + assert.equal(childBeforeAccept.turnItems[0]?.title, "Finished command"); + const accepted = yield* sink.writeIfSubagentUnchanged({ expected: subagent, - expectedNode: claimedNode, + expectedNodes: [claimedNode], + expectedTurnItems: [completedChildItem], events: [ { id: EventId.make("subagent-guard:fail-subagent-accepted"), @@ -289,5 +392,8 @@ it.effect("rejects writeIfSubagentUnchanged when expectedNode changed to another assert.equal(afterAccept.subagents[0]?.status, "failed"); assert.equal(afterAccept.nodes[0]?.runId, run2Id); assert.equal(afterAccept.nodes[0]?.status, "running"); + const childAfterAccept = yield* projections.getThreadProjection(childThreadId); + assert.equal(childAfterAccept.turnItems[0]?.status, "completed"); + assert.equal(childAfterAccept.turnItems[0]?.title, "Finished command"); }).pipe(Effect.provide(layer)), ); diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index 8327b8813ae66..70c5b2148ebcf 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -194,10 +194,11 @@ export interface EventSinkV2Shape { }, EventSinkV2Error >; - /** Commits only if the stored subagent (and optional node) still equal expected. */ + /** Commits only if the stored subagent (and optional nodes and turn items) still equal expected. */ readonly writeIfSubagentUnchanged: (input: { readonly expected: OrchestrationV2Subagent; - readonly expectedNode?: OrchestrationV2ExecutionNode; + readonly expectedNodes?: ReadonlyArray; + readonly expectedTurnItems?: ReadonlyArray; readonly events: ReadonlyArray; }) => Effect.Effect< { @@ -788,8 +789,7 @@ const layerBase: Layer.Layer< expectedJson, }, ]; - if (input.expectedNode !== undefined) { - const expectedNode = input.expectedNode; + for (const expectedNode of input.expectedNodes ?? []) { checks.push({ read: sql<{ readonly payload_json: string }>` SELECT payload_json @@ -800,6 +800,17 @@ const layerBase: Layer.Layer< expectedJson: yield* ProjectionStore.encodeNodePayload(expectedNode), }); } + for (const expectedTurnItem of input.expectedTurnItems ?? []) { + checks.push({ + read: sql<{ readonly payload_json: string }>` + SELECT payload_json + FROM orchestration_v2_projection_turn_items + WHERE turn_item_id = ${expectedTurnItem.id} + LIMIT 1 + `, + expectedJson: yield* ProjectionStore.encodeTurnItemPayload(expectedTurnItem), + }); + } return yield* writeIfStoredPayloadsMatch(checks, input.events); }); diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index e3ac0c6f35930..6f12f11670683 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -3638,12 +3638,20 @@ it.effect.each([ { ended: false, rootLive: true, settledStatus: "completed" as const }, // Moving child C to another run keeps nested subagent G owned by this run for lost-work cleanup. { ended: true, rootLive: true, handedOffWithNested: true }, + // Losing the runtime while both C and nested G are open isolates G from C's batch while keeping C's own child work. + { ended: true, rootLive: true, nestedOpen: true }, // A legacy subagent claimed by a newer run under the same ID is not failed by the older run's cascade. { ended: false, rootLive: true, claimedByOtherRun: true }, // Even if only the shared node has been claimed by a newer run, the older run skips the subagent. { ended: false, rootLive: true, nodeClaimedByOtherRun: true }, - // If the shared node is claimed between reload and conditional write, expectedNode rejects the write. + // If the shared node is claimed between reload and conditional write, expectedNodes rejects the write. { ended: false, rootLive: true, nodeClaimedBeforeWrite: true }, + // If only the shared subagent turn item is claimed before write, expectedTurnItems rejects and retry skips the subagent. + { ended: false, rootLive: true, itemClaimedBeforeWrite: true }, + // If the node completes before write while the subagent stays running, cleanup reloads and settles without rewriting the node. + { ended: true, rootLive: true, nodeCompletedBeforeWrite: true }, + // If a child command item completes before write, expectedTurnItems rejects the stale write and retry preserves the completed item. + { ended: true, rootLive: true, childItemCompletedBeforeWrite: true }, ] as const)( "ends a subagent that outlived its root when the provider stream fails (%o)", ({ @@ -3653,9 +3661,13 @@ it.effect.each([ rootLive, settledStatus, handedOffWithNested, + nestedOpen, claimedByOtherRun, nodeClaimedByOtherRun, nodeClaimedBeforeWrite, + itemClaimedBeforeWrite, + nodeCompletedBeforeWrite, + childItemCompletedBeforeWrite, }) => Effect.gen(function* () { const providerInstanceId = ProviderInstanceId.make("codex"); @@ -3664,7 +3676,16 @@ it.effect.each([ const grandchildItemId = TurnItemId.make( "turn-item:stream-error-outlived-subagent:grandchild", ); - const { finalEvents } = yield* captureRootRunTermination({ + const childWorkNodeId = NodeId.make("node:stream-error-outlived-subagent:child-work"); + let writeAttempts = 0; + const batchWriteIds: Array<{ + readonly subagentId: NodeId; + readonly nodeIds: ReadonlyArray; + readonly turnItemIds: ReadonlyArray; + readonly expectedNodeIds: ReadonlyArray; + readonly expectedTurnItemIds: ReadonlyArray; + }> = []; + const { ids, stored, finalEvents } = yield* captureRootRunTermination({ key: "stream-error-outlived-subagent", shouldFinalizeRun: () => Effect.succeed(true), events: (ids) => { @@ -3689,7 +3710,10 @@ it.effect.each([ if ( claimedByOtherRun === true || nodeClaimedByOtherRun === true || - nodeClaimedBeforeWrite === true + nodeClaimedBeforeWrite === true || + itemClaimedBeforeWrite === true || + nodeCompletedBeforeWrite === true || + childItemCompletedBeforeWrite === true ) { events.push({ type: "node.updated", @@ -3697,13 +3721,50 @@ it.effect.each([ node: makeRunOwnedSubagentNodeFixture({ ids, status: "running" }), }); } - if (handedOffWithNested === true) { + if (childItemCompletedBeforeWrite === true) { + events.push({ + type: "turn_item.updated", + driver, + turnItem: makeLinkedChildTurnItemFixture({ + ids, + driver, + type: "command_execution", + }), + }); + } + if (handedOffWithNested === true || nestedOpen === true) { events.push( { type: "node.updated", driver, node: makeRunOwnedSubagentNodeFixture({ ids, status: "running" }), }, + ...(nestedOpen === true + ? ([ + { + type: "node.updated", + driver, + node: { + ...makeRunOwnedSubagentNodeFixture({ ids, status: "running" }), + id: childWorkNodeId, + threadId: ids.childThreadId, + runId: null, + kind: "root_turn", + parentNodeId: baseSubagent.id, + rootNodeId: childWorkNodeId, + }, + }, + { + type: "turn_item.updated", + driver, + turnItem: makeLinkedChildTurnItemFixture({ + ids, + driver, + type: "command_execution", + }), + }, + ] as const) + : []), { type: "subagent.updated", driver, @@ -3741,11 +3802,15 @@ it.effect.each([ childThreadId: grandchildThreadId, }, }, - { - type: "subagent.updated", - driver, - subagent: { ...baseSubagent, runId: RunId.make("run:other-owner") }, - }, + ...(handedOffWithNested === true + ? ([ + { + type: "subagent.updated", + driver, + subagent: { ...baseSubagent, runId: RunId.make("run:other-owner") }, + }, + ] as const) + : []), ); } if (rootLive !== true) { @@ -3766,13 +3831,61 @@ it.effect.each([ ), ); }, - ...(nodeClaimedBeforeWrite === true + ...(nodeClaimedBeforeWrite === true || + itemClaimedBeforeWrite === true || + nodeCompletedBeforeWrite === true || + childItemCompletedBeforeWrite === true || + nestedOpen === true ? { beforeSubagentWrite: (payload, stored) => Effect.sync(() => { - const node = stored.nodes.get(payload.expected.id); - if (node !== undefined) { - stored.nodes.set(node.id, { ...node, runId: RunId.make("run:other-owner") }); + writeAttempts++; + batchWriteIds.push({ + subagentId: payload.expected.id, + nodeIds: payload.events.flatMap((event) => + event.type === "node.updated" ? [event.payload.id] : [], + ), + turnItemIds: payload.events.flatMap((event) => + event.type === "turn-item.updated" ? [event.payload.id] : [], + ), + expectedNodeIds: (payload.expectedNodes ?? []).map((node) => node.id), + expectedTurnItemIds: (payload.expectedTurnItems ?? []).map((item) => item.id), + }); + if (writeAttempts > 1) return; + if (nodeClaimedBeforeWrite === true) { + const node = stored.nodes.get(payload.expected.id); + if (node !== undefined) { + stored.nodes.set(node.id, { ...node, runId: RunId.make("run:other-owner") }); + } + } + if (itemClaimedBeforeWrite === true) { + const subagentItem = [...stored.turnItems.values()].find( + (item) => item.type === "subagent" && item.subagentId === payload.expected.id, + ); + if (subagentItem !== undefined) { + stored.turnItems.set(subagentItem.id, { + ...subagentItem, + runId: RunId.make("run:other-owner"), + }); + } + } + if (nodeCompletedBeforeWrite === true) { + const node = stored.nodes.get(payload.expected.id); + if (node !== undefined) { + stored.nodes.set(node.id, { ...node, status: "completed" }); + } + } + if (childItemCompletedBeforeWrite === true) { + const childItem = [...stored.turnItems.values()].find( + (item) => item.type === "command_execution", + ); + if (childItem !== undefined) { + stored.turnItems.set(childItem.id, { + ...childItem, + status: "completed", + title: "Completed output", + }); + } } }), } @@ -3808,6 +3921,11 @@ it.effect.each([ } : {}), }); + const detailedIds = + handedOffWithNested === true || + nestedOpen === true || + nodeCompletedBeforeWrite === true || + childItemCompletedBeforeWrite === true; const expected = handedOffWithNested === true ? [ @@ -3816,34 +3934,65 @@ it.effect.each([ `node.updated:failed:${grandchildId}`, `turn-item.updated:failed:${grandchildItemId}`, ] - : rootLive === true - ? ["run.updated:failed"] - : ended - ? ["run.updated:waiting", "subagent.updated:failed", "turn-item.updated:failed"] - : ["run.updated:waiting"]; + : nestedOpen === true + ? [ + "run.updated:failed", + `subagent.updated:failed:${ids.subagentNodeId}`, + `node.updated:failed:${ids.subagentNodeId}`, + `turn-item.updated:failed:${ids.itemId}`, + `node.updated:failed:${childWorkNodeId}`, + `turn-item.updated:failed:${ids.childItemId}`, + `subagent.updated:failed:${grandchildId}`, + `node.updated:failed:${grandchildId}`, + `turn-item.updated:failed:${grandchildItemId}`, + ] + : nodeCompletedBeforeWrite === true + ? [ + "run.updated:failed", + `subagent.updated:failed:${ids.subagentNodeId}`, + `turn-item.updated:failed:${ids.itemId}`, + ] + : childItemCompletedBeforeWrite === true + ? [ + "run.updated:failed", + `subagent.updated:failed:${ids.subagentNodeId}`, + `node.updated:failed:${ids.subagentNodeId}`, + `turn-item.updated:failed:${ids.itemId}`, + ] + : rootLive === true + ? ["run.updated:failed"] + : ended + ? ["run.updated:waiting", "subagent.updated:failed", "turn-item.updated:failed"] + : ["run.updated:waiting"]; assert.deepEqual( finalEvents.flatMap((event) => { if (event.type === "run.updated") return [`${event.type}:${event.payload.status}`]; if (event.type === "subagent.updated") { return [ - handedOffWithNested === true + detailedIds ? `${event.type}:${event.payload.status}:${event.payload.id}` : `${event.type}:${event.payload.status}`, ]; } if ( - (handedOffWithNested === true || + (detailedIds || claimedByOtherRun === true || nodeClaimedByOtherRun === true || - nodeClaimedBeforeWrite === true) && + nodeClaimedBeforeWrite === true || + itemClaimedBeforeWrite === true) && event.type === "node.updated" && - event.payload.kind === "subagent" + event.payload.id !== ids.rootNodeId ) { return [`${event.type}:${event.payload.status}:${event.payload.id}`]; } - if (event.type === "turn-item.updated" && event.payload.type === "subagent") { + if ( + event.type === "turn-item.updated" && + (event.payload.type === "subagent" || + ((nestedOpen === true || childItemCompletedBeforeWrite === true) && + event.payload.type === "command_execution")) + ) { return [ - handedOffWithNested === true + detailedIds ? `${event.type}:${event.payload.status}:${event.payload.id}` : `${event.type}:${event.payload.status}`, ]; @@ -3852,6 +4001,49 @@ it.effect.each([ }), expected, ); + if (nestedOpen === true) { + assert.deepEqual(batchWriteIds, [ + { + subagentId: ids.subagentNodeId, + nodeIds: [ids.subagentNodeId, childWorkNodeId], + turnItemIds: [ids.itemId, ids.childItemId], + expectedNodeIds: [ids.subagentNodeId, childWorkNodeId], + expectedTurnItemIds: [ids.itemId, ids.childItemId], + }, + { + subagentId: grandchildId, + nodeIds: [grandchildId], + turnItemIds: [grandchildItemId], + expectedNodeIds: [grandchildId], + expectedTurnItemIds: [grandchildItemId], + }, + ]); + } + if (nodeClaimedBeforeWrite === true || itemClaimedBeforeWrite === true) { + assert.equal(writeAttempts, 1); + assert.equal(stored.subagents.get(ids.subagentNodeId)?.status, "running"); + assert.equal(stored.nodes.get(ids.subagentNodeId)?.status, "running"); + assert.equal(stored.turnItems.get(ids.itemId)?.status, "running"); + if (itemClaimedBeforeWrite === true) { + assert.equal(stored.turnItems.get(ids.itemId)?.runId, RunId.make("run:other-owner")); + } + } + if (nodeCompletedBeforeWrite === true) { + assert.equal(writeAttempts, 2); + assert.deepEqual(batchWriteIds[1]?.nodeIds, []); + assert.deepEqual(batchWriteIds[1]?.expectedNodeIds, [ids.subagentNodeId]); + assert.equal(stored.nodes.get(ids.subagentNodeId)?.status, "completed"); + assert.equal(stored.subagents.get(ids.subagentNodeId)?.status, "failed"); + assert.equal(stored.turnItems.get(ids.itemId)?.status, "failed"); + } + if (childItemCompletedBeforeWrite === true) { + assert.equal(writeAttempts, 2); + assert.deepEqual(batchWriteIds[0]?.expectedTurnItemIds, [ids.itemId, ids.childItemId]); + assert.deepEqual(batchWriteIds[1]?.turnItemIds, [ids.itemId]); + assert.equal(stored.turnItems.get(ids.childItemId)?.status, "completed"); + assert.equal(stored.turnItems.get(ids.childItemId)?.title, "Completed output"); + assert.equal(stored.subagents.get(ids.subagentNodeId)?.status, "failed"); + } }), ); @@ -4416,7 +4608,7 @@ function captureRootRunTermination(input: { yield* captureFinalEvents(payload.events); return { committed: true, storedEvents: [] }; }), - // Committed only against the row that was read, as the real conditional writes are. + // Committed only against the rows that were read, as the real conditional writes are. writeIfSubagentUnchanged: (payload) => Effect.gen(function* () { if (input.beforeSubagentWrite !== undefined) { @@ -4424,8 +4616,12 @@ function captureRootRunTermination(input: { } if ( stored.subagents.get(payload.expected.id) !== payload.expected || - (payload.expectedNode !== undefined && - stored.nodes.get(payload.expectedNode.id) !== payload.expectedNode) + (payload.expectedNodes ?? []).some( + (expectedNode) => stored.nodes.get(expectedNode.id) !== expectedNode, + ) || + (payload.expectedTurnItems ?? []).some( + (expectedItem) => stored.turnItems.get(expectedItem.id) !== expectedItem, + ) ) { return { committed: false, storedEvents: [] }; } @@ -4444,11 +4640,18 @@ function captureRootRunTermination(input: { return { committed: true, storedEvents: [] }; }), writeIfTurnItemUnchanged: (payload) => - stored.turnItems.get(payload.expected.id) === payload.expected - ? captureFinalEvents(payload.events).pipe( - Effect.as({ committed: true, storedEvents: [] }), - ) - : Effect.succeed({ committed: false, storedEvents: [] }), + Effect.gen(function* () { + if (stored.turnItems.get(payload.expected.id) !== payload.expected) { + return { committed: false, storedEvents: [] }; + } + for (const event of payload.events) { + if (event.type === "turn-item.updated") { + stored.turnItems.set(event.payload.id, event.payload); + } + } + yield* captureFinalEvents(payload.events); + return { committed: true, storedEvents: [] }; + }), }), IdAllocator.layer, ThreadCommandExecutor.layer, @@ -4588,6 +4791,8 @@ function captureRootRunTermination(input: { }).pipe(Effect.provide(layerTest)); return { + ids, + stored, previewActivityEnded, written: yield* Ref.get(writtenItems), finalEvents: yield* Ref.get(finalEvents), diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index d94816cd1147f..e81199c6fee74 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -1357,8 +1357,12 @@ export const layer: Layer.Layer< const open = yield* Ref.get(openRunOwnedSubagents); const subagents = new Map(); const nodes = new Map(); + // Owned node per tracked subagent, even if settled, so a concurrent claim still conflicts. + const guardedSubagentNodes = new Map(); const turnItems = new Map(); - if (input.loadUnsettledWork === undefined) return { subagents, nodes, turnItems }; + if (input.loadUnsettledWork === undefined) { + return { subagents, nodes, guardedSubagentNodes, turnItems }; + } const trackedSubagentItemIds = new Set( [...open.turnItems.values()].map((item) => item.id), ); @@ -1369,28 +1373,29 @@ export const layer: Layer.Layer< .filter((record) => record.threadId === threadId) .map((record) => record.id), ); - const foreignOwnedNodeIds = new Set(); + const foreignOwnedSubagentIds = new Set(); for (const node of stored.nodes) { if (!isOwnedByRun(node, input.run)) { - foreignOwnedNodeIds.add(node.id); + foreignOwnedSubagentIds.add(node.id); continue; } + if (open.subagents.has(node.id)) { + guardedSubagentNodes.set(node.id, node); + } if (open.nodes.has(node.id) && isOpenExecutionNodeStatus(node.status)) { nodes.set(node.id, node); } } - for (const subagent of stored.subagents) { + for (const item of stored.turnItems) { + if (isSettledTurnItemStatus(item.status)) continue; if ( - open.subagents.has(subagent.id) && - !foreignOwnedNodeIds.has(subagent.id) && - isOwnedByRun(subagent, input.run) && - !isSettledSubagentStatus(subagent.status) + trackedSubagentItemIds.has(item.id) && + item.type === "subagent" && + !isOwnedByRun(item, input.run) ) { - subagents.set(subagent.id, subagent); + foreignOwnedSubagentIds.add(item.subagentId); + continue; } - } - for (const item of stored.turnItems) { - if (isSettledTurnItemStatus(item.status)) continue; const trackedRootToolCall = openRootToolCalls.has(item.id) && item.threadId === input.run.threadId && @@ -1407,8 +1412,18 @@ export const layer: Layer.Layer< turnItems.set(item.id, item); } } + for (const subagent of stored.subagents) { + if ( + open.subagents.has(subagent.id) && + !foreignOwnedSubagentIds.has(subagent.id) && + isOwnedByRun(subagent, input.run) && + !isSettledSubagentStatus(subagent.status) + ) { + subagents.set(subagent.id, subagent); + } + } } - return { subagents, nodes, turnItems }; + return { subagents, nodes, guardedSubagentNodes, turnItems }; }); const loadLostRootToolCalls = loadTrackedWork([input.run.threadId]).pipe( Effect.map(({ turnItems }) => @@ -1416,7 +1431,7 @@ export const layer: Layer.Layer< ), ); // Subagents and tool calls that outlived the root run die with the provider runtime. Stop - // can settle them meanwhile, so each write holds only while its stored row is unchanged. + // can settle them meanwhile, so each write holds only while its stored rows are unchanged. const endLostWorkOnce = Effect.fn("RunExecutionService.endLostWork")(function* ( completedAt: DateTime.Utc, options: { readonly includeToolCalls: boolean }, @@ -1430,66 +1445,116 @@ export const layer: Layer.Layer< yield* Ref.set(openRunOwnedSubagents, emptyOpenRunOwnedSubagentProjection()); return; } - const stored = yield* loadTrackedWork([ + const trackedThreadIds = [ input.run.threadId, ...[ ...open.subagents.values(), ...open.nodes.values(), ...open.childTurnItems.values(), ].map((record) => record.threadId), - ]); - for (const subagent of stored.subagents.values()) { - const subagentItemId = open.turnItems.get(subagent.id)?.id; - const subagentItem = - subagentItemId === undefined ? undefined : stored.turnItems.get(subagentItemId); - const events = yield* cascadeTerminalizeRunOwnedSubagents({ - run: input.run, - open: { - subagents: new Map([[subagent.id, subagent]]), - turnItems: new Map( - subagentItem?.type === "subagent" ? [[subagent.id, subagentItem]] : [], + ]; + const maxConflictPasses = 4; + let conflictedSubagents = 0; + let conflictedToolCalls = 0; + for (let pass = 0; pass < maxConflictPasses; pass++) { + const stored = yield* loadTrackedWork(trackedThreadIds); + conflictedSubagents = 0; + conflictedToolCalls = 0; + for (const subagent of stored.subagents.values()) { + const subagentItemId = open.turnItems.get(subagent.id)?.id; + const rawSubagentItem = + subagentItemId === undefined ? undefined : stored.turnItems.get(subagentItemId); + const subagentItem = + rawSubagentItem?.type === "subagent" ? rawSubagentItem : undefined; + const batchChildTurnItems = new Map( + [...stored.turnItems].filter( + ([id, item]) => + open.childTurnItems.has(id) && + id !== subagentItem?.id && + (item.threadId === subagent.childThreadId || + (item.type === "subagent" && item.subagentId === subagent.id)) && + !( + item.type === "subagent" && + item.subagentId !== subagent.id && + open.subagents.has(item.subagentId) + ), ), - childTurnItems: new Map( - [...stored.turnItems].filter( - ([id, item]) => - open.childTurnItems.has(id) && - (item.threadId === subagent.childThreadId || - (item.type === "subagent" && item.subagentId === subagent.id)), - ), + ); + const batchNodes = new Map( + [...stored.nodes].filter( + ([id, node]) => + id === subagent.id || + (subagent.childThreadId !== null && + node.threadId === subagent.childThreadId && + !open.subagents.has(id)), ), - nodes: new Map( - [...stored.nodes].filter( - ([id, node]) => id === subagent.id || node.threadId === subagent.childThreadId, + ); + const events = yield* cascadeTerminalizeRunOwnedSubagents({ + run: input.run, + open: { + subagents: new Map([[subagent.id, subagent]]), + turnItems: new Map( + subagentItem === undefined ? [] : [[subagent.id, subagentItem]], ), - ), - linkedChildThreadIds: new Set( - subagent.childThreadId === null ? [] : [subagent.childThreadId], - ), - }, - status: "failed", - completedAt, - allocateEventId, - }); - const expectedNode = stored.nodes.get(subagent.id); - yield* eventSink.writeIfSubagentUnchanged({ - expected: subagent, - ...(expectedNode === undefined ? {} : { expectedNode }), - events, + childTurnItems: batchChildTurnItems, + nodes: batchNodes, + linkedChildThreadIds: new Set( + subagent.childThreadId === null ? [] : [subagent.childThreadId], + ), + }, + status: "failed", + completedAt, + allocateEventId, + }); + const guardedNode = stored.guardedSubagentNodes.get(subagent.id); + const expectedNodes = + guardedNode !== undefined && !batchNodes.has(guardedNode.id) + ? [...batchNodes.values(), guardedNode] + : [...batchNodes.values()]; + const expectedTurnItems = [ + ...(subagentItem === undefined ? [] : [subagentItem]), + ...batchChildTurnItems.values(), + ]; + const result = yield* eventSink.writeIfSubagentUnchanged({ + expected: subagent, + expectedNodes, + expectedTurnItems, + events, + }); + if (!result.committed) { + conflictedSubagents++; + } + } + if (options.includeToolCalls) { + for (const toolCall of stored.turnItems.values()) { + if (!openRootToolCalls.has(toolCall.id)) continue; + const result = yield* eventSink.writeIfTurnItemUnchanged({ + expected: toolCall, + events: yield* interruptLostToolCalls({ + run: input.run, + toolCalls: [toolCall], + completedAt, + allocateEventId, + }), + }); + if (!result.committed) { + conflictedToolCalls++; + } + } + } + if (conflictedSubagents === 0 && conflictedToolCalls === 0) { + break; + } + } + if (conflictedSubagents > 0 || conflictedToolCalls > 0) { + yield* Effect.logWarning("orchestration V2 lost-work cleanup conflict", { + runId: input.run.id, + attemptId: input.attempt.id, + conflictedSubagents, + conflictedToolCalls, }); } if (options.includeToolCalls) { - for (const toolCall of stored.turnItems.values()) { - if (!openRootToolCalls.has(toolCall.id)) continue; - yield* eventSink.writeIfTurnItemUnchanged({ - expected: toolCall, - events: yield* interruptLostToolCalls({ - run: input.run, - toolCalls: [toolCall], - completedAt, - allocateEventId, - }), - }); - } openRootToolCalls.clear(); } yield* Ref.set(openRunOwnedSubagents, emptyOpenRunOwnedSubagentProjection());