From 6b34318298c12c9e6b0327badd03790e02c46390 Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Tue, 17 Nov 2026 22:36:31 +0100 Subject: [PATCH 1/5] remote delegate: record and follow a launched child even if the call is interrupted (#386 PR 1) Co-Authored-By: Claude Opus 5.5 --- apps/server/src/peer/RemoteDelegation.test.ts | 130 +++++++++++- apps/server/src/peer/RemoteDelegation.ts | 192 +++++++++--------- 2 files changed, 226 insertions(+), 96 deletions(-) diff --git a/apps/server/src/peer/RemoteDelegation.test.ts b/apps/server/src/peer/RemoteDelegation.test.ts index eeef7bf05a36e..2665894237549 100644 --- a/apps/server/src/peer/RemoteDelegation.test.ts +++ b/apps/server/src/peer/RemoteDelegation.test.ts @@ -25,6 +25,7 @@ import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; @@ -647,6 +648,11 @@ const makeLaptop = ( kept?: Context.Context>>, /** Runs as each follow attempt here starts, on its follower's fiber. */ beforeFollow: Effect.Effect = Effect.void, + /** Run on a delegating call's fiber: before it looks for the project there, and before it records its task. */ + pauses: { + readonly beforeResolve?: Effect.Effect; + readonly beforeRecord?: Effect.Effect; + } = {}, ) => Effect.gen(function* () { const linking = kept ?? (yield* layerLinkingEnvironment(laptop).pipe(Layer.build)); @@ -684,6 +690,11 @@ const makeLaptop = ( ...threads, getThreadEventSequence: (threadId: ThreadId) => beforeFollow.pipe(Effect.andThen(threads.getThreadEventSequence(threadId))), + dispatch: (command) => + (command.type === "delegated_task.remote.request" + ? (pauses.beforeRecord ?? Effect.void) + : Effect.void + ).pipe(Effect.andThen(threads.dispatch(command))), }; }), ), @@ -702,10 +713,12 @@ const makeLaptop = ( Layer.provide( Layer.mock(RepositoryIdentityResolver.RepositoryIdentityResolver)({ resolve: (cwd) => - Effect.succeed( - cwd === "/home/me/app" - ? ({ canonicalKey: "github.com/acme/app" } as RepositoryIdentity) - : null, + (pauses.beforeResolve ?? Effect.void).pipe( + Effect.as( + cwd === "/home/me/app" + ? ({ canonicalKey: "github.com/acme/app" } as RepositoryIdentity) + : null, + ), ), }), ), @@ -1309,6 +1322,115 @@ it.effect("refuses worktree isolation for a task there before launching anything ).pipe(Effect.provide(NodeHttpServer.layerTest)), ); +/** A point a delegating call stops at until the test lets it go on. */ +const makePause = Effect.gen(function* () { + const reached = yield* Deferred.make(); + const release = yield* Deferred.make(); + return { + reached, + release, + hold: Deferred.succeed(reached, undefined).pipe(Effect.andThen(Deferred.await(release))), + }; +}); + +/** + * Delegates to the box and interrupts the call at `pause`, as the parent's provider does when + * its turn moves on mid-call (a message steered into it), then lets the call go on. + */ +const delegateInterruptedAt = ( + laptopEnv: Effect.Success>, + pause: Effect.Success, +) => + Effect.gen(function* () { + const call = yield* delegateToBox(laptopEnv).pipe(Effect.forkChild); + yield* Deferred.await(pause.reached); + call.interruptUnsafe(); + yield* Deferred.succeed(pause.release, undefined); + return yield* Fiber.await(call); + }); + +it.effect.each(["after the launch there answered", "while the launch there is answered"] as const)( + "a delegate call interrupted %s still records and follows its task", + (step) => + Effect.scoped( + Effect.gen(function* () { + const thread = yield* makeBoxThread; + const b = yield* serveBox(thread); + const offers = yield* SubscriptionRef.make< + ReadonlyArray + >([]); + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); + const pause = yield* makePause; + const a = yield* makeLaptop( + database, + offers, + undefined, + Effect.void, + step === "after the launch there answered" ? { beforeRecord: pause.hold } : {}, + ); + if (step === "while the launch there is answered") { + yield* Ref.set(thread.afterLaunch, pause.hold); + } + yield* linkTo(a.links, b, "full-access"); + yield* seedParent(a); + + const exit = yield* delegateInterruptedAt(a, pause); + // The caller hears nothing; the thread there is still this parent's task. + expect(Exit.hasInterrupts(exit)).toBe(true); + const launched = yield* Ref.get(thread.launched); + expect(launched).not.toBeNull(); + const { subagents } = yield* a.orchestrator.getThreadProjection(parentThreadId); + expect(subagents).toHaveLength(1); + const taskId = subagents[0]!.id; + expect(subagents[0]!.remoteChild?.threadId).toBe(launched); + + // The same call again (same clientRequestId) finds that task, not a second one. + const retried = yield* delegateToBox(a); + expect(retried.taskId).toBe(taskId); + expect((yield* a.orchestrator.getThreadProjection(parentThreadId)).subagents).toHaveLength( + 1, + ); + + // Its follower runs, so its end there reaches the parent here, once. + yield* Ref.set(thread.reply, "Done after all."); + yield* Ref.set(thread.status, "completed"); + yield* Deferred.succeed(thread.finished, undefined); + expect(yield* taskResult(a, taskId)).toMatchObject({ + status: "completed", + result: "Done after all.", + }); + const parentRun = (yield* a.orchestrator.getThreadProjection(parentThreadId)).runs.find( + (run) => run.id === parentRunId, + ); + expect(parentRun?.delegatedCompletion?.delivery?.taskIds).toEqual([taskId]); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); + +it.effect("a delegate call interrupted before its launch there launches and records nothing", () => + Effect.scoped( + Effect.gen(function* () { + const thread = yield* makeBoxThread; + const b = yield* serveBox(thread); + const offers = yield* SubscriptionRef.make< + ReadonlyArray + >([]); + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); + const pause = yield* makePause; + const a = yield* makeLaptop(database, offers, undefined, Effect.void, { + beforeResolve: pause.hold, + }); + yield* linkTo(a.links, b, "full-access"); + yield* seedParent(a); + + const exit = yield* delegateInterruptedAt(a, pause); + expect(Exit.hasInterrupts(exit)).toBe(true); + expect(yield* Ref.get(thread.launched)).toBeNull(); + expect((yield* a.orchestrator.getThreadProjection(parentThreadId)).subagents).toEqual([]); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); + it.effect("follows an open task again after the laptop restarts", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/peer/RemoteDelegation.ts b/apps/server/src/peer/RemoteDelegation.ts index 932b012969e36..77fc9bbaf0b64 100644 --- a/apps/server/src/peer/RemoteDelegation.ts +++ b/apps/server/src/peer/RemoteDelegation.ts @@ -344,101 +344,109 @@ const make = Effect.gen(function* () { const key = input.clientRequestId ?? (yield* random.uuidV4()); const task = taskPrompt(input); const title = input.title ?? parent.thread.title; - const launched = yield* forwarding.call( - scope, - ProjectToolkit.tools.t3_thread_launch, - target.environmentId, - { - projectId, - title, - modelSelection: { instanceId, model, ...(options === undefined ? {} : { options }) }, - ...(runtimeMode === undefined ? {} : { runtimeMode }), - ...(interactionMode === undefined ? {} : { interactionMode }), - message: task, - clientRequestId: `delegate:${scope.thread.threadId}:${key}`, - // The thread there shows it is this thread's subagent, with a way back. - delegatedFrom: { - environmentId: hereId, - threadId: parent.thread.id, - title: parent.thread.title, - }, - }, - ); - const link = yield* links.get(target.environmentId).pipe( - Effect.map(Option.getOrUndefined), - Effect.orElseSucceed(() => undefined), - ); - const commandId = CommandId.make( - `command:mcp:${encodeURIComponent(scope.requestNamespace)}:remote-delegate:${encodeURIComponent(key)}`, - ); - const remoteChild: OrchestrationV2RemoteTaskChild = { - environmentId: target.environmentId, - threadId: launched.threadId, - label: link?.label ?? target.environmentId, - title, - ...(launched.runId === null ? {} : { runId: launched.runId }), - }; - const recorded = yield* threads - .dispatch({ - type: "delegated_task.remote.request", - commandId, - parentThreadId: scope.thread.threadId, - parentRunId: parentRun.id, - parentNodeId, - task, - ...(input.title === undefined ? {} : { title: input.title }), - // The driver runs there; this side only labels the task with it. - driver: ProviderDriverKind.make("remote"), - modelSelection: launched.modelSelection, - remoteChild, - completionWake: input.mode === "wait" ? "settled_only" : "always", - }) - .pipe( - // The cause goes to the log; the caller learns only that recording failed. - Effect.tapError((cause) => - Effect.logWarning("remote-delegation.record-failed", { + // Once the launch there may have been sent, the task is recorded and followed even if + // this call is interrupted (the parent's turn moved on): otherwise the thread there runs + // with nothing here to follow it or report its end. Everything above stays cancellable. + return yield* Effect.uninterruptible( + Effect.gen(function* () { + const launched = yield* forwarding.call( + scope, + ProjectToolkit.tools.t3_thread_launch, + target.environmentId, + { + projectId, + title, + modelSelection: { instanceId, model, ...(options === undefined ? {} : { options }) }, + ...(runtimeMode === undefined ? {} : { runtimeMode }), + ...(interactionMode === undefined ? {} : { interactionMode }), + message: task, + clientRequestId: `delegate:${scope.thread.threadId}:${key}`, + // The thread there shows it is this thread's subagent, with a way back. + delegatedFrom: { + environmentId: hereId, + threadId: parent.thread.id, + title: parent.thread.title, + }, + }, + ); + const link = yield* links.get(target.environmentId).pipe( + Effect.map(Option.getOrUndefined), + Effect.orElseSucceed(() => undefined), + ); + const commandId = CommandId.make( + `command:mcp:${encodeURIComponent(scope.requestNamespace)}:remote-delegate:${encodeURIComponent(key)}`, + ); + const remoteChild: OrchestrationV2RemoteTaskChild = { + environmentId: target.environmentId, + threadId: launched.threadId, + label: link?.label ?? target.environmentId, + title, + ...(launched.runId === null ? {} : { runId: launched.runId }), + }; + const recorded = yield* threads + .dispatch({ + type: "delegated_task.remote.request", + commandId, parentThreadId: scope.thread.threadId, - cause, - }), - ), - Effect.mapError( - () => - new OrchestratorMcpFailure({ - code: "orchestration_error", - message: "Unable to record the delegated task.", - }), - ), - // Nothing here follows a thread the task was never recorded for, so stop it there. - // A crash between the launch and this record still leaves it running there - // until a retry with the same clientRequestId records it. - Effect.tapError(() => - stopThere(followScope(scope.thread, remoteChild), remoteChild, "stop").pipe( - Effect.ignoreCause({ log: true }), - ), - ), - ); - const taskEvent = recorded.storedEvents.find( - (stored) => stored.event.type === "subagent.updated", - ); - const taskId = - taskEvent?.event.type === "subagent.updated" - ? taskEvent.event.payload.id - : // A replayed request recorded the task the first time. - yield* threads.getThreadRecords(scope.thread.threadId, ["subagents"]).pipe( - Effect.map( - ({ subagents }) => - subagents.find((task) => task.remoteChild?.threadId === launched.threadId)?.id, + parentRunId: parentRun.id, + parentNodeId, + task, + ...(input.title === undefined ? {} : { title: input.title }), + // The driver runs there; this side only labels the task with it. + driver: ProviderDriverKind.make("remote"), + modelSelection: launched.modelSelection, + remoteChild, + completionWake: input.mode === "wait" ? "settled_only" : "always", + }) + .pipe( + // The cause goes to the log; the caller learns only that recording failed. + Effect.tapError((cause) => + Effect.logWarning("remote-delegation.record-failed", { + parentThreadId: scope.thread.threadId, + cause, + }), + ), + Effect.mapError( + () => + new OrchestratorMcpFailure({ + code: "orchestration_error", + message: "Unable to record the delegated task.", + }), + ), + // Nothing here follows a thread the task was never recorded for, so stop it there. + // A crash between the launch and this record still leaves it running there + // until a retry with the same clientRequestId records it. + Effect.tapError(() => + stopThere(followScope(scope.thread, remoteChild), remoteChild, "stop").pipe( + Effect.ignoreCause({ log: true }), + ), ), - Effect.orElseSucceed(() => undefined), ); - if (taskId === undefined) { - return yield* new OrchestratorMcpFailure({ - code: "orchestration_error", - message: "The delegated task was not recorded.", - }); - } - yield* followOne(scope.thread.threadId, taskId).pipe(Effect.forkIn(followers)); - return { taskId }; + const taskEvent = recorded.storedEvents.find( + (stored) => stored.event.type === "subagent.updated", + ); + const taskId = + taskEvent?.event.type === "subagent.updated" + ? taskEvent.event.payload.id + : // A replayed request recorded the task the first time. + yield* threads.getThreadRecords(scope.thread.threadId, ["subagents"]).pipe( + Effect.map( + ({ subagents }) => + subagents.find((task) => task.remoteChild?.threadId === launched.threadId) + ?.id, + ), + Effect.orElseSucceed(() => undefined), + ); + if (taskId === undefined) { + return yield* new OrchestratorMcpFailure({ + code: "orchestration_error", + message: "The delegated task was not recorded.", + }); + } + yield* followOne(scope.thread.threadId, taskId).pipe(Effect.forkIn(followers)); + return { taskId }; + }), + ); }, ); From 02769e84898a005f93b2bdf4ad83baa35c3827cd Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Tue, 17 Nov 2026 21:58:38 +0000 Subject: [PATCH 2/5] remote delegate: seam-sized fix, fork-owned tests, stop a recorded launch with no task (#386) - RemoteDelegation.ts: the launch, record and follow run as one uninterruptible generator (`launchAndRecord`) at its old indentation plus two, so the upstream file gains 10 specific lines instead of a re-wrapped block (the seam check refuses generic lines like `});`). The reads before the launch stay cancellable. - A record that succeeds but names no task here now logs remote-delegation.record-found-no-task and stops the thread there, as a failed record does. - Tests move out of upstream's RemoteDelegation.test.ts (back to main's copy) into apps/server/src/fork/remoteDelegationAbortSafe.test.ts. - .github/fork-seams.json lists the 10 lines. Co-Authored-By: Claude Opus 5.5 --- .github/fork-seams.json | 15 + .../fork/remoteDelegationAbortSafe.test.ts | 301 ++++++++++++++++++ apps/server/src/peer/RemoteDelegation.test.ts | 130 +------- apps/server/src/peer/RemoteDelegation.ts | 200 ++++++------ 4 files changed, 421 insertions(+), 225 deletions(-) create mode 100644 apps/server/src/fork/remoteDelegationAbortSafe.test.ts diff --git a/.github/fork-seams.json b/.github/fork-seams.json index 62c7913a1e012..c8a48615b2c30 100644 --- a/.github/fork-seams.json +++ b/.github/fork-seams.json @@ -90,6 +90,21 @@ ": { unit: tracked.unit, slice: ownedSlice };", "})," ] + }, + "apps/server/src/peer/RemoteDelegation.ts": { + "why": "#386: a remote delegate_task interrupted after its launch there (the parent's turn moved on) still records and follows its task: the launch, record and follow run as one uninterruptible step. A launch recorded with no task here is stopped there with a warning, as a failed record is. Tested in apps/server/src/fork/remoteDelegationAbortSafe.test.ts. Upstream PR pingdotgg/t3code#30249 (record before launch) also changes this code; when it lands, check whether this seam is still needed.", + "added": [ + "// Fork: from the launch on, the task is recorded and followed even if this call is", + "// interrupted (the parent's turn moved on), else the thread there runs with nothing here", + "// following it or reporting its end. The reads above stay cancellable.", + "const launchAndRecord = function* () {", + "}; // Fork: end of launchAndRecord.", + "return yield* Effect.uninterruptible(Effect.gen(launchAndRecord));", + "// Fork: nothing here follows it, so stop it there, as a failed record does.", + "yield* Effect.logWarning(\"remote-delegation.record-found-no-task\", remoteChild);", + "const unfollowed = stopThere(followScope(scope.thread, remoteChild), remoteChild, \"stop\");", + "yield* unfollowed.pipe(Effect.ignoreCause({ log: true }));" + ] } }, "//pending": "Upstream files whose fork lines predate counting every upstream file, each with its ceiling: the most fork lines it may hold against upstream (counted from upstream, as fork-follow runs the check). To move into fork files or pin as seams; lower a ceiling when lines move out and take a file off when none are left. A pull request may change them but not grow them.", diff --git a/apps/server/src/fork/remoteDelegationAbortSafe.test.ts b/apps/server/src/fork/remoteDelegationAbortSafe.test.ts new file mode 100644 index 0000000000000..98e5fce525339 --- /dev/null +++ b/apps/server/src/fork/remoteDelegationAbortSafe.test.ts @@ -0,0 +1,301 @@ +/** + * Fork: a remote `delegate_task` interrupted once its launch there may have been sent still + * records and follows its task; interrupted before, it leaves nothing. A recorded launch with no + * task here is stopped there. Upstream's `RemoteDelegation.test.ts` covers the rest. + */ +import { assert, it } from "@effect/vitest"; +import { + type EnvironmentId, + NodeId, + type OrchestratorMcpDelegateTaskInput, + ProjectId, + ProviderInstanceId, + RunId, + ThreadId, +} from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Deferred from "effect/Deferred"; +import * as SecureRandom from "effect/crypto/SecureRandom"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; +import * as Logger from "effect/Logger"; + +import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; +import type * as McpInvocationContext from "../mcp/McpInvocationContext.ts"; +import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; +import * as ThreadManagement from "../orchestration-v2/ThreadManagementService.ts"; +import * as PeerForwarding from "../peer/PeerForwarding.ts"; +import * as PeerLinks from "../peer/PeerLinks.ts"; +import * as RemoteDelegation from "../peer/RemoteDelegation.ts"; +import * as ProjectService from "../project/ProjectService.ts"; +import * as RepositoryIdentityResolver from "../project/RepositoryIdentityResolver.ts"; + +const box = "environment-box" as EnvironmentId; +const parentThreadId = ThreadId.make("thread:parent"); +const childThreadId = ThreadId.make("thread:box-child"); +const taskId = NodeId.make("node:task"); +const instanceId = ProviderInstanceId.make("codex"); +const modelSelection = { instanceId, model: "gpt-5.4" }; +const repository = "github.com/acme/app"; + +const scope: McpInvocationContext.McpThreadInvocationScope = { + environmentId: "environment-here" as EnvironmentId, + requestNamespace: "provider-session:parent", + thread: { + threadId: parentThreadId, + providerSessionId: "provider-session:parent", + providerInstanceId: instanceId, + }, + client: undefined, + capabilities: new Set(["orchestration"]), + issuedAt: 1, +}; + +const input = { + task: "Review the change on the box.", + target: { environmentId: box, providerInstanceId: instanceId, model: "gpt-5.4" }, + mode: "async", + clientRequestId: "review-on-box", +} as OrchestratorMcpDelegateTaskInput as Parameters< + RemoteDelegation.RemoteDelegation["Service"]["delegate"] +>[1]; + +/** A point a delegating call stops at until the test lets it go on. */ +const makePause = Effect.gen(function* () { + const reached = yield* Deferred.make(); + const release = yield* Deferred.make(); + return { + reached, + release, + hold: Deferred.succeed(reached, undefined).pipe(Effect.andThen(Deferred.await(release))), + }; +}); +type Pause = Effect.Success; + +/** + * The delegation service with everything around it faked: the box (its project list and launch), + * the records here, and the follower, which only notes the task it follows and then waits. + */ +const makeHarness = ( + options: { + /** Holds the read of the box's projects, the last remote read before the launch. */ + readonly beforeLaunch?: Pause; + /** Holds the launch's answer: the request reached the box, the reply has not come. */ + readonly duringLaunch?: Pause; + /** Holds the record here, after the launch answered. */ + readonly beforeRecord?: Pause; + /** The record succeeds, yet names no task, and the parent's task list has none. */ + readonly recordWithoutTask?: boolean; + } = {}, +) => + Effect.gen(function* () { + const launches: Array = []; + const records: Array = []; + const followed: Array = []; + const stops: Array = []; + const warnings: Array = []; + const following = yield* Deferred.make(); + + const forwarding = Layer.mock(PeerForwarding.PeerForwarding)({ + call: ((_scope: unknown, tool: { readonly name: string }, _env: unknown, params: never) => { + if (tool.name === "t3_project_list") { + return (options.beforeLaunch?.hold ?? Effect.void).pipe( + Effect.as({ + projects: [ + { + id: ProjectId.make("project:box"), + title: "app", + repositoryIdentity: { canonicalKey: repository }, + }, + ], + nextCursor: null, + }), + ); + } + if (tool.name === "t3_thread_launch") { + return Effect.sync(() => { + launches.push((params as { readonly clientRequestId: string }).clientRequestId); + }).pipe( + Effect.andThen(options.duringLaunch?.hold ?? Effect.void), + Effect.as({ threadId: childThreadId, runId: null, modelSelection }), + ); + } + return Effect.die(`unexpected call ${tool.name}`); + }) as never, + interruptThread: (_scope, _env, request) => + Effect.sync(() => { + stops.push(request.threadId); + return {} as never; + }), + }); + + const threads = Layer.mock(ThreadManagement.ThreadManagementService)({ + // A follower's first step: here it notes the task it follows, then waits for good. + getThreadEventSequence: () => + Effect.sync(() => followed.push(taskId)).pipe( + Effect.andThen(Deferred.succeed(following, undefined)), + Effect.andThen(Effect.never), + ), + getThreadRecords: ((threadId: ThreadId) => + Effect.succeed({ + thread: { + id: threadId, + title: "Parent", + projectId: ProjectId.make("project:here"), + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + }, + runs: [ + { + id: RunId.make("run:parent"), + status: "running", + ordinal: 1, + rootNodeId: NodeId.make("node:parent-root"), + providerInstanceId: instanceId, + }, + ], + subagents: + records.length === 0 || options.recordWithoutTask + ? [] + : [{ id: taskId, remoteChild: { threadId: childThreadId } }], + })) as never, + dispatch: (command) => + (command.type === "delegated_task.remote.request" + ? (options.beforeRecord?.hold ?? Effect.void) + : Effect.void + ).pipe( + Effect.andThen( + Effect.sync(() => { + records.push(command.type); + return { + storedEvents: options.recordWithoutTask + ? [] + : [{ event: { type: "subagent.updated", payload: { id: taskId } } }], + } as never; + }), + ), + ), + }); + + const remote = yield* Layer.build( + RemoteDelegation.layer.pipe( + Layer.provide( + Layer.mergeAll( + forwarding, + threads, + Layer.mock(PeerLinks.PeerLinks)({ get: () => Effect.succeedNone }), + Layer.mock(ProjectionStore.ProjectionStoreV2)({}), + Layer.mock(ProjectService.ProjectService)({ + getById: () => Effect.succeedSome({ workspaceRoot: "/srv/app" } as never), + }), + Layer.mock(RepositoryIdentityResolver.RepositoryIdentityResolver)({ + resolve: () => Effect.succeed({ canonicalKey: repository } as never), + }), + Layer.mock(ServerEnvironment.ServerEnvironment)({ + getEnvironmentId: Effect.succeed("environment-here" as EnvironmentId), + }), + Layer.succeed(SecureRandom.SecureRandom, { + uuidV4: Effect.succeed("00000000-0000-4000-8000-000000000000"), + } as never), + ), + ), + ), + ).pipe(Effect.map(Context.get(RemoteDelegation.RemoteDelegation))); + // Warnings the call logs, on its own fiber. + const logged = Logger.layer( + [ + Logger.make(({ message, logLevel }) => { + if (logLevel === "Warn") warnings.push(String([message].flat()[0])); + }), + ], + { mergeWithExisting: false }, + ); + + return { + delegate: remote.delegate(scope, input).pipe(Effect.provide(logged)), + launches, + records, + following, + followed, + stops, + warnings, + }; + }); + +/** + * Starts a delegate call, interrupts it at `pause` as the parent's provider does when its turn + * moves on mid-call, then lets the held step go on. + */ +const interruptedAt = (delegate: Effect.Effect, pause: Pause) => + Effect.gen(function* () { + const call = yield* delegate.pipe(Effect.forkChild); + yield* Deferred.await(pause.reached); + call.interruptUnsafe(); + yield* Deferred.succeed(pause.release, undefined); + return yield* Fiber.await(call); + }); + +it.effect.each(["beforeRecord", "duringLaunch"] as const)( + "a delegate call interrupted %s still records and follows its task", + (step) => + Effect.scoped( + Effect.gen(function* () { + const pause = yield* makePause; + const harness = yield* makeHarness({ [step]: pause }); + + const exit = yield* interruptedAt(harness.delegate, pause); + // The caller hears nothing back; the thread there is still this parent's task. + assert.isTrue(Exit.hasInterrupts(exit)); + assert.deepStrictEqual(harness.launches, [`delegate:${parentThreadId}:review-on-box`]); + assert.deepStrictEqual(harness.records, ["delegated_task.remote.request"]); + yield* Deferred.await(harness.following); + assert.deepStrictEqual(harness.followed, [taskId]); + assert.deepStrictEqual(harness.stops, []); + }), + ), +); + +it.effect("a delegate call interrupted before its launch there launches and records nothing", () => + Effect.scoped( + Effect.gen(function* () { + const pause = yield* makePause; + const harness = yield* makeHarness({ beforeLaunch: pause }); + + const exit = yield* interruptedAt(harness.delegate, pause); + assert.isTrue(Exit.hasInterrupts(exit)); + assert.deepStrictEqual(harness.launches, []); + assert.deepStrictEqual(harness.records, []); + assert.deepStrictEqual(harness.followed, []); + }), + ), +); + +it.effect("an uninterrupted delegate call launches, records and follows once", () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeHarness(); + const result = yield* harness.delegate; + assert.deepStrictEqual(result, { taskId }); + yield* Deferred.await(harness.following); + assert.strictEqual(harness.launches.length, 1); + assert.deepStrictEqual(harness.followed, [taskId]); + }), + ), +); + +it.effect("a recorded launch with no task here to follow is stopped there, with a warning", () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeHarness({ recordWithoutTask: true }); + const exit = yield* Effect.exit(harness.delegate); + assert.isTrue(Exit.isFailure(exit)); + assert.include(JSON.stringify(exit), "The delegated task was not recorded."); + assert.deepStrictEqual(harness.followed, []); + assert.deepStrictEqual(harness.stops, [childThreadId]); + assert.include(harness.warnings, "remote-delegation.record-found-no-task"); + }), + ), +); diff --git a/apps/server/src/peer/RemoteDelegation.test.ts b/apps/server/src/peer/RemoteDelegation.test.ts index 2665894237549..eeef7bf05a36e 100644 --- a/apps/server/src/peer/RemoteDelegation.test.ts +++ b/apps/server/src/peer/RemoteDelegation.test.ts @@ -25,7 +25,6 @@ import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; -import * as Exit from "effect/Exit"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; @@ -648,11 +647,6 @@ const makeLaptop = ( kept?: Context.Context>>, /** Runs as each follow attempt here starts, on its follower's fiber. */ beforeFollow: Effect.Effect = Effect.void, - /** Run on a delegating call's fiber: before it looks for the project there, and before it records its task. */ - pauses: { - readonly beforeResolve?: Effect.Effect; - readonly beforeRecord?: Effect.Effect; - } = {}, ) => Effect.gen(function* () { const linking = kept ?? (yield* layerLinkingEnvironment(laptop).pipe(Layer.build)); @@ -690,11 +684,6 @@ const makeLaptop = ( ...threads, getThreadEventSequence: (threadId: ThreadId) => beforeFollow.pipe(Effect.andThen(threads.getThreadEventSequence(threadId))), - dispatch: (command) => - (command.type === "delegated_task.remote.request" - ? (pauses.beforeRecord ?? Effect.void) - : Effect.void - ).pipe(Effect.andThen(threads.dispatch(command))), }; }), ), @@ -713,12 +702,10 @@ const makeLaptop = ( Layer.provide( Layer.mock(RepositoryIdentityResolver.RepositoryIdentityResolver)({ resolve: (cwd) => - (pauses.beforeResolve ?? Effect.void).pipe( - Effect.as( - cwd === "/home/me/app" - ? ({ canonicalKey: "github.com/acme/app" } as RepositoryIdentity) - : null, - ), + Effect.succeed( + cwd === "/home/me/app" + ? ({ canonicalKey: "github.com/acme/app" } as RepositoryIdentity) + : null, ), }), ), @@ -1322,115 +1309,6 @@ it.effect("refuses worktree isolation for a task there before launching anything ).pipe(Effect.provide(NodeHttpServer.layerTest)), ); -/** A point a delegating call stops at until the test lets it go on. */ -const makePause = Effect.gen(function* () { - const reached = yield* Deferred.make(); - const release = yield* Deferred.make(); - return { - reached, - release, - hold: Deferred.succeed(reached, undefined).pipe(Effect.andThen(Deferred.await(release))), - }; -}); - -/** - * Delegates to the box and interrupts the call at `pause`, as the parent's provider does when - * its turn moves on mid-call (a message steered into it), then lets the call go on. - */ -const delegateInterruptedAt = ( - laptopEnv: Effect.Success>, - pause: Effect.Success, -) => - Effect.gen(function* () { - const call = yield* delegateToBox(laptopEnv).pipe(Effect.forkChild); - yield* Deferred.await(pause.reached); - call.interruptUnsafe(); - yield* Deferred.succeed(pause.release, undefined); - return yield* Fiber.await(call); - }); - -it.effect.each(["after the launch there answered", "while the launch there is answered"] as const)( - "a delegate call interrupted %s still records and follows its task", - (step) => - Effect.scoped( - Effect.gen(function* () { - const thread = yield* makeBoxThread; - const b = yield* serveBox(thread); - const offers = yield* SubscriptionRef.make< - ReadonlyArray - >([]); - const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); - const pause = yield* makePause; - const a = yield* makeLaptop( - database, - offers, - undefined, - Effect.void, - step === "after the launch there answered" ? { beforeRecord: pause.hold } : {}, - ); - if (step === "while the launch there is answered") { - yield* Ref.set(thread.afterLaunch, pause.hold); - } - yield* linkTo(a.links, b, "full-access"); - yield* seedParent(a); - - const exit = yield* delegateInterruptedAt(a, pause); - // The caller hears nothing; the thread there is still this parent's task. - expect(Exit.hasInterrupts(exit)).toBe(true); - const launched = yield* Ref.get(thread.launched); - expect(launched).not.toBeNull(); - const { subagents } = yield* a.orchestrator.getThreadProjection(parentThreadId); - expect(subagents).toHaveLength(1); - const taskId = subagents[0]!.id; - expect(subagents[0]!.remoteChild?.threadId).toBe(launched); - - // The same call again (same clientRequestId) finds that task, not a second one. - const retried = yield* delegateToBox(a); - expect(retried.taskId).toBe(taskId); - expect((yield* a.orchestrator.getThreadProjection(parentThreadId)).subagents).toHaveLength( - 1, - ); - - // Its follower runs, so its end there reaches the parent here, once. - yield* Ref.set(thread.reply, "Done after all."); - yield* Ref.set(thread.status, "completed"); - yield* Deferred.succeed(thread.finished, undefined); - expect(yield* taskResult(a, taskId)).toMatchObject({ - status: "completed", - result: "Done after all.", - }); - const parentRun = (yield* a.orchestrator.getThreadProjection(parentThreadId)).runs.find( - (run) => run.id === parentRunId, - ); - expect(parentRun?.delegatedCompletion?.delivery?.taskIds).toEqual([taskId]); - }), - ).pipe(Effect.provide(NodeHttpServer.layerTest)), -); - -it.effect("a delegate call interrupted before its launch there launches and records nothing", () => - Effect.scoped( - Effect.gen(function* () { - const thread = yield* makeBoxThread; - const b = yield* serveBox(thread); - const offers = yield* SubscriptionRef.make< - ReadonlyArray - >([]); - const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); - const pause = yield* makePause; - const a = yield* makeLaptop(database, offers, undefined, Effect.void, { - beforeResolve: pause.hold, - }); - yield* linkTo(a.links, b, "full-access"); - yield* seedParent(a); - - const exit = yield* delegateInterruptedAt(a, pause); - expect(Exit.hasInterrupts(exit)).toBe(true); - expect(yield* Ref.get(thread.launched)).toBeNull(); - expect((yield* a.orchestrator.getThreadProjection(parentThreadId)).subagents).toEqual([]); - }), - ).pipe(Effect.provide(NodeHttpServer.layerTest)), -); - it.effect("follows an open task again after the laptop restarts", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/peer/RemoteDelegation.ts b/apps/server/src/peer/RemoteDelegation.ts index 77fc9bbaf0b64..a0cff4d2bebb3 100644 --- a/apps/server/src/peer/RemoteDelegation.ts +++ b/apps/server/src/peer/RemoteDelegation.ts @@ -344,109 +344,111 @@ const make = Effect.gen(function* () { const key = input.clientRequestId ?? (yield* random.uuidV4()); const task = taskPrompt(input); const title = input.title ?? parent.thread.title; - // Once the launch there may have been sent, the task is recorded and followed even if - // this call is interrupted (the parent's turn moved on): otherwise the thread there runs - // with nothing here to follow it or report its end. Everything above stays cancellable. - return yield* Effect.uninterruptible( - Effect.gen(function* () { - const launched = yield* forwarding.call( - scope, - ProjectToolkit.tools.t3_thread_launch, - target.environmentId, - { - projectId, - title, - modelSelection: { instanceId, model, ...(options === undefined ? {} : { options }) }, - ...(runtimeMode === undefined ? {} : { runtimeMode }), - ...(interactionMode === undefined ? {} : { interactionMode }), - message: task, - clientRequestId: `delegate:${scope.thread.threadId}:${key}`, - // The thread there shows it is this thread's subagent, with a way back. - delegatedFrom: { - environmentId: hereId, - threadId: parent.thread.id, - title: parent.thread.title, - }, - }, - ); - const link = yield* links.get(target.environmentId).pipe( - Effect.map(Option.getOrUndefined), - Effect.orElseSucceed(() => undefined), - ); - const commandId = CommandId.make( - `command:mcp:${encodeURIComponent(scope.requestNamespace)}:remote-delegate:${encodeURIComponent(key)}`, - ); - const remoteChild: OrchestrationV2RemoteTaskChild = { - environmentId: target.environmentId, - threadId: launched.threadId, - label: link?.label ?? target.environmentId, + // Fork: from the launch on, the task is recorded and followed even if this call is + // interrupted (the parent's turn moved on), else the thread there runs with nothing here + // following it or reporting its end. The reads above stay cancellable. + const launchAndRecord = function* () { + const launched = yield* forwarding.call( + scope, + ProjectToolkit.tools.t3_thread_launch, + target.environmentId, + { + projectId, title, - ...(launched.runId === null ? {} : { runId: launched.runId }), - }; - const recorded = yield* threads - .dispatch({ - type: "delegated_task.remote.request", - commandId, - parentThreadId: scope.thread.threadId, - parentRunId: parentRun.id, - parentNodeId, - task, - ...(input.title === undefined ? {} : { title: input.title }), - // The driver runs there; this side only labels the task with it. - driver: ProviderDriverKind.make("remote"), - modelSelection: launched.modelSelection, - remoteChild, - completionWake: input.mode === "wait" ? "settled_only" : "always", - }) - .pipe( - // The cause goes to the log; the caller learns only that recording failed. - Effect.tapError((cause) => - Effect.logWarning("remote-delegation.record-failed", { - parentThreadId: scope.thread.threadId, - cause, + modelSelection: { instanceId, model, ...(options === undefined ? {} : { options }) }, + ...(runtimeMode === undefined ? {} : { runtimeMode }), + ...(interactionMode === undefined ? {} : { interactionMode }), + message: task, + clientRequestId: `delegate:${scope.thread.threadId}:${key}`, + // The thread there shows it is this thread's subagent, with a way back. + delegatedFrom: { + environmentId: hereId, + threadId: parent.thread.id, + title: parent.thread.title, + }, + }, + ); + const link = yield* links.get(target.environmentId).pipe( + Effect.map(Option.getOrUndefined), + Effect.orElseSucceed(() => undefined), + ); + const commandId = CommandId.make( + `command:mcp:${encodeURIComponent(scope.requestNamespace)}:remote-delegate:${encodeURIComponent(key)}`, + ); + const remoteChild: OrchestrationV2RemoteTaskChild = { + environmentId: target.environmentId, + threadId: launched.threadId, + label: link?.label ?? target.environmentId, + title, + ...(launched.runId === null ? {} : { runId: launched.runId }), + }; + const recorded = yield* threads + .dispatch({ + type: "delegated_task.remote.request", + commandId, + parentThreadId: scope.thread.threadId, + parentRunId: parentRun.id, + parentNodeId, + task, + ...(input.title === undefined ? {} : { title: input.title }), + // The driver runs there; this side only labels the task with it. + driver: ProviderDriverKind.make("remote"), + modelSelection: launched.modelSelection, + remoteChild, + completionWake: input.mode === "wait" ? "settled_only" : "always", + }) + .pipe( + // The cause goes to the log; the caller learns only that recording failed. + Effect.tapError((cause) => + Effect.logWarning("remote-delegation.record-failed", { + parentThreadId: scope.thread.threadId, + cause, + }), + ), + Effect.mapError( + () => + new OrchestratorMcpFailure({ + code: "orchestration_error", + message: "Unable to record the delegated task.", }), + ), + // Nothing here follows a thread the task was never recorded for, so stop it there. + // A crash between the launch and this record still leaves it running there + // until a retry with the same clientRequestId records it. + Effect.tapError(() => + stopThere(followScope(scope.thread, remoteChild), remoteChild, "stop").pipe( + Effect.ignoreCause({ log: true }), ), - Effect.mapError( - () => - new OrchestratorMcpFailure({ - code: "orchestration_error", - message: "Unable to record the delegated task.", - }), - ), - // Nothing here follows a thread the task was never recorded for, so stop it there. - // A crash between the launch and this record still leaves it running there - // until a retry with the same clientRequestId records it. - Effect.tapError(() => - stopThere(followScope(scope.thread, remoteChild), remoteChild, "stop").pipe( - Effect.ignoreCause({ log: true }), - ), - ), - ); - const taskEvent = recorded.storedEvents.find( - (stored) => stored.event.type === "subagent.updated", + ), ); - const taskId = - taskEvent?.event.type === "subagent.updated" - ? taskEvent.event.payload.id - : // A replayed request recorded the task the first time. - yield* threads.getThreadRecords(scope.thread.threadId, ["subagents"]).pipe( - Effect.map( - ({ subagents }) => - subagents.find((task) => task.remoteChild?.threadId === launched.threadId) - ?.id, - ), - Effect.orElseSucceed(() => undefined), - ); - if (taskId === undefined) { - return yield* new OrchestratorMcpFailure({ - code: "orchestration_error", - message: "The delegated task was not recorded.", - }); - } - yield* followOne(scope.thread.threadId, taskId).pipe(Effect.forkIn(followers)); - return { taskId }; - }), - ); + const taskEvent = recorded.storedEvents.find( + (stored) => stored.event.type === "subagent.updated", + ); + const taskId = + taskEvent?.event.type === "subagent.updated" + ? taskEvent.event.payload.id + : // A replayed request recorded the task the first time. + yield* threads.getThreadRecords(scope.thread.threadId, ["subagents"]).pipe( + Effect.map( + ({ subagents }) => + subagents.find((task) => task.remoteChild?.threadId === launched.threadId)?.id, + ), + Effect.orElseSucceed(() => undefined), + ); + if (taskId === undefined) { + // Fork: nothing here follows it, so stop it there, as a failed record does. + yield* Effect.logWarning("remote-delegation.record-found-no-task", remoteChild); + const unfollowed = stopThere(followScope(scope.thread, remoteChild), remoteChild, "stop"); + yield* unfollowed.pipe(Effect.ignoreCause({ log: true })); + return yield* new OrchestratorMcpFailure({ + code: "orchestration_error", + message: "The delegated task was not recorded.", + }); + } + yield* followOne(scope.thread.threadId, taskId).pipe(Effect.forkIn(followers)); + return { taskId }; + }; // Fork: end of launchAndRecord. + return yield* Effect.uninterruptible(Effect.gen(launchAndRecord)); }, ); From b30d198f52aa36d5bda2f54c7cbff97e72cf254b Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Tue, 17 Nov 2026 22:48:54 +0000 Subject: [PATCH 3/5] remote delegate: test failing stops and the stop after a refused record; seam note on #30249 (#386) - A record that is refused (also while interrupted) stops the launched thread there and follows nothing; a stop that fails there does not replace the record's error. These cover the two mutations the state lens found surviving (stop failure not ignored; no stop after a failed record). - The follower stub now notes the task id it reads, so the tests check which task is followed. - The seam note says upstream #30249 stops the child after an interrupted launch, while this seam keeps and follows it, so bringing it in through fork-follow is a deliberate choice. Co-Authored-By: Claude Opus 5.5 --- .github/fork-seams.json | 2 +- .../fork/remoteDelegationAbortSafe.test.ts | 143 +++++++++++++----- 2 files changed, 109 insertions(+), 36 deletions(-) diff --git a/.github/fork-seams.json b/.github/fork-seams.json index 176d777dbbb8d..7cd3c03668610 100644 --- a/.github/fork-seams.json +++ b/.github/fork-seams.json @@ -92,7 +92,7 @@ ] }, "apps/server/src/peer/RemoteDelegation.ts": { - "why": "#386: a remote delegate_task interrupted after its launch there (the parent's turn moved on) still records and follows its task: the launch, record and follow run as one uninterruptible step. A launch recorded with no task here is stopped there with a warning, as a failed record is. Tested in apps/server/src/fork/remoteDelegationAbortSafe.test.ts. Upstream PR pingdotgg/t3code#30249 (record before launch) also changes this code; when it lands, check whether this seam is still needed.", + "why": "#386: a remote delegate_task interrupted after its launch there (the parent's turn moved on) still records and follows its task: the launch, record and follow run as one uninterruptible step. A launch recorded with no task here is stopped there with a warning, as a failed record is. Tested in apps/server/src/fork/remoteDelegationAbortSafe.test.ts. Upstream PR pingdotgg/t3code#30249 (record before launch) rewrites this block, and its intent differs: after an interrupted, uncertain launch it stops the child there, while this seam keeps the child and follows it. Whoever brings #30249 in through fork-follow must choose on purpose between stopping and keeping (and expect a merge conflict on the whole re-indented block), then drop this seam or keep it.", "added": [ "// Fork: from the launch on, the task is recorded and followed even if this call is", "// interrupted (the parent's turn moved on), else the thread there runs with nothing here", diff --git a/apps/server/src/fork/remoteDelegationAbortSafe.test.ts b/apps/server/src/fork/remoteDelegationAbortSafe.test.ts index 98e5fce525339..a0e937b23d175 100644 --- a/apps/server/src/fork/remoteDelegationAbortSafe.test.ts +++ b/apps/server/src/fork/remoteDelegationAbortSafe.test.ts @@ -8,6 +8,7 @@ import { type EnvironmentId, NodeId, type OrchestratorMcpDelegateTaskInput, + OrchestratorMcpFailure, ProjectId, ProviderInstanceId, RunId, @@ -24,6 +25,7 @@ import * as Logger from "effect/Logger"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; import type * as McpInvocationContext from "../mcp/McpInvocationContext.ts"; +import * as Orchestrator from "../orchestration-v2/Orchestrator.ts"; import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; import * as ThreadManagement from "../orchestration-v2/ThreadManagementService.ts"; import * as PeerForwarding from "../peer/PeerForwarding.ts"; @@ -53,6 +55,8 @@ const scope: McpInvocationContext.McpThreadInvocationScope = { issuedAt: 1, }; +const stopFailed = "The box did not answer the stop."; + const input = { task: "Review the change on the box.", target: { environmentId: box, providerInstanceId: instanceId, model: "gpt-5.4" }, @@ -88,6 +92,10 @@ const makeHarness = ( readonly beforeRecord?: Pause; /** The record succeeds, yet names no task, and the parent's task list has none. */ readonly recordWithoutTask?: boolean; + /** The record here is refused, as when the parent's run was stopped meanwhile. */ + readonly recordFails?: boolean; + /** The stop sent there fails, as when the box stopped answering. */ + readonly stopFails?: boolean; } = {}, ) => Effect.gen(function* () { @@ -124,49 +132,70 @@ const makeHarness = ( } return Effect.die(`unexpected call ${tool.name}`); }) as never, - interruptThread: (_scope, _env, request) => - Effect.sync(() => { - stops.push(request.threadId); - return {} as never; - }), + interruptThread: ((_scope: unknown, _env: unknown, request: { threadId: ThreadId }) => + Effect.sync(() => stops.push(request.threadId)).pipe( + Effect.andThen( + options.stopFails + ? Effect.fail( + new OrchestratorMcpFailure({ code: "orchestration_error", message: stopFailed }), + ) + : Effect.succeed({}), + ), + )) as never, }); const threads = Layer.mock(ThreadManagement.ThreadManagementService)({ - // A follower's first step: here it notes the task it follows, then waits for good. - getThreadEventSequence: () => - Effect.sync(() => followed.push(taskId)).pipe( - Effect.andThen(Deferred.succeed(following, undefined)), - Effect.andThen(Effect.never), - ), - getThreadRecords: ((threadId: ThreadId) => - Effect.succeed({ - thread: { - id: threadId, - title: "Parent", - projectId: ProjectId.make("project:here"), - modelSelection, - runtimeMode: "full-access", - interactionMode: "default", - }, - runs: [ - { - id: RunId.make("run:parent"), - status: "running", - ordinal: 1, - rootNodeId: NodeId.make("node:parent-root"), - providerInstanceId: instanceId, - }, - ], - subagents: - records.length === 0 || options.recordWithoutTask - ? [] - : [{ id: taskId, remoteChild: { threadId: childThreadId } }], - })) as never, + getThreadEventSequence: () => Effect.succeed(0) as never, + getThreadRecords: (( + threadId: ThreadId, + _parts: unknown, + only?: { readonly subagentIds?: ReadonlyArray }, + ) => + // A follower's first read of its task: here it notes which task it follows, then waits + // for good. + only?.subagentIds !== undefined + ? Effect.sync(() => followed.push(...(only.subagentIds ?? []))).pipe( + Effect.andThen(Deferred.succeed(following, undefined)), + Effect.andThen(Effect.never), + ) + : Effect.succeed({ + thread: { + id: threadId, + title: "Parent", + projectId: ProjectId.make("project:here"), + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + }, + runs: [ + { + id: RunId.make("run:parent"), + status: "running", + ordinal: 1, + rootNodeId: NodeId.make("node:parent-root"), + providerInstanceId: instanceId, + }, + ], + subagents: + records.length === 0 || options.recordWithoutTask + ? [] + : [{ id: taskId, remoteChild: { threadId: childThreadId } }], + })) as never, dispatch: (command) => (command.type === "delegated_task.remote.request" ? (options.beforeRecord?.hold ?? Effect.void) : Effect.void ).pipe( + Effect.andThen( + command.type === "delegated_task.remote.request" && options.recordFails + ? Effect.fail( + new Orchestrator.OrchestratorCommandRejectedError({ + commandId: command.commandId, + commandType: command.type, + }), + ) + : Effect.void, + ), Effect.andThen( Effect.sync(() => { records.push(command.type); @@ -299,3 +328,47 @@ it.effect("a recorded launch with no task here to follow is stopped there, with }), ), ); + +it.effect.each([false, true])( + "a launch whose record is refused is stopped there and not followed (interrupted: %s)", + (interrupted) => + Effect.scoped( + Effect.gen(function* () { + const pause = yield* makePause; + const harness = yield* makeHarness({ + recordFails: true, + ...(interrupted ? { beforeRecord: pause } : {}), + }); + const exit = interrupted + ? yield* interruptedAt(harness.delegate, pause) + : yield* Effect.exit(harness.delegate); + assert.isTrue(Exit.isFailure(exit)); + if (!interrupted) + assert.include(JSON.stringify(exit), "Unable to record the delegated task."); + assert.deepStrictEqual(harness.launches, [`delegate:${parentThreadId}:review-on-box`]); + assert.deepStrictEqual(harness.records, []); + assert.deepStrictEqual(harness.stops, [childThreadId]); + assert.deepStrictEqual(harness.followed, []); + assert.include(harness.warnings, "remote-delegation.record-failed"); + }), + ), +); + +it.effect.each([ + ["refused", { recordFails: true }, "Unable to record the delegated task."], + ["found no task", { recordWithoutTask: true }, "The delegated task was not recorded."], +] as const)( + "a stop that fails there after a record %s does not replace the record's error", + ([, record, message]) => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeHarness({ ...record, stopFails: true }); + const exit = yield* Effect.exit(harness.delegate); + assert.isTrue(Exit.isFailure(exit)); + assert.include(JSON.stringify(exit), message); + assert.notInclude(JSON.stringify(exit), stopFailed); + assert.deepStrictEqual(harness.stops, [childThreadId]); + assert.deepStrictEqual(harness.followed, []); + }), + ), +); From 8da128308afd44d77c4a22c79fff58cf4b1f0b4d Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Tue, 17 Nov 2026 23:03:05 +0000 Subject: [PATCH 4/5] remote delegate: keep upstream indentation in the launch-and-record block The block keeps upstream's indentation under a prettier-ignore, wrapped by Effect.gen(...).pipe(Effect.uninterruptible), so the PR re-indents no upstream lines (#380). Seam entry lines updated to match. Co-Authored-By: Claude Opus 5.5 --- .github/fork-seams.json | 6 +- apps/server/src/peer/RemoteDelegation.ts | 196 +++++++++++------------ 2 files changed, 101 insertions(+), 101 deletions(-) diff --git a/.github/fork-seams.json b/.github/fork-seams.json index 7cd3c03668610..0d579f5e88da8 100644 --- a/.github/fork-seams.json +++ b/.github/fork-seams.json @@ -97,9 +97,9 @@ "// Fork: from the launch on, the task is recorded and followed even if this call is", "// interrupted (the parent's turn moved on), else the thread there runs with nothing here", "// following it or reporting its end. The reads above stay cancellable.", - "const launchAndRecord = function* () {", - "}; // Fork: end of launchAndRecord.", - "return yield* Effect.uninterruptible(Effect.gen(launchAndRecord));", + "// prettier-ignore", + "return yield* Effect.gen(function* () { // Fork: start of launchAndRecord.", + "}).pipe(Effect.uninterruptible); // Fork: end of launchAndRecord.", "// Fork: nothing here follows it, so stop it there, as a failed record does.", "yield* Effect.logWarning(\"remote-delegation.record-found-no-task\", remoteChild);", "const unfollowed = stopThere(followScope(scope.thread, remoteChild), remoteChild, \"stop\");", diff --git a/apps/server/src/peer/RemoteDelegation.ts b/apps/server/src/peer/RemoteDelegation.ts index a0cff4d2bebb3..592f5f6727ae2 100644 --- a/apps/server/src/peer/RemoteDelegation.ts +++ b/apps/server/src/peer/RemoteDelegation.ts @@ -347,108 +347,108 @@ const make = Effect.gen(function* () { // Fork: from the launch on, the task is recorded and followed even if this call is // interrupted (the parent's turn moved on), else the thread there runs with nothing here // following it or reporting its end. The reads above stay cancellable. - const launchAndRecord = function* () { - const launched = yield* forwarding.call( - scope, - ProjectToolkit.tools.t3_thread_launch, - target.environmentId, - { - projectId, - title, - modelSelection: { instanceId, model, ...(options === undefined ? {} : { options }) }, - ...(runtimeMode === undefined ? {} : { runtimeMode }), - ...(interactionMode === undefined ? {} : { interactionMode }), - message: task, - clientRequestId: `delegate:${scope.thread.threadId}:${key}`, - // The thread there shows it is this thread's subagent, with a way back. - delegatedFrom: { - environmentId: hereId, - threadId: parent.thread.id, - title: parent.thread.title, - }, - }, - ); - const link = yield* links.get(target.environmentId).pipe( - Effect.map(Option.getOrUndefined), - Effect.orElseSucceed(() => undefined), - ); - const commandId = CommandId.make( - `command:mcp:${encodeURIComponent(scope.requestNamespace)}:remote-delegate:${encodeURIComponent(key)}`, - ); - const remoteChild: OrchestrationV2RemoteTaskChild = { - environmentId: target.environmentId, - threadId: launched.threadId, - label: link?.label ?? target.environmentId, + // prettier-ignore + return yield* Effect.gen(function* () { // Fork: start of launchAndRecord. + const launched = yield* forwarding.call( + scope, + ProjectToolkit.tools.t3_thread_launch, + target.environmentId, + { + projectId, title, - ...(launched.runId === null ? {} : { runId: launched.runId }), - }; - const recorded = yield* threads - .dispatch({ - type: "delegated_task.remote.request", - commandId, - parentThreadId: scope.thread.threadId, - parentRunId: parentRun.id, - parentNodeId, - task, - ...(input.title === undefined ? {} : { title: input.title }), - // The driver runs there; this side only labels the task with it. - driver: ProviderDriverKind.make("remote"), - modelSelection: launched.modelSelection, - remoteChild, - completionWake: input.mode === "wait" ? "settled_only" : "always", - }) - .pipe( - // The cause goes to the log; the caller learns only that recording failed. - Effect.tapError((cause) => - Effect.logWarning("remote-delegation.record-failed", { - parentThreadId: scope.thread.threadId, - cause, + modelSelection: { instanceId, model, ...(options === undefined ? {} : { options }) }, + ...(runtimeMode === undefined ? {} : { runtimeMode }), + ...(interactionMode === undefined ? {} : { interactionMode }), + message: task, + clientRequestId: `delegate:${scope.thread.threadId}:${key}`, + // The thread there shows it is this thread's subagent, with a way back. + delegatedFrom: { + environmentId: hereId, + threadId: parent.thread.id, + title: parent.thread.title, + }, + }, + ); + const link = yield* links.get(target.environmentId).pipe( + Effect.map(Option.getOrUndefined), + Effect.orElseSucceed(() => undefined), + ); + const commandId = CommandId.make( + `command:mcp:${encodeURIComponent(scope.requestNamespace)}:remote-delegate:${encodeURIComponent(key)}`, + ); + const remoteChild: OrchestrationV2RemoteTaskChild = { + environmentId: target.environmentId, + threadId: launched.threadId, + label: link?.label ?? target.environmentId, + title, + ...(launched.runId === null ? {} : { runId: launched.runId }), + }; + const recorded = yield* threads + .dispatch({ + type: "delegated_task.remote.request", + commandId, + parentThreadId: scope.thread.threadId, + parentRunId: parentRun.id, + parentNodeId, + task, + ...(input.title === undefined ? {} : { title: input.title }), + // The driver runs there; this side only labels the task with it. + driver: ProviderDriverKind.make("remote"), + modelSelection: launched.modelSelection, + remoteChild, + completionWake: input.mode === "wait" ? "settled_only" : "always", + }) + .pipe( + // The cause goes to the log; the caller learns only that recording failed. + Effect.tapError((cause) => + Effect.logWarning("remote-delegation.record-failed", { + parentThreadId: scope.thread.threadId, + cause, + }), + ), + Effect.mapError( + () => + new OrchestratorMcpFailure({ + code: "orchestration_error", + message: "Unable to record the delegated task.", }), + ), + // Nothing here follows a thread the task was never recorded for, so stop it there. + // A crash between the launch and this record still leaves it running there + // until a retry with the same clientRequestId records it. + Effect.tapError(() => + stopThere(followScope(scope.thread, remoteChild), remoteChild, "stop").pipe( + Effect.ignoreCause({ log: true }), ), - Effect.mapError( - () => - new OrchestratorMcpFailure({ - code: "orchestration_error", - message: "Unable to record the delegated task.", - }), - ), - // Nothing here follows a thread the task was never recorded for, so stop it there. - // A crash between the launch and this record still leaves it running there - // until a retry with the same clientRequestId records it. - Effect.tapError(() => - stopThere(followScope(scope.thread, remoteChild), remoteChild, "stop").pipe( - Effect.ignoreCause({ log: true }), - ), - ), - ); - const taskEvent = recorded.storedEvents.find( - (stored) => stored.event.type === "subagent.updated", + ), ); - const taskId = - taskEvent?.event.type === "subagent.updated" - ? taskEvent.event.payload.id - : // A replayed request recorded the task the first time. - yield* threads.getThreadRecords(scope.thread.threadId, ["subagents"]).pipe( - Effect.map( - ({ subagents }) => - subagents.find((task) => task.remoteChild?.threadId === launched.threadId)?.id, - ), - Effect.orElseSucceed(() => undefined), - ); - if (taskId === undefined) { - // Fork: nothing here follows it, so stop it there, as a failed record does. - yield* Effect.logWarning("remote-delegation.record-found-no-task", remoteChild); - const unfollowed = stopThere(followScope(scope.thread, remoteChild), remoteChild, "stop"); - yield* unfollowed.pipe(Effect.ignoreCause({ log: true })); - return yield* new OrchestratorMcpFailure({ - code: "orchestration_error", - message: "The delegated task was not recorded.", - }); - } - yield* followOne(scope.thread.threadId, taskId).pipe(Effect.forkIn(followers)); - return { taskId }; - }; // Fork: end of launchAndRecord. - return yield* Effect.uninterruptible(Effect.gen(launchAndRecord)); + const taskEvent = recorded.storedEvents.find( + (stored) => stored.event.type === "subagent.updated", + ); + const taskId = + taskEvent?.event.type === "subagent.updated" + ? taskEvent.event.payload.id + : // A replayed request recorded the task the first time. + yield* threads.getThreadRecords(scope.thread.threadId, ["subagents"]).pipe( + Effect.map( + ({ subagents }) => + subagents.find((task) => task.remoteChild?.threadId === launched.threadId)?.id, + ), + Effect.orElseSucceed(() => undefined), + ); + if (taskId === undefined) { + // Fork: nothing here follows it, so stop it there, as a failed record does. + yield* Effect.logWarning("remote-delegation.record-found-no-task", remoteChild); + const unfollowed = stopThere(followScope(scope.thread, remoteChild), remoteChild, "stop"); + yield* unfollowed.pipe(Effect.ignoreCause({ log: true })); + return yield* new OrchestratorMcpFailure({ + code: "orchestration_error", + message: "The delegated task was not recorded.", + }); + } + yield* followOne(scope.thread.threadId, taskId).pipe(Effect.forkIn(followers)); + return { taskId }; + }).pipe(Effect.uninterruptible); // Fork: end of launchAndRecord. }, ); From d6f5082c05b3971a204ce0cd642410641093db3d Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Tue, 17 Nov 2026 23:06:51 +0000 Subject: [PATCH 5/5] fork-seams: RemoteDelegation entry no longer mentions a re-indented block Co-Authored-By: Claude Opus 5.5 --- .github/fork-seams.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/fork-seams.json b/.github/fork-seams.json index 0d579f5e88da8..35a80268c7538 100644 --- a/.github/fork-seams.json +++ b/.github/fork-seams.json @@ -92,7 +92,7 @@ ] }, "apps/server/src/peer/RemoteDelegation.ts": { - "why": "#386: a remote delegate_task interrupted after its launch there (the parent's turn moved on) still records and follows its task: the launch, record and follow run as one uninterruptible step. A launch recorded with no task here is stopped there with a warning, as a failed record is. Tested in apps/server/src/fork/remoteDelegationAbortSafe.test.ts. Upstream PR pingdotgg/t3code#30249 (record before launch) rewrites this block, and its intent differs: after an interrupted, uncertain launch it stops the child there, while this seam keeps the child and follows it. Whoever brings #30249 in through fork-follow must choose on purpose between stopping and keeping (and expect a merge conflict on the whole re-indented block), then drop this seam or keep it.", + "why": "#386: a remote delegate_task interrupted after its launch there (the parent's turn moved on) still records and follows its task: the launch, record and follow run as one uninterruptible step. A launch recorded with no task here is stopped there with a warning, as a failed record is. Tested in apps/server/src/fork/remoteDelegationAbortSafe.test.ts. Upstream PR pingdotgg/t3code#30249 (record before launch) rewrites this block, and its intent differs: after an interrupted, uncertain launch it stops the child there, while this seam keeps the child and follows it. Whoever brings #30249 in through fork-follow must choose on purpose between stopping and keeping (and expect a merge conflict on this block), then drop this seam or keep it.", "added": [ "// Fork: from the launch on, the task is recorded and followed even if this call is", "// interrupted (the parent's turn moved on), else the thread there runs with nothing here",