From f943506b4cb867ba92e8fcefd7e770c6b3f7cfcb Mon Sep 17 00:00:00 2001 From: Kameron Smith <112618179+nullStack65@users.noreply.github.com> Date: Wed, 18 Nov 2026 05:12:43 +0000 Subject: [PATCH 1/3] feat(placement): bind auto-placed delegated tasks before launching anywhere delegate_task gains placement:"auto" for work that needs nothing from the parent's checkout. The environment comes from this server's and authenticated linked environments' own orchestrator_capabilities answers (host load, exact provider instance and model, mode within the link) and the peer's project list. It is bound to the caller-scoped request key in the existing command receipt store before any launch, and every retry, lost answer or restart reuses that binding instead of choosing again. A named target or a worktree/shared isolation never reaches capacity placement, and the existing local and remote paths still run all their checks. create_threads no longer ignores target.environmentId: its threads share the parent's checkout, so another environment is refused explicitly. The ARC workflow now typechecks @t3tools/shared and runs the placement, remote delegation and orchestrator MCP suites. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/closura-arc-ci.yml | 8 +- .../src/mcp/OrchestratorMcpService.test.ts | 16 ++ apps/server/src/mcp/OrchestratorMcpService.ts | 102 ++++++++++- .../ThreadManagementService.ts | 56 ++++++ apps/server/src/peer/RemoteDelegation.test.ts | 93 +++++++++- apps/server/src/peer/RemoteDelegation.ts | 163 +++++++++++++++++- .../src/relay/AgentAwarenessRelay.test.ts | 2 + packages/contracts/src/orchestratorMcp.ts | 6 + 8 files changed, 432 insertions(+), 14 deletions(-) diff --git a/.github/workflows/closura-arc-ci.yml b/.github/workflows/closura-arc-ci.yml index 79a20c8239977..8050491b53e79 100644 --- a/.github/workflows/closura-arc-ci.yml +++ b/.github/workflows/closura-arc-ci.yml @@ -132,11 +132,12 @@ jobs: run: | set -euo pipefail vp run --filter @t3tools/contracts typecheck + vp run --filter @t3tools/shared typecheck vp run --filter @t3tools/client-runtime typecheck vp run --filter @t3tools/provider-core typecheck vp run --filter t3 typecheck - - name: Test AssemblyAI dictation and native inbox behavior + - name: Test AssemblyAI dictation, native inbox and thread placement behavior shell: bash run: | set -euo pipefail @@ -146,7 +147,10 @@ jobs: packages/contracts/src/threadMetadataMcp.test.ts \ apps/web/src/components/threadActionMenu.logic.test.ts \ apps/server/src/orchestration-v2/ThreadLaunchService.test.ts \ - apps/server/src/orchestration-v2/SubagentProjection.test.ts + apps/server/src/orchestration-v2/SubagentProjection.test.ts \ + packages/shared/src/threadPlacement.test.ts \ + apps/server/src/peer/RemoteDelegation.test.ts \ + apps/server/src/mcp/OrchestratorMcpService.test.ts - name: Build native T3 server shell: bash diff --git a/apps/server/src/mcp/OrchestratorMcpService.test.ts b/apps/server/src/mcp/OrchestratorMcpService.test.ts index 6a969e21e3abd..97907e2d50b2c 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.test.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.test.ts @@ -2906,6 +2906,22 @@ describe("OrchestratorMcpService provider resolution", () => { assert.equal(child.providerInstanceId, selection.instanceId); assert.equal(child.model, selection.model); assert.deepEqual(yield* service.createThreads(scope, request), first); + // Its threads share this checkout, so another environment is refused, not ignored. + const elsewhere = yield* service + .createThreads(scope, { + threads: [ + { + target: { + ...request.threads[0]!.target, + environmentId: EnvironmentId.make("environment-elsewhere"), + }, + }, + ], + clientRequestId: "elsewhere", + }) + .pipe(Effect.flip); + assert.equal(elsewhere.code, "invalid_request"); + assert.include(elsewhere.message, "t3_thread_launch"); // The creation receipt exists, but this prompt has not been accepted yet. const withPrompt = { diff --git a/apps/server/src/mcp/OrchestratorMcpService.ts b/apps/server/src/mcp/OrchestratorMcpService.ts index b86a564c19f9d..c5602e77ac6ab 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.ts @@ -1,5 +1,6 @@ import { CommandId, + EnvironmentId, type RunId, isProviderAvailable, MessageId, @@ -71,6 +72,10 @@ import { backgroundWorkHoldsCompletion } from "@t3tools/shared/orchestrationV2Pe import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError"; import { resolveProjectSettings } from "@t3tools/shared/projectSettings"; import { formatThreadLink } from "@t3tools/shared/threadLinks"; +import { + chooseThreadPlacement, + type ThreadPlacementDecision, +} from "@t3tools/shared/threadPlacement"; import { threadPullRequestUpdatedAtMs } from "@t3tools/shared/threadPullRequests"; import { normalizeThreadPullRequestKey, @@ -2224,7 +2229,57 @@ const make = Effect.gen(function* () { return task; }); - return OrchestratorMcpService.of({ + /** Where a placed task may run, or why it may not. */ + const admitPlacement = (decision: ThreadPlacementDecision, conflict: string) => + decision.accepted + ? Effect.succeed(EnvironmentId.make(decision.environmentId)) + : Effect.fail( + failure( + "invalid_request", + decision.reason === "conflicting-locality" + ? conflict + : `No environment can take this (${decision.reason}).`, + ), + ); + + /** The environment bound to an auto-placed task, for the parent's model or the exact one named. */ + const placeTask = Effect.fn("OrchestratorMcpService.placeTask")(function* ( + callerScope: McpInvocationScope, + input: OrchestratorMcpDelegateTaskInput, + key: string, + ) { + yield* requireCapability(callerScope); + const { scope, parent } = yield* loadThreadCaller(callerScope, "delegate_task"); + const target = input.target; + if (target?.driverKind !== undefined && target.providerInstanceId === undefined) { + return yield* failure( + "invalid_request", + 'With placement "auto", name target.providerInstanceId instead of driverKind; every environment matches it exactly.', + ); + } + const inherited = parent.thread.modelSelection; + const instanceId = target?.providerInstanceId ?? inherited.instanceId; + const model = + target?.model ?? (instanceId === inherited.instanceId ? inherited.model : undefined); + if (model === undefined) { + return yield* failure( + "target_required", + 'With placement "auto" and another providerInstanceId, pass target.model.', + ); + } + return yield* remoteDelegation.place(scope, { + key, + projectId: parent.thread.projectId, + modelSelection: { instanceId, model }, + runtimeMode: + input.runtimeMode === undefined || input.runtimeMode === "inherit" + ? undefined + : input.runtimeMode, + here: yield* service.capabilities(callerScope, { search: model }), + }); + }); + + const service: OrchestratorMcpServiceShape = OrchestratorMcpService.of({ scheduleTask: Effect.fn("OrchestratorMcpService.scheduleTask")(function* (scope, input) { const { parent, limits } = yield* loadCaller(scope, ["runs"]); const projectId = yield* resolveProjectTarget(parent, input.projectId); @@ -2642,16 +2697,33 @@ const make = Effect.gen(function* () { }; }), delegateTask: Effect.fn("OrchestratorMcpService.delegateTask")(function* (callerScope, input) { - const environmentId = - input.target === undefined - ? undefined - : PeerForwarding.remoteTarget(callerScope, input.target.environmentId); - if (input.target !== undefined && environmentId !== undefined) { + // One key for the placement, the launch and the record, so a retry finds all three. + const clientRequestId = yield* requestKey(input.clientRequestId); + const named = input.target?.environmentId; + // A named target or the parent's checkout decides; only work needing neither is placed. + const placed = + input.placement === "auto" && named === undefined && input.isolation === undefined + ? yield* placeTask(callerScope, input, clientRequestId) + : yield* admitPlacement( + chooseThreadPlacement( + { + originEnvironmentId: callerScope.environmentId, + pinnedEnvironmentId: named ?? callerScope.environmentId, + requiresOrigin: input.isolation === "worktree", + }, + [], + 0, + ), + 'A task in a linked environment runs in that project\'s checkout; isolation "worktree" is not supported there. Omit isolation, or use t3_thread_launch with environmentId and a worktree workspaceStrategy.', + ); + const environmentId = PeerForwarding.remoteTarget(callerScope, placed); + if (environmentId !== undefined) { yield* requireCapability(callerScope); const scope = yield* requireThreadScope(callerScope, "delegate_task"); // The task's child runs as an ordinary thread there; this thread keeps the task. const { taskId } = yield* remoteDelegation.delegate(scope, { ...input, + clientRequestId, target: { ...input.target, environmentId }, }); return yield* input.mode === "wait" @@ -2682,7 +2754,7 @@ const make = Effect.gen(function* () { ); } const providers = yield* loadProviders; - const key = yield* requestKey(input.clientRequestId); + const key = clientRequestId; const commandId = stableCommandId({ scope, requestKey: key, @@ -3021,6 +3093,21 @@ const make = Effect.gen(function* () { createThreads: Effect.fn("OrchestratorMcpService.createThreads")( function* (callerScope, input) { const { scope, parent } = yield* loadThreadCaller(callerScope, "create_threads"); + // Each thread shares this thread's checkout, so another environment cannot take it. + for (const request of input.threads) { + yield* admitPlacement( + chooseThreadPlacement( + { + originEnvironmentId: callerScope.environmentId, + pinnedEnvironmentId: request.target?.environmentId, + requiresOrigin: true, + }, + [], + 0, + ), + "create_threads starts threads in this checkout. To start one in a linked environment, use t3_thread_launch with that environmentId.", + ); + } const parentRun = ThreadManagementService.latestActiveRun(parent); if ( parentRun === undefined || @@ -3604,6 +3691,7 @@ const make = Effect.gen(function* () { } satisfies OrchestratorMcpThreadInterruptResult; }), }); + return service; }); export const layer: Layer.Layer< diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index 611d9a3244f24..e1d06cdc4d2fe 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -12,6 +12,7 @@ import type { import { type ChatAttachment, CommandId, + EnvironmentId, isDelegatedAppThread, isListedThreadLineage, MessageId, @@ -472,6 +473,19 @@ export interface ThreadManagementServiceShape { }) => Effect.Effect; /** Receipt replays start no work; skip enablement checks with the same command id so settings cannot reject accepted retries. */ readonly hasReceipt: (commandId: CommandId) => Effect.Effect; + /** The environment a placement intent was bound to, if it was. */ + readonly placementOf: ( + commandId: CommandId, + ) => Effect.Effect, CommandReceiptStore.CommandReceiptStoreV2Error>; + /** + * Binds a placement intent to an environment unless it is bound already, and + * returns the binding that holds: the first writer's, read back from storage. + */ + readonly bindPlacement: (input: { + readonly commandId: CommandId; + readonly threadId: ThreadId; + readonly environmentId: EnvironmentId; + }) => Effect.Effect; readonly waitForThread: ( input: ThreadManagementWaitInput, ) => Effect.Effect; @@ -604,6 +618,8 @@ const FORCE_STOP_REASON = "Force stopped by user"; const FORCE_STOP_RELEASE_WAIT = "10 seconds"; const isForceStopThreadError = Schema.is(OrchestrationV2ForceStopThreadError); +const PLACEMENT_RECEIPT_TYPE = "placement.bind:"; + const make = Effect.gen(function* () { const threadDispatch = yield* ThreadCommandExecutor.ThreadCommandExecutor; const orchestrator = yield* Orchestrator.OrchestratorV2; @@ -651,6 +667,44 @@ const make = Effect.gen(function* () { Effect.map(Option.isSome), ); + // A placement binding is a receipt with no events, so it lives and is kept like every + // other command's; its type names the environment. + const placementOf: ThreadManagementServiceShape["placementOf"] = (commandId) => + receipts + .getByCommandId(commandId) + .pipe( + Effect.map( + Option.flatMap((receipt) => + receipt.commandType.startsWith(PLACEMENT_RECEIPT_TYPE) + ? Option.some( + EnvironmentId.make(receipt.commandType.slice(PLACEMENT_RECEIPT_TYPE.length)), + ) + : Option.none(), + ), + ), + ); + + const bindPlacement: ThreadManagementServiceShape["bindPlacement"] = Effect.fn( + "orchestrationV2.threadManagement.bindPlacement", + )(function* (input) { + yield* receipts.insertIfAbsent({ + commandId: input.commandId, + threadId: input.threadId, + commandType: `${PLACEMENT_RECEIPT_TYPE}${input.environmentId}`, + acceptedAt: yield* DateTime.now, + resultSequence: 0, + status: "accepted", + error: null, + }); + const bound = yield* placementOf(input.commandId); + if (Option.isNone(bound)) { + return yield* new CommandReceiptStore.CommandReceiptStoreReadError({ + commandId: input.commandId, + }); + } + return bound.value; + }); + const ensureLegacyTranscript = Effect.fn( "orchestrationV2.threadManagement.ensureLegacyTranscript", )(function* (threadId: ThreadId) { @@ -1768,6 +1822,8 @@ const make = Effect.gen(function* () { assertProviderInstanceEnabled, isProviderInstanceEnabledForProject, hasReceipt, + placementOf, + bindPlacement, waitForThread, settleAfterRun, settleThread, diff --git a/apps/server/src/peer/RemoteDelegation.test.ts b/apps/server/src/peer/RemoteDelegation.test.ts index 90c6eb83baae5..1db346981651e 100644 --- a/apps/server/src/peer/RemoteDelegation.test.ts +++ b/apps/server/src/peer/RemoteDelegation.test.ts @@ -16,6 +16,7 @@ import { type Project, type RepositoryIdentity, RunId, + type HostResourcesSnapshot, type ServerProvider, ThreadId, TurnItemId, @@ -58,6 +59,7 @@ import * as ProjectCloneTracker from "../project/ProjectCloneTracker.ts"; import * as ProjectService from "../project/ProjectService.ts"; import * as RepositoryIdentityResolver from "../project/RepositoryIdentityResolver.ts"; import * as ProviderRegistry from "../provider/ProviderRegistry.ts"; +import * as HostResources from "../resourceTelemetry/HostResources.ts"; import * as ScheduledTaskService from "../scheduledTasks/ScheduledTaskService.ts"; import * as SecretRequests from "../secrets/SecretRequests.ts"; import * as ServerSettings from "../serverSettings.ts"; @@ -141,8 +143,19 @@ interface BoxThread { readonly sent: Ref.Ref>; /** Runs as the launch there is answered, before the laptop hears back. */ readonly afterLaunch: Ref.Ref>; + /** The box's load as its `orchestrator_capabilities` reports it. */ + readonly resources: Ref.Ref; } +const GiB = 1024 ** 3; +const loadOf = (cpuUtilization: number): HostResourcesSnapshot => ({ + sampledAt: 0, + cpuUtilization, + cpuCount: 8, + availableMemoryBytes: 8 * GiB, + totalMemoryBytes: 16 * GiB, +}); + const makeBoxThread = Effect.gen(function* () { return { launched: yield* Ref.make(null), @@ -179,6 +192,7 @@ const makeBoxThread = Effect.gen(function* () { releaseStop: yield* Deferred.make(), sent: yield* Ref.make>([]), afterLaunch: yield* Ref.make>(Effect.void), + resources: yield* Ref.make(loadOf(0.1)), } satisfies BoxThread; }); @@ -549,7 +563,12 @@ const boxToolkitLayer = (thread: BoxThread) => { Layer.provide(DocumentText.layer.pipe(Layer.provideMerge(NodeServices.layer))), Layer.provide(NodeCrypto.layer), Layer.provide(layerBoxThreads), - Layer.provide(layerBoxLaunches), + Layer.provide( + Layer.merge( + layerBoxLaunches, + Layer.succeed(HostResources.HostResources, { read: Ref.get(thread.resources) }), + ), + ), Layer.provide(Layer.mock(RemoteDelegation.RemoteDelegation)({})), Layer.provide(Layer.mock(PeerForwarding.PeerForwarding)({})), Layer.provide(Layer.mock(PeerLinkRequests.PeerLinkRequests)({})), @@ -715,8 +734,15 @@ const makeLaptop = ( Layer.provide(Layer.mock(PeerLinkRequests.PeerLinkRequests)({})), Layer.provide(NodeCrypto.layer), Layer.provideMerge(layerThreads), - Layer.provide(Layer.mock(ProviderRegistry.ProviderRegistry)({})), - Layer.provide(Layer.mock(ProviderAdapterRegistry.ProviderAdapterRegistryV2)({})), + // The laptop lists no provider of its own, so it never runs the box's model. + Layer.provide( + Layer.mock(ProviderRegistry.ProviderRegistry)({ getProviders: Effect.succeed([]) }), + ), + Layer.provide( + Layer.mock(ProviderAdapterRegistry.ProviderAdapterRegistryV2)({ + list: () => Effect.succeed([]), + }), + ), Layer.provide(Layer.mock(ScheduledTaskService.ScheduledTaskService)({})), Layer.provide(Layer.mock(ProjectService.ProjectService)({})), Layer.provide(Layer.mock(ProjectCloneTracker.ProjectCloneTracker)({})), @@ -1342,6 +1368,67 @@ it.effect("follows an open task again after the laptop restarts", () => ).pipe(Effect.provide(NodeHttpServer.layerTest)), ); +it.effect("an auto-placed task keeps its environment after a lost answer and a restart", () => + Effect.scoped( + Effect.gen(function* () { + const thread = yield* makeBoxThread; + const b = yield* serveBox(thread); + const offers = yield* SubscriptionRef.make< + ReadonlyArray + >([]); + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); + // Its link outlives the restart, so the box knows it as the same link session. + const linking = yield* layerLinkingEnvironment(laptop).pipe(Layer.build); + const placeTask = ( + laptopEnv: Effect.Success>, + clientRequestId: string, + ) => + laptopEnv.call("delegate_task", { + task: "Run the suite wherever there is room.", + placement: "auto", + target: { providerInstanceId: boxClaude.instanceId, model: "claude-sonnet-4-6" }, + clientRequestId, + }); + // The box takes the launch with room to spare, but its answer never arrives. + yield* Ref.set(thread.afterLaunch, Effect.die("connection lost")); + const lost = yield* Effect.scoped( + Effect.gen(function* () { + const before = yield* makeLaptop(database, offers, linking); + yield* linkTo(before.links, b, "full-access"); + yield* seedParent(before); + return yield* placeTask(before, "auto-suite"); + }), + ); + expect(lost.isError).toBe(true); + const launchedThere = yield* Ref.get(thread.launched); + expect(launchedThere).not.toBeNull(); + + // The box fills up while the laptop restarts: choosing afresh finds nowhere. + yield* Ref.set(thread.resources, loadOf(0.99)); + yield* Ref.set(thread.afterLaunch, Effect.void); + const after = yield* makeLaptop(database, offers, linking); + const fresh = yield* placeTask(after, "another-suite"); + expect(fresh.isError).toBe(true); + expect(JSON.stringify(fresh.content)).toContain("no-eligible-environment"); + expect(yield* Ref.get(thread.launched)).toBe(launchedThere); + + // The retry keeps the binding: the same thread there, recorded once here. + const retried = yield* placeTask(after, "auto-suite"); + expect(retried.isError, JSON.stringify(retried.content)).toBe(false); + expect(retried.structuredContent).toMatchObject({ + remoteChild: { environmentId: box.environmentId, threadId: launchedThere }, + }); + expect((yield* Ref.get(thread.launchedWith))?.modelSelection).toEqual({ + instanceId: boxClaude.instanceId, + model: "claude-sonnet-4-6", + }); + expect( + (yield* after.orchestrator.getThreadProjection(parentThreadId)).subagents, + ).toHaveLength(1); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); + it.effect("fails the task when the box revokes the link, and cancel interrupts it there", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/peer/RemoteDelegation.ts b/apps/server/src/peer/RemoteDelegation.ts index 8fcd240e70577..73f12bdbdfd58 100644 --- a/apps/server/src/peer/RemoteDelegation.ts +++ b/apps/server/src/peer/RemoteDelegation.ts @@ -1,6 +1,8 @@ import { CommandId, - type EnvironmentId, + EnvironmentId, + isRuntimeModeAbove, + type ModelSelection, NodeId, OrchestratorMcpCapabilitiesInput, OrchestratorMcpCapabilitiesResult, @@ -13,13 +15,20 @@ import { type OrchestratorMcpThreadTimelineItem, type OrchestrationV2RemoteTaskChild, type OrchestrationV2Subagent, + type PeerLinkSummary, type ProjectId, type ProviderInstanceId, ProviderDriverKind, type RunId, + type RuntimeMode, type ThreadId, } from "@t3tools/contracts"; +import { + chooseThreadPlacement, + type ThreadPlacementCandidate, +} from "@t3tools/shared/threadPlacement"; import * as Cause from "effect/Cause"; +import * as Clock from "effect/Clock"; import * as Context from "effect/Context"; import * as SecureRandom from "effect/crypto/SecureRandom"; import * as DateTime from "effect/DateTime"; @@ -96,6 +105,22 @@ export class RemoteDelegation extends Context.Service< }; }, ) => Effect.Effect<{ readonly taskId: NodeId }, OrchestratorMcpFailure>; + /** + * Where a `placement: "auto"` task runs: the environment its key is bound to, + * or the one chosen now from this environment's and authenticated links' own + * answers, bound before anything starts. A retry never chooses again. + */ + readonly place: ( + scope: ThreadScope, + input: { + readonly key: string; + readonly projectId: ProjectId; + readonly modelSelection: Pick; + readonly runtimeMode: RuntimeMode | undefined; + /** This environment's own `orchestrator_capabilities`, searched for the model. */ + readonly here: OrchestratorMcpCapabilitiesResult; + }, + ) => Effect.Effect; /** * A `t3_thread_send` there that started or queued a run on the thread of * one of this thread's tasks that has its result: like a follow-up to a @@ -137,6 +162,24 @@ export class RemoteDelegation extends Context.Service< } >()("t3/peer/RemoteDelegation") {} +/** The exact instance and model there can run a child task, in the mode asked for. */ +const runsModel = ( + capabilities: OrchestratorMcpCapabilitiesResult, + requested: { + readonly modelSelection: Pick; + readonly runtimeMode: RuntimeMode | undefined; + }, +) => + capabilities.providers.some( + (provider) => + provider.providerInstanceId === requested.modelSelection.instanceId && + provider.canRunChildTask && + provider.models.some((model) => model.id === requested.modelSelection.model) && + (requested.runtimeMode === undefined || + provider.runtimeModes === undefined || + provider.runtimeModes.includes(requested.runtimeMode)), + ); + /** The other side refused because the thread there runs above this caller's modes. */ const isModeRefusal = (error: OrchestratorMcpFailure) => error.code === "runtime_mode_escalation_denied" || @@ -262,6 +305,122 @@ const make = Effect.gen(function* () { return chosen.providerInstanceId; }); + /** A linked environment as a placement candidate, from what it answered just now. */ + const peerCandidate = Effect.fn("RemoteDelegation.peerCandidate")( + function* ( + scope: ThreadScope, + link: PeerLinkSummary, + input: Parameters[1], + ) { + const capabilities = yield* forwarding.call( + scope, + RemoteCapabilitiesTool, + link.environmentId, + { + search: input.modelSelection.model, + }, + ); + const receivedAt = yield* Clock.currentTimeMillis; + const projectAvailable = yield* resolveRemoteProject( + scope, + link.environmentId, + input.projectId, + undefined, + ).pipe( + Effect.as(true), + Effect.orElseSucceed(() => false), + ); + return { + environmentId: link.environmentId, + resources: capabilities.hostResources ?? null, + receivedAt, + weight: 1, + operable: true, + projectAvailable, + modelAvailable: runsModel(capabilities, input), + } satisfies ThreadPlacementCandidate; + }, + // One that does not answer, or refuses this caller, is not a candidate. + (effect, _scope, link) => + effect.pipe( + Effect.tapError((cause) => + Effect.logDebug("remote-delegation.placement-candidate-unavailable", { + environmentId: link.environmentId, + cause, + }), + ), + Effect.option, + ), + ); + + const place: RemoteDelegation["Service"]["place"] = Effect.fn("RemoteDelegation.place")( + function* (scope, input) { + const bindingId = CommandId.make( + `command:mcp:${encodeURIComponent(scope.requestNamespace)}:placement:delegate-task:${encodeURIComponent(input.key)}`, + ); + const bindingFailure = () => + new OrchestratorMcpFailure({ + code: "orchestration_error", + message: + "Unable to record where this task runs, so nothing was started. Retry with the same clientRequestId.", + }); + // A bound intent keeps its environment, even one that has since filled up or + // stopped answering: its first attempt may have started the task there. + const bound = yield* threads.placementOf(bindingId).pipe(Effect.mapError(bindingFailure)); + if (Option.isSome(bound)) return bound.value; + + const linked = (yield* links.enabled) + ? yield* links.list.pipe(Effect.orElseSucceed(() => [])) + : []; + const peers = yield* Effect.forEach( + linked.filter( + (link) => + link.environmentId !== hereId && + link.status === "reachable" && + link.access !== "read-only" && + (input.runtimeMode === undefined || + !isRuntimeModeAbove(input.runtimeMode, link.access)), + ), + (link) => peerCandidate(scope, link, input), + { concurrency: "unbounded" }, + ); + const now = yield* Clock.currentTimeMillis; + const decision = chooseThreadPlacement( + { originEnvironmentId: hereId }, + [ + { + environmentId: hereId, + resources: input.here.hostResources ?? null, + receivedAt: now, + weight: 1, + operable: true, + projectAvailable: true, + modelAvailable: runsModel(input.here, input), + }, + ...peers.flatMap((peer) => (Option.isSome(peer) ? [peer.value] : [])), + ], + now, + ); + yield* Effect.annotateCurrentSpan({ + "placement.reason": decision.reason, + "placement.candidates": peers.length + 1, + }); + if (!decision.accepted) { + return yield* new OrchestratorMcpFailure({ + code: "target_required", + message: `No environment can take this task now (${decision.reason}): none answered with spare capacity, this repository's project and ${input.modelSelection.instanceId}/${input.modelSelection.model}. Nothing was started. Pass target.environmentId, or omit placement to run it here.`, + }); + } + return yield* threads + .bindPlacement({ + commandId: bindingId, + threadId: scope.thread.threadId, + environmentId: EnvironmentId.make(decision.environmentId), + }) + .pipe(Effect.mapError(bindingFailure)); + }, + ); + const delegate: RemoteDelegation["Service"]["delegate"] = Effect.fn("RemoteDelegation.delegate")( function* (scope, input) { const parent = yield* threads.getThreadRecords(scope.thread.threadId, ["runs"]).pipe( @@ -1034,7 +1193,7 @@ const make = Effect.gen(function* () { }).pipe(Effect.ignoreCause({ log: true })), ); - return RemoteDelegation.of({ delegate, followUp, cancel, runStatus, start }); + return RemoteDelegation.of({ delegate, place, followUp, cancel, runStatus, start }); }); /** A defect, logged, as a failure, so a retry covers it. */ diff --git a/apps/server/src/relay/AgentAwarenessRelay.test.ts b/apps/server/src/relay/AgentAwarenessRelay.test.ts index 10388e99fe7c5..6f9b798c8ef6a 100644 --- a/apps/server/src/relay/AgentAwarenessRelay.test.ts +++ b/apps/server/src/relay/AgentAwarenessRelay.test.ts @@ -252,6 +252,8 @@ const makeTestRelay = Effect.fnUntraced(function* ( assertProviderInstanceEnabled: unused, isProviderInstanceEnabledForProject: unused, hasReceipt: unused, + placementOf: unused, + bindPlacement: unused, waitForThread: unused, settleAfterRun: unused, settleThread: unused, diff --git a/packages/contracts/src/orchestratorMcp.ts b/packages/contracts/src/orchestratorMcp.ts index c30c4fea2e785..e7b3827987b8b 100644 --- a/packages/contracts/src/orchestratorMcp.ts +++ b/packages/contracts/src/orchestratorMcp.ts @@ -180,6 +180,12 @@ export const OrchestratorMcpDelegateTaskInput = Schema.Struct({ "'shared' (default): parent checkout. 'worktree': new prepared checkout from the parent commit, for a separate checkout or PR per child. Uncommitted edits stay out.", }), ), + placement: Schema.optional( + Schema.Literals(["auto"]).annotate({ + description: + "'auto': for work that needs none of your checkout's changes. Runs here or on a linked environment with spare capacity, this repository's project and your exact provider and model; there, in that project's checkout. Ignored with target.environmentId or isolation. Reuse clientRequestId on retry to keep the environment chosen.", + }), + ), mode: Schema.optional( Schema.Literals(["async", "wait"]).annotate({ description: From 6e8414980fa9991535b329d7e8771ec983f5f9a7 Mon Sep 17 00:00:00 2001 From: Kameron Smith <112618179+nullStack65@users.noreply.github.com> Date: Wed, 18 Nov 2026 05:35:14 +0000 Subject: [PATCH 2/3] fix(placement): no duplicate child after re-link; keep parent modes and project under auto Review round 1 on the placement binding. - A remote launch binds the link session that took it (the link's per-link secret name, not a timestamp) and, once answered, the child thread there, in the existing receipt store. A retry under a replaced link no longer launches a second child: it returns the task already recorded for that child, or refuses with how to find the thread there. Covers auto and named targets. - placement:"auto" resolves the parent's effective runtime and interaction modes (or narrower ones asked for) before choosing, excludes links whose access is below them, and forwards both explicitly so no destination narrows them silently. - placement:"auto" refuses target.projectId before any binding or launch. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/mcp/OrchestratorMcpService.ts | 31 ++++- .../ThreadManagementService.ts | 43 +++--- apps/server/src/peer/PeerLinks.ts | 12 ++ apps/server/src/peer/RemoteDelegation.test.ts | 130 ++++++++++++++++-- apps/server/src/peer/RemoteDelegation.ts | 106 +++++++++++--- .../src/relay/AgentAwarenessRelay.test.ts | 4 +- 6 files changed, 262 insertions(+), 64 deletions(-) diff --git a/apps/server/src/mcp/OrchestratorMcpService.ts b/apps/server/src/mcp/OrchestratorMcpService.ts index c5602e77ac6ab..a4405d02c0ed6 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.ts @@ -2251,6 +2251,13 @@ const make = Effect.gen(function* () { yield* requireCapability(callerScope); const { scope, parent } = yield* loadThreadCaller(callerScope, "delegate_task"); const target = input.target; + // Placement qualifies the project with this repository wherever it runs, and no other. + if (target?.projectId !== undefined) { + return yield* failure( + "invalid_request", + 'With placement "auto", omit target.projectId: the task runs in the project with this repository wherever it is placed. To use another project, pass target.environmentId with it.', + ); + } if (target?.driverKind !== undefined && target.providerInstanceId === undefined) { return yield* failure( "invalid_request", @@ -2267,16 +2274,21 @@ const make = Effect.gen(function* () { 'With placement "auto" and another providerInstanceId, pass target.model.', ); } - return yield* remoteDelegation.place(scope, { + // The parent's modes, or narrower ones asked for, are chosen with and sent explicitly, + // so no environment qualifies or runs the task at less than they allow. + const runtimeMode = yield* resolveRuntimeMode(parent.thread.runtimeMode, input.runtimeMode); + const interactionMode = yield* resolveInteractionMode( + parent.thread.interactionMode, + input.interactionMode, + ); + const environmentId = yield* remoteDelegation.place(scope, { key, projectId: parent.thread.projectId, modelSelection: { instanceId, model }, - runtimeMode: - input.runtimeMode === undefined || input.runtimeMode === "inherit" - ? undefined - : input.runtimeMode, + runtimeMode, here: yield* service.capabilities(callerScope, { search: model }), }); + return { environmentId, runtimeMode, interactionMode }; }); const service: OrchestratorMcpServiceShape = OrchestratorMcpService.of({ @@ -2701,9 +2713,13 @@ const make = Effect.gen(function* () { const clientRequestId = yield* requestKey(input.clientRequestId); const named = input.target?.environmentId; // A named target or the parent's checkout decides; only work needing neither is placed. - const placed = + const automatic = input.placement === "auto" && named === undefined && input.isolation === undefined ? yield* placeTask(callerScope, input, clientRequestId) + : undefined; + const placed = + automatic !== undefined + ? automatic.environmentId : yield* admitPlacement( chooseThreadPlacement( { @@ -2723,6 +2739,9 @@ const make = Effect.gen(function* () { // The task's child runs as an ordinary thread there; this thread keeps the task. const { taskId } = yield* remoteDelegation.delegate(scope, { ...input, + ...(automatic === undefined + ? {} + : { runtimeMode: automatic.runtimeMode, interactionMode: automatic.interactionMode }), clientRequestId, target: { ...input.target, environmentId }, }); diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index e1d06cdc4d2fe..9ebba141e0808 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -12,7 +12,6 @@ import type { import { type ChatAttachment, CommandId, - EnvironmentId, isDelegatedAppThread, isListedThreadLineage, MessageId, @@ -473,19 +472,19 @@ export interface ThreadManagementServiceShape { }) => Effect.Effect; /** Receipt replays start no work; skip enablement checks with the same command id so settings cannot reject accepted retries. */ readonly hasReceipt: (commandId: CommandId) => Effect.Effect; - /** The environment a placement intent was bound to, if it was. */ - readonly placementOf: ( + /** What a caller's intent was bound to, such as the environment chosen for it, if it was. */ + readonly intentOf: ( commandId: CommandId, - ) => Effect.Effect, CommandReceiptStore.CommandReceiptStoreV2Error>; + ) => Effect.Effect, CommandReceiptStore.CommandReceiptStoreV2Error>; /** - * Binds a placement intent to an environment unless it is bound already, and - * returns the binding that holds: the first writer's, read back from storage. + * Binds a caller's intent to a value unless it is bound already, and returns + * the binding that holds: the first writer's, read back from storage. */ - readonly bindPlacement: (input: { + readonly bindIntent: (input: { readonly commandId: CommandId; readonly threadId: ThreadId; - readonly environmentId: EnvironmentId; - }) => Effect.Effect; + readonly value: string; + }) => Effect.Effect; readonly waitForThread: ( input: ThreadManagementWaitInput, ) => Effect.Effect; @@ -618,7 +617,7 @@ const FORCE_STOP_REASON = "Force stopped by user"; const FORCE_STOP_RELEASE_WAIT = "10 seconds"; const isForceStopThreadError = Schema.is(OrchestrationV2ForceStopThreadError); -const PLACEMENT_RECEIPT_TYPE = "placement.bind:"; +const INTENT_RECEIPT_TYPE = "intent.bind:"; const make = Effect.gen(function* () { const threadDispatch = yield* ThreadCommandExecutor.ThreadCommandExecutor; @@ -667,36 +666,34 @@ const make = Effect.gen(function* () { Effect.map(Option.isSome), ); - // A placement binding is a receipt with no events, so it lives and is kept like every - // other command's; its type names the environment. - const placementOf: ThreadManagementServiceShape["placementOf"] = (commandId) => + // A binding is a receipt with no events, so it lives and is kept like every other + // command's; its type carries the value. + const intentOf: ThreadManagementServiceShape["intentOf"] = (commandId) => receipts .getByCommandId(commandId) .pipe( Effect.map( Option.flatMap((receipt) => - receipt.commandType.startsWith(PLACEMENT_RECEIPT_TYPE) - ? Option.some( - EnvironmentId.make(receipt.commandType.slice(PLACEMENT_RECEIPT_TYPE.length)), - ) + receipt.commandType.startsWith(INTENT_RECEIPT_TYPE) + ? Option.some(receipt.commandType.slice(INTENT_RECEIPT_TYPE.length)) : Option.none(), ), ), ); - const bindPlacement: ThreadManagementServiceShape["bindPlacement"] = Effect.fn( - "orchestrationV2.threadManagement.bindPlacement", + const bindIntent: ThreadManagementServiceShape["bindIntent"] = Effect.fn( + "orchestrationV2.threadManagement.bindIntent", )(function* (input) { yield* receipts.insertIfAbsent({ commandId: input.commandId, threadId: input.threadId, - commandType: `${PLACEMENT_RECEIPT_TYPE}${input.environmentId}`, + commandType: `${INTENT_RECEIPT_TYPE}${input.value}`, acceptedAt: yield* DateTime.now, resultSequence: 0, status: "accepted", error: null, }); - const bound = yield* placementOf(input.commandId); + const bound = yield* intentOf(input.commandId); if (Option.isNone(bound)) { return yield* new CommandReceiptStore.CommandReceiptStoreReadError({ commandId: input.commandId, @@ -1822,8 +1819,8 @@ const make = Effect.gen(function* () { assertProviderInstanceEnabled, isProviderInstanceEnabledForProject, hasReceipt, - placementOf, - bindPlacement, + intentOf, + bindIntent, waitForThread, settleAfterRun, settleThread, diff --git a/apps/server/src/peer/PeerLinks.ts b/apps/server/src/peer/PeerLinks.ts index 6fddf407cf4d4..e9168a8c7fa81 100644 --- a/apps/server/src/peer/PeerLinks.ts +++ b/apps/server/src/peer/PeerLinks.ts @@ -97,6 +97,13 @@ export class PeerLinks extends Context.Service< readonly enabled: Effect.Effect; readonly link: (input: PeerLinkCreateInput) => Effect.Effect; readonly list: Effect.Effect, PeerLinkError>; + /** + * Names the link's session there without revealing its token. Linking again + * starts a new session there, so this changes exactly when that does. + */ + readonly session: ( + environmentId: EnvironmentId, + ) => Effect.Effect, PeerLinkError>; /** The stored link, without asking the peer whether it answers. */ readonly get: ( environmentId: EnvironmentId, @@ -712,10 +719,15 @@ const make = Effect.gen(function* () { Effect.ignoreCause({ log: true }), ); + // Each link stores its token under a fresh random name, so the name identifies the session. + const session: PeerLinks["Service"]["session"] = (environmentId) => + readRow(environmentId).pipe(Effect.map(Option.map((row) => row.secret_name))); + return PeerLinks.of({ enabled, link, list, + session, get, unlink, resolve, diff --git a/apps/server/src/peer/RemoteDelegation.test.ts b/apps/server/src/peer/RemoteDelegation.test.ts index 1db346981651e..d68f6df11bc1a 100644 --- a/apps/server/src/peer/RemoteDelegation.test.ts +++ b/apps/server/src/peer/RemoteDelegation.test.ts @@ -791,6 +791,7 @@ const parentScope: McpInvocationContext.McpInvocationScope = { const seedParent = ( laptopEnv: Effect.Success>, parentModelSelection: ModelSelection = modelSelection, + interactionMode: "default" | "plan" = "default", ) => Effect.gen(function* () { const now = yield* DateTime.now; @@ -805,7 +806,7 @@ const seedParent = ( title: "Laptop parent", modelSelection: parentModelSelection, runtimeMode: "full-access", - interactionMode: "default", + interactionMode, branch: null, worktreePath: null, }); @@ -919,6 +920,19 @@ const delegateToBox = (laptopEnv: Effect.Success>) }), ); +/** A task placed by capacity, for the box's Claude model, which the laptop does not run. */ +const autoTask = ( + laptopEnv: Effect.Success>, + clientRequestId: string, + target: Record = {}, +) => + laptopEnv.call("delegate_task", { + task: "Run the suite wherever there is room.", + placement: "auto", + target: { providerInstanceId: boxClaude.instanceId, model: "claude-sonnet-4-6", ...target }, + clientRequestId, + }); + /** Waits until the task here has the result its thread there ended with. */ const taskResult = (laptopEnv: Effect.Success>, taskId: NodeId) => laptopEnv.orchestrator @@ -1379,16 +1393,6 @@ it.effect("an auto-placed task keeps its environment after a lost answer and a r const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); // Its link outlives the restart, so the box knows it as the same link session. const linking = yield* layerLinkingEnvironment(laptop).pipe(Layer.build); - const placeTask = ( - laptopEnv: Effect.Success>, - clientRequestId: string, - ) => - laptopEnv.call("delegate_task", { - task: "Run the suite wherever there is room.", - placement: "auto", - target: { providerInstanceId: boxClaude.instanceId, model: "claude-sonnet-4-6" }, - clientRequestId, - }); // The box takes the launch with room to spare, but its answer never arrives. yield* Ref.set(thread.afterLaunch, Effect.die("connection lost")); const lost = yield* Effect.scoped( @@ -1396,7 +1400,7 @@ it.effect("an auto-placed task keeps its environment after a lost answer and a r const before = yield* makeLaptop(database, offers, linking); yield* linkTo(before.links, b, "full-access"); yield* seedParent(before); - return yield* placeTask(before, "auto-suite"); + return yield* autoTask(before, "auto-suite"); }), ); expect(lost.isError).toBe(true); @@ -1407,13 +1411,13 @@ it.effect("an auto-placed task keeps its environment after a lost answer and a r yield* Ref.set(thread.resources, loadOf(0.99)); yield* Ref.set(thread.afterLaunch, Effect.void); const after = yield* makeLaptop(database, offers, linking); - const fresh = yield* placeTask(after, "another-suite"); + const fresh = yield* autoTask(after, "another-suite"); expect(fresh.isError).toBe(true); expect(JSON.stringify(fresh.content)).toContain("no-eligible-environment"); expect(yield* Ref.get(thread.launched)).toBe(launchedThere); // The retry keeps the binding: the same thread there, recorded once here. - const retried = yield* placeTask(after, "auto-suite"); + const retried = yield* autoTask(after, "auto-suite"); expect(retried.isError, JSON.stringify(retried.content)).toBe(false); expect(retried.structuredContent).toMatchObject({ remoteChild: { environmentId: box.environmentId, threadId: launchedThere }, @@ -1429,6 +1433,104 @@ it.effect("an auto-placed task keeps its environment after a lost answer and a r ).pipe(Effect.provide(NodeHttpServer.layerTest)), ); +it.effect("a launch whose answer was lost is never started again under a new link", () => + Effect.scoped( + Effect.gen(function* () { + const thread = yield* makeBoxThread; + const b = yield* serveBox(thread); + const offers = yield* SubscriptionRef.make< + ReadonlyArray + >([]); + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); + const a = yield* makeLaptop(database, offers); + yield* linkTo(a.links, b, "full-access"); + yield* seedParent(a); + const tasks = () => + a.orchestrator.getThreadProjection(parentThreadId).pipe(Effect.map((p) => p.subagents)); + + // The box starts the thread, but its answer is lost; then the box is linked again. + yield* Ref.set(thread.afterLaunch, Effect.die("connection lost")); + expect((yield* autoTask(a, "relinked")).isError).toBe(true); + const lostThere = yield* Ref.get(thread.launched); + expect(lostThere).not.toBeNull(); + yield* Ref.set(thread.afterLaunch, Effect.void); + yield* linkTo(a.links, b, "full-access"); + // The new session would name a new thread there, so the retry starts nothing. + const retried = yield* autoTask(a, "relinked"); + expect(retried.isError).toBe(true); + expect(JSON.stringify(retried.content)).toContain("linked again"); + expect(yield* Ref.get(thread.launched)).toBe(lostThere); + expect(yield* tasks()).toHaveLength(0); + + // A task already recorded is found again after a new link, without a second launch. + const recorded = yield* autoTask(a, "recorded"); + expect(recorded.isError, JSON.stringify(recorded.content)).toBe(false); + const recordedThere = yield* Ref.get(thread.launched); + yield* linkTo(a.links, b, "full-access"); + const again = yield* autoTask(a, "recorded"); + expect(again.isError, JSON.stringify(again.content)).toBe(false); + expect((again.structuredContent as { taskId: NodeId }).taskId).toBe( + (recorded.structuredContent as { taskId: NodeId }).taskId, + ); + expect(yield* Ref.get(thread.launched)).toBe(recordedThere); + expect(yield* tasks()).toHaveLength(1); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); + +it.effect("an auto-placed task runs at the parent's modes, never a narrower link's", () => + Effect.scoped( + Effect.gen(function* () { + const thread = yield* makeBoxThread; + const b = yield* serveBox(thread); + const offers = yield* SubscriptionRef.make< + ReadonlyArray + >([]); + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); + const a = yield* makeLaptop(database, offers); + yield* seedParent(a, modelSelection, "plan"); + // The parent runs with full access; this link allows less, so the box is not eligible. + yield* linkTo(a.links, b, "auto-accept-edits"); + const narrow = yield* autoTask(a, "narrow"); + expect(narrow.isError).toBe(true); + expect(JSON.stringify(narrow.content)).toContain("no-eligible-environment"); + expect(yield* Ref.get(thread.launched)).toBeNull(); + + yield* linkTo(a.links, b, "full-access"); + const wide = yield* autoTask(a, "wide"); + expect(wide.isError, JSON.stringify(wide.content)).toBe(false); + expect(yield* Ref.get(thread.launchedWith)).toEqual({ + modelSelection: { instanceId: boxClaude.instanceId, model: "claude-sonnet-4-6" }, + runtimeMode: "full-access", + interactionMode: "plan", + }); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); + +it.effect("an auto-placed task refuses a project placement did not choose", () => + Effect.scoped( + Effect.gen(function* () { + const thread = yield* makeBoxThread; + const b = yield* serveBox(thread); + const offers = yield* SubscriptionRef.make< + ReadonlyArray + >([]); + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); + const a = yield* makeLaptop(database, offers); + yield* linkTo(a.links, b, "full-access"); + yield* seedParent(a); + // Placement qualifies the project with this repository, not another one there. + const other = yield* autoTask(a, "other-project", { + projectId: ProjectId.make("project:box-other-3"), + }); + expect(other.isError).toBe(true); + expect(JSON.stringify(other.content)).toContain("invalid_request"); + expect(yield* Ref.get(thread.launched)).toBeNull(); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); + it.effect("fails the task when the box revokes the link, and cancel interrupts it there", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/peer/RemoteDelegation.ts b/apps/server/src/peer/RemoteDelegation.ts index 73f12bdbdfd58..31241de5c1393 100644 --- a/apps/server/src/peer/RemoteDelegation.ts +++ b/apps/server/src/peer/RemoteDelegation.ts @@ -116,7 +116,8 @@ export class RemoteDelegation extends Context.Service< readonly key: string; readonly projectId: ProjectId; readonly modelSelection: Pick; - readonly runtimeMode: RuntimeMode | undefined; + /** The parent's effective mode or a narrower one asked for, sent explicitly. */ + readonly runtimeMode: RuntimeMode; /** This environment's own `orchestrator_capabilities`, searched for the model. */ readonly here: OrchestratorMcpCapabilitiesResult; }, @@ -167,7 +168,7 @@ const runsModel = ( capabilities: OrchestratorMcpCapabilitiesResult, requested: { readonly modelSelection: Pick; - readonly runtimeMode: RuntimeMode | undefined; + readonly runtimeMode: RuntimeMode; }, ) => capabilities.providers.some( @@ -175,8 +176,7 @@ const runsModel = ( provider.providerInstanceId === requested.modelSelection.instanceId && provider.canRunChildTask && provider.models.some((model) => model.id === requested.modelSelection.model) && - (requested.runtimeMode === undefined || - provider.runtimeModes === undefined || + (provider.runtimeModes === undefined || provider.runtimeModes.includes(requested.runtimeMode)), ); @@ -366,8 +366,8 @@ const make = Effect.gen(function* () { }); // A bound intent keeps its environment, even one that has since filled up or // stopped answering: its first attempt may have started the task there. - const bound = yield* threads.placementOf(bindingId).pipe(Effect.mapError(bindingFailure)); - if (Option.isSome(bound)) return bound.value; + const bound = yield* threads.intentOf(bindingId).pipe(Effect.mapError(bindingFailure)); + if (Option.isSome(bound)) return EnvironmentId.make(bound.value); const linked = (yield* links.enabled) ? yield* links.list.pipe(Effect.orElseSucceed(() => [])) @@ -378,8 +378,7 @@ const make = Effect.gen(function* () { link.environmentId !== hereId && link.status === "reachable" && link.access !== "read-only" && - (input.runtimeMode === undefined || - !isRuntimeModeAbove(input.runtimeMode, link.access)), + !isRuntimeModeAbove(input.runtimeMode, link.access), ), (link) => peerCandidate(scope, link, input), { concurrency: "unbounded" }, @@ -412,12 +411,12 @@ const make = Effect.gen(function* () { }); } return yield* threads - .bindPlacement({ + .bindIntent({ commandId: bindingId, threadId: scope.thread.threadId, - environmentId: EnvironmentId.make(decision.environmentId), + value: decision.environmentId, }) - .pipe(Effect.mapError(bindingFailure)); + .pipe(Effect.map(EnvironmentId.make), Effect.mapError(bindingFailure)); }, ); @@ -501,6 +500,78 @@ const make = Effect.gen(function* () { // Both are scoped to the provider session, as every key a caller passes is: // the forwarder scopes the launch's, and the record's names it. const key = input.clientRequestId ?? (yield* random.uuidV4()); + const keyed = (operation: string) => + CommandId.make( + `command:mcp:${encodeURIComponent(scope.requestNamespace)}:${operation}:${encodeURIComponent(key)}`, + ); + const commandId = keyed("remote-delegate"); + const link = yield* links.get(target.environmentId).pipe( + Effect.map(Option.getOrUndefined), + Effect.orElseSucceed(() => undefined), + ); + const label = link?.label ?? target.environmentId; + // The other side names the thread from its link session and the key, so a + // retry after linking again would start a second one. The session that took + // the first launch and the thread it answered with are kept here. + const childId = keyed("remote-child"); + const linkSession = yield* links.session(target.environmentId).pipe( + Effect.map(Option.getOrUndefined), + Effect.mapError( + () => + new OrchestratorMcpFailure({ + code: "orchestration_error", + message: `Unable to read the link to ${label}, so nothing was started. Retry with the same clientRequestId.`, + }), + ), + ); + // Without a link the launch itself is refused, so there is nothing to bind. + if (linkSession !== undefined) { + const session = `${target.environmentId}@${linkSession}`; + const launchedUnder = yield* threads + .bindIntent({ + commandId: keyed("remote-launch"), + threadId: parent.thread.id, + value: session, + }) + .pipe( + Effect.mapError( + () => + new OrchestratorMcpFailure({ + code: "orchestration_error", + message: + "Unable to record the launch there, so nothing was started. Retry with the same clientRequestId.", + }), + ), + ); + if (launchedUnder !== session) { + const childThere = yield* threads.intentOf(childId).pipe( + Effect.map(Option.getOrUndefined), + Effect.orElseSucceed(() => undefined), + ); + const recordedTask = + childThere === undefined + ? undefined + : yield* threads.getThreadRecords(scope.thread.threadId, ["subagents"]).pipe( + Effect.map(({ subagents }) => + subagents.find( + (task) => + task.remoteChild?.environmentId === target.environmentId && + task.remoteChild.threadId === childThere, + ), + ), + Effect.orElseSucceed(() => undefined), + ); + if (recordedTask !== undefined) return { taskId: recordedTask.id }; + return yield* new OrchestratorMcpFailure({ + code: "orchestration_error", + message: `${label} was linked again after an earlier attempt with this clientRequestId launched the task there, and the new link cannot find that thread. Nothing new was started. ${ + childThere === undefined + ? "t3_thread_list with this environmentId finds it: it names this thread as its parent." + : `Its thread there is ${childThere}.` + } To start the task again, use a new clientRequestId.`, + }); + } + } const task = taskPrompt(input); const title = input.title ?? parent.thread.title; const launched = yield* forwarding.call( @@ -523,17 +594,14 @@ const make = Effect.gen(function* () { }, }, ); - const link = yield* links.get(target.environmentId).pipe( - Effect.map(Option.getOrUndefined), - Effect.orElseSucceed(() => undefined), - ); - const commandId = CommandId.make( - `command:mcp:${encodeURIComponent(scope.requestNamespace)}:remote-delegate:${encodeURIComponent(key)}`, - ); + // Losing this only makes a retry after linking again refuse instead of finding the task. + yield* threads + .bindIntent({ commandId: childId, threadId: parent.thread.id, value: launched.threadId }) + .pipe(Effect.ignoreCause({ log: true })); const remoteChild: OrchestrationV2RemoteTaskChild = { environmentId: target.environmentId, threadId: launched.threadId, - label: link?.label ?? target.environmentId, + label, title, ...(launched.runId === null ? {} : { runId: launched.runId }), }; diff --git a/apps/server/src/relay/AgentAwarenessRelay.test.ts b/apps/server/src/relay/AgentAwarenessRelay.test.ts index 6f9b798c8ef6a..0287c00e5f905 100644 --- a/apps/server/src/relay/AgentAwarenessRelay.test.ts +++ b/apps/server/src/relay/AgentAwarenessRelay.test.ts @@ -252,8 +252,8 @@ const makeTestRelay = Effect.fnUntraced(function* ( assertProviderInstanceEnabled: unused, isProviderInstanceEnabledForProject: unused, hasReceipt: unused, - placementOf: unused, - bindPlacement: unused, + intentOf: unused, + bindIntent: unused, waitForThread: unused, settleAfterRun: unused, settleThread: unused, From e361cfbcffbdce471f49a7fde45d44b6d4882a9f Mon Sep 17 00:00:00 2001 From: Kameron Smith <112618179+nullStack65@users.noreply.github.com> Date: Wed, 18 Nov 2026 05:51:00 +0000 Subject: [PATCH 3/3] fix(placement): send a remote launch only under the link session it is bound to Continuation of review finding P1 under the root disposition (https://github.com/nullStack65/t3code/pull/68#issuecomment-6970222770). The launch checked its link session in one read and the transport resolved the link again for the send, so a link replaced in between still sent it under the new session and started a second child there. PeerLinks.resolve now returns the incarnation of the same stored link whose token it returns; PeerForwarding.call and PeerMcpClient.call carry an optional expectedIncarnation and refuse before any handshake or request when it does not match. RemoteDelegation sends its launch only under the incarnation it bound, refuses outright when there is no link to bind to, and never rebinds a key. Its refusal no longer suggests retrying with a new key. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/peer/PeerForwarding.ts | 16 ++- apps/server/src/peer/PeerLinks.ts | 3 + apps/server/src/peer/PeerMcpClient.test.ts | 2 + apps/server/src/peer/PeerMcpClient.ts | 15 +++ apps/server/src/peer/RemoteDelegation.test.ts | 113 ++++++++++++++++++ apps/server/src/peer/RemoteDelegation.ts | 99 ++++++++------- 6 files changed, 200 insertions(+), 48 deletions(-) diff --git a/apps/server/src/peer/PeerForwarding.ts b/apps/server/src/peer/PeerForwarding.ts index 080f701c45cb2..132e3b13b8b28 100644 --- a/apps/server/src/peer/PeerForwarding.ts +++ b/apps/server/src/peer/PeerForwarding.ts @@ -47,12 +47,14 @@ export class PeerForwarding extends Context.Service< /** * Runs `tool` in the linked environment `environmentId` with `params`. A * `t3_thread_wait` goes in pieces short enough to survive any route. + * `expectedIncarnation` sends it only under that link session. */ readonly call: ( scope: Scope, tool: T, environmentId: EnvironmentId, params: Tool.Parameters, + options?: { readonly expectedIncarnation?: string }, ) => Effect.Effect, OrchestratorMcpFailure>; /** `t3_thread_read` there. */ readonly readThread: ( @@ -155,6 +157,7 @@ const make = Effect.gen(function* () { success: Schema.Top, environmentId: EnvironmentId, params: unknown, + expectedIncarnation?: string, ) { const { caller, limits } = yield* forwardingCaller(scope); const encoded = (yield* Schema.encodeUnknownEffect(parameters as Schema.Encoder)( @@ -187,6 +190,7 @@ const make = Effect.gen(function* () { arguments: args, limits, ...(sender === undefined ? {} : { sender }), + ...(expectedIncarnation === undefined ? {} : { expectedIncarnation }), success: success as Schema.Decoder, }); }); @@ -237,7 +241,7 @@ const make = Effect.gen(function* () { Effect.flatMap((enabled) => (enabled ? Effect.void : Effect.fail(linkedEnvironmentsOff))), ); - const call: PeerForwarding["Service"]["call"] = (scope, tool, environmentId, params) => { + const call: PeerForwarding["Service"]["call"] = (scope, tool, environmentId, params, options) => { const forwarded: Effect.Effect< Tool.Success, OrchestratorMcpFailure @@ -247,7 +251,15 @@ const make = Effect.gen(function* () { environmentId, params as OrchestratorMcpThreadWaitInput, ) as Effect.Effect, OrchestratorMcpFailure>) - : forward(scope, tool.name, tool.parametersSchema, tool.successSchema, environmentId, params); + : forward( + scope, + tool.name, + tool.parametersSchema, + tool.successSchema, + environmentId, + params, + options?.expectedIncarnation, + ); return Effect.andThen(requireEnabled, forwarded); }; diff --git a/apps/server/src/peer/PeerLinks.ts b/apps/server/src/peer/PeerLinks.ts index e9168a8c7fa81..1858de216c39c 100644 --- a/apps/server/src/peer/PeerLinks.ts +++ b/apps/server/src/peer/PeerLinks.ts @@ -79,6 +79,8 @@ export interface ResolvedPeerLink { /** The first address that answered as this peer. */ readonly url: string; readonly token: string; + /** The session `token` belongs to, from the same stored link: see `PeerLinks.session`. */ + readonly incarnation: string; } export const LINKED_ENVIRONMENTS_OFF = @@ -697,6 +699,7 @@ const make = Effect.gen(function* () { link: yield* withLabel(link, peer.label), url, token: new TextDecoder().decode(stored.value), + incarnation: row.value.secret_name, }; }, ); diff --git a/apps/server/src/peer/PeerMcpClient.test.ts b/apps/server/src/peer/PeerMcpClient.test.ts index 8fd65e7e2d796..70b610dfa2916 100644 --- a/apps/server/src/peer/PeerMcpClient.test.ts +++ b/apps/server/src/peer/PeerMcpClient.test.ts @@ -70,6 +70,7 @@ const probeThrough = Effect.fn(function* (http: HttpClient.HttpClient) { }, url: "https://peer.example", token: "fixture", + incarnation: "fixture-session", }), recordOutcome: () => Effect.void, }), @@ -299,6 +300,7 @@ it.effect("renews a missing peer session once and reuses its replacement", () => }, url: "https://peer.example", token: "fixture", + incarnation: "fixture-session", }), recordOutcome: (_, outcome) => Effect.sync(() => outcomes.push(outcome.error)), }), diff --git a/apps/server/src/peer/PeerMcpClient.ts b/apps/server/src/peer/PeerMcpClient.ts index 8c438ad702b53..6e920f8bfad52 100644 --- a/apps/server/src/peer/PeerMcpClient.ts +++ b/apps/server/src/peer/PeerMcpClient.ts @@ -47,6 +47,8 @@ export class PeerMcpClient extends Context.Service< readonly limits: PeerCallLimits; /** The thread here the call is made for, sent as `T3-Sender-Thread`. */ readonly sender?: OrchestrationV2LinkedSender; + /** Sends only under this link session, refusing before contacting the peer otherwise. */ + readonly expectedIncarnation?: string; readonly success: S; }) => Effect.Effect; } @@ -169,6 +171,19 @@ const make = Effect.gen(function* () { const resolved = yield* links .resolve(input.environmentId) .pipe(Effect.mapError(fromLinkError)); + // The peer names work after the session of the token sent, so work bound to one + // session must never go out under another. + if ( + input.expectedIncarnation !== undefined && + input.expectedIncarnation !== resolved.incarnation + ) { + return Result.fail( + new OrchestratorMcpFailure({ + code: "orchestration_error", + message: `${resolved.link.label} was linked again while this call was being sent, so nothing was sent.`, + }), + ); + } const limitHeader = { [McpInvocationContext.MODE_LIMIT_HEADER]: `${input.limits.runtimeMode}/${input.limits.interactionMode}`, }; diff --git a/apps/server/src/peer/RemoteDelegation.test.ts b/apps/server/src/peer/RemoteDelegation.test.ts index d68f6df11bc1a..ac480f51382d0 100644 --- a/apps/server/src/peer/RemoteDelegation.test.ts +++ b/apps/server/src/peer/RemoteDelegation.test.ts @@ -666,6 +666,13 @@ const makeLaptop = ( kept?: Context.Context>>, /** Runs as each follow attempt here starts, on its follower's fiber. */ beforeFollow: Effect.Effect = Effect.void, + /** Interleavings for delegation only: the transport keeps the real links. */ + hooks: { + /** Runs once delegation has bound an intent, before it acts on it. */ + readonly afterBindIntent?: (commandId: string) => Effect.Effect; + /** Runs once delegation has read the link session it binds a launch to. */ + readonly afterSession?: (session: Option.Option) => Effect.Effect; + } = {}, ) => Effect.gen(function* () { const linking = kept ?? (yield* layerLinkingEnvironment(laptop).pipe(Layer.build)); @@ -703,6 +710,27 @@ const makeLaptop = ( ...threads, getThreadEventSequence: (threadId: ThreadId) => beforeFollow.pipe(Effect.andThen(threads.getThreadEventSequence(threadId))), + bindIntent: (input: Parameters[0]) => + threads + .bindIntent(input) + .pipe( + Effect.tap(() => hooks.afterBindIntent?.(input.commandId) ?? Effect.void), + ), + }; + }), + ), + ), + Layer.provide( + Layer.effect( + PeerLinks.PeerLinks, + Effect.gen(function* () { + const links = yield* PeerLinks.PeerLinks; + return { + ...links, + session: (environmentId: Parameters[0]) => + links + .session(environmentId) + .pipe(Effect.tap((session) => hooks.afterSession?.(session) ?? Effect.void)), }; }), ), @@ -1478,6 +1506,91 @@ it.effect("a launch whose answer was lost is never started again under a new lin ).pipe(Effect.provide(NodeHttpServer.layerTest)), ); +it.effect("a link replaced between the launch check and the send starts no second child", () => + Effect.scoped( + Effect.gen(function* () { + const thread = yield* makeBoxThread; + const b = yield* serveBox(thread); + const offers = yield* SubscriptionRef.make< + ReadonlyArray + >([]); + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); + const relinkOnce = yield* Ref.make | null>(null); + const a = yield* makeLaptop(database, offers, undefined, Effect.void, { + // The box is linked again after the launch is checked against its link, before it is sent. + afterBindIntent: (commandId) => + commandId.includes(":remote-launch:") + ? Ref.getAndSet(relinkOnce, null).pipe( + Effect.flatMap((relink) => relink ?? Effect.void), + ) + : Effect.void, + }); + yield* linkTo(a.links, b, "full-access"); + yield* seedParent(a); + const tasks = () => + a.orchestrator.getThreadProjection(parentThreadId).pipe(Effect.map((p) => p.subagents)); + + yield* Ref.set(thread.afterLaunch, Effect.die("connection lost")); + expect((yield* autoTask(a, "raced")).isError).toBe(true); + const lostThere = yield* Ref.get(thread.launched); + expect(lostThere).not.toBeNull(); + yield* Ref.set(thread.afterLaunch, Effect.void); + yield* Ref.set( + relinkOnce, + linkTo(a.links, b, "full-access").pipe(Effect.asVoid, Effect.orDie), + ); + + // The retry passes the check under the first link, but the send would use the new one. + const raced = yield* autoTask(a, "raced"); + expect(raced.isError).toBe(true); + expect(yield* Ref.get(thread.launched)).toBe(lostThere); + expect(yield* tasks()).toHaveLength(0); + // The key stays bound to the first link, so retrying it starts nothing either. + expect((yield* autoTask(a, "raced")).isError).toBe(true); + expect(yield* Ref.get(thread.launched)).toBe(lostThere); + expect(yield* tasks()).toHaveLength(0); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); + +it.effect("a launch with no link to bind to starts nothing, even if a link appears", () => + Effect.scoped( + Effect.gen(function* () { + const thread = yield* makeBoxThread; + const b = yield* serveBox(thread); + const offers = yield* SubscriptionRef.make< + ReadonlyArray + >([]); + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); + const linkOnce = yield* Ref.make | null>(null); + const a = yield* makeLaptop(database, offers, undefined, Effect.void, { + // The box is linked just after delegation finds no link to bind the launch to. + afterSession: (session) => + Option.isNone(session) + ? Ref.getAndSet(linkOnce, null).pipe(Effect.flatMap((link) => link ?? Effect.void)) + : Effect.void, + }); + yield* seedParent(a); + yield* Ref.set(linkOnce, linkTo(a.links, b, "full-access").pipe(Effect.asVoid, Effect.orDie)); + + // A named project there means nothing reaches the box before the launch itself. + const unbound = yield* a.call("delegate_task", { + task: "Run the full suite on the box.", + target: { + environmentId: box.environmentId, + projectId: boxProject, + providerInstanceId: instanceId, + model: "gpt-5.4", + }, + clientRequestId: "unbound", + }); + expect(unbound.isError).toBe(true); + expect(yield* Ref.get(thread.launched)).toBeNull(); + expect((yield* a.orchestrator.getThreadProjection(parentThreadId)).subagents).toHaveLength(0); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); + it.effect("an auto-placed task runs at the parent's modes, never a narrower link's", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/peer/RemoteDelegation.ts b/apps/server/src/peer/RemoteDelegation.ts index 31241de5c1393..5ef7b4a690414 100644 --- a/apps/server/src/peer/RemoteDelegation.ts +++ b/apps/server/src/peer/RemoteDelegation.ts @@ -524,53 +524,58 @@ const make = Effect.gen(function* () { }), ), ); - // Without a link the launch itself is refused, so there is nothing to bind. - if (linkSession !== undefined) { - const session = `${target.environmentId}@${linkSession}`; - const launchedUnder = yield* threads - .bindIntent({ - commandId: keyed("remote-launch"), - threadId: parent.thread.id, - value: session, - }) - .pipe( - Effect.mapError( - () => - new OrchestratorMcpFailure({ - code: "orchestration_error", - message: - "Unable to record the launch there, so nothing was started. Retry with the same clientRequestId.", - }), - ), - ); - if (launchedUnder !== session) { - const childThere = yield* threads.intentOf(childId).pipe( - Effect.map(Option.getOrUndefined), - Effect.orElseSucceed(() => undefined), - ); - const recordedTask = - childThere === undefined - ? undefined - : yield* threads.getThreadRecords(scope.thread.threadId, ["subagents"]).pipe( - Effect.map(({ subagents }) => - subagents.find( - (task) => - task.remoteChild?.environmentId === target.environmentId && - task.remoteChild.threadId === childThere, - ), + // A launch is only sent bound to the link session it goes out under; with no link + // there is none, and one appearing before the send must not carry it unbound. + if (linkSession === undefined) { + return yield* new OrchestratorMcpFailure({ + code: "invalid_request", + message: `This environment is not linked to ${label}, so nothing was started. t3_environment_links lists the linked ones.`, + }); + } + const session = `${target.environmentId}@${linkSession}`; + const launchedUnder = yield* threads + .bindIntent({ + commandId: keyed("remote-launch"), + threadId: parent.thread.id, + value: session, + }) + .pipe( + Effect.mapError( + () => + new OrchestratorMcpFailure({ + code: "orchestration_error", + message: + "Unable to record the launch there, so nothing was started. Retry with the same clientRequestId.", + }), + ), + ); + if (launchedUnder !== session) { + const childThere = yield* threads.intentOf(childId).pipe( + Effect.map(Option.getOrUndefined), + Effect.orElseSucceed(() => undefined), + ); + const recordedTask = + childThere === undefined + ? undefined + : yield* threads.getThreadRecords(scope.thread.threadId, ["subagents"]).pipe( + Effect.map(({ subagents }) => + subagents.find( + (task) => + task.remoteChild?.environmentId === target.environmentId && + task.remoteChild.threadId === childThere, ), - Effect.orElseSucceed(() => undefined), - ); - if (recordedTask !== undefined) return { taskId: recordedTask.id }; - return yield* new OrchestratorMcpFailure({ - code: "orchestration_error", - message: `${label} was linked again after an earlier attempt with this clientRequestId launched the task there, and the new link cannot find that thread. Nothing new was started. ${ - childThere === undefined - ? "t3_thread_list with this environmentId finds it: it names this thread as its parent." - : `Its thread there is ${childThere}.` - } To start the task again, use a new clientRequestId.`, - }); - } + ), + Effect.orElseSucceed(() => undefined), + ); + if (recordedTask !== undefined) return { taskId: recordedTask.id }; + return yield* new OrchestratorMcpFailure({ + code: "orchestration_error", + message: `${label} was linked again after an earlier attempt with this clientRequestId launched the task there, and the new link cannot find that thread. Nothing new was started. ${ + childThere === undefined + ? "t3_thread_list with this environmentId finds it: it names this thread as its parent." + : `Its thread there is ${childThere}.` + } This clientRequestId stays bound to the earlier link. Check that thread there before deciding the work still needs doing.`, + }); } const task = taskPrompt(input); const title = input.title ?? parent.thread.title; @@ -593,6 +598,8 @@ const make = Effect.gen(function* () { title: parent.thread.title, }, }, + // Sent only under the session bound above, never one linked since. + { expectedIncarnation: linkSession }, ); // Losing this only makes a retry after linking again refuse instead of finding the task. yield* threads