diff --git a/.github/workflows/closura-arc-ci.yml b/.github/workflows/closura-arc-ci.yml index 79a20c8239977..8050491b53e79 100644 --- a/.github/workflows/closura-arc-ci.yml +++ b/.github/workflows/closura-arc-ci.yml @@ -132,11 +132,12 @@ jobs: run: | set -euo pipefail vp run --filter @t3tools/contracts typecheck + vp run --filter @t3tools/shared typecheck vp run --filter @t3tools/client-runtime typecheck vp run --filter @t3tools/provider-core typecheck vp run --filter t3 typecheck - - name: Test AssemblyAI dictation and native inbox behavior + - name: Test AssemblyAI dictation, native inbox and thread placement behavior shell: bash run: | set -euo pipefail @@ -146,7 +147,10 @@ jobs: packages/contracts/src/threadMetadataMcp.test.ts \ apps/web/src/components/threadActionMenu.logic.test.ts \ apps/server/src/orchestration-v2/ThreadLaunchService.test.ts \ - apps/server/src/orchestration-v2/SubagentProjection.test.ts + apps/server/src/orchestration-v2/SubagentProjection.test.ts \ + packages/shared/src/threadPlacement.test.ts \ + apps/server/src/peer/RemoteDelegation.test.ts \ + apps/server/src/mcp/OrchestratorMcpService.test.ts - name: Build native T3 server shell: bash diff --git a/apps/server/src/mcp/OrchestratorMcpService.test.ts b/apps/server/src/mcp/OrchestratorMcpService.test.ts index 6a969e21e3abd..97907e2d50b2c 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.test.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.test.ts @@ -2906,6 +2906,22 @@ describe("OrchestratorMcpService provider resolution", () => { assert.equal(child.providerInstanceId, selection.instanceId); assert.equal(child.model, selection.model); assert.deepEqual(yield* service.createThreads(scope, request), first); + // Its threads share this checkout, so another environment is refused, not ignored. + const elsewhere = yield* service + .createThreads(scope, { + threads: [ + { + target: { + ...request.threads[0]!.target, + environmentId: EnvironmentId.make("environment-elsewhere"), + }, + }, + ], + clientRequestId: "elsewhere", + }) + .pipe(Effect.flip); + assert.equal(elsewhere.code, "invalid_request"); + assert.include(elsewhere.message, "t3_thread_launch"); // The creation receipt exists, but this prompt has not been accepted yet. const withPrompt = { diff --git a/apps/server/src/mcp/OrchestratorMcpService.ts b/apps/server/src/mcp/OrchestratorMcpService.ts index b86a564c19f9d..a4405d02c0ed6 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.ts @@ -1,5 +1,6 @@ import { CommandId, + EnvironmentId, type RunId, isProviderAvailable, MessageId, @@ -71,6 +72,10 @@ import { backgroundWorkHoldsCompletion } from "@t3tools/shared/orchestrationV2Pe import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError"; import { resolveProjectSettings } from "@t3tools/shared/projectSettings"; import { formatThreadLink } from "@t3tools/shared/threadLinks"; +import { + chooseThreadPlacement, + type ThreadPlacementDecision, +} from "@t3tools/shared/threadPlacement"; import { threadPullRequestUpdatedAtMs } from "@t3tools/shared/threadPullRequests"; import { normalizeThreadPullRequestKey, @@ -2224,7 +2229,69 @@ const make = Effect.gen(function* () { return task; }); - return OrchestratorMcpService.of({ + /** Where a placed task may run, or why it may not. */ + const admitPlacement = (decision: ThreadPlacementDecision, conflict: string) => + decision.accepted + ? Effect.succeed(EnvironmentId.make(decision.environmentId)) + : Effect.fail( + failure( + "invalid_request", + decision.reason === "conflicting-locality" + ? conflict + : `No environment can take this (${decision.reason}).`, + ), + ); + + /** The environment bound to an auto-placed task, for the parent's model or the exact one named. */ + const placeTask = Effect.fn("OrchestratorMcpService.placeTask")(function* ( + callerScope: McpInvocationScope, + input: OrchestratorMcpDelegateTaskInput, + key: string, + ) { + yield* requireCapability(callerScope); + const { scope, parent } = yield* loadThreadCaller(callerScope, "delegate_task"); + const target = input.target; + // Placement qualifies the project with this repository wherever it runs, and no other. + if (target?.projectId !== undefined) { + return yield* failure( + "invalid_request", + 'With placement "auto", omit target.projectId: the task runs in the project with this repository wherever it is placed. To use another project, pass target.environmentId with it.', + ); + } + if (target?.driverKind !== undefined && target.providerInstanceId === undefined) { + return yield* failure( + "invalid_request", + 'With placement "auto", name target.providerInstanceId instead of driverKind; every environment matches it exactly.', + ); + } + const inherited = parent.thread.modelSelection; + const instanceId = target?.providerInstanceId ?? inherited.instanceId; + const model = + target?.model ?? (instanceId === inherited.instanceId ? inherited.model : undefined); + if (model === undefined) { + return yield* failure( + "target_required", + 'With placement "auto" and another providerInstanceId, pass target.model.', + ); + } + // The parent's modes, or narrower ones asked for, are chosen with and sent explicitly, + // so no environment qualifies or runs the task at less than they allow. + const runtimeMode = yield* resolveRuntimeMode(parent.thread.runtimeMode, input.runtimeMode); + const interactionMode = yield* resolveInteractionMode( + parent.thread.interactionMode, + input.interactionMode, + ); + const environmentId = yield* remoteDelegation.place(scope, { + key, + projectId: parent.thread.projectId, + modelSelection: { instanceId, model }, + runtimeMode, + here: yield* service.capabilities(callerScope, { search: model }), + }); + return { environmentId, runtimeMode, interactionMode }; + }); + + const service: OrchestratorMcpServiceShape = OrchestratorMcpService.of({ scheduleTask: Effect.fn("OrchestratorMcpService.scheduleTask")(function* (scope, input) { const { parent, limits } = yield* loadCaller(scope, ["runs"]); const projectId = yield* resolveProjectTarget(parent, input.projectId); @@ -2642,16 +2709,40 @@ const make = Effect.gen(function* () { }; }), delegateTask: Effect.fn("OrchestratorMcpService.delegateTask")(function* (callerScope, input) { - const environmentId = - input.target === undefined - ? undefined - : PeerForwarding.remoteTarget(callerScope, input.target.environmentId); - if (input.target !== undefined && environmentId !== undefined) { + // One key for the placement, the launch and the record, so a retry finds all three. + const clientRequestId = yield* requestKey(input.clientRequestId); + const named = input.target?.environmentId; + // A named target or the parent's checkout decides; only work needing neither is placed. + const automatic = + input.placement === "auto" && named === undefined && input.isolation === undefined + ? yield* placeTask(callerScope, input, clientRequestId) + : undefined; + const placed = + automatic !== undefined + ? automatic.environmentId + : yield* admitPlacement( + chooseThreadPlacement( + { + originEnvironmentId: callerScope.environmentId, + pinnedEnvironmentId: named ?? callerScope.environmentId, + requiresOrigin: input.isolation === "worktree", + }, + [], + 0, + ), + 'A task in a linked environment runs in that project\'s checkout; isolation "worktree" is not supported there. Omit isolation, or use t3_thread_launch with environmentId and a worktree workspaceStrategy.', + ); + const environmentId = PeerForwarding.remoteTarget(callerScope, placed); + if (environmentId !== undefined) { yield* requireCapability(callerScope); const scope = yield* requireThreadScope(callerScope, "delegate_task"); // The task's child runs as an ordinary thread there; this thread keeps the task. const { taskId } = yield* remoteDelegation.delegate(scope, { ...input, + ...(automatic === undefined + ? {} + : { runtimeMode: automatic.runtimeMode, interactionMode: automatic.interactionMode }), + clientRequestId, target: { ...input.target, environmentId }, }); return yield* input.mode === "wait" @@ -2682,7 +2773,7 @@ const make = Effect.gen(function* () { ); } const providers = yield* loadProviders; - const key = yield* requestKey(input.clientRequestId); + const key = clientRequestId; const commandId = stableCommandId({ scope, requestKey: key, @@ -3021,6 +3112,21 @@ const make = Effect.gen(function* () { createThreads: Effect.fn("OrchestratorMcpService.createThreads")( function* (callerScope, input) { const { scope, parent } = yield* loadThreadCaller(callerScope, "create_threads"); + // Each thread shares this thread's checkout, so another environment cannot take it. + for (const request of input.threads) { + yield* admitPlacement( + chooseThreadPlacement( + { + originEnvironmentId: callerScope.environmentId, + pinnedEnvironmentId: request.target?.environmentId, + requiresOrigin: true, + }, + [], + 0, + ), + "create_threads starts threads in this checkout. To start one in a linked environment, use t3_thread_launch with that environmentId.", + ); + } const parentRun = ThreadManagementService.latestActiveRun(parent); if ( parentRun === undefined || @@ -3604,6 +3710,7 @@ const make = Effect.gen(function* () { } satisfies OrchestratorMcpThreadInterruptResult; }), }); + return service; }); export const layer: Layer.Layer< diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index 611d9a3244f24..9ebba141e0808 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -472,6 +472,19 @@ export interface ThreadManagementServiceShape { }) => Effect.Effect; /** Receipt replays start no work; skip enablement checks with the same command id so settings cannot reject accepted retries. */ readonly hasReceipt: (commandId: CommandId) => Effect.Effect; + /** What a caller's intent was bound to, such as the environment chosen for it, if it was. */ + readonly intentOf: ( + commandId: CommandId, + ) => Effect.Effect, CommandReceiptStore.CommandReceiptStoreV2Error>; + /** + * Binds a caller's intent to a value unless it is bound already, and returns + * the binding that holds: the first writer's, read back from storage. + */ + readonly bindIntent: (input: { + readonly commandId: CommandId; + readonly threadId: ThreadId; + readonly value: string; + }) => Effect.Effect; readonly waitForThread: ( input: ThreadManagementWaitInput, ) => Effect.Effect; @@ -604,6 +617,8 @@ const FORCE_STOP_REASON = "Force stopped by user"; const FORCE_STOP_RELEASE_WAIT = "10 seconds"; const isForceStopThreadError = Schema.is(OrchestrationV2ForceStopThreadError); +const INTENT_RECEIPT_TYPE = "intent.bind:"; + const make = Effect.gen(function* () { const threadDispatch = yield* ThreadCommandExecutor.ThreadCommandExecutor; const orchestrator = yield* Orchestrator.OrchestratorV2; @@ -651,6 +666,42 @@ const make = Effect.gen(function* () { Effect.map(Option.isSome), ); + // A binding is a receipt with no events, so it lives and is kept like every other + // command's; its type carries the value. + const intentOf: ThreadManagementServiceShape["intentOf"] = (commandId) => + receipts + .getByCommandId(commandId) + .pipe( + Effect.map( + Option.flatMap((receipt) => + receipt.commandType.startsWith(INTENT_RECEIPT_TYPE) + ? Option.some(receipt.commandType.slice(INTENT_RECEIPT_TYPE.length)) + : Option.none(), + ), + ), + ); + + const bindIntent: ThreadManagementServiceShape["bindIntent"] = Effect.fn( + "orchestrationV2.threadManagement.bindIntent", + )(function* (input) { + yield* receipts.insertIfAbsent({ + commandId: input.commandId, + threadId: input.threadId, + commandType: `${INTENT_RECEIPT_TYPE}${input.value}`, + acceptedAt: yield* DateTime.now, + resultSequence: 0, + status: "accepted", + error: null, + }); + const bound = yield* intentOf(input.commandId); + if (Option.isNone(bound)) { + return yield* new CommandReceiptStore.CommandReceiptStoreReadError({ + commandId: input.commandId, + }); + } + return bound.value; + }); + const ensureLegacyTranscript = Effect.fn( "orchestrationV2.threadManagement.ensureLegacyTranscript", )(function* (threadId: ThreadId) { @@ -1768,6 +1819,8 @@ const make = Effect.gen(function* () { assertProviderInstanceEnabled, isProviderInstanceEnabledForProject, hasReceipt, + intentOf, + bindIntent, waitForThread, settleAfterRun, settleThread, diff --git a/apps/server/src/peer/PeerForwarding.ts b/apps/server/src/peer/PeerForwarding.ts index 080f701c45cb2..132e3b13b8b28 100644 --- a/apps/server/src/peer/PeerForwarding.ts +++ b/apps/server/src/peer/PeerForwarding.ts @@ -47,12 +47,14 @@ export class PeerForwarding extends Context.Service< /** * Runs `tool` in the linked environment `environmentId` with `params`. A * `t3_thread_wait` goes in pieces short enough to survive any route. + * `expectedIncarnation` sends it only under that link session. */ readonly call: ( scope: Scope, tool: T, environmentId: EnvironmentId, params: Tool.Parameters, + options?: { readonly expectedIncarnation?: string }, ) => Effect.Effect, OrchestratorMcpFailure>; /** `t3_thread_read` there. */ readonly readThread: ( @@ -155,6 +157,7 @@ const make = Effect.gen(function* () { success: Schema.Top, environmentId: EnvironmentId, params: unknown, + expectedIncarnation?: string, ) { const { caller, limits } = yield* forwardingCaller(scope); const encoded = (yield* Schema.encodeUnknownEffect(parameters as Schema.Encoder)( @@ -187,6 +190,7 @@ const make = Effect.gen(function* () { arguments: args, limits, ...(sender === undefined ? {} : { sender }), + ...(expectedIncarnation === undefined ? {} : { expectedIncarnation }), success: success as Schema.Decoder, }); }); @@ -237,7 +241,7 @@ const make = Effect.gen(function* () { Effect.flatMap((enabled) => (enabled ? Effect.void : Effect.fail(linkedEnvironmentsOff))), ); - const call: PeerForwarding["Service"]["call"] = (scope, tool, environmentId, params) => { + const call: PeerForwarding["Service"]["call"] = (scope, tool, environmentId, params, options) => { const forwarded: Effect.Effect< Tool.Success, OrchestratorMcpFailure @@ -247,7 +251,15 @@ const make = Effect.gen(function* () { environmentId, params as OrchestratorMcpThreadWaitInput, ) as Effect.Effect, OrchestratorMcpFailure>) - : forward(scope, tool.name, tool.parametersSchema, tool.successSchema, environmentId, params); + : forward( + scope, + tool.name, + tool.parametersSchema, + tool.successSchema, + environmentId, + params, + options?.expectedIncarnation, + ); return Effect.andThen(requireEnabled, forwarded); }; diff --git a/apps/server/src/peer/PeerLinks.ts b/apps/server/src/peer/PeerLinks.ts index 6fddf407cf4d4..1858de216c39c 100644 --- a/apps/server/src/peer/PeerLinks.ts +++ b/apps/server/src/peer/PeerLinks.ts @@ -79,6 +79,8 @@ export interface ResolvedPeerLink { /** The first address that answered as this peer. */ readonly url: string; readonly token: string; + /** The session `token` belongs to, from the same stored link: see `PeerLinks.session`. */ + readonly incarnation: string; } export const LINKED_ENVIRONMENTS_OFF = @@ -97,6 +99,13 @@ export class PeerLinks extends Context.Service< readonly enabled: Effect.Effect; readonly link: (input: PeerLinkCreateInput) => Effect.Effect; readonly list: Effect.Effect, PeerLinkError>; + /** + * Names the link's session there without revealing its token. Linking again + * starts a new session there, so this changes exactly when that does. + */ + readonly session: ( + environmentId: EnvironmentId, + ) => Effect.Effect, PeerLinkError>; /** The stored link, without asking the peer whether it answers. */ readonly get: ( environmentId: EnvironmentId, @@ -690,6 +699,7 @@ const make = Effect.gen(function* () { link: yield* withLabel(link, peer.label), url, token: new TextDecoder().decode(stored.value), + incarnation: row.value.secret_name, }; }, ); @@ -712,10 +722,15 @@ const make = Effect.gen(function* () { Effect.ignoreCause({ log: true }), ); + // Each link stores its token under a fresh random name, so the name identifies the session. + const session: PeerLinks["Service"]["session"] = (environmentId) => + readRow(environmentId).pipe(Effect.map(Option.map((row) => row.secret_name))); + return PeerLinks.of({ enabled, link, list, + session, get, unlink, resolve, diff --git a/apps/server/src/peer/PeerMcpClient.test.ts b/apps/server/src/peer/PeerMcpClient.test.ts index 8fd65e7e2d796..70b610dfa2916 100644 --- a/apps/server/src/peer/PeerMcpClient.test.ts +++ b/apps/server/src/peer/PeerMcpClient.test.ts @@ -70,6 +70,7 @@ const probeThrough = Effect.fn(function* (http: HttpClient.HttpClient) { }, url: "https://peer.example", token: "fixture", + incarnation: "fixture-session", }), recordOutcome: () => Effect.void, }), @@ -299,6 +300,7 @@ it.effect("renews a missing peer session once and reuses its replacement", () => }, url: "https://peer.example", token: "fixture", + incarnation: "fixture-session", }), recordOutcome: (_, outcome) => Effect.sync(() => outcomes.push(outcome.error)), }), diff --git a/apps/server/src/peer/PeerMcpClient.ts b/apps/server/src/peer/PeerMcpClient.ts index 8c438ad702b53..6e920f8bfad52 100644 --- a/apps/server/src/peer/PeerMcpClient.ts +++ b/apps/server/src/peer/PeerMcpClient.ts @@ -47,6 +47,8 @@ export class PeerMcpClient extends Context.Service< readonly limits: PeerCallLimits; /** The thread here the call is made for, sent as `T3-Sender-Thread`. */ readonly sender?: OrchestrationV2LinkedSender; + /** Sends only under this link session, refusing before contacting the peer otherwise. */ + readonly expectedIncarnation?: string; readonly success: S; }) => Effect.Effect; } @@ -169,6 +171,19 @@ const make = Effect.gen(function* () { const resolved = yield* links .resolve(input.environmentId) .pipe(Effect.mapError(fromLinkError)); + // The peer names work after the session of the token sent, so work bound to one + // session must never go out under another. + if ( + input.expectedIncarnation !== undefined && + input.expectedIncarnation !== resolved.incarnation + ) { + return Result.fail( + new OrchestratorMcpFailure({ + code: "orchestration_error", + message: `${resolved.link.label} was linked again while this call was being sent, so nothing was sent.`, + }), + ); + } const limitHeader = { [McpInvocationContext.MODE_LIMIT_HEADER]: `${input.limits.runtimeMode}/${input.limits.interactionMode}`, }; diff --git a/apps/server/src/peer/RemoteDelegation.test.ts b/apps/server/src/peer/RemoteDelegation.test.ts index 90c6eb83baae5..ac480f51382d0 100644 --- a/apps/server/src/peer/RemoteDelegation.test.ts +++ b/apps/server/src/peer/RemoteDelegation.test.ts @@ -16,6 +16,7 @@ import { type Project, type RepositoryIdentity, RunId, + type HostResourcesSnapshot, type ServerProvider, ThreadId, TurnItemId, @@ -58,6 +59,7 @@ import * as ProjectCloneTracker from "../project/ProjectCloneTracker.ts"; import * as ProjectService from "../project/ProjectService.ts"; import * as RepositoryIdentityResolver from "../project/RepositoryIdentityResolver.ts"; import * as ProviderRegistry from "../provider/ProviderRegistry.ts"; +import * as HostResources from "../resourceTelemetry/HostResources.ts"; import * as ScheduledTaskService from "../scheduledTasks/ScheduledTaskService.ts"; import * as SecretRequests from "../secrets/SecretRequests.ts"; import * as ServerSettings from "../serverSettings.ts"; @@ -141,8 +143,19 @@ interface BoxThread { readonly sent: Ref.Ref>; /** Runs as the launch there is answered, before the laptop hears back. */ readonly afterLaunch: Ref.Ref>; + /** The box's load as its `orchestrator_capabilities` reports it. */ + readonly resources: Ref.Ref; } +const GiB = 1024 ** 3; +const loadOf = (cpuUtilization: number): HostResourcesSnapshot => ({ + sampledAt: 0, + cpuUtilization, + cpuCount: 8, + availableMemoryBytes: 8 * GiB, + totalMemoryBytes: 16 * GiB, +}); + const makeBoxThread = Effect.gen(function* () { return { launched: yield* Ref.make(null), @@ -179,6 +192,7 @@ const makeBoxThread = Effect.gen(function* () { releaseStop: yield* Deferred.make(), sent: yield* Ref.make>([]), afterLaunch: yield* Ref.make>(Effect.void), + resources: yield* Ref.make(loadOf(0.1)), } satisfies BoxThread; }); @@ -549,7 +563,12 @@ const boxToolkitLayer = (thread: BoxThread) => { Layer.provide(DocumentText.layer.pipe(Layer.provideMerge(NodeServices.layer))), Layer.provide(NodeCrypto.layer), Layer.provide(layerBoxThreads), - Layer.provide(layerBoxLaunches), + Layer.provide( + Layer.merge( + layerBoxLaunches, + Layer.succeed(HostResources.HostResources, { read: Ref.get(thread.resources) }), + ), + ), Layer.provide(Layer.mock(RemoteDelegation.RemoteDelegation)({})), Layer.provide(Layer.mock(PeerForwarding.PeerForwarding)({})), Layer.provide(Layer.mock(PeerLinkRequests.PeerLinkRequests)({})), @@ -647,6 +666,13 @@ const makeLaptop = ( kept?: Context.Context>>, /** Runs as each follow attempt here starts, on its follower's fiber. */ beforeFollow: Effect.Effect = Effect.void, + /** Interleavings for delegation only: the transport keeps the real links. */ + hooks: { + /** Runs once delegation has bound an intent, before it acts on it. */ + readonly afterBindIntent?: (commandId: string) => Effect.Effect; + /** Runs once delegation has read the link session it binds a launch to. */ + readonly afterSession?: (session: Option.Option) => Effect.Effect; + } = {}, ) => Effect.gen(function* () { const linking = kept ?? (yield* layerLinkingEnvironment(laptop).pipe(Layer.build)); @@ -684,6 +710,27 @@ const makeLaptop = ( ...threads, getThreadEventSequence: (threadId: ThreadId) => beforeFollow.pipe(Effect.andThen(threads.getThreadEventSequence(threadId))), + bindIntent: (input: Parameters[0]) => + threads + .bindIntent(input) + .pipe( + Effect.tap(() => hooks.afterBindIntent?.(input.commandId) ?? Effect.void), + ), + }; + }), + ), + ), + Layer.provide( + Layer.effect( + PeerLinks.PeerLinks, + Effect.gen(function* () { + const links = yield* PeerLinks.PeerLinks; + return { + ...links, + session: (environmentId: Parameters[0]) => + links + .session(environmentId) + .pipe(Effect.tap((session) => hooks.afterSession?.(session) ?? Effect.void)), }; }), ), @@ -715,8 +762,15 @@ const makeLaptop = ( Layer.provide(Layer.mock(PeerLinkRequests.PeerLinkRequests)({})), Layer.provide(NodeCrypto.layer), Layer.provideMerge(layerThreads), - Layer.provide(Layer.mock(ProviderRegistry.ProviderRegistry)({})), - Layer.provide(Layer.mock(ProviderAdapterRegistry.ProviderAdapterRegistryV2)({})), + // The laptop lists no provider of its own, so it never runs the box's model. + Layer.provide( + Layer.mock(ProviderRegistry.ProviderRegistry)({ getProviders: Effect.succeed([]) }), + ), + Layer.provide( + Layer.mock(ProviderAdapterRegistry.ProviderAdapterRegistryV2)({ + list: () => Effect.succeed([]), + }), + ), Layer.provide(Layer.mock(ScheduledTaskService.ScheduledTaskService)({})), Layer.provide(Layer.mock(ProjectService.ProjectService)({})), Layer.provide(Layer.mock(ProjectCloneTracker.ProjectCloneTracker)({})), @@ -765,6 +819,7 @@ const parentScope: McpInvocationContext.McpInvocationScope = { const seedParent = ( laptopEnv: Effect.Success>, parentModelSelection: ModelSelection = modelSelection, + interactionMode: "default" | "plan" = "default", ) => Effect.gen(function* () { const now = yield* DateTime.now; @@ -779,7 +834,7 @@ const seedParent = ( title: "Laptop parent", modelSelection: parentModelSelection, runtimeMode: "full-access", - interactionMode: "default", + interactionMode, branch: null, worktreePath: null, }); @@ -893,6 +948,19 @@ const delegateToBox = (laptopEnv: Effect.Success>) }), ); +/** A task placed by capacity, for the box's Claude model, which the laptop does not run. */ +const autoTask = ( + laptopEnv: Effect.Success>, + clientRequestId: string, + target: Record = {}, +) => + laptopEnv.call("delegate_task", { + task: "Run the suite wherever there is room.", + placement: "auto", + target: { providerInstanceId: boxClaude.instanceId, model: "claude-sonnet-4-6", ...target }, + clientRequestId, + }); + /** Waits until the task here has the result its thread there ended with. */ const taskResult = (laptopEnv: Effect.Success>, taskId: NodeId) => laptopEnv.orchestrator @@ -1342,6 +1410,240 @@ it.effect("follows an open task again after the laptop restarts", () => ).pipe(Effect.provide(NodeHttpServer.layerTest)), ); +it.effect("an auto-placed task keeps its environment after a lost answer and a restart", () => + Effect.scoped( + Effect.gen(function* () { + const thread = yield* makeBoxThread; + const b = yield* serveBox(thread); + const offers = yield* SubscriptionRef.make< + ReadonlyArray + >([]); + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); + // Its link outlives the restart, so the box knows it as the same link session. + const linking = yield* layerLinkingEnvironment(laptop).pipe(Layer.build); + // The box takes the launch with room to spare, but its answer never arrives. + yield* Ref.set(thread.afterLaunch, Effect.die("connection lost")); + const lost = yield* Effect.scoped( + Effect.gen(function* () { + const before = yield* makeLaptop(database, offers, linking); + yield* linkTo(before.links, b, "full-access"); + yield* seedParent(before); + return yield* autoTask(before, "auto-suite"); + }), + ); + expect(lost.isError).toBe(true); + const launchedThere = yield* Ref.get(thread.launched); + expect(launchedThere).not.toBeNull(); + + // The box fills up while the laptop restarts: choosing afresh finds nowhere. + yield* Ref.set(thread.resources, loadOf(0.99)); + yield* Ref.set(thread.afterLaunch, Effect.void); + const after = yield* makeLaptop(database, offers, linking); + const fresh = yield* autoTask(after, "another-suite"); + expect(fresh.isError).toBe(true); + expect(JSON.stringify(fresh.content)).toContain("no-eligible-environment"); + expect(yield* Ref.get(thread.launched)).toBe(launchedThere); + + // The retry keeps the binding: the same thread there, recorded once here. + const retried = yield* autoTask(after, "auto-suite"); + expect(retried.isError, JSON.stringify(retried.content)).toBe(false); + expect(retried.structuredContent).toMatchObject({ + remoteChild: { environmentId: box.environmentId, threadId: launchedThere }, + }); + expect((yield* Ref.get(thread.launchedWith))?.modelSelection).toEqual({ + instanceId: boxClaude.instanceId, + model: "claude-sonnet-4-6", + }); + expect( + (yield* after.orchestrator.getThreadProjection(parentThreadId)).subagents, + ).toHaveLength(1); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); + +it.effect("a launch whose answer was lost is never started again under a new link", () => + Effect.scoped( + Effect.gen(function* () { + const thread = yield* makeBoxThread; + const b = yield* serveBox(thread); + const offers = yield* SubscriptionRef.make< + ReadonlyArray + >([]); + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); + const a = yield* makeLaptop(database, offers); + yield* linkTo(a.links, b, "full-access"); + yield* seedParent(a); + const tasks = () => + a.orchestrator.getThreadProjection(parentThreadId).pipe(Effect.map((p) => p.subagents)); + + // The box starts the thread, but its answer is lost; then the box is linked again. + yield* Ref.set(thread.afterLaunch, Effect.die("connection lost")); + expect((yield* autoTask(a, "relinked")).isError).toBe(true); + const lostThere = yield* Ref.get(thread.launched); + expect(lostThere).not.toBeNull(); + yield* Ref.set(thread.afterLaunch, Effect.void); + yield* linkTo(a.links, b, "full-access"); + // The new session would name a new thread there, so the retry starts nothing. + const retried = yield* autoTask(a, "relinked"); + expect(retried.isError).toBe(true); + expect(JSON.stringify(retried.content)).toContain("linked again"); + expect(yield* Ref.get(thread.launched)).toBe(lostThere); + expect(yield* tasks()).toHaveLength(0); + + // A task already recorded is found again after a new link, without a second launch. + const recorded = yield* autoTask(a, "recorded"); + expect(recorded.isError, JSON.stringify(recorded.content)).toBe(false); + const recordedThere = yield* Ref.get(thread.launched); + yield* linkTo(a.links, b, "full-access"); + const again = yield* autoTask(a, "recorded"); + expect(again.isError, JSON.stringify(again.content)).toBe(false); + expect((again.structuredContent as { taskId: NodeId }).taskId).toBe( + (recorded.structuredContent as { taskId: NodeId }).taskId, + ); + expect(yield* Ref.get(thread.launched)).toBe(recordedThere); + expect(yield* tasks()).toHaveLength(1); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); + +it.effect("a link replaced between the launch check and the send starts no second child", () => + Effect.scoped( + Effect.gen(function* () { + const thread = yield* makeBoxThread; + const b = yield* serveBox(thread); + const offers = yield* SubscriptionRef.make< + ReadonlyArray + >([]); + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); + const relinkOnce = yield* Ref.make | null>(null); + const a = yield* makeLaptop(database, offers, undefined, Effect.void, { + // The box is linked again after the launch is checked against its link, before it is sent. + afterBindIntent: (commandId) => + commandId.includes(":remote-launch:") + ? Ref.getAndSet(relinkOnce, null).pipe( + Effect.flatMap((relink) => relink ?? Effect.void), + ) + : Effect.void, + }); + yield* linkTo(a.links, b, "full-access"); + yield* seedParent(a); + const tasks = () => + a.orchestrator.getThreadProjection(parentThreadId).pipe(Effect.map((p) => p.subagents)); + + yield* Ref.set(thread.afterLaunch, Effect.die("connection lost")); + expect((yield* autoTask(a, "raced")).isError).toBe(true); + const lostThere = yield* Ref.get(thread.launched); + expect(lostThere).not.toBeNull(); + yield* Ref.set(thread.afterLaunch, Effect.void); + yield* Ref.set( + relinkOnce, + linkTo(a.links, b, "full-access").pipe(Effect.asVoid, Effect.orDie), + ); + + // The retry passes the check under the first link, but the send would use the new one. + const raced = yield* autoTask(a, "raced"); + expect(raced.isError).toBe(true); + expect(yield* Ref.get(thread.launched)).toBe(lostThere); + expect(yield* tasks()).toHaveLength(0); + // The key stays bound to the first link, so retrying it starts nothing either. + expect((yield* autoTask(a, "raced")).isError).toBe(true); + expect(yield* Ref.get(thread.launched)).toBe(lostThere); + expect(yield* tasks()).toHaveLength(0); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); + +it.effect("a launch with no link to bind to starts nothing, even if a link appears", () => + Effect.scoped( + Effect.gen(function* () { + const thread = yield* makeBoxThread; + const b = yield* serveBox(thread); + const offers = yield* SubscriptionRef.make< + ReadonlyArray + >([]); + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); + const linkOnce = yield* Ref.make | null>(null); + const a = yield* makeLaptop(database, offers, undefined, Effect.void, { + // The box is linked just after delegation finds no link to bind the launch to. + afterSession: (session) => + Option.isNone(session) + ? Ref.getAndSet(linkOnce, null).pipe(Effect.flatMap((link) => link ?? Effect.void)) + : Effect.void, + }); + yield* seedParent(a); + yield* Ref.set(linkOnce, linkTo(a.links, b, "full-access").pipe(Effect.asVoid, Effect.orDie)); + + // A named project there means nothing reaches the box before the launch itself. + const unbound = yield* a.call("delegate_task", { + task: "Run the full suite on the box.", + target: { + environmentId: box.environmentId, + projectId: boxProject, + providerInstanceId: instanceId, + model: "gpt-5.4", + }, + clientRequestId: "unbound", + }); + expect(unbound.isError).toBe(true); + expect(yield* Ref.get(thread.launched)).toBeNull(); + expect((yield* a.orchestrator.getThreadProjection(parentThreadId)).subagents).toHaveLength(0); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); + +it.effect("an auto-placed task runs at the parent's modes, never a narrower link's", () => + Effect.scoped( + Effect.gen(function* () { + const thread = yield* makeBoxThread; + const b = yield* serveBox(thread); + const offers = yield* SubscriptionRef.make< + ReadonlyArray + >([]); + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); + const a = yield* makeLaptop(database, offers); + yield* seedParent(a, modelSelection, "plan"); + // The parent runs with full access; this link allows less, so the box is not eligible. + yield* linkTo(a.links, b, "auto-accept-edits"); + const narrow = yield* autoTask(a, "narrow"); + expect(narrow.isError).toBe(true); + expect(JSON.stringify(narrow.content)).toContain("no-eligible-environment"); + expect(yield* Ref.get(thread.launched)).toBeNull(); + + yield* linkTo(a.links, b, "full-access"); + const wide = yield* autoTask(a, "wide"); + expect(wide.isError, JSON.stringify(wide.content)).toBe(false); + expect(yield* Ref.get(thread.launchedWith)).toEqual({ + modelSelection: { instanceId: boxClaude.instanceId, model: "claude-sonnet-4-6" }, + runtimeMode: "full-access", + interactionMode: "plan", + }); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); + +it.effect("an auto-placed task refuses a project placement did not choose", () => + Effect.scoped( + Effect.gen(function* () { + const thread = yield* makeBoxThread; + const b = yield* serveBox(thread); + const offers = yield* SubscriptionRef.make< + ReadonlyArray + >([]); + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); + const a = yield* makeLaptop(database, offers); + yield* linkTo(a.links, b, "full-access"); + yield* seedParent(a); + // Placement qualifies the project with this repository, not another one there. + const other = yield* autoTask(a, "other-project", { + projectId: ProjectId.make("project:box-other-3"), + }); + expect(other.isError).toBe(true); + expect(JSON.stringify(other.content)).toContain("invalid_request"); + expect(yield* Ref.get(thread.launched)).toBeNull(); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); + it.effect("fails the task when the box revokes the link, and cancel interrupts it there", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/peer/RemoteDelegation.ts b/apps/server/src/peer/RemoteDelegation.ts index 8fcd240e70577..5ef7b4a690414 100644 --- a/apps/server/src/peer/RemoteDelegation.ts +++ b/apps/server/src/peer/RemoteDelegation.ts @@ -1,6 +1,8 @@ import { CommandId, - type EnvironmentId, + EnvironmentId, + isRuntimeModeAbove, + type ModelSelection, NodeId, OrchestratorMcpCapabilitiesInput, OrchestratorMcpCapabilitiesResult, @@ -13,13 +15,20 @@ import { type OrchestratorMcpThreadTimelineItem, type OrchestrationV2RemoteTaskChild, type OrchestrationV2Subagent, + type PeerLinkSummary, type ProjectId, type ProviderInstanceId, ProviderDriverKind, type RunId, + type RuntimeMode, type ThreadId, } from "@t3tools/contracts"; +import { + chooseThreadPlacement, + type ThreadPlacementCandidate, +} from "@t3tools/shared/threadPlacement"; import * as Cause from "effect/Cause"; +import * as Clock from "effect/Clock"; import * as Context from "effect/Context"; import * as SecureRandom from "effect/crypto/SecureRandom"; import * as DateTime from "effect/DateTime"; @@ -96,6 +105,23 @@ export class RemoteDelegation extends Context.Service< }; }, ) => Effect.Effect<{ readonly taskId: NodeId }, OrchestratorMcpFailure>; + /** + * Where a `placement: "auto"` task runs: the environment its key is bound to, + * or the one chosen now from this environment's and authenticated links' own + * answers, bound before anything starts. A retry never chooses again. + */ + readonly place: ( + scope: ThreadScope, + input: { + readonly key: string; + readonly projectId: ProjectId; + readonly modelSelection: Pick; + /** The parent's effective mode or a narrower one asked for, sent explicitly. */ + readonly runtimeMode: RuntimeMode; + /** This environment's own `orchestrator_capabilities`, searched for the model. */ + readonly here: OrchestratorMcpCapabilitiesResult; + }, + ) => Effect.Effect; /** * A `t3_thread_send` there that started or queued a run on the thread of * one of this thread's tasks that has its result: like a follow-up to a @@ -137,6 +163,23 @@ export class RemoteDelegation extends Context.Service< } >()("t3/peer/RemoteDelegation") {} +/** The exact instance and model there can run a child task, in the mode asked for. */ +const runsModel = ( + capabilities: OrchestratorMcpCapabilitiesResult, + requested: { + readonly modelSelection: Pick; + readonly runtimeMode: RuntimeMode; + }, +) => + capabilities.providers.some( + (provider) => + provider.providerInstanceId === requested.modelSelection.instanceId && + provider.canRunChildTask && + provider.models.some((model) => model.id === requested.modelSelection.model) && + (provider.runtimeModes === undefined || + provider.runtimeModes.includes(requested.runtimeMode)), + ); + /** The other side refused because the thread there runs above this caller's modes. */ const isModeRefusal = (error: OrchestratorMcpFailure) => error.code === "runtime_mode_escalation_denied" || @@ -262,6 +305,121 @@ const make = Effect.gen(function* () { return chosen.providerInstanceId; }); + /** A linked environment as a placement candidate, from what it answered just now. */ + const peerCandidate = Effect.fn("RemoteDelegation.peerCandidate")( + function* ( + scope: ThreadScope, + link: PeerLinkSummary, + input: Parameters[1], + ) { + const capabilities = yield* forwarding.call( + scope, + RemoteCapabilitiesTool, + link.environmentId, + { + search: input.modelSelection.model, + }, + ); + const receivedAt = yield* Clock.currentTimeMillis; + const projectAvailable = yield* resolveRemoteProject( + scope, + link.environmentId, + input.projectId, + undefined, + ).pipe( + Effect.as(true), + Effect.orElseSucceed(() => false), + ); + return { + environmentId: link.environmentId, + resources: capabilities.hostResources ?? null, + receivedAt, + weight: 1, + operable: true, + projectAvailable, + modelAvailable: runsModel(capabilities, input), + } satisfies ThreadPlacementCandidate; + }, + // One that does not answer, or refuses this caller, is not a candidate. + (effect, _scope, link) => + effect.pipe( + Effect.tapError((cause) => + Effect.logDebug("remote-delegation.placement-candidate-unavailable", { + environmentId: link.environmentId, + cause, + }), + ), + Effect.option, + ), + ); + + const place: RemoteDelegation["Service"]["place"] = Effect.fn("RemoteDelegation.place")( + function* (scope, input) { + const bindingId = CommandId.make( + `command:mcp:${encodeURIComponent(scope.requestNamespace)}:placement:delegate-task:${encodeURIComponent(input.key)}`, + ); + const bindingFailure = () => + new OrchestratorMcpFailure({ + code: "orchestration_error", + message: + "Unable to record where this task runs, so nothing was started. Retry with the same clientRequestId.", + }); + // A bound intent keeps its environment, even one that has since filled up or + // stopped answering: its first attempt may have started the task there. + const bound = yield* threads.intentOf(bindingId).pipe(Effect.mapError(bindingFailure)); + if (Option.isSome(bound)) return EnvironmentId.make(bound.value); + + const linked = (yield* links.enabled) + ? yield* links.list.pipe(Effect.orElseSucceed(() => [])) + : []; + const peers = yield* Effect.forEach( + linked.filter( + (link) => + link.environmentId !== hereId && + link.status === "reachable" && + link.access !== "read-only" && + !isRuntimeModeAbove(input.runtimeMode, link.access), + ), + (link) => peerCandidate(scope, link, input), + { concurrency: "unbounded" }, + ); + const now = yield* Clock.currentTimeMillis; + const decision = chooseThreadPlacement( + { originEnvironmentId: hereId }, + [ + { + environmentId: hereId, + resources: input.here.hostResources ?? null, + receivedAt: now, + weight: 1, + operable: true, + projectAvailable: true, + modelAvailable: runsModel(input.here, input), + }, + ...peers.flatMap((peer) => (Option.isSome(peer) ? [peer.value] : [])), + ], + now, + ); + yield* Effect.annotateCurrentSpan({ + "placement.reason": decision.reason, + "placement.candidates": peers.length + 1, + }); + if (!decision.accepted) { + return yield* new OrchestratorMcpFailure({ + code: "target_required", + message: `No environment can take this task now (${decision.reason}): none answered with spare capacity, this repository's project and ${input.modelSelection.instanceId}/${input.modelSelection.model}. Nothing was started. Pass target.environmentId, or omit placement to run it here.`, + }); + } + return yield* threads + .bindIntent({ + commandId: bindingId, + threadId: scope.thread.threadId, + value: decision.environmentId, + }) + .pipe(Effect.map(EnvironmentId.make), Effect.mapError(bindingFailure)); + }, + ); + const delegate: RemoteDelegation["Service"]["delegate"] = Effect.fn("RemoteDelegation.delegate")( function* (scope, input) { const parent = yield* threads.getThreadRecords(scope.thread.threadId, ["runs"]).pipe( @@ -342,6 +500,83 @@ const make = Effect.gen(function* () { // Both are scoped to the provider session, as every key a caller passes is: // the forwarder scopes the launch's, and the record's names it. const key = input.clientRequestId ?? (yield* random.uuidV4()); + const keyed = (operation: string) => + CommandId.make( + `command:mcp:${encodeURIComponent(scope.requestNamespace)}:${operation}:${encodeURIComponent(key)}`, + ); + const commandId = keyed("remote-delegate"); + const link = yield* links.get(target.environmentId).pipe( + Effect.map(Option.getOrUndefined), + Effect.orElseSucceed(() => undefined), + ); + const label = link?.label ?? target.environmentId; + // The other side names the thread from its link session and the key, so a + // retry after linking again would start a second one. The session that took + // the first launch and the thread it answered with are kept here. + const childId = keyed("remote-child"); + const linkSession = yield* links.session(target.environmentId).pipe( + Effect.map(Option.getOrUndefined), + Effect.mapError( + () => + new OrchestratorMcpFailure({ + code: "orchestration_error", + message: `Unable to read the link to ${label}, so nothing was started. Retry with the same clientRequestId.`, + }), + ), + ); + // A launch is only sent bound to the link session it goes out under; with no link + // there is none, and one appearing before the send must not carry it unbound. + if (linkSession === undefined) { + return yield* new OrchestratorMcpFailure({ + code: "invalid_request", + message: `This environment is not linked to ${label}, so nothing was started. t3_environment_links lists the linked ones.`, + }); + } + const session = `${target.environmentId}@${linkSession}`; + const launchedUnder = yield* threads + .bindIntent({ + commandId: keyed("remote-launch"), + threadId: parent.thread.id, + value: session, + }) + .pipe( + Effect.mapError( + () => + new OrchestratorMcpFailure({ + code: "orchestration_error", + message: + "Unable to record the launch there, so nothing was started. Retry with the same clientRequestId.", + }), + ), + ); + if (launchedUnder !== session) { + const childThere = yield* threads.intentOf(childId).pipe( + Effect.map(Option.getOrUndefined), + Effect.orElseSucceed(() => undefined), + ); + const recordedTask = + childThere === undefined + ? undefined + : yield* threads.getThreadRecords(scope.thread.threadId, ["subagents"]).pipe( + Effect.map(({ subagents }) => + subagents.find( + (task) => + task.remoteChild?.environmentId === target.environmentId && + task.remoteChild.threadId === childThere, + ), + ), + Effect.orElseSucceed(() => undefined), + ); + if (recordedTask !== undefined) return { taskId: recordedTask.id }; + return yield* new OrchestratorMcpFailure({ + code: "orchestration_error", + message: `${label} was linked again after an earlier attempt with this clientRequestId launched the task there, and the new link cannot find that thread. Nothing new was started. ${ + childThere === undefined + ? "t3_thread_list with this environmentId finds it: it names this thread as its parent." + : `Its thread there is ${childThere}.` + } This clientRequestId stays bound to the earlier link. Check that thread there before deciding the work still needs doing.`, + }); + } const task = taskPrompt(input); const title = input.title ?? parent.thread.title; const launched = yield* forwarding.call( @@ -363,18 +598,17 @@ const make = Effect.gen(function* () { title: parent.thread.title, }, }, + // Sent only under the session bound above, never one linked since. + { expectedIncarnation: linkSession }, ); - const link = yield* links.get(target.environmentId).pipe( - Effect.map(Option.getOrUndefined), - Effect.orElseSucceed(() => undefined), - ); - const commandId = CommandId.make( - `command:mcp:${encodeURIComponent(scope.requestNamespace)}:remote-delegate:${encodeURIComponent(key)}`, - ); + // Losing this only makes a retry after linking again refuse instead of finding the task. + yield* threads + .bindIntent({ commandId: childId, threadId: parent.thread.id, value: launched.threadId }) + .pipe(Effect.ignoreCause({ log: true })); const remoteChild: OrchestrationV2RemoteTaskChild = { environmentId: target.environmentId, threadId: launched.threadId, - label: link?.label ?? target.environmentId, + label, title, ...(launched.runId === null ? {} : { runId: launched.runId }), }; @@ -1034,7 +1268,7 @@ const make = Effect.gen(function* () { }).pipe(Effect.ignoreCause({ log: true })), ); - return RemoteDelegation.of({ delegate, followUp, cancel, runStatus, start }); + return RemoteDelegation.of({ delegate, place, followUp, cancel, runStatus, start }); }); /** A defect, logged, as a failure, so a retry covers it. */ diff --git a/apps/server/src/relay/AgentAwarenessRelay.test.ts b/apps/server/src/relay/AgentAwarenessRelay.test.ts index 10388e99fe7c5..0287c00e5f905 100644 --- a/apps/server/src/relay/AgentAwarenessRelay.test.ts +++ b/apps/server/src/relay/AgentAwarenessRelay.test.ts @@ -252,6 +252,8 @@ const makeTestRelay = Effect.fnUntraced(function* ( assertProviderInstanceEnabled: unused, isProviderInstanceEnabledForProject: unused, hasReceipt: unused, + intentOf: unused, + bindIntent: unused, waitForThread: unused, settleAfterRun: unused, settleThread: unused, diff --git a/packages/contracts/src/orchestratorMcp.ts b/packages/contracts/src/orchestratorMcp.ts index c30c4fea2e785..e7b3827987b8b 100644 --- a/packages/contracts/src/orchestratorMcp.ts +++ b/packages/contracts/src/orchestratorMcp.ts @@ -180,6 +180,12 @@ export const OrchestratorMcpDelegateTaskInput = Schema.Struct({ "'shared' (default): parent checkout. 'worktree': new prepared checkout from the parent commit, for a separate checkout or PR per child. Uncommitted edits stay out.", }), ), + placement: Schema.optional( + Schema.Literals(["auto"]).annotate({ + description: + "'auto': for work that needs none of your checkout's changes. Runs here or on a linked environment with spare capacity, this repository's project and your exact provider and model; there, in that project's checkout. Ignored with target.environmentId or isolation. Reuse clientRequestId on retry to keep the environment chosen.", + }), + ), mode: Schema.optional( Schema.Literals(["async", "wait"]).annotate({ description: