From 42b343970c200cb3922b31b44e63852ef16f211c Mon Sep 17 00:00:00 2001 From: Kameron Smith <112618179+nullStack65@users.noreply.github.com> Date: Wed, 18 Nov 2026 13:16:24 +0000 Subject: [PATCH 1/5] feat(placement): admit a web draft's new thread on its server before it runs elsewhere The web and desktop "Auto balance" chose a draft's machine in the client from the load it saw, with no server checking that this sign-in may launch there, that the machine runs the exact model in the requested mode, or binding the choice before anything went there. A new protected `orchestration.admitThreadPlacement` RPC (operate scope) on the draft's server decides through the shared placement policy, over that server's own authenticated links narrowed to machines this client may launch on, and binds the destination before any effect. The binding is keyed by this server's id and the launch's command id; the session that first admitted it owns it. Another, expired or revoked session is refused before any write; a bound operation is reconciled on every send without rescoring or reading fresh evidence. Machines listed in the new `automaticPlacementExcludedEnvironmentIds` server setting (empty by default) are dropped before any link is contacted, in this path and in MCP automatic placement; work bound to one is refused. The draft stores its launch operation, intent and asked server before asking, and the binding before it moves; the launch then goes out under those ids over the client's own connection there, with attachments prepared only after that. Changed work is refused while the operation is open. Model: claude-opus-5-5 Task: CLO-866 universal native thread placement, shared native web/desktop admission (clo866-post-pr71-shared-client-admission-20261118-v1) Agent: Claude Code in T3 Code, existing CLO-866 source owner thread (Opus 5.5) Co-Authored-By: Claude Opus 5.5 --- .github/workflows/closura-arc-ci.yml | 4 +- apps/server/src/auth/RpcAuthorization.ts | 1 + .../src/mcp/toolkits/project/handlers.ts | 33 +- .../src/observability/RpcInstrumentation.ts | 1 + apps/server/src/peer/PeerForwarding.ts | 118 ++++- apps/server/src/peer/PeerLinks.ts | 20 + apps/server/src/peer/RemoteDelegation.test.ts | 197 +++++++++ apps/server/src/peer/RemoteDelegation.ts | 389 +++++++++++++---- apps/server/src/peer/placementEvidence.ts | 42 ++ apps/server/src/ws.test.ts | 220 ++++++++++ apps/server/src/ws.ts | 77 ++++ .../components/chat/draftPlacement.test.ts | 409 ++++++++++++++++++ .../web/src/components/chat/draftPlacement.ts | 219 ++++++++++ .../src/components/chat/useComposerSend.ts | 30 +- ...seDraftEnvironmentRouting.browser.test.tsx | 58 ++- .../chat/useDraftEnvironmentRouting.ts | 125 ++++-- apps/web/src/composerDraftStore.ts | 134 ++++++ .../client-runtime/src/state/orchestration.ts | 5 + .../contracts/src/clientRpcPermissions.ts | 1 + packages/contracts/src/orchestrationV2.ts | 59 +++ packages/contracts/src/rpc.ts | 11 + packages/contracts/src/settings.ts | 8 + 22 files changed, 1983 insertions(+), 178 deletions(-) create mode 100644 apps/server/src/peer/placementEvidence.ts create mode 100644 apps/web/src/components/chat/draftPlacement.test.ts create mode 100644 apps/web/src/components/chat/draftPlacement.ts diff --git a/.github/workflows/closura-arc-ci.yml b/.github/workflows/closura-arc-ci.yml index f5a9e0ce4161..5adaff1cb593 100644 --- a/.github/workflows/closura-arc-ci.yml +++ b/.github/workflows/closura-arc-ci.yml @@ -155,7 +155,9 @@ jobs: apps/web/src/composerDraftStore.test.ts \ apps/web/src/components/chat/composerSend.logic.test.ts \ apps/web/src/components/chat/chatViewSend.test.ts \ - apps/web/src/components/chat/composerImagePersistence.test.ts + apps/web/src/components/chat/composerImagePersistence.test.ts \ + apps/web/src/components/chat/draftPlacement.test.ts \ + apps/server/src/ws.test.ts - name: Build native T3 server shell: bash diff --git a/apps/server/src/auth/RpcAuthorization.ts b/apps/server/src/auth/RpcAuthorization.ts index 7925fd869799..9599adbba2bd 100644 --- a/apps/server/src/auth/RpcAuthorization.ts +++ b/apps/server/src/auth/RpcAuthorization.ts @@ -58,6 +58,7 @@ export const RPC_REQUIRED_SCOPES = { [ORCHESTRATION_V2_WS_METHODS.getThreadProjection]: AuthOrchestrationReadScope, [ORCHESTRATION_V2_WS_METHODS.getTurnItem]: AuthOrchestrationReadScope, [ORCHESTRATION_V2_WS_METHODS.launchThread]: AuthOrchestrationOperateScope, + [ORCHESTRATION_V2_WS_METHODS.admitThreadPlacement]: AuthOrchestrationOperateScope, [ORCHESTRATION_V2_WS_METHODS.subscribeShell]: AuthOrchestrationReadScope, [ORCHESTRATION_V2_WS_METHODS.subscribeThread]: AuthOrchestrationReadScope, [ORCHESTRATION_V2_WS_METHODS.subscribeCommandOutput]: AuthOrchestrationReadScope, diff --git a/apps/server/src/mcp/toolkits/project/handlers.ts b/apps/server/src/mcp/toolkits/project/handlers.ts index 57a7c7952061..f66e13636d0f 100644 --- a/apps/server/src/mcp/toolkits/project/handlers.ts +++ b/apps/server/src/mcp/toolkits/project/handlers.ts @@ -2,18 +2,16 @@ import { agentLaunchInboxRouting, CommandId, type EnvironmentId, - isProviderAvailable, MessageId, ThreadId, OrchestratorMcpFailure, ProjectId, type ProviderInteractionMode, - resolveProjectProviderInstanceEnabled, type RuntimeMode, } from "@t3tools/contracts"; import type * as Tool from "effect/ai/Tool"; import * as RemoteDelegation from "../../../peer/RemoteDelegation.ts"; -import * as HostResources from "../../../resourceTelemetry/HostResources.ts"; +import { hereEvidence } from "../../../peer/placementEvidence.ts"; import * as Digest from "effect/crypto/Digest"; import * as Effect from "effect/Effect"; import * as Hex from "effect/encoding/Hex"; @@ -30,8 +28,8 @@ import { mutateProjectWithClone } from "../../../project/ProjectMutation.ts"; import * as ManagedProjectFolders from "../../../project/ManagedProjectFolders.ts"; import * as Repositories from "../../../sourceControl/SourceControlRepositoryService.ts"; import * as GitVcsDriver from "../../../vcs/GitVcsDriver.ts"; -import * as ServerSettings from "../../../serverSettings.ts"; import * as ProviderRegistry from "../../../provider/ProviderRegistry.ts"; +import * as ServerSettings from "../../../serverSettings.ts"; import * as DateTime from "effect/DateTime"; import { resolveNewThreadModelSelection } from "@t3tools/shared/usageLimits"; import { resolveProjectSettings } from "@t3tools/shared/projectSettings"; @@ -198,15 +196,11 @@ const launchPlacedThere = Effect.fn("mcp.launchPlacedThere")(function* ( message: 'With placement "auto", pass clientRequestId and modelSelection: a retry must reach the environment chosen, with the exact model.', }); - const providers = yield* (yield* ProviderRegistry.ProviderRegistry).getProviders; - const settings = yield* ServerSettings.ServerSettingsService.pipe( - Effect.flatMap((service) => service.getSettings), - Effect.mapError(unavailable), - ); - const provider = providers.find( - (candidate) => candidate.instanceId === modelSelection.instanceId, - ); - const hostResources = yield* Effect.serviceOption(HostResources.HostResources); + const here = yield* hereEvidence({ + projectId: local.projectId, + modelSelection, + runtimeMode: local.runtimeMode, + }).pipe(Effect.mapError(unavailable)); const environmentId = yield* delegation.value.place(scope, { operation: "thread-launch", key, @@ -214,18 +208,7 @@ const launchPlacedThere = Effect.fn("mcp.launchPlacedThere")(function* ( projectId: local.projectId, modelSelection, runtimeMode: local.runtimeMode, - here: { - resources: Option.isSome(hostResources) ? yield* hostResources.value.read : null, - modelAvailable: - provider !== undefined && - provider.enabled && - isProviderAvailable(provider) && - resolveProjectProviderInstanceEnabled(settings, local.projectId, provider.instanceId) && - provider.models.some((model) => model.slug === modelSelection.model) && - (provider.supportedRuntimeModes === undefined || - provider.supportedRuntimeModes.length === 0 || - provider.supportedRuntimeModes.includes(local.runtimeMode)), - }, + here, }); return environmentId === scope.environmentId ? undefined : yield* sendThere(environmentId, key); }); diff --git a/apps/server/src/observability/RpcInstrumentation.ts b/apps/server/src/observability/RpcInstrumentation.ts index 4549ee8a33c1..6ffd88b2ab78 100644 --- a/apps/server/src/observability/RpcInstrumentation.ts +++ b/apps/server/src/observability/RpcInstrumentation.ts @@ -31,6 +31,7 @@ const RPC_AGGREGATES = { [ORCHESTRATION_V2_WS_METHODS.getOlderSettledThreads]: "orchestration", [ORCHESTRATION_V2_WS_METHODS.getThreadProjection]: "orchestrationV2", [ORCHESTRATION_V2_WS_METHODS.launchThread]: "orchestration", + [ORCHESTRATION_V2_WS_METHODS.admitThreadPlacement]: "orchestration", [ORCHESTRATION_V2_WS_METHODS.subscribeShell]: "orchestrationV2", [ORCHESTRATION_V2_WS_METHODS.subscribeThread]: "orchestrationV2", [ORCHESTRATION_V2_WS_METHODS.subscribeCommandOutput]: "orchestrationV2", diff --git a/apps/server/src/peer/PeerForwarding.ts b/apps/server/src/peer/PeerForwarding.ts index 132e3b13b8b2..365fb55f206c 100644 --- a/apps/server/src/peer/PeerForwarding.ts +++ b/apps/server/src/peer/PeerForwarding.ts @@ -10,6 +10,7 @@ import { OrchestratorMcpThreadWaitResult, type PeerLinkError, RuntimeMode, + type ThreadId, } from "@t3tools/contracts"; import * as Context from "effect/Context"; import * as Digest from "effect/crypto/Digest"; @@ -24,7 +25,7 @@ import type * as Tool from "effect/ai/Tool"; import { runtimeModeRank } from "../mcp/delegatedTaskModes.ts"; import { assertNotLinked } from "../mcp/linkOrigin.ts"; import * as McpInvocationContext from "../mcp/McpInvocationContext.ts"; -import { readCaller, unavailable } from "../mcp/threadAccess.ts"; +import { type CallerLimits, readCaller, unavailable } from "../mcp/threadAccess.ts"; import * as ThreadManagement from "../orchestration-v2/ThreadManagementService.ts"; import * as PeerLinks from "./PeerLinks.ts"; import * as PeerMcpClient from "./PeerMcpClient.ts"; @@ -35,6 +36,29 @@ const MAX_WAIT_TIMEOUT_MS = 45_000; type Scope = McpInvocationContext.McpInvocationScope; +/** + * A client signed in to this server itself, rather than an agent over MCP: it has no thread + * here, and its requests carry the modes it asked for, already within this server's ceilings. + */ +export interface NativeCaller { + readonly limits: CallerLimits; +} + +/** What a native caller may ask a linked environment: evidence for placement, never an effect. */ +const NATIVE_EVIDENCE_TOOLS: ReadonlySet = new Set([ + "orchestrator_capabilities", + "t3_project_list", +]); + +/** Who a forwarded call is made for, as it travels: its limits, retry-key scope and thread. */ +interface ForwardedCaller { + readonly requestNamespace: string | undefined; + readonly sender: + | { readonly environmentId: EnvironmentId; readonly threadId: ThreadId; readonly title: string } + | undefined; + readonly limits: CallerLimits; +} + /** * Runs T3 MCP tools in a linked environment for an agent here. The tool's own * declaration checks the agent here first; the linked environment then holds @@ -74,6 +98,16 @@ export class PeerForwarding extends Context.Service< environmentId: EnvironmentId, input: OrchestratorMcpThreadWaitInput, ) => Effect.Effect; + /** + * A read-only evidence tool (`orchestrator_capabilities`, `t3_project_list`) there for a + * native caller, under the link's credential narrowed to the caller's modes. + */ + readonly callNative: ( + caller: NativeCaller, + tool: T, + environmentId: EnvironmentId, + params: Tool.Parameters, + ) => Effect.Effect, OrchestratorMcpFailure>; /** The environments this one links to, as an agent sees them. */ readonly links: ( scope: Scope, @@ -149,8 +183,20 @@ const make = Effect.gen(function* () { }); }); - /** Tool and contract schemas are plain data: they encode and decode without services. */ - const forward = Effect.fn("PeerForwarding.forward")(function* ( + const mcpCaller = (scope: Scope) => + forwardingCaller(scope).pipe( + Effect.map(({ caller, limits }): ForwardedCaller => ({ + requestNamespace: scope.requestNamespace, + // So a message sent there names the thread that sent it, even to a client that cannot reach here. + sender: + caller === undefined + ? undefined + : { environmentId: scope.environmentId, threadId: caller.id, title: caller.title }, + limits, + })), + ); + + const forward = ( scope: Scope, toolName: string, parameters: Schema.Top, @@ -158,8 +204,29 @@ const make = Effect.gen(function* () { environmentId: EnvironmentId, params: unknown, expectedIncarnation?: string, + ) => + Effect.flatMap(mcpCaller(scope), (caller) => + forwardAs( + caller, + toolName, + parameters, + success, + environmentId, + params, + expectedIncarnation, + ), + ); + + /** Tool and contract schemas are plain data: they encode and decode without services. */ + const forwardAs = Effect.fn("PeerForwarding.forward")(function* ( + { requestNamespace, sender, limits }: ForwardedCaller, + toolName: string, + parameters: Schema.Top, + success: Schema.Top, + environmentId: EnvironmentId, + params: unknown, + expectedIncarnation?: string, ) { - const { caller, limits } = yield* forwardingCaller(scope); const encoded = (yield* Schema.encodeUnknownEffect(parameters as Schema.Encoder)( params, ).pipe(Effect.mapError(unavailable))) as Record; @@ -173,17 +240,14 @@ const make = Effect.gen(function* () { }); } if (typeof args.clientRequestId === "string") { - args.clientRequestId = yield* scopedRequestId(scope.requestNamespace, args.clientRequestId); + if (requestNamespace === undefined) { + return yield* new OrchestratorMcpFailure({ + code: "invalid_request", + message: "This caller cannot send a request with a retry key to a linked environment.", + }); + } + args.clientRequestId = yield* scopedRequestId(requestNamespace, args.clientRequestId); } - // So a message sent there names the thread that sent it, even to a client that cannot reach here. - const sender = - caller === undefined - ? undefined - : { - environmentId: scope.environmentId, - threadId: caller.id, - title: caller.title, - }; return yield* peers.call({ environmentId, tool: toolName, @@ -290,8 +354,34 @@ const make = Effect.gen(function* () { }, ); + const callNative: PeerForwarding["Service"]["callNative"] = ( + caller, + tool, + environmentId, + params, + ) => + NATIVE_EVIDENCE_TOOLS.has(tool.name) + ? Effect.andThen( + requireEnabled, + forwardAs>( + { requestNamespace: undefined, sender: undefined, limits: caller.limits }, + tool.name, + tool.parametersSchema, + tool.successSchema, + environmentId, + params, + ), + ) + : Effect.fail( + new OrchestratorMcpFailure({ + code: "capability_denied", + message: `${tool.name} is not placement evidence, so it is not sent for a native caller.`, + }), + ); + return PeerForwarding.of({ call, + callNative, readThread, interruptThread, waitForThread, diff --git a/apps/server/src/peer/PeerLinks.ts b/apps/server/src/peer/PeerLinks.ts index 1858de216c39..8371591dde05 100644 --- a/apps/server/src/peer/PeerLinks.ts +++ b/apps/server/src/peer/PeerLinks.ts @@ -99,6 +99,13 @@ export class PeerLinks extends Context.Service< readonly enabled: Effect.Effect; readonly link: (input: PeerLinkCreateInput) => Effect.Effect; readonly list: Effect.Effect, PeerLinkError>; + /** + * Machines automatic placement never uses or contacts (server settings). Unreadable settings + * fail, so placement contacts nobody rather than everyone. + */ + readonly placementExclusions: Effect.Effect, PeerLinkError>; + /** The stored links, read here only: no peer is asked whether it answers. */ + readonly stored: Effect.Effect, PeerLinkError>; /** * Names the link's session there without revealing its token. Linking again * starts a new session there, so this changes exactly when that does. @@ -241,6 +248,13 @@ const make = Effect.gen(function* () { Effect.map((settings) => settings.linkedEnvironmentsEnabled), Effect.orElseSucceed(() => false), ); + const placementExclusions = serverSettings.getSettings.pipe( + Effect.map( + (settings): ReadonlySet => + new Set(settings.automaticPlacementExcludedEnvironmentIds), + ), + Effect.catch(failStorage), + ); /** A fresh name for one link's token; the row records it. */ const newSecretName = random.randomBytes(16).pipe( @@ -619,6 +633,10 @@ const make = Effect.gen(function* () { }); }); + const stored: PeerLinks["Service"]["stored"] = Effect.flatMap(readRows, (rows) => + Effect.forEach(rows, toLink), + ); + const get: PeerLinks["Service"]["get"] = (environmentId) => readRow(environmentId).pipe( Effect.flatMap( @@ -730,6 +748,8 @@ const make = Effect.gen(function* () { enabled, link, list, + stored, + placementExclusions, session, get, unlink, diff --git a/apps/server/src/peer/RemoteDelegation.test.ts b/apps/server/src/peer/RemoteDelegation.test.ts index 42f5919c5032..511744e45b2a 100644 --- a/apps/server/src/peer/RemoteDelegation.test.ts +++ b/apps/server/src/peer/RemoteDelegation.test.ts @@ -17,6 +17,7 @@ import { type RepositoryIdentity, RunId, type HostResourcesSnapshot, + OrchestrationV2ThreadPlacementAdmitError, type ServerProvider, ThreadId, TurnItemId, @@ -31,6 +32,7 @@ import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; +import * as Result from "effect/Result"; import * as Schedule from "effect/Schedule"; import * as Stream from "effect/Stream"; import * as SubscriptionRef from "effect/SubscriptionRef"; @@ -3033,3 +3035,198 @@ it.effect("a local task runs in this thread's project", () => }), ).pipe(Effect.provide(NodeHttpServer.layerTest)), ); + +/** A native client's new-thread launch, admitted on the laptop for the box's Claude model. */ +const nativeAdmission = ( + laptopEnv: Effect.Success>, + options: { + readonly commandId: string; + readonly sessionId?: string; + readonly stillAuthorized?: Effect.Effect; + readonly model?: string; + readonly here?: Effect.Effect< + { readonly resources: HostResourcesSnapshot | null; readonly modelAvailable: boolean }, + OrchestrationV2ThreadPlacementAdmitError + >; + }, +) => + laptopEnv.remote + .admit( + { + sessionId: options.sessionId ?? "web-session", + stillAuthorized: options.stillAuthorized ?? Effect.succeed(true), + }, + { + commandId: CommandId.make(options.commandId), + threadId: ThreadId.make(`thread:${options.commandId}`), + projectId: laptopProject.id, + modelSelection: { + instanceId: boxClaude.instanceId, + model: options.model ?? "claude-sonnet-4-6", + }, + runtimeMode: "full-access", + interactionMode: "default", + candidateEnvironmentIds: [laptop.environmentId, box.environmentId], + // The laptop cannot run the model, so only the box can take it. + here: options.here ?? Effect.succeed({ resources: loadOf(0.1), modelAvailable: false }), + }, + ) + .pipe(Effect.result); + +const nativePlacementBinding = (commandId: string) => + CommandId.make( + `command:native:${encodeURIComponent(laptop.environmentId)}:placement:thread-launch:${encodeURIComponent(commandId)}`, + ); + +it.effect( + "a native launch is bound to the box with its session, and only that session resumes it, as bound", + () => + Effect.scoped( + Effect.gen(function* () { + const thread = yield* makeBoxThread; + const b = yield* serveBox(thread); + const offers = yield* SubscriptionRef.make< + ReadonlyArray + >([]); + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); + const laptopEnv = yield* makeLaptop(database, offers); + yield* linkTo(laptopEnv.links, b, "full-access"); + + const admitted = yield* nativeAdmission(laptopEnv, { commandId: "web-launch" }); + expect(admitted).toEqual( + Result.succeed({ + environmentId: box.environmentId, + projectId: boxProject, + reason: "capacity", + resumed: false, + }), + ); + // Admission sends no launch: the client launches there over its own connection. + expect(yield* Ref.get(thread.launched)).toBeNull(); + + // The box fills up and the request changes; this machine's evidence cannot even be read. + // Its binding still holds for the session that made it, without reading any of that. + yield* Ref.set(thread.resources, loadOf(0.99)); + const hereRead = yield* Ref.make(0); + const resumed = yield* nativeAdmission(laptopEnv, { + commandId: "web-launch", + model: "a-model-nowhere", + here: Ref.update(hereRead, (count) => count + 1).pipe( + Effect.andThen( + Effect.fail( + new OrchestrationV2ThreadPlacementAdmitError({ + commandId: CommandId.make("web-launch"), + reason: "unavailable", + detail: "providers unreadable", + }), + ), + ), + ), + }); + expect(resumed).toEqual( + Result.succeed({ + environmentId: box.environmentId, + projectId: boxProject, + reason: "capacity", + resumed: true, + }), + ); + expect(yield* Ref.get(hereRead)).toBe(0); + + // Another sign-in neither adopts it nor starts it afresh. + const other = yield* nativeAdmission(laptopEnv, { + commandId: "web-launch", + sessionId: "replaced-session", + }); + expect(Result.isFailure(other) && other.failure.reason).toBe("other-session"); + + // A session no longer signed in binds nothing, so nothing of it stays to be adopted. + yield* Ref.set(thread.resources, loadOf(0.1)); + const signedOut = yield* nativeAdmission(laptopEnv, { + commandId: "signed-out", + stillAuthorized: Effect.succeed(false), + }); + expect(Result.isFailure(signedOut) && signedOut.failure.reason).toBe("unavailable"); + const later = yield* nativeAdmission(laptopEnv, { + commandId: "signed-out", + sessionId: "another-session", + }); + expect(Result.isSuccess(later)).toBe(true); + + // Signed out while the machines were asked: the choice is not bound. + const checks = yield* Ref.make(0); + const revokedMidway = yield* nativeAdmission(laptopEnv, { + commandId: "revoked-midway", + stillAuthorized: Ref.updateAndGet(checks, (count) => count + 1).pipe( + Effect.map((count) => count === 1), + ), + }); + expect(Result.isFailure(revokedMidway) && revokedMidway.failure.reason).toBe("unavailable"); + expect(yield* Ref.get(checks)).toBe(2); + expect(yield* laptopEnv.threads.intentOf(nativePlacementBinding("revoked-midway"))).toEqual( + Option.none(), + ); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); + +it.effect( + "automatic placement never contacts an excluded machine, and refuses a launch bound to one", + () => + Effect.scoped( + Effect.gen(function* () { + const thread = yield* makeBoxThread; + const b = yield* serveBox(thread); + const offers = yield* SubscriptionRef.make< + ReadonlyArray + >([]); + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); + const laptopEnv = yield* makeLaptop(database, offers); + yield* linkTo(laptopEnv.links, b, "full-access"); + // Bound before the box is excluded: both the native and the MCP operations. + expect( + Result.isSuccess(yield* nativeAdmission(laptopEnv, { commandId: "bound-before" })), + ).toBe(true); + const placeHere = (key: string, here = { resources: loadOf(0.1), modelAvailable: false }) => + laptopEnv.remote + .place(outsideScope, { + operation: "thread-launch", + key, + threadId: ThreadId.make(`thread:${key}`), + projectId: laptopProject.id, + modelSelection: { instanceId: boxClaude.instanceId, model: "claude-sonnet-4-6" }, + runtimeMode: "full-access", + here, + }) + .pipe(Effect.result); + expect(yield* placeHere("mcp-bound-before")).toEqual(Result.succeed(box.environmentId)); + + yield* laptopEnv.settings.updateSettings({ + automaticPlacementExcludedEnvironmentIds: [box.environmentId], + }); + const contact = Effect.all([Ref.get(b.probes), Ref.get(b.bearers)]); + const before = yield* contact; + + // The box still answers with room to spare, yet nobody asks it: this machine is chosen. + const native = yield* nativeAdmission(laptopEnv, { + commandId: "fresh", + here: Effect.succeed({ resources: loadOf(0.5), modelAvailable: true }), + }); + expect(Result.isSuccess(native) && native.success.environmentId).toBe(laptop.environmentId); + expect( + yield* placeHere("mcp-fresh", { resources: loadOf(0.5), modelAvailable: true }), + ).toEqual(Result.succeed(laptop.environmentId)); + + // Work bound there earlier is refused, never rescored elsewhere or sent there. + const boundNative = yield* nativeAdmission(laptopEnv, { commandId: "bound-before" }); + expect(Result.isFailure(boundNative) && boundNative.failure.reason).toBe("excluded"); + const boundMcp = yield* placeHere("mcp-bound-before"); + expect(Result.isFailure(boundMcp) && boundMcp.failure.code).toBe("target_required"); + // An excluded machine alone leaves nowhere eligible. + const nowhere = yield* nativeAdmission(laptopEnv, { commandId: "nowhere" }); + expect(Result.isFailure(nowhere) && nowhere.failure.reason).toBe("no-eligible-environment"); + + expect(yield* contact).toEqual(before); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); diff --git a/apps/server/src/peer/RemoteDelegation.ts b/apps/server/src/peer/RemoteDelegation.ts index e30b65c590d2..ce1f9854c37f 100644 --- a/apps/server/src/peer/RemoteDelegation.ts +++ b/apps/server/src/peer/RemoteDelegation.ts @@ -15,7 +15,9 @@ import { type OrchestratorMcpThreadTimelineItem, type OrchestrationV2RemoteTaskChild, type OrchestrationV2Subagent, - type PeerLinkSummary, + type PeerLink, + OrchestrationV2ThreadPlacementAdmitError, + type ProviderInteractionMode, type ProjectId, type ProviderInstanceId, ProviderDriverKind, @@ -38,6 +40,7 @@ import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Schedule from "effect/Schedule"; +import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import * as Tool from "effect/ai/Tool"; @@ -61,6 +64,43 @@ import * as PeerForwarding from "./PeerForwarding.ts"; import * as PeerLinks from "./PeerLinks.ts"; /** How long one follow of a remote task waits on its thread there before checking again. */ +/** What placement weighs a candidate on, gathered by this server itself. */ +export interface PlacementEvidence { + /** The project here; elsewhere, that machine's project with its repository. */ + readonly projectId: ProjectId; + readonly modelSelection: Pick; + /** The effective mode, sent explicitly wherever the work runs. */ + readonly runtimeMode: RuntimeMode; + /** This environment's own load, and whether it runs the exact model in that mode. */ + readonly here: { + readonly resources: HostResourcesSnapshot | null; + readonly modelAvailable: boolean; + }; +} + +const NativePlacement = Schema.Struct({ + environmentId: Schema.String, + projectId: Schema.String, + reason: Schema.Literal("capacity"), +}); +export interface NativePlacement { + readonly environmentId: EnvironmentId; + readonly projectId: ProjectId; + readonly reason: "capacity"; +} +const NativePlacementJson = Schema.fromJsonString(NativePlacement); +const encodeNativePlacementJson = Schema.encodeSync(NativePlacementJson); +const decodeNativePlacementJson = Schema.decodeEffect(NativePlacementJson); +const encodeNativePlacement = (placement: NativePlacement) => encodeNativePlacementJson(placement); +const decodeNativePlacement = (value: string) => + decodeNativePlacementJson(value).pipe( + Effect.map((placement): NativePlacement => ({ + environmentId: EnvironmentId.make(placement.environmentId), + projectId: placement.projectId as ProjectId, + reason: placement.reason, + })), + ); + const FOLLOW_WAIT_MS = 10 * 60 * 1_000; /** A linked environment that stops answering is retried, backing off to this. */ const FOLLOW_MAX_BACKOFF = Duration.minutes(5); @@ -119,17 +159,36 @@ export class RemoteDelegation extends Context.Service< readonly key: string; /** The thread the binding is recorded against. */ readonly threadId: ThreadId; - readonly projectId: ProjectId; - readonly modelSelection: Pick; - /** The effective mode, sent explicitly wherever the work runs. */ - readonly runtimeMode: RuntimeMode; - /** This environment's own load, and whether it runs the exact model in that mode. */ - readonly here: { - readonly resources: HostResourcesSnapshot | null; - readonly modelAvailable: boolean; - }; - }, + } & PlacementEvidence, ) => Effect.Effect; + /** + * Where a native client's new-thread launch runs: the environment its operation was first + * bound to, or one chosen now from this environment and authenticated links the client may + * launch on, bound with the session that admitted it before anything is sent. Another + * session's operation is refused, never adopted or admitted afresh. + */ + readonly admit: ( + caller: { + readonly sessionId: string; + /** Whether that session is still signed in with operate access; asked before each write. */ + readonly stillAuthorized: Effect.Effect; + }, + input: Omit & { + /** This environment's evidence, read only when nothing is bound yet. */ + readonly here: Effect.Effect< + PlacementEvidence["here"], + OrchestrationV2ThreadPlacementAdmitError + >; + readonly commandId: CommandId; + readonly threadId: ThreadId; + readonly interactionMode: ProviderInteractionMode; + /** Narrows the choice to machines the client may launch on; never widens it. */ + readonly candidateEnvironmentIds: ReadonlyArray; + }, + ) => Effect.Effect< + NativePlacement & { readonly resumed: boolean }, + OrchestrationV2ThreadPlacementAdmitError + >; /** The environment a caller's key was placed in, if it was. */ readonly placed: ( scope: McpInvocationContext.McpInvocationScope, @@ -257,7 +316,7 @@ const make = Effect.gen(function* () { /** The peer's project with this thread's repository, or the one the caller named. */ const resolveRemoteProject = Effect.fn("RemoteDelegation.resolveRemoteProject")(function* ( - scope: McpInvocationContext.McpInvocationScope, + ask: Ask, environmentId: EnvironmentId, parentProjectId: ProjectId, requested: ProjectId | undefined, @@ -282,11 +341,14 @@ const make = Effect.gen(function* () { const matches: Array<{ readonly id: ProjectId; readonly title: string }> = []; let cursor: number | null = 0; while (cursor !== null) { - const listed: Tool.Success = - yield* forwarding.call(scope, ProjectToolkit.tools.t3_project_list, environmentId, { + const listed: Tool.Success = yield* ask( + ProjectToolkit.tools.t3_project_list, + environmentId, + { cursor, limit: 100, - }); + }, + ); matches.push( ...listed.projects.filter((project) => project.repositoryIdentity?.canonicalKey === key), ); @@ -341,43 +403,52 @@ const make = Effect.gen(function* () { return chosen.providerInstanceId; }); + /** One linked environment's answer for placement, asked over whichever caller's forwarding. */ + type Ask = ( + tool: T, + environmentId: EnvironmentId, + params: Tool.Parameters, + ) => Effect.Effect, OrchestratorMcpFailure>; + const askFor = + (scope: McpInvocationContext.McpInvocationScope): Ask => + (tool, environmentId, params) => + forwarding.call(scope, tool, environmentId, params); + const askNative = + (caller: PeerForwarding.NativeCaller): Ask => + (tool, environmentId, params) => + forwarding.callNative(caller, tool, environmentId, params); + /** A linked environment as a placement candidate, from what it answered just now. */ const peerCandidate = Effect.fn("RemoteDelegation.peerCandidate")( - function* ( - scope: McpInvocationContext.McpInvocationScope, - link: PeerLinkSummary, - input: Parameters[1], - ) { - const capabilities = yield* forwarding.call( - scope, - RemoteCapabilitiesTool, - link.environmentId, - { - search: input.modelSelection.model, - }, - ); + function* (ask: Ask, link: PeerLink, input: PlacementEvidence) { + const capabilities = yield* ask(RemoteCapabilitiesTool, link.environmentId, { + search: input.modelSelection.model, + }); const receivedAt = yield* Clock.currentTimeMillis; - const projectAvailable = yield* resolveRemoteProject( - scope, + const projectId = yield* resolveRemoteProject( + ask, link.environmentId, input.projectId, undefined, ).pipe( - Effect.as(true), - Effect.orElseSucceed(() => false), + Effect.asSome, + Effect.orElseSucceed(() => Option.none()), ); return { - environmentId: link.environmentId, - resources: capabilities.hostResources ?? null, - receivedAt, - weight: 1, - operable: true, - projectAvailable, - modelAvailable: runsModel(capabilities, input), - } satisfies ThreadPlacementCandidate; + projectId, + candidate: { + environmentId: link.environmentId, + resources: capabilities.hostResources ?? null, + receivedAt, + weight: 1, + operable: true, + projectAvailable: Option.isSome(projectId), + modelAvailable: runsModel(capabilities, input), + } satisfies ThreadPlacementCandidate, + }; }, // One that does not answer, or refuses this caller, is not a candidate. - (effect, _scope, link) => + (effect, _ask, link) => effect.pipe( Effect.tapError((cause) => Effect.logDebug("remote-delegation.placement-candidate-unavailable", { @@ -389,6 +460,96 @@ const make = Effect.gen(function* () { ), ); + const exclusions = links.placementExclusions.pipe( + Effect.mapError( + () => + new OrchestratorMcpFailure({ + code: "orchestration_error", + message: + "Unable to read which machines automatic placement may use, so nothing was started. Retry with the same clientRequestId.", + }), + ), + ); + + /** + * Linked environments placement may ask, from what is stored here only: an excluded one, or + * one this work could not use, is dropped before any peer is contacted. `restrictTo` narrows + * them to the ones a caller can launch on. + */ + const askableLinks = Effect.fn("RemoteDelegation.askableLinks")(function* (input: { + readonly runtimeMode: RuntimeMode; + readonly excluded: ReadonlySet; + readonly restrictTo?: ReadonlyArray | undefined; + }) { + if (!(yield* links.enabled)) return []; + const usable = (environmentId: EnvironmentId) => + environmentId !== hereId && !input.excluded.has(environmentId); + const stored = + input.restrictTo === undefined + ? yield* links.stored.pipe(Effect.orElseSucceed((): ReadonlyArray => [])) + : (yield* Effect.forEach(input.restrictTo.filter(usable), (environmentId) => + links.get(environmentId).pipe(Effect.orElseSucceed(() => Option.none())), + )).flatMap((link) => (Option.isSome(link) ? [link.value] : [])); + const now = yield* DateTime.now; + return stored.filter( + (link) => + usable(link.environmentId) && + link.access !== "read-only" && + !isRuntimeModeAbove(input.runtimeMode, link.access) && + DateTime.isGreaterThan(link.expiresAt, now), + ); + }); + + /** The shared choice: this environment and the askable links, from their own answers. */ + const choosePlacement = Effect.fn("RemoteDelegation.choosePlacement")(function* ( + ask: Ask, + input: PlacementEvidence & { + readonly excluded: ReadonlySet; + readonly restrictTo?: ReadonlyArray | undefined; + }, + ) { + const askable = yield* askableLinks(input); + const peers = (yield* Effect.forEach(askable, (link) => peerCandidate(ask, link, input), { + concurrency: "unbounded", + })).flatMap((peer) => (Option.isSome(peer) ? [peer.value] : [])); + const now = yield* Clock.currentTimeMillis; + const hereCandidate = + !input.excluded.has(hereId) && + (input.restrictTo === undefined || input.restrictTo.includes(hereId)); + const decision = chooseThreadPlacement( + { originEnvironmentId: hereId }, + [ + ...(hereCandidate + ? [ + { + environmentId: hereId, + resources: input.here.resources, + receivedAt: now, + weight: 1, + operable: true, + projectAvailable: true, + modelAvailable: input.here.modelAvailable, + }, + ] + : []), + ...peers.map((peer) => peer.candidate), + ], + now, + ); + yield* Effect.annotateCurrentSpan({ + "placement.reason": decision.reason, + "placement.candidates": askable.length + (hereCandidate ? 1 : 0), + }); + const projectId = !decision.accepted + ? undefined + : decision.environmentId === hereId + ? input.projectId + : peers + .find((peer) => peer.candidate.environmentId === decision.environmentId) + ?.projectId.pipe(Option.getOrUndefined); + return { decision, projectId }; + }); + const placementBinding = ( scope: McpInvocationContext.McpInvocationScope, input: { readonly operation: string; readonly key: string }, @@ -403,10 +564,25 @@ const make = Effect.gen(function* () { "Unable to record where this task runs, so nothing was started. Retry with the same clientRequestId.", }); + /** A bound environment automatic placement may no longer use: nothing is sent there or anywhere. */ + const boundExcluded = (environmentId: EnvironmentId) => + new OrchestratorMcpFailure({ + code: "target_required", + message: `This request is bound to environment ${environmentId}, which automatic placement may no longer use, so nothing was sent. Reconcile it there, or start new work with a new clientRequestId.`, + }); + const placed: RemoteDelegation["Service"]["placed"] = (scope, input) => - threads - .intentOf(placementBinding(scope, input)) - .pipe(Effect.map(Option.map(EnvironmentId.make)), Effect.mapError(bindingFailure)); + threads.intentOf(placementBinding(scope, input)).pipe( + Effect.map(Option.map(EnvironmentId.make)), + Effect.mapError(bindingFailure), + Effect.tap((bound) => + Option.isSome(bound) && bound.value !== hereId + ? Effect.flatMap(exclusions, (excluded) => + excluded.has(bound.value) ? Effect.fail(boundExcluded(bound.value)) : Effect.void, + ) + : Effect.void, + ), + ); const place: RemoteDelegation["Service"]["place"] = Effect.fn("RemoteDelegation.place")( function* (scope, input) { @@ -415,41 +591,9 @@ const make = Effect.gen(function* () { // stopped answering: its first attempt may have started the task there. const bound = yield* placed(scope, input); if (Option.isSome(bound)) return bound.value; - - const linked = (yield* links.enabled) - ? yield* links.list.pipe(Effect.orElseSucceed(() => [])) - : []; - const peers = yield* Effect.forEach( - linked.filter( - (link) => - link.environmentId !== hereId && - link.status === "reachable" && - link.access !== "read-only" && - !isRuntimeModeAbove(input.runtimeMode, link.access), - ), - (link) => peerCandidate(scope, link, input), - { concurrency: "unbounded" }, - ); - const now = yield* Clock.currentTimeMillis; - const decision = chooseThreadPlacement( - { originEnvironmentId: hereId }, - [ - { - environmentId: hereId, - resources: input.here.resources, - receivedAt: now, - weight: 1, - operable: true, - projectAvailable: true, - modelAvailable: input.here.modelAvailable, - }, - ...peers.flatMap((peer) => (Option.isSome(peer) ? [peer.value] : [])), - ], - now, - ); - yield* Effect.annotateCurrentSpan({ - "placement.reason": decision.reason, - "placement.candidates": peers.length + 1, + const { decision } = yield* choosePlacement(askFor(scope), { + ...input, + excluded: yield* exclusions, }); if (!decision.accepted) { return yield* new OrchestratorMcpFailure({ @@ -467,6 +611,88 @@ const make = Effect.gen(function* () { }, ); + /** + * Native admission keys hang off this server's own id and the launch's command id. Its + * owner, the session that first admitted it, is bound before anything is chosen. + */ + const nativeBinding = (kind: "placement" | "placement-owner", commandId: CommandId) => + CommandId.make( + `command:native:${encodeURIComponent(hereId)}:${kind}:thread-launch:${encodeURIComponent(commandId)}`, + ); + + const admit: RemoteDelegation["Service"]["admit"] = Effect.fn("RemoteDelegation.admit")( + function* (caller, input) { + const refuse = (reason: OrchestrationV2ThreadPlacementAdmitError["reason"], detail: string) => + new OrchestrationV2ThreadPlacementAdmitError({ + commandId: input.commandId, + reason, + detail, + }); + const unavailableHere = () => + refuse( + "unavailable", + "Unable to read or record where this launch runs, so nothing was bound or started. Send it again.", + ); + const signedOut = () => + refuse( + "unavailable", + "This sign-in can no longer launch threads here, so nothing was bound or started.", + ); + if (!(yield* caller.stillAuthorized)) return yield* signedOut(); + const owner = yield* threads + .bindIntent({ + commandId: nativeBinding("placement-owner", input.commandId), + threadId: input.threadId, + value: caller.sessionId, + }) + .pipe(Effect.mapError(unavailableHere)); + if (owner !== caller.sessionId) { + return yield* refuse( + "other-session", + "This launch was started from another sign-in, so it is not continued or started again from this one. Open it where it was sent from, or start a new draft.", + ); + } + const excluded = yield* links.placementExclusions.pipe(Effect.mapError(unavailableHere)); + const placementId = nativeBinding("placement", input.commandId); + const decode = (value: string) => + decodeNativePlacement(value).pipe(Effect.mapError(unavailableHere)); + // The first admission's answer holds, whatever this one asks: it may already have launched. + const bound = yield* threads.intentOf(placementId).pipe(Effect.mapError(unavailableHere)); + if (Option.isSome(bound)) { + const placement = yield* decode(bound.value); + if (placement.environmentId !== hereId && excluded.has(placement.environmentId)) { + return yield* refuse( + "excluded", + `This launch is bound to environment ${placement.environmentId}, which automatic placement may no longer use, so nothing was sent. Reconcile it there, or start a new draft.`, + ); + } + return { ...placement, resumed: true }; + } + const { decision, projectId } = yield* choosePlacement( + askNative({ + limits: { runtimeMode: input.runtimeMode, interactionMode: input.interactionMode }, + }), + { ...input, here: yield* input.here, excluded, restrictTo: input.candidateEnvironmentIds }, + ); + if (!decision.accepted || projectId === undefined) { + return yield* refuse( + "no-eligible-environment", + `No machine you can launch on answered with spare capacity, this repository's project and ${input.modelSelection.instanceId}/${input.modelSelection.model} in ${input.runtimeMode}. Nothing was started.`, + ); + } + if (!(yield* caller.stillAuthorized)) return yield* signedOut(); + const value = encodeNativePlacement({ + environmentId: EnvironmentId.make(decision.environmentId), + projectId, + reason: "capacity", + }); + const held = yield* threads + .bindIntent({ commandId: placementId, threadId: input.threadId, value }) + .pipe(Effect.mapError(unavailableHere)); + return { ...(yield* decode(held)), resumed: held !== value }; + }, + ); + /** * The other side names the thread from its link session and the key. The first launch's * session and answered thread are bound here, and it is sent only under that session; a @@ -548,7 +774,7 @@ const make = Effect.gen(function* () { Effect.orElseSucceed(() => input.environmentId), ); const projectId = yield* resolveRemoteProject( - scope, + askFor(scope), input.environmentId, input.projectId, undefined, @@ -644,7 +870,7 @@ const make = Effect.gen(function* () { ? undefined : yield* resolveInteractionMode(parent.thread.interactionMode, input.interactionMode); const projectId = yield* resolveRemoteProject( - scope, + askFor(scope), target.environmentId, parent.thread.projectId, target.projectId, @@ -1379,6 +1605,7 @@ const make = Effect.gen(function* () { return RemoteDelegation.of({ delegate, place, + admit, placed, launch, followUp, diff --git a/apps/server/src/peer/placementEvidence.ts b/apps/server/src/peer/placementEvidence.ts new file mode 100644 index 000000000000..763f89523683 --- /dev/null +++ b/apps/server/src/peer/placementEvidence.ts @@ -0,0 +1,42 @@ +import { + isProviderAvailable, + type ModelSelection, + type ProjectId, + resolveProjectProviderInstanceEnabled, + type RuntimeMode, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; + +import * as ProviderRegistry from "../provider/ProviderRegistry.ts"; +import * as HostResources from "../resourceTelemetry/HostResources.ts"; +import * as ServerSettings from "../serverSettings.ts"; + +/** + * This environment as a placement candidate: its own load, and whether it runs the exact + * provider instance and model for the project in the requested mode. + */ +export const hereEvidence = Effect.fn("placement.hereEvidence")(function* (input: { + readonly projectId: ProjectId; + readonly modelSelection: Pick; + readonly runtimeMode: RuntimeMode; +}) { + const providers = yield* (yield* ProviderRegistry.ProviderRegistry).getProviders; + const settings = yield* (yield* ServerSettings.ServerSettingsService).getSettings; + const provider = providers.find( + (candidate) => candidate.instanceId === input.modelSelection.instanceId, + ); + const hostResources = yield* Effect.serviceOption(HostResources.HostResources); + return { + resources: Option.isSome(hostResources) ? yield* hostResources.value.read : null, + modelAvailable: + provider !== undefined && + provider.enabled && + isProviderAvailable(provider) && + resolveProjectProviderInstanceEnabled(settings, input.projectId, provider.instanceId) && + provider.models.some((model) => model.slug === input.modelSelection.model) && + (provider.supportedRuntimeModes === undefined || + provider.supportedRuntimeModes.length === 0 || + provider.supportedRuntimeModes.includes(input.runtimeMode)), + }; +}); diff --git a/apps/server/src/ws.test.ts b/apps/server/src/ws.test.ts index f8ad4d3229ba..d79969d62a98 100644 --- a/apps/server/src/ws.test.ts +++ b/apps/server/src/ws.test.ts @@ -11,6 +11,9 @@ import { AuthOrchestrationOperateScope, AuthOrchestrationReadScope, AuthSessionId, + CommandId, + ORCHESTRATION_V2_WS_METHODS, + OrchestrationV2ThreadPlacementAdmitError, AuthSourceControlWriteScope, ORCHESTRATION_PROTOCOL_VERSION, ORCHESTRATION_PROTOCOL_QUERY_PARAM, @@ -130,6 +133,7 @@ import * as ServerRuntimeStartup from "./serverRuntimeStartup.ts"; import * as ServerSelfUpdate from "./cloud/selfUpdate.ts"; import * as NpmRelease from "./cloud/npmRelease.ts"; import * as SessionStore from "./auth/SessionStore.ts"; +import * as RemoteDelegation from "./peer/RemoteDelegation.ts"; import * as SourceControlDiscovery from "./sourceControl/SourceControlDiscovery.ts"; import * as SourceControlRepositoryService from "./sourceControl/SourceControlRepositoryService.ts"; import * as SourceControlSignIn from "./sourceControl/SourceControlSignIn.ts"; @@ -2621,3 +2625,219 @@ it.effect("ACP session RPCs list a thread missing its history and repair it on i ), ), ); + +it.effect.each([ + { + label: "an unexpired session with operate access", + expiresInMs: 3_600_000, + listed: true, + admitted: true, + }, + // Listed sessions include an expired one whose socket is still connected. + { + label: "an expired session its socket still holds", + expiresInMs: -60_000, + listed: true, + admitted: false, + }, + { label: "a revoked session", expiresInMs: 3_600_000, listed: false, admitted: false }, +] as const)( + "a native placement admission writes only for $label", + ({ expiresInMs, listed, admitted }) => + Effect.gen(function* () { + const sessionId = AuthSessionId.make("web-session"); + const now = yield* DateTime.now; + const asked: Array<{ readonly sessionId: string; readonly authorized: boolean }> = []; + const exits = yield* Queue.unbounded>(); + const services = Layer.mergeAll( + layerUnusedConfigRpcServices, + Layer.mock(ExternalLauncher.ExternalLauncher)({}), + Layer.mock(ProviderRegistry.ProviderRegistry)({}), + Layer.mock(UsageLimitSources.UsageLimitSources)({}), + Layer.mock(RemoteOpenTargets.RemoteOpenTargets)({}), + Layer.mock(DirectEndpoints.DirectEndpoints)({}), + Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({ namedProjectsRoot: "/unused" }), + Layer.mock(ScratchThreadPromotion.ScratchThreadPromotion)({}), + Layer.mock(ServerEnvironment.ServerEnvironment)({}), + Layer.mock(EnvironmentAuth.EnvironmentAuth)({}), + Layer.mock(ServerRuntimeStartup.ServerRuntimeStartup)({ + enqueueCommand: (effect) => effect, + }), + Layer.mock(SessionStore.SessionStore)({ + cookieName: "admission-test", + legacyCookieName: undefined, + listActive: () => + Effect.succeed( + listed + ? [ + { + sessionId, + subject: "browser", + scopes: [AuthOrchestrationOperateScope], + method: "browser-session-cookie" as const, + client: { deviceType: "desktop" as const }, + issuedAt: now, + expiresAt: DateTime.add(now, { milliseconds: expiresInMs }), + lastConnectedAt: now, + connected: true, + current: true, + }, + ] + : [], + ), + }), + // The delegation itself is covered with real links; here, what the socket hands it. + Layer.mock(RemoteDelegation.RemoteDelegation)({ + admit: (caller, input) => + caller.stillAuthorized.pipe( + Effect.tap((authorized) => + Effect.sync(() => asked.push({ sessionId: caller.sessionId, authorized })), + ), + Effect.flatMap((authorized) => + authorized + ? Effect.succeed({ + environmentId: EnvironmentId.make("environment-box"), + projectId: input.projectId, + reason: "capacity" as const, + resumed: false, + }) + : Effect.fail( + new OrchestrationV2ThreadPlacementAdmitError({ + commandId: input.commandId, + reason: "unavailable", + detail: "signed out", + }), + ), + ), + ), + }), + ); + const server = yield* RpcServer.makeNoSerialization(WsRpcGroup, { + ...WS_RPC_SERVER_OPTIONS, + onFromServer: (response) => + response._tag === "Exit" ? Queue.offer(exits, response.exit) : Effect.void, + }).pipe( + Effect.provide( + layerWsRpc( + { + sessionId, + subject: "browser", + method: "browser-session-cookie", + scopes: [AuthOrchestrationOperateScope], + }, + {}, + yield* ServerBrowser.ServerBrowser, + () => Effect.void, + { secretReferences: true }, + ).pipe( + Layer.provide(services), + Layer.merge(RpcAuthorization.layer([AuthOrchestrationOperateScope])), + ), + ), + ); + yield* server.write(0, { + _tag: "Request", + id: RpcMessage.RequestId("admit"), + tag: ORCHESTRATION_V2_WS_METHODS.admitThreadPlacement, + payload: { + commandId: CommandId.make("web-launch"), + threadId: ThreadId.make("thread:web-launch"), + projectId: ProjectId.make("project:app"), + modelSelection: { + instanceId: ProviderInstanceId.make("claudeAgent"), + model: "claude-sonnet-4-6", + }, + runtimeMode: "full-access", + interactionMode: "default", + candidateEnvironmentIds: [EnvironmentId.make("environment-box")], + }, + headers: Headers.empty, + }); + const exit = yield* Queue.take(exits); + assert.equal(Exit.isSuccess(exit), admitted); + // The socket's own session asks, never one the request names. + assert.deepEqual(asked, [{ sessionId, authorized: admitted }]); + }).pipe( + Effect.provide( + Layer.mergeAll( + Layer.fresh(layerConfigSources), + Layer.mock(PreviewAutomationBroker.PreviewAutomationBroker)({}), + Layer.mock(ServerBrowser.ServerBrowser)({}), + ), + ), + ), +); + +it.effect("a native placement admission needs operate access before anything is asked", () => + Effect.gen(function* () { + let asked = 0; + const exits = yield* Queue.unbounded>(); + const services = Layer.mergeAll( + layerUnusedConfigRpcServices, + Layer.mock(ExternalLauncher.ExternalLauncher)({}), + Layer.mock(ProviderRegistry.ProviderRegistry)({}), + Layer.mock(UsageLimitSources.UsageLimitSources)({}), + Layer.mock(RemoteOpenTargets.RemoteOpenTargets)({}), + Layer.mock(DirectEndpoints.DirectEndpoints)({}), + Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({ namedProjectsRoot: "/unused" }), + Layer.mock(ScratchThreadPromotion.ScratchThreadPromotion)({}), + Layer.mock(ServerEnvironment.ServerEnvironment)({}), + Layer.mock(EnvironmentAuth.EnvironmentAuth)({}), + Layer.mock(RemoteDelegation.RemoteDelegation)({ + admit: () => + Effect.sync(() => { + asked += 1; + }).pipe(Effect.andThen(Effect.die("admitted without operate access"))), + }), + ); + const server = yield* RpcServer.makeNoSerialization(WsRpcGroup, { + ...WS_RPC_SERVER_OPTIONS, + onFromServer: (response) => + response._tag === "Exit" ? Queue.offer(exits, response.exit) : Effect.void, + }).pipe( + Effect.provide( + layerWsRpc( + { + sessionId: AuthSessionId.make("read-only"), + subject: "browser", + method: "browser-session-cookie", + scopes: [AuthAccessReadScope], + }, + {}, + yield* ServerBrowser.ServerBrowser, + () => Effect.void, + { secretReferences: true }, + ).pipe(Layer.provide(services), Layer.merge(RpcAuthorization.layer([AuthAccessReadScope]))), + ), + ); + yield* server.write(0, { + _tag: "Request", + id: RpcMessage.RequestId("admit"), + tag: ORCHESTRATION_V2_WS_METHODS.admitThreadPlacement, + payload: { + commandId: CommandId.make("web-launch"), + threadId: ThreadId.make("thread:web-launch"), + projectId: ProjectId.make("project:app"), + modelSelection: { + instanceId: ProviderInstanceId.make("claudeAgent"), + model: "claude-sonnet-4-6", + }, + runtimeMode: "full-access", + interactionMode: "default", + candidateEnvironmentIds: [], + }, + headers: Headers.empty, + }); + const exit = yield* Queue.take(exits); + assert.equal(Exit.isFailure(exit), true); + assert.equal(asked, 0); + }).pipe( + Effect.provide( + Layer.mergeAll( + Layer.fresh(layerConfigSources), + Layer.mock(PreviewAutomationBroker.PreviewAutomationBroker)({}), + Layer.mock(ServerBrowser.ServerBrowser)({}), + ), + ), + ), +); diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index f9979dcbf7f4..feb2ef6ac65a 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -64,6 +64,9 @@ import { OrchestrationV2GetShellSnapshotError, OrchestrationV2GetThreadProjectionError, OrchestrationV2ThreadLaunchError, + OrchestrationV2ThreadPlacementAdmitError, + type OrchestrationV2ThreadPlacementAdmitInput, + managedRuntimeModeDenial, type OrchestrationProjectShell, type OrchestrationV2ShellSnapshot, type OrchestrationV2ShellStreamItem, @@ -175,6 +178,9 @@ import * as ThreadSearch from "./orchestration-v2/ThreadSearch.ts"; import * as OrchestrationEventStore from "./persistence/OrchestrationEventStore.ts"; import { userFacingDispatchErrorMessage } from "./orchestration-v2/UserFacingErrors.ts"; import * as ProviderRegistry from "./provider/ProviderRegistry.ts"; +import { hereEvidence } from "./peer/placementEvidence.ts"; +import * as RemoteDelegation from "./peer/RemoteDelegation.ts"; +import * as ManagedPolicy from "./policy/ManagedPolicy.ts"; import * as SecretCheck from "./secrets/SecretCheck.ts"; import * as AcpRegistrySessions from "./provider/AcpRegistrySessions.ts"; import * as ProviderInstanceRegistry from "./provider/ProviderInstanceRegistry.ts"; @@ -1580,6 +1586,64 @@ export const layerWsRpc = ( const processDiagnostics = yield* ProcessDiagnostics.ProcessDiagnostics; const triage = yield* TriageService.TriageService; const hostResources = yield* HostResources.HostResources; + const remoteDelegation = yield* Effect.serviceOption(RemoteDelegation.RemoteDelegation); + const managedPolicy = yield* Effect.serviceOption(ManagedPolicy.ManagedPolicy); + const placementContext = yield* Effect.context< + | ProviderRegistry.ProviderRegistry + | ServerSettings.ServerSettingsService + | HostResources.HostResources + >(); + /** + * Where this session's new-thread launch runs, decided and bound here before anything is + * sent. Its session must still be signed in with operate access at each write. + */ + const admitThreadPlacement = Effect.fn("ws.admitThreadPlacement")(function* ( + input: OrchestrationV2ThreadPlacementAdmitInput, + ) { + const unavailableHere = (detail: string) => + new OrchestrationV2ThreadPlacementAdmitError({ + commandId: input.commandId, + reason: "unavailable", + detail, + }); + if (Option.isNone(remoteDelegation)) { + return yield* unavailableHere( + "This server cannot place launches, so nothing was started.", + ); + } + // Read only for a new admission: a bound operation resumes whatever this machine runs now. + const here = hereEvidence(input).pipe( + Effect.provide(placementContext), + Effect.map((evidence) => ({ + ...evidence, + // The launch here would refuse a mode above this machine's managed ceiling. + modelAvailable: + evidence.modelAvailable && + managedRuntimeModeDenial( + Option.isSome(managedPolicy) ? managedPolicy.value.summary : undefined, + input.runtimeMode, + ) === undefined, + })), + Effect.mapError(() => + unavailableHere("Unable to read this machine's providers, so nothing was started."), + ), + ); + // Listed sessions include expired ones a socket still holds; only an unexpired one counts. + const stillAuthorized = Effect.gen(function* () { + const active = yield* sessions.listActive(); + const now = yield* DateTime.now; + return active.some( + (session) => + session.sessionId === currentSessionId && + DateTime.isGreaterThan(session.expiresAt, now) && + session.scopes.includes(AuthOrchestrationOperateScope), + ); + }).pipe(Effect.orElseSucceed(() => false)); + return yield* remoteDelegation.value.admit( + { sessionId: currentSessionId, stillAuthorized }, + { ...input, here }, + ); + }); const processResourceMonitor = yield* ProcessResourceMonitor.ProcessResourceMonitor; const resourceTelemetry = yield* ResourceTelemetry.ResourceTelemetry; const relayClient = yield* RelayClient.RelayClient; @@ -2148,6 +2212,19 @@ export const layerWsRpc = ( ), ), ), + [ORCHESTRATION_V2_WS_METHODS.admitThreadPlacement]: (input) => + startup.enqueueCommand(admitThreadPlacement(input)).pipe( + Effect.mapError((error) => + error._tag === "OrchestrationV2ThreadPlacementAdmitError" + ? error + : new OrchestrationV2ThreadPlacementAdmitError({ + commandId: input.commandId, + reason: "unavailable", + detail: + "This server is not ready to place launches yet, so nothing was started.", + }), + ), + ), [ORCHESTRATION_V2_WS_METHODS.launchThread]: (input) => Effect.annotateCurrentSpan({ "orchestration_v2.command_id": input.commandId, diff --git a/apps/web/src/components/chat/draftPlacement.test.ts b/apps/web/src/components/chat/draftPlacement.test.ts new file mode 100644 index 000000000000..1288e169059f --- /dev/null +++ b/apps/web/src/components/chat/draftPlacement.test.ts @@ -0,0 +1,409 @@ +import { + EnvironmentId, + ProjectId, + ProviderInstanceId, + ThreadId, + type OrchestrationV2ThreadPlacementAdmitInput, + OrchestrationV2ThreadPlacementAdmitError, + type OrchestrationV2ThreadPlacementAdmitResult, +} from "@t3tools/contracts"; +import { scopeProjectRef, scopeThreadRef } from "@t3tools/client-runtime/environment"; +import * as Cause from "effect/Cause"; +import { AsyncResult } from "effect/reactivity"; +import { describe, expect, it, vi } from "vite-plus/test"; +import { createMemoryStorage } from "../../lib/storage"; + +const origin = EnvironmentId.make("environment-laptop"); +const box = EnvironmentId.make("environment-box"); +const projectHere = ProjectId.make("project:laptop"); +const projectThere = ProjectId.make("project:box"); +const request = { + modelSelection: { + instanceId: ProviderInstanceId.make("claudeAgent"), + model: "claude-sonnet-4-6", + }, + runtimeMode: "full-access", + interactionMode: "default", + multipleModels: false, +} as const; +const KEY = "t3code:composer-drafts:v1"; + +const storageFaults = { drops: false }; + +async function openTab() { + return { + store: await import("../../composerDraftStore"), + placement: await import("./draftPlacement"), + }; +} +type Tab = Awaited>; + +async function withStorage(run: (memory: ReturnType) => Promise) { + const memory = createMemoryStorage(); + vi.stubGlobal("localStorage", { + getItem: (name: string) => memory.getItem(name), + removeItem: (name: string) => memory.removeItem(name), + setItem: (name: string, value: string) => { + if (!storageFaults.drops) memory.setItem(name, value); + }, + }); + vi.stubGlobal("window", new EventTarget()); + vi.useFakeTimers(); + vi.resetModules(); + try { + await run(memory); + } finally { + storageFaults.drops = false; + vi.useRealTimers(); + vi.unstubAllGlobals(); + vi.resetModules(); + } +} + +function composeDraft(tab: Tab, name: string) { + const draftId = tab.store.DraftId.make(`draft-${name}`); + const store = tab.store.useComposerDraftStore.getState(); + store.setProjectDraftThreadId(scopeProjectRef(origin, projectHere), draftId, { + environmentSelection: "auto", + loadBalancedEnvironmentId: origin, + }); + store.setPrompt(draftId, "Run the suite"); + return draftId; +} + +/** What storage holds of the draft's placement operation right now. */ +const storedOperation = (memory: ReturnType, draftId: string) => + JSON.parse((memory.getItem(KEY) as string | null) ?? "{}")?.state?.draftThreadsByThreadKey?.[ + draftId + ]?.placementOperation; + +type Admit = (target: { + readonly environmentId: EnvironmentId; + readonly input: OrchestrationV2ThreadPlacementAdmitInput; +}) => Promise>; + +const admittedTo = + ( + seen: Array<{ environmentId: EnvironmentId; commandId: string; stored: unknown }>, + memory: ReturnType, + draftId: string, + ): Admit => + async ({ environmentId, input }) => { + seen.push({ + environmentId, + commandId: input.commandId, + stored: storedOperation(memory, draftId), + }); + return AsyncResult.success({ + environmentId: box, + projectId: projectThere, + reason: "capacity", + resumed: seen.length > 1, + }); + }; + +type PlacementRequest = Parameters[0]["request"]; + +const place = ( + tab: Tab, + draftId: string, + admit: Admit, + options: { + readonly canLaunchOn?: () => boolean; + readonly hasAttachments?: boolean; + readonly request?: PlacementRequest; + } = {}, +) => + tab.placement.placeDraftBeforeSend({ + draftId: tab.store.DraftId.make(draftId), + request: options.request ?? request, + candidateEnvironmentIds: [origin, box], + hasAttachments: options.hasAttachments ?? false, + admit: admit as never, + canLaunchOn: options.canLaunchOn ?? (() => true), + composerUnchanged: () => true, + }); + +/** The origin refuses with one of its typed reasons. */ +const refusing = + (reason: OrchestrationV2ThreadPlacementAdmitError["reason"], asked?: Array): Admit => + async ({ input }) => { + asked?.push(input.commandId); + return AsyncResult.failure( + Cause.fail( + new OrchestrationV2ThreadPlacementAdmitError({ + commandId: input.commandId, + reason, + detail: `refused: ${reason}`, + }), + ), + ); + }; + +describe("placing an automatic draft before its send", () => { + it("stores the operation before asking, and where it was bound before anything moves", async () => { + await withStorage(async (memory) => { + const tab = await openTab(); + const draftId = composeDraft(tab, "ordered"); + const seen: Array<{ environmentId: EnvironmentId; commandId: string; stored: unknown }> = []; + const outcome = await place(tab, draftId, admittedTo(seen, memory, draftId)); + expect(outcome).toEqual({ type: "placed", environmentId: box, projectId: projectThere }); + // The server was asked only once storage held the operation, without a binding yet. + expect(seen).toHaveLength(1); + expect(seen[0]?.environmentId).toBe(origin); + expect(seen[0]?.stored).toMatchObject({ + commandId: seen[0]?.commandId, + admittedOn: origin, + admittedProjectId: projectHere, + }); + expect((seen[0]?.stored as { admission?: unknown } | undefined)?.admission).toBeUndefined(); + // Where it was bound is stored before the draft moves there. + expect(storedOperation(memory, draftId)).toMatchObject({ + commandId: seen[0]?.commandId, + admission: { environmentId: box, projectId: projectThere }, + }); + // Placing moves nothing itself: the routing hook retargets after this answer. + expect( + tab.store.useComposerDraftStore.getState().getDraftSession(tab.store.DraftId.make(draftId)) + ?.environmentId, + ).toBe(origin); + }); + }); + + it("asks the same server for the same operation after a lost answer and a reload", async () => { + await withStorage(async (memory) => { + const tab = await openTab(); + const draftId = composeDraft(tab, "retried"); + const asked: Array<{ environmentId: EnvironmentId; commandId: string }> = []; + // The server bound it, but its answer was lost. + const lost: Admit = async ({ environmentId, input }) => { + asked.push({ environmentId, commandId: input.commandId }); + return AsyncResult.failure(Cause.die("connection lost")); + }; + expect((await place(tab, draftId, lost)).type).toBe("failed"); + const reserved = storedOperation(memory, draftId); + expect(reserved?.admission).toBeUndefined(); + + vi.clearAllTimers(); + vi.resetModules(); + const reloaded = await openTab(); + const seen: Array<{ environmentId: EnvironmentId; commandId: string; stored: unknown }> = []; + expect((await place(reloaded, draftId, admittedTo(seen, memory, draftId))).type).toBe( + "placed", + ); + expect( + [...asked, ...seen].map(({ environmentId, commandId }) => [environmentId, commandId]), + ).toEqual([ + [origin, reserved.commandId], + [origin, reserved.commandId], + ]); + // Bound and stored, a retry still asks that server under this sign-in for the same one. + const again: Array<{ environmentId: EnvironmentId; commandId: string; stored: unknown }> = []; + expect((await place(reloaded, draftId, admittedTo(again, memory, draftId))).type).toBe( + "placed", + ); + expect(again.map(({ environmentId, commandId }) => [environmentId, commandId])).toEqual([ + [origin, reserved.commandId], + ]); + }); + }); + + it("asks nowhere when storage drops the operation, and keeps a known refusal's operation", async () => { + await withStorage(async (memory) => { + const tab = await openTab(); + const draftId = composeDraft(tab, "unsaved"); + const seen: Array<{ environmentId: EnvironmentId; commandId: string; stored: unknown }> = []; + storageFaults.drops = true; + const unsaved = await place(tab, draftId, admittedTo(seen, memory, draftId)); + storageFaults.drops = false; + expect(unsaved.type).toBe("failed"); + expect(seen).toHaveLength(0); + expect( + tab.store.useComposerDraftStore.getState().getDraftSession(tab.store.DraftId.make(draftId)) + ?.placementOperation ?? null, + ).toBeNull(); + + const refused: Admit = async ({ input }) => + AsyncResult.failure( + Cause.fail( + new OrchestrationV2ThreadPlacementAdmitError({ + commandId: input.commandId, + reason: "other-session", + detail: "This launch was started from another sign-in.", + }), + ), + ); + const outcome = await place(tab, draftId, refused); + expect(outcome).toMatchObject({ + type: "failed", + description: "This launch was started from another sign-in.", + }); + // The operation stays, so it is never admitted afresh under a new identity. + expect(storedOperation(memory, draftId)?.commandId).toBeDefined(); + }); + }); + + it("launches nowhere when this client's grant there is gone by the time it is bound", async () => { + await withStorage(async (memory) => { + const tab = await openTab(); + const draftId = composeDraft(tab, "revoked"); + const seen: Array<{ environmentId: EnvironmentId; commandId: string; stored: unknown }> = []; + const outcome = await place(tab, draftId, admittedTo(seen, memory, draftId), { + canLaunchOn: () => false, + }); + expect(outcome.type).toBe("failed"); + // Its binding is kept for a later retry; the draft stays where it was. + expect(storedOperation(memory, draftId)?.admission).toEqual({ + environmentId: box, + projectId: projectThere, + }); + expect( + tab.store.useComposerDraftStore.getState().getDraftSession(tab.store.DraftId.make(draftId)) + ?.environmentId, + ).toBe(origin); + }); + }); + + it.each(["other-session", "excluded", "unavailable"] as const)( + "never follows a stored binding its server now refuses (%s), after a reload", + async (reason) => { + await withStorage(async (memory) => { + const tab = await openTab(); + const draftId = composeDraft(tab, `reconciled-${reason}`); + const seen: Array<{ environmentId: EnvironmentId; commandId: string; stored: unknown }> = + []; + expect((await place(tab, draftId, admittedTo(seen, memory, draftId))).type).toBe("placed"); + const bound = storedOperation(memory, draftId); + vi.clearAllTimers(); + vi.resetModules(); + const reloaded = await openTab(); + // Another sign-in now, the machine excluded since, or this sign-in gone: refused there. + const asked: Array = []; + const outcome = await place(reloaded, draftId, refusing(reason, asked)); + expect(outcome.type).toBe("failed"); + expect(asked).toEqual([bound.commandId]); + // The operation and its binding stay as they were, for reconciling where it runs. + expect(storedOperation(memory, draftId)).toEqual(bound); + expect( + reloaded.store.useComposerDraftStore + .getState() + .getDraftSession(reloaded.store.DraftId.make(draftId))?.environmentId, + ).toBe(origin); + }); + }, + ); + + it("refuses changed work while an operation is open, without asking or replacing it", async () => { + await withStorage(async (memory) => { + const tab = await openTab(); + const draftId = composeDraft(tab, "changed"); + const lost: Admit = async () => AsyncResult.failure(Cause.die("connection lost")); + expect((await place(tab, draftId, lost)).type).toBe("failed"); + const reserved = storedOperation(memory, draftId); + vi.clearAllTimers(); + vi.resetModules(); + const reloaded = await openTab(); + const asked: Array = []; + const changed: ReadonlyArray = [ + { + ...request, + modelSelection: { + instanceId: ProviderInstanceId.make("claudeAgent"), + model: "claude-opus-5-5", + }, + }, + { ...request, runtimeMode: "approval-required" }, + ]; + for (const changedRequest of changed) { + const outcome = await place(reloaded, draftId, refusing("unavailable", asked), { + request: changedRequest, + }); + expect(outcome).toMatchObject({ + type: "failed", + title: "Your last send's placement is still open", + }); + } + expect(asked).toEqual([]); + expect(storedOperation(memory, draftId)).toEqual(reserved); + // Sent unchanged, it is the same operation again. + const seen: Array<{ environmentId: EnvironmentId; commandId: string; stored: unknown }> = []; + expect((await place(reloaded, draftId, admittedTo(seen, memory, draftId))).type).toBe( + "placed", + ); + expect(seen.map(({ commandId }) => commandId)).toEqual([reserved.commandId]); + }); + }); + + it("starts new work only after its server refused the last one without binding it", async () => { + await withStorage(async (memory) => { + const tab = await openTab(); + const draftId = composeDraft(tab, "released"); + const asked: Array = []; + expect((await place(tab, draftId, refusing("no-eligible-environment", asked))).type).toBe( + "failed", + ); + expect(storedOperation(memory, draftId) ?? null).toBeNull(); + const seen: Array<{ environmentId: EnvironmentId; commandId: string; stored: unknown }> = []; + expect((await place(tab, draftId, admittedTo(seen, memory, draftId))).type).toBe("placed"); + expect(seen[0]?.commandId).not.toBe(asked[0]); + }); + }); + + it("keeps attachments on the machine they were added on, rather than moving them", async () => { + await withStorage(async (memory) => { + const tab = await openTab(); + const draftId = composeDraft(tab, "attached"); + const seen: Array<{ environmentId: EnvironmentId; commandId: string; stored: unknown }> = []; + const outcome = await place(tab, draftId, admittedTo(seen, memory, draftId), { + hasAttachments: true, + }); + expect(outcome).toMatchObject({ type: "failed" }); + expect( + tab.store.useComposerDraftStore.getState().getDraftSession(tab.store.DraftId.make(draftId)) + ?.environmentId, + ).toBe(origin); + }); + }); + + it("reconciles a held operation first, whatever automatic routing allows new work", async () => { + await withStorage(async () => { + const tab = await openTab(); + const held = { placementOperation: heldOperation, promotedTo: null }; + const fresh = { placementOperation: null, promotedTo: null }; + const multiple: PlacementRequest = { ...request, multipleModels: true }; + const cases: ReadonlyArray< + readonly [typeof held | typeof fresh, boolean, boolean, PlacementRequest, boolean] + > = [ + [held, false, true, multiple, true], + [held, true, false, request, true], + [fresh, true, false, request, true], + [fresh, false, false, request, false], + [fresh, true, true, request, false], + [fresh, true, false, multiple, false], + ]; + for (const [draft, automatic, hasAttachments, asked, places] of cases) { + expect( + tab.placement.placesBeforeSend({ draft, automatic, hasAttachments, request: asked }), + ).toBe(places); + } + // A send already bound for its launch is not placed again. + expect( + tab.placement.placesBeforeSend({ + draft: { ...held, promotedTo: scopeThreadRef(box, ThreadId.make("thread")) }, + automatic: true, + hasAttachments: false, + request, + }), + ).toBe(false); + }); + }); +}); + +const heldOperation = { + commandId: "command", + messageId: "message", + createdAt: "2026-11-18T12:00:00.000Z", + admittedOn: origin, + admittedProjectId: projectHere, + intent: "{}", +}; diff --git a/apps/web/src/components/chat/draftPlacement.ts b/apps/web/src/components/chat/draftPlacement.ts new file mode 100644 index 000000000000..35eec77efb27 --- /dev/null +++ b/apps/web/src/components/chat/draftPlacement.ts @@ -0,0 +1,219 @@ +import type { + EnvironmentId, + ModelSelection, + OrchestrationV2ThreadPlacementAdmitInput, + OrchestrationV2ThreadPlacementAdmitResult, + ProjectId, + ProviderInteractionMode, + RuntimeMode, +} from "@t3tools/contracts"; +import { CommandId } from "@t3tools/contracts"; +import { + type AtomCommandResult, + squashAtomCommandFailure, +} from "@t3tools/client-runtime/state/runtime"; + +import { + type DraftId, + type DraftPlacementOperation, + type DraftSessionState, + recordDraftPlacementAdmission, + releaseDraftPlacement, + reserveDraftPlacement, + useComposerDraftStore, +} from "../../composerDraftStore"; +import { newCommandId, newMessageId } from "~/lib/utils"; + +/** What a send asks for, read from the composer at send. */ +export interface DraftPlacementRequest { + readonly modelSelection: ModelSelection; + readonly runtimeMode: RuntimeMode; + readonly interactionMode: ProviderInteractionMode; + /** A multiple-model send prepares its own worktrees here, so it is not placed elsewhere. */ + readonly multipleModels: boolean; +} + +/** + * The work a placement is for, fixed when its operation is reserved: the exact model with its + * options, the modes, the logical project and the workspace. A different one is new work. + */ +export const draftPlacementIntent = ( + request: DraftPlacementRequest, + draft: Pick< + DraftSessionState, + "logicalProjectKey" | "branch" | "worktreePath" | "envMode" | "startFromOrigin" + >, +) => + JSON.stringify({ + modelSelection: request.modelSelection, + runtimeMode: request.runtimeMode, + interactionMode: request.interactionMode, + multipleModels: request.multipleModels, + logicalProjectKey: draft.logicalProjectKey, + branch: draft.branch, + worktreePath: draft.worktreePath, + envMode: draft.envMode, + startFromOrigin: draft.startFromOrigin, + }); + +/** + * Whether a send goes through placement first: always when its draft holds an operation already + * (reserved, uncertain or bound), whatever automatic routing says now; otherwise only new work + * automatic routing may place. + */ +export const placesBeforeSend = (input: { + readonly draft: Pick | null; + readonly automatic: boolean; + readonly hasAttachments: boolean; + readonly request: DraftPlacementRequest; +}) => + input.draft !== null && + !input.draft.promotedTo && + (Boolean(input.draft.placementOperation) || + (input.automatic && !input.hasAttachments && !input.request.multipleModels)); + +export type DraftPlacementOutcome = + | { + readonly type: "placed"; + readonly environmentId: EnvironmentId; + readonly projectId: ProjectId; + } + | { readonly type: "failed"; readonly title: string; readonly description: string } + /** The draft changed while it was being placed: its send is dropped, its binding kept. */ + | { readonly type: "abandoned" }; + +const unsaved = (title: string): DraftPlacementOutcome => ({ + type: "failed", + title, + description: "Nothing was started. Free up browser storage, then send again.", +}); + +const stillOpen = (description: string): DraftPlacementOutcome => ({ + type: "failed", + title: "Your last send's placement is still open", + description, +}); + +const refusalReason = (error: unknown) => + typeof error === "object" && + error !== null && + "_tag" in error && + error._tag === "OrchestrationV2ThreadPlacementAdmitError" && + "reason" in error + ? String(error.reason) + : undefined; + +/** + * Where a draft's new thread runs: its operation and intent are stored first; the server asked is + * asked again under this sign-in every time, even once bound, and its binding is never rescored. + * Changed work is refused while the operation is open. Nothing is uploaded or launched here. + */ +export async function placeDraftBeforeSend(input: { + readonly draftId: DraftId; + readonly request: DraftPlacementRequest; + /** The machines this client may launch on; they narrow the server's choice. */ + readonly candidateEnvironmentIds: ReadonlyArray; + /** The composer holds attachments, which are prepared for the machine it is on. */ + readonly hasAttachments: boolean; + readonly admit: (target: { + readonly environmentId: EnvironmentId; + readonly input: OrchestrationV2ThreadPlacementAdmitInput; + }) => Promise>; + /** Whether this client may still launch there, read when it is needed. */ + readonly canLaunchOn: (environmentId: EnvironmentId) => boolean; + /** Whether the composer is still what this send read. */ + readonly composerUnchanged: () => boolean; +}): Promise { + const { draftId } = input; + const session = useComposerDraftStore.getState().getDraftSession(draftId); + if (!session) return { type: "abandoned" }; + const intent = draftPlacementIntent(input.request, session); + const existing = session.placementOperation ?? null; + if (existing) { + if (existing.intent !== intent) + return stillOpen( + "It may already be running with what you sent then. Send that unchanged to finish it, or start a new draft for different work.", + ); + if ( + session.environmentId !== existing.admittedOn && + session.environmentId !== existing.admission?.environmentId + ) + return stillOpen( + "It was placed from another machine than this draft is on now. Switch back to send it, or start a new draft.", + ); + } + const reserved: DraftPlacementOperation = existing ?? { + commandId: newCommandId(), + messageId: newMessageId(), + createdAt: new Date().toISOString(), + admittedOn: session.environmentId, + admittedProjectId: session.projectId, + intent, + }; + if (!existing && !reserveDraftPlacement(draftId, reserved)) + return unsaved("Could not save this draft before sending"); + const result = await input.admit({ + environmentId: reserved.admittedOn, + input: { + commandId: CommandId.make(reserved.commandId), + threadId: session.threadId, + projectId: reserved.admittedProjectId, + modelSelection: input.request.modelSelection, + runtimeMode: input.request.runtimeMode, + interactionMode: input.request.interactionMode, + candidateEnvironmentIds: input.candidateEnvironmentIds, + }, + }); + if (result._tag !== "Success") { + const error = squashAtomCommandFailure(result); + // Refused before anything was bound: the next send is new work, under a new identity. + if (refusalReason(error) === "no-eligible-environment") + releaseDraftPlacement(draftId, reserved.commandId); + return { + type: "failed", + title: "Could not place this draft", + description: + error instanceof Error && error.message.length > 0 + ? error.message + : "Nothing was started. Send it again.", + }; + } + const admission = { + environmentId: result.value.environmentId, + projectId: result.value.projectId, + }; + if (reserved.admission) { + // Its server answers with the binding it holds; anything else is never followed. + if ( + reserved.admission.environmentId !== admission.environmentId || + reserved.admission.projectId !== admission.projectId + ) + return stillOpen( + "Its machine answered with another destination than it was bound to, so nothing was sent.", + ); + } else if (!recordDraftPlacementAdmission(draftId, reserved.commandId, admission)) { + return unsaved("Could not save where this draft runs"); + } + const current = useComposerDraftStore.getState().getDraftSession(draftId); + if ( + !input.composerUnchanged() || + current?.placementOperation?.commandId !== reserved.commandId || + current.environmentId !== session.environmentId || + current.projectId !== session.projectId + ) + return { type: "abandoned" }; + // Attachments are prepared for the machine the draft is on, so they never move with it. + if (input.hasAttachments && admission.environmentId !== session.environmentId) + return stillOpen( + "It runs on another machine than its attachments were added on. Remove them and attach them there, or start a new draft.", + ); + // The grant to launch there is this client's own, checked again now that it is needed. + if (!input.canLaunchOn(admission.environmentId)) { + return { + type: "failed", + title: "This machine no longer accepts your launches", + description: "Its sign-in changed while the draft was being placed. Nothing was started.", + }; + } + return { type: "placed", ...admission }; +} diff --git a/apps/web/src/components/chat/useComposerSend.ts b/apps/web/src/components/chat/useComposerSend.ts index ab6988099e26..47b75471f359 100644 --- a/apps/web/src/components/chat/useComposerSend.ts +++ b/apps/web/src/components/chat/useComposerSend.ts @@ -911,15 +911,26 @@ export function useComposerSend(input: ComposerSendInput) { return; } const retriedLaunch = launchPlan?.type === "retry" ? launchPlan.launch : null; + // A new launch admitted to this machine goes out as the operation reserved for its admission. + const placed = draftSessionForSend?.placementOperation; + const reservedLaunch = + retriedLaunch === null && + placed?.admission?.environmentId === environmentId && + placed.admission.projectId === activeProject.id + ? placed + : null; const messageIdForSend = retriedLaunch ? MessageId.make(retriedLaunch.messageId) - : newMessageId(); - const messageCreatedAt = retriedLaunch?.createdAt ?? new Date().toISOString(); + : reservedLaunch + ? MessageId.make(reservedLaunch.messageId) + : newMessageId(); + const messageCreatedAt = + retriedLaunch?.createdAt ?? reservedLaunch?.createdAt ?? new Date().toISOString(); const launchOperation: PendingDraftLaunch | undefined = launchRequest === null ? undefined : (retriedLaunch ?? { - commandId: newCommandId(), + commandId: reservedLaunch?.commandId ?? newCommandId(), messageId: messageIdForSend, createdAt: messageCreatedAt, request: launchRequest, @@ -1691,10 +1702,19 @@ export function useComposerSend(input: ComposerSendInput) { const onSendLatest = useStableCallback(onSend); const onSendWithRouting = async (...args: Parameters) => { args[0]?.preventDefault(); + const sendContext = composerRef.current?.getSendContext(); const refresh = - args[3] || sendInFlightRef.current || readPendingEnvironmentChange() !== null + args[3] || + sendInFlightRef.current || + readPendingEnvironmentChange() !== null || + sendContext === undefined ? null - : refreshBeforeSend(); + : refreshBeforeSend({ + modelSelection: sendContext.selectedModelSelection, + runtimeMode, + interactionMode: DEFAULT_INTERACTION_MODE, + multipleModels: (sendContext.multipleModelSelections?.length ?? 0) > 0, + }); if (refresh === null) return onSend(...args); sendInFlightRef.current = true; const ready = await refresh.finally(() => { diff --git a/apps/web/src/components/chat/useDraftEnvironmentRouting.browser.test.tsx b/apps/web/src/components/chat/useDraftEnvironmentRouting.browser.test.tsx index b4ddb847d082..b97334991a7a 100644 --- a/apps/web/src/components/chat/useDraftEnvironmentRouting.browser.test.tsx +++ b/apps/web/src/components/chat/useDraftEnvironmentRouting.browser.test.tsx @@ -24,11 +24,26 @@ import { DraftId, useComposerDraftStore } from "../../composerDraftStore"; import { useDraftEnvironmentRouting } from "./useDraftEnvironmentRouting"; import { appAtomRegistry, AppAtomRegistryProvider } from "../../rpc/atomRegistry"; import { draftFanoutSelectionsAtom } from "./chatViewSend"; +import type { DraftPlacementOutcome, placeDraftBeforeSend } from "./draftPlacement"; + +type PlaceInput = Parameters[0]; +const sendRequest = { + modelSelection: createModelSelection(ProviderInstanceId.make("codex"), "gpt-6.1-sol"), + runtimeMode: "full-access", + interactionMode: "default", + multipleModels: false, +} as const; const resources = vi.hoisted(() => ({ projects: [] as EnvironmentProject[], pickedEnvironmentId: null as EnvironmentId | null, refreshSelection: vi.fn<(ids: readonly EnvironmentId[]) => Promise>(), + // The server's admission is covered in draftPlacement.test.ts; here, what the hook does with it. + place: vi.fn<(input: PlaceInput) => Promise>(), +})); +vi.mock("./draftPlacement", async (importOriginal) => ({ + ...(await importOriginal()), + placeDraftBeforeSend: resources.place, })); vi.mock("../../hooks/useLoadBalancedEnvironment", () => ({ useLoadBalancedEnvironment: () => ({ @@ -50,6 +65,7 @@ vi.mock("../../hooks/useScratchProject", () => ({ useScratchProject: () => ({}) function setup() { resources.refreshSelection.mockReset(); + resources.place.mockReset(); resources.projects = []; resources.pickedEnvironmentId = null; const a = EnvironmentId.make("machine-a"); @@ -123,7 +139,11 @@ function setup() { projectGroupingSettings: DEFAULT_UNIFIED_SETTINGS, sendInFlightRef: { current: false }, }; - resources.refreshSelection.mockImplementation(async (ids) => (ids.includes(b) ? b : null)); + resources.place.mockImplementation(async () => ({ + type: "placed", + environmentId: b, + projectId, + })); return { a, b, draftId, input, store }; } @@ -164,7 +184,7 @@ function RoutingProbe({ input, onPrepared }: HarnessProps) { {draft?.environmentId}