diff --git a/.github/workflows/closura-arc-ci.yml b/.github/workflows/closura-arc-ci.yml index f5a9e0ce4161..1e7d9a1c7d87 100644 --- a/.github/workflows/closura-arc-ci.yml +++ b/.github/workflows/closura-arc-ci.yml @@ -155,7 +155,12 @@ jobs: apps/web/src/composerDraftStore.test.ts \ apps/web/src/components/chat/composerSend.logic.test.ts \ apps/web/src/components/chat/chatViewSend.test.ts \ - apps/web/src/components/chat/composerImagePersistence.test.ts + apps/web/src/components/chat/composerImagePersistence.test.ts \ + apps/web/src/components/chat/draftPlacement.test.ts \ + apps/web/src/components/chat/draftAttachmentUploads.test.ts \ + apps/web/src/components/chat/draftUploadReconcile.test.ts \ + apps/web/src/lib/attachmentUploadQueue.test.ts \ + apps/server/src/ws.test.ts - name: Build native T3 server shell: bash diff --git a/apps/server/src/auth/RpcAuthorization.ts b/apps/server/src/auth/RpcAuthorization.ts index 7925fd869799..9599adbba2bd 100644 --- a/apps/server/src/auth/RpcAuthorization.ts +++ b/apps/server/src/auth/RpcAuthorization.ts @@ -58,6 +58,7 @@ export const RPC_REQUIRED_SCOPES = { [ORCHESTRATION_V2_WS_METHODS.getThreadProjection]: AuthOrchestrationReadScope, [ORCHESTRATION_V2_WS_METHODS.getTurnItem]: AuthOrchestrationReadScope, [ORCHESTRATION_V2_WS_METHODS.launchThread]: AuthOrchestrationOperateScope, + [ORCHESTRATION_V2_WS_METHODS.admitThreadPlacement]: AuthOrchestrationOperateScope, [ORCHESTRATION_V2_WS_METHODS.subscribeShell]: AuthOrchestrationReadScope, [ORCHESTRATION_V2_WS_METHODS.subscribeThread]: AuthOrchestrationReadScope, [ORCHESTRATION_V2_WS_METHODS.subscribeCommandOutput]: AuthOrchestrationReadScope, diff --git a/apps/server/src/mcp/toolkits/project/handlers.ts b/apps/server/src/mcp/toolkits/project/handlers.ts index 57a7c7952061..f66e13636d0f 100644 --- a/apps/server/src/mcp/toolkits/project/handlers.ts +++ b/apps/server/src/mcp/toolkits/project/handlers.ts @@ -2,18 +2,16 @@ import { agentLaunchInboxRouting, CommandId, type EnvironmentId, - isProviderAvailable, MessageId, ThreadId, OrchestratorMcpFailure, ProjectId, type ProviderInteractionMode, - resolveProjectProviderInstanceEnabled, type RuntimeMode, } from "@t3tools/contracts"; import type * as Tool from "effect/ai/Tool"; import * as RemoteDelegation from "../../../peer/RemoteDelegation.ts"; -import * as HostResources from "../../../resourceTelemetry/HostResources.ts"; +import { hereEvidence } from "../../../peer/placementEvidence.ts"; import * as Digest from "effect/crypto/Digest"; import * as Effect from "effect/Effect"; import * as Hex from "effect/encoding/Hex"; @@ -30,8 +28,8 @@ import { mutateProjectWithClone } from "../../../project/ProjectMutation.ts"; import * as ManagedProjectFolders from "../../../project/ManagedProjectFolders.ts"; import * as Repositories from "../../../sourceControl/SourceControlRepositoryService.ts"; import * as GitVcsDriver from "../../../vcs/GitVcsDriver.ts"; -import * as ServerSettings from "../../../serverSettings.ts"; import * as ProviderRegistry from "../../../provider/ProviderRegistry.ts"; +import * as ServerSettings from "../../../serverSettings.ts"; import * as DateTime from "effect/DateTime"; import { resolveNewThreadModelSelection } from "@t3tools/shared/usageLimits"; import { resolveProjectSettings } from "@t3tools/shared/projectSettings"; @@ -198,15 +196,11 @@ const launchPlacedThere = Effect.fn("mcp.launchPlacedThere")(function* ( message: 'With placement "auto", pass clientRequestId and modelSelection: a retry must reach the environment chosen, with the exact model.', }); - const providers = yield* (yield* ProviderRegistry.ProviderRegistry).getProviders; - const settings = yield* ServerSettings.ServerSettingsService.pipe( - Effect.flatMap((service) => service.getSettings), - Effect.mapError(unavailable), - ); - const provider = providers.find( - (candidate) => candidate.instanceId === modelSelection.instanceId, - ); - const hostResources = yield* Effect.serviceOption(HostResources.HostResources); + const here = yield* hereEvidence({ + projectId: local.projectId, + modelSelection, + runtimeMode: local.runtimeMode, + }).pipe(Effect.mapError(unavailable)); const environmentId = yield* delegation.value.place(scope, { operation: "thread-launch", key, @@ -214,18 +208,7 @@ const launchPlacedThere = Effect.fn("mcp.launchPlacedThere")(function* ( projectId: local.projectId, modelSelection, runtimeMode: local.runtimeMode, - here: { - resources: Option.isSome(hostResources) ? yield* hostResources.value.read : null, - modelAvailable: - provider !== undefined && - provider.enabled && - isProviderAvailable(provider) && - resolveProjectProviderInstanceEnabled(settings, local.projectId, provider.instanceId) && - provider.models.some((model) => model.slug === modelSelection.model) && - (provider.supportedRuntimeModes === undefined || - provider.supportedRuntimeModes.length === 0 || - provider.supportedRuntimeModes.includes(local.runtimeMode)), - }, + here, }); return environmentId === scope.environmentId ? undefined : yield* sendThere(environmentId, key); }); diff --git a/apps/server/src/observability/RpcInstrumentation.ts b/apps/server/src/observability/RpcInstrumentation.ts index 4549ee8a33c1..6ffd88b2ab78 100644 --- a/apps/server/src/observability/RpcInstrumentation.ts +++ b/apps/server/src/observability/RpcInstrumentation.ts @@ -31,6 +31,7 @@ const RPC_AGGREGATES = { [ORCHESTRATION_V2_WS_METHODS.getOlderSettledThreads]: "orchestration", [ORCHESTRATION_V2_WS_METHODS.getThreadProjection]: "orchestrationV2", [ORCHESTRATION_V2_WS_METHODS.launchThread]: "orchestration", + [ORCHESTRATION_V2_WS_METHODS.admitThreadPlacement]: "orchestration", [ORCHESTRATION_V2_WS_METHODS.subscribeShell]: "orchestrationV2", [ORCHESTRATION_V2_WS_METHODS.subscribeThread]: "orchestrationV2", [ORCHESTRATION_V2_WS_METHODS.subscribeCommandOutput]: "orchestrationV2", diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index 9ebba141e080..4d9b95b20d94 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -472,6 +472,10 @@ export interface ThreadManagementServiceShape { }) => Effect.Effect; /** Receipt replays start no work; skip enablement checks with the same command id so settings cannot reject accepted retries. */ readonly hasReceipt: (commandId: CommandId) => Effect.Effect; + /** The thread a command's launch here created, when its `thread.create` was accepted. */ + readonly acceptedLaunchOf: ( + commandId: CommandId, + ) => Effect.Effect, CommandReceiptStore.CommandReceiptStoreV2Error>; /** What a caller's intent was bound to, such as the environment chosen for it, if it was. */ readonly intentOf: ( commandId: CommandId, @@ -666,6 +670,19 @@ const make = Effect.gen(function* () { Effect.map(Option.isSome), ); + const acceptedLaunchOf: ThreadManagementServiceShape["acceptedLaunchOf"] = (commandId) => + receipts + .getByCommandId(commandId) + .pipe( + Effect.map( + Option.flatMap((receipt) => + receipt.status === "accepted" && receipt.commandType === "thread.create" + ? Option.some(receipt.threadId) + : Option.none(), + ), + ), + ); + // A binding is a receipt with no events, so it lives and is kept like every other // command's; its type carries the value. const intentOf: ThreadManagementServiceShape["intentOf"] = (commandId) => @@ -1819,6 +1836,7 @@ const make = Effect.gen(function* () { assertProviderInstanceEnabled, isProviderInstanceEnabledForProject, hasReceipt, + acceptedLaunchOf, intentOf, bindIntent, waitForThread, diff --git a/apps/server/src/peer/PeerForwarding.ts b/apps/server/src/peer/PeerForwarding.ts index 132e3b13b8b2..365fb55f206c 100644 --- a/apps/server/src/peer/PeerForwarding.ts +++ b/apps/server/src/peer/PeerForwarding.ts @@ -10,6 +10,7 @@ import { OrchestratorMcpThreadWaitResult, type PeerLinkError, RuntimeMode, + type ThreadId, } from "@t3tools/contracts"; import * as Context from "effect/Context"; import * as Digest from "effect/crypto/Digest"; @@ -24,7 +25,7 @@ import type * as Tool from "effect/ai/Tool"; import { runtimeModeRank } from "../mcp/delegatedTaskModes.ts"; import { assertNotLinked } from "../mcp/linkOrigin.ts"; import * as McpInvocationContext from "../mcp/McpInvocationContext.ts"; -import { readCaller, unavailable } from "../mcp/threadAccess.ts"; +import { type CallerLimits, readCaller, unavailable } from "../mcp/threadAccess.ts"; import * as ThreadManagement from "../orchestration-v2/ThreadManagementService.ts"; import * as PeerLinks from "./PeerLinks.ts"; import * as PeerMcpClient from "./PeerMcpClient.ts"; @@ -35,6 +36,29 @@ const MAX_WAIT_TIMEOUT_MS = 45_000; type Scope = McpInvocationContext.McpInvocationScope; +/** + * A client signed in to this server itself, rather than an agent over MCP: it has no thread + * here, and its requests carry the modes it asked for, already within this server's ceilings. + */ +export interface NativeCaller { + readonly limits: CallerLimits; +} + +/** What a native caller may ask a linked environment: evidence for placement, never an effect. */ +const NATIVE_EVIDENCE_TOOLS: ReadonlySet = new Set([ + "orchestrator_capabilities", + "t3_project_list", +]); + +/** Who a forwarded call is made for, as it travels: its limits, retry-key scope and thread. */ +interface ForwardedCaller { + readonly requestNamespace: string | undefined; + readonly sender: + | { readonly environmentId: EnvironmentId; readonly threadId: ThreadId; readonly title: string } + | undefined; + readonly limits: CallerLimits; +} + /** * Runs T3 MCP tools in a linked environment for an agent here. The tool's own * declaration checks the agent here first; the linked environment then holds @@ -74,6 +98,16 @@ export class PeerForwarding extends Context.Service< environmentId: EnvironmentId, input: OrchestratorMcpThreadWaitInput, ) => Effect.Effect; + /** + * A read-only evidence tool (`orchestrator_capabilities`, `t3_project_list`) there for a + * native caller, under the link's credential narrowed to the caller's modes. + */ + readonly callNative: ( + caller: NativeCaller, + tool: T, + environmentId: EnvironmentId, + params: Tool.Parameters, + ) => Effect.Effect, OrchestratorMcpFailure>; /** The environments this one links to, as an agent sees them. */ readonly links: ( scope: Scope, @@ -149,8 +183,20 @@ const make = Effect.gen(function* () { }); }); - /** Tool and contract schemas are plain data: they encode and decode without services. */ - const forward = Effect.fn("PeerForwarding.forward")(function* ( + const mcpCaller = (scope: Scope) => + forwardingCaller(scope).pipe( + Effect.map(({ caller, limits }): ForwardedCaller => ({ + requestNamespace: scope.requestNamespace, + // So a message sent there names the thread that sent it, even to a client that cannot reach here. + sender: + caller === undefined + ? undefined + : { environmentId: scope.environmentId, threadId: caller.id, title: caller.title }, + limits, + })), + ); + + const forward = ( scope: Scope, toolName: string, parameters: Schema.Top, @@ -158,8 +204,29 @@ const make = Effect.gen(function* () { environmentId: EnvironmentId, params: unknown, expectedIncarnation?: string, + ) => + Effect.flatMap(mcpCaller(scope), (caller) => + forwardAs( + caller, + toolName, + parameters, + success, + environmentId, + params, + expectedIncarnation, + ), + ); + + /** Tool and contract schemas are plain data: they encode and decode without services. */ + const forwardAs = Effect.fn("PeerForwarding.forward")(function* ( + { requestNamespace, sender, limits }: ForwardedCaller, + toolName: string, + parameters: Schema.Top, + success: Schema.Top, + environmentId: EnvironmentId, + params: unknown, + expectedIncarnation?: string, ) { - const { caller, limits } = yield* forwardingCaller(scope); const encoded = (yield* Schema.encodeUnknownEffect(parameters as Schema.Encoder)( params, ).pipe(Effect.mapError(unavailable))) as Record; @@ -173,17 +240,14 @@ const make = Effect.gen(function* () { }); } if (typeof args.clientRequestId === "string") { - args.clientRequestId = yield* scopedRequestId(scope.requestNamespace, args.clientRequestId); + if (requestNamespace === undefined) { + return yield* new OrchestratorMcpFailure({ + code: "invalid_request", + message: "This caller cannot send a request with a retry key to a linked environment.", + }); + } + args.clientRequestId = yield* scopedRequestId(requestNamespace, args.clientRequestId); } - // So a message sent there names the thread that sent it, even to a client that cannot reach here. - const sender = - caller === undefined - ? undefined - : { - environmentId: scope.environmentId, - threadId: caller.id, - title: caller.title, - }; return yield* peers.call({ environmentId, tool: toolName, @@ -290,8 +354,34 @@ const make = Effect.gen(function* () { }, ); + const callNative: PeerForwarding["Service"]["callNative"] = ( + caller, + tool, + environmentId, + params, + ) => + NATIVE_EVIDENCE_TOOLS.has(tool.name) + ? Effect.andThen( + requireEnabled, + forwardAs>( + { requestNamespace: undefined, sender: undefined, limits: caller.limits }, + tool.name, + tool.parametersSchema, + tool.successSchema, + environmentId, + params, + ), + ) + : Effect.fail( + new OrchestratorMcpFailure({ + code: "capability_denied", + message: `${tool.name} is not placement evidence, so it is not sent for a native caller.`, + }), + ); + return PeerForwarding.of({ call, + callNative, readThread, interruptThread, waitForThread, diff --git a/apps/server/src/peer/PeerLinks.ts b/apps/server/src/peer/PeerLinks.ts index 1858de216c39..8371591dde05 100644 --- a/apps/server/src/peer/PeerLinks.ts +++ b/apps/server/src/peer/PeerLinks.ts @@ -99,6 +99,13 @@ export class PeerLinks extends Context.Service< readonly enabled: Effect.Effect; readonly link: (input: PeerLinkCreateInput) => Effect.Effect; readonly list: Effect.Effect, PeerLinkError>; + /** + * Machines automatic placement never uses or contacts (server settings). Unreadable settings + * fail, so placement contacts nobody rather than everyone. + */ + readonly placementExclusions: Effect.Effect, PeerLinkError>; + /** The stored links, read here only: no peer is asked whether it answers. */ + readonly stored: Effect.Effect, PeerLinkError>; /** * Names the link's session there without revealing its token. Linking again * starts a new session there, so this changes exactly when that does. @@ -241,6 +248,13 @@ const make = Effect.gen(function* () { Effect.map((settings) => settings.linkedEnvironmentsEnabled), Effect.orElseSucceed(() => false), ); + const placementExclusions = serverSettings.getSettings.pipe( + Effect.map( + (settings): ReadonlySet => + new Set(settings.automaticPlacementExcludedEnvironmentIds), + ), + Effect.catch(failStorage), + ); /** A fresh name for one link's token; the row records it. */ const newSecretName = random.randomBytes(16).pipe( @@ -619,6 +633,10 @@ const make = Effect.gen(function* () { }); }); + const stored: PeerLinks["Service"]["stored"] = Effect.flatMap(readRows, (rows) => + Effect.forEach(rows, toLink), + ); + const get: PeerLinks["Service"]["get"] = (environmentId) => readRow(environmentId).pipe( Effect.flatMap( @@ -730,6 +748,8 @@ const make = Effect.gen(function* () { enabled, link, list, + stored, + placementExclusions, session, get, unlink, diff --git a/apps/server/src/peer/RemoteDelegation.test.ts b/apps/server/src/peer/RemoteDelegation.test.ts index 42f5919c5032..ac63ce91cdd4 100644 --- a/apps/server/src/peer/RemoteDelegation.test.ts +++ b/apps/server/src/peer/RemoteDelegation.test.ts @@ -17,6 +17,7 @@ import { type RepositoryIdentity, RunId, type HostResourcesSnapshot, + OrchestrationV2ThreadPlacementAdmitError, type ServerProvider, ThreadId, TurnItemId, @@ -31,6 +32,7 @@ import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; +import * as Result from "effect/Result"; import * as Schedule from "effect/Schedule"; import * as Stream from "effect/Stream"; import * as SubscriptionRef from "effect/SubscriptionRef"; @@ -3033,3 +3035,412 @@ it.effect("a local task runs in this thread's project", () => }), ).pipe(Effect.provide(NodeHttpServer.layerTest)), ); + +/** A native client's new-thread launch, admitted on the laptop for the box's Claude model. */ +const nativeAdmission = ( + laptopEnv: Effect.Success>, + options: { + readonly commandId: string; + readonly sessionId?: string; + readonly stillAuthorized?: Effect.Effect; + readonly model?: string; + readonly here?: Effect.Effect< + { readonly resources: HostResourcesSnapshot | null; readonly modelAvailable: boolean }, + OrchestrationV2ThreadPlacementAdmitError + >; + }, +) => + laptopEnv.remote + .admit( + { + sessionId: options.sessionId ?? "web-session", + stillAuthorized: options.stillAuthorized ?? Effect.succeed(true), + }, + { + commandId: CommandId.make(options.commandId), + threadId: ThreadId.make(`thread:${options.commandId}`), + projectId: laptopProject.id, + modelSelection: { + instanceId: boxClaude.instanceId, + model: options.model ?? "claude-sonnet-4-6", + }, + runtimeMode: "full-access", + interactionMode: "default", + candidateEnvironmentIds: [laptop.environmentId, box.environmentId], + // The laptop cannot run the model, so only the box can take it. + here: options.here ?? Effect.succeed({ resources: loadOf(0.1), modelAvailable: false }), + }, + ) + .pipe(Effect.result); + +const nativePlacementBinding = (commandId: string) => + CommandId.make( + `command:native:${encodeURIComponent(laptop.environmentId)}:placement:thread-launch:${encodeURIComponent(commandId)}`, + ); + +it.effect( + "a native launch is bound to the box with its session, and only that session resumes it, as bound", + () => + Effect.scoped( + Effect.gen(function* () { + const thread = yield* makeBoxThread; + const b = yield* serveBox(thread); + const offers = yield* SubscriptionRef.make< + ReadonlyArray + >([]); + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); + const laptopEnv = yield* makeLaptop(database, offers); + yield* linkTo(laptopEnv.links, b, "full-access"); + + const admitted = yield* nativeAdmission(laptopEnv, { commandId: "web-launch" }); + expect(admitted).toEqual( + Result.succeed({ + environmentId: box.environmentId, + projectId: boxProject, + reason: "capacity", + resumed: false, + }), + ); + // Admission sends no launch: the client launches there over its own connection. + expect(yield* Ref.get(thread.launched)).toBeNull(); + + // The box fills up and the request changes; this machine's evidence cannot even be read. + // Its binding still holds for the session that made it, without reading any of that. + yield* Ref.set(thread.resources, loadOf(0.99)); + const hereRead = yield* Ref.make(0); + const resumed = yield* nativeAdmission(laptopEnv, { + commandId: "web-launch", + model: "a-model-nowhere", + here: Ref.update(hereRead, (count) => count + 1).pipe( + Effect.andThen( + Effect.fail( + new OrchestrationV2ThreadPlacementAdmitError({ + commandId: CommandId.make("web-launch"), + reason: "unavailable", + detail: "providers unreadable", + }), + ), + ), + ), + }); + expect(resumed).toEqual( + Result.succeed({ + environmentId: box.environmentId, + projectId: boxProject, + reason: "capacity", + resumed: true, + }), + ); + expect(yield* Ref.get(hereRead)).toBe(0); + + // Another sign-in neither adopts it nor starts it afresh. + const other = yield* nativeAdmission(laptopEnv, { + commandId: "web-launch", + sessionId: "replaced-session", + }); + expect(Result.isFailure(other) && other.failure.reason).toBe("other-session"); + + // A session no longer signed in binds nothing, so nothing of it stays to be adopted. + yield* Ref.set(thread.resources, loadOf(0.1)); + const signedOut = yield* nativeAdmission(laptopEnv, { + commandId: "signed-out", + stillAuthorized: Effect.succeed(false), + }); + expect(Result.isFailure(signedOut) && signedOut.failure.reason).toBe("unavailable"); + const later = yield* nativeAdmission(laptopEnv, { + commandId: "signed-out", + sessionId: "another-session", + }); + expect(Result.isSuccess(later)).toBe(true); + + // Signed out while the machines were asked: the choice is not bound. + const checks = yield* Ref.make(0); + const revokedMidway = yield* nativeAdmission(laptopEnv, { + commandId: "revoked-midway", + stillAuthorized: Ref.updateAndGet(checks, (count) => count + 1).pipe( + Effect.map((count) => count === 1), + ), + }); + expect(Result.isFailure(revokedMidway) && revokedMidway.failure.reason).toBe("unavailable"); + expect(yield* Ref.get(checks)).toBe(2); + expect(yield* laptopEnv.threads.intentOf(nativePlacementBinding("revoked-midway"))).toEqual( + Option.none(), + ); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); + +it.effect( + "automatic placement never contacts an excluded machine, and refuses a launch bound to one", + () => + Effect.scoped( + Effect.gen(function* () { + const thread = yield* makeBoxThread; + const b = yield* serveBox(thread); + const offers = yield* SubscriptionRef.make< + ReadonlyArray + >([]); + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); + const laptopEnv = yield* makeLaptop(database, offers); + yield* linkTo(laptopEnv.links, b, "full-access"); + // Bound before the box is excluded: both the native and the MCP operations. + expect( + Result.isSuccess(yield* nativeAdmission(laptopEnv, { commandId: "bound-before" })), + ).toBe(true); + const placeHere = (key: string, here = { resources: loadOf(0.1), modelAvailable: false }) => + laptopEnv.remote + .place(outsideScope, { + operation: "thread-launch", + key, + threadId: ThreadId.make(`thread:${key}`), + projectId: laptopProject.id, + modelSelection: { instanceId: boxClaude.instanceId, model: "claude-sonnet-4-6" }, + runtimeMode: "full-access", + here, + }) + .pipe(Effect.result); + expect(yield* placeHere("mcp-bound-before")).toEqual(Result.succeed(box.environmentId)); + + yield* laptopEnv.settings.updateSettings({ + automaticPlacementExcludedEnvironmentIds: [box.environmentId], + }); + const contact = Effect.all([Ref.get(b.probes), Ref.get(b.bearers)]); + const before = yield* contact; + + // The box still answers with room to spare, yet nobody asks it: this machine is chosen. + const native = yield* nativeAdmission(laptopEnv, { + commandId: "fresh", + here: Effect.succeed({ resources: loadOf(0.5), modelAvailable: true }), + }); + expect(Result.isSuccess(native) && native.success.environmentId).toBe(laptop.environmentId); + expect( + yield* placeHere("mcp-fresh", { resources: loadOf(0.5), modelAvailable: true }), + ).toEqual(Result.succeed(laptop.environmentId)); + + // Work bound there earlier is refused, never rescored elsewhere or sent there. + const boundNative = yield* nativeAdmission(laptopEnv, { commandId: "bound-before" }); + expect(Result.isFailure(boundNative) && boundNative.failure.reason).toBe("excluded"); + const boundMcp = yield* placeHere("mcp-bound-before"); + expect(Result.isFailure(boundMcp) && boundMcp.failure.code).toBe("target_required"); + // An excluded machine alone leaves nowhere eligible. + const nowhere = yield* nativeAdmission(laptopEnv, { commandId: "nowhere" }); + expect(Result.isFailure(nowhere) && nowhere.failure.reason).toBe("no-eligible-environment"); + + expect(yield* contact).toEqual(before); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); + +it.effect( + "PR74-R1-F1: a launch admitted here, then excluded, is refused unless it was launched here", + () => + Effect.scoped( + Effect.gen(function* () { + const thread = yield* makeBoxThread; + const b = yield* serveBox(thread); + const offers = yield* SubscriptionRef.make< + ReadonlyArray + >([]); + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); + const laptopEnv = yield* makeLaptop(database, offers); + yield* linkTo(laptopEnv.links, b, "full-access"); + yield* seedParent(laptopEnv); + // The box is full, so this machine is chosen: admission only, nothing launched here. + yield* Ref.set(thread.resources, loadOf(0.99)); + const runsHere = Effect.succeed({ resources: loadOf(0.1), modelAvailable: true }); + const admitHere = (commandId: string, threadId?: ThreadId) => + laptopEnv.remote + .admit( + { sessionId: "web-session", stillAuthorized: Effect.succeed(true) }, + { + commandId: CommandId.make(commandId), + threadId: threadId ?? ThreadId.make(`thread:${commandId}`), + projectId: laptopProject.id, + modelSelection: { instanceId: boxClaude.instanceId, model: "claude-sonnet-4-6" }, + runtimeMode: "full-access", + interactionMode: "default", + candidateEnvironmentIds: [laptop.environmentId, box.environmentId], + here: runsHere, + }, + ) + .pipe(Effect.result); + const placeHere = (key: string) => + laptopEnv.remote + .place(outsideScope, { + operation: "thread-launch", + key, + threadId: ThreadId.make(`thread:${key}`), + projectId: laptopProject.id, + modelSelection: { instanceId: boxClaude.instanceId, model: "claude-sonnet-4-6" }, + runtimeMode: "full-access", + here: { resources: loadOf(0.1), modelAvailable: true }, + }) + .pipe(Effect.result); + const pending = yield* admitHere("local-pending"); + expect(Result.isSuccess(pending) && pending.success.environmentId).toBe( + laptop.environmentId, + ); + // Admitted here, then actually launched here as itself: `thread.create` under the same + // command id for the same thread, through the real orchestrator and its receipt. + const acceptedThread = ThreadId.make("thread:local-accepted"); + const accepted = yield* admitHere("local-accepted", acceptedThread); + expect(Result.isSuccess(accepted) && accepted.success.environmentId).toBe( + laptop.environmentId, + ); + const launched = yield* laptopEnv.orchestrator + .dispatch({ + type: "thread.create", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make("local-accepted"), + threadId: acceptedThread, + projectId: ProjectId.make("project:laptop"), + title: "Placed here", + modelSelection: { instanceId: boxClaude.instanceId, model: "claude-sonnet-4-6" }, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + }) + .pipe(Effect.result); + expect(Result.isSuccess(launched)).toBe(true); + expect(yield* placeHere("mcp-local-pending")).toEqual(Result.succeed(laptop.environmentId)); + + yield* laptopEnv.settings.updateSettings({ + automaticPlacementExcludedEnvironmentIds: [laptop.environmentId], + }); + // Not launched yet: refused, its binding kept as it was, never placed again. + const refused = yield* admitHere("local-pending"); + expect(Result.isFailure(refused) && refused.failure.reason).toBe("excluded"); + const refusedMcp = yield* placeHere("mcp-local-pending"); + expect(Result.isFailure(refusedMcp) && refusedMcp.failure.code).toBe("target_required"); + yield* laptopEnv.settings.updateSettings({ automaticPlacementExcludedEnvironmentIds: [] }); + const kept = yield* admitHere("local-pending"); + expect(Result.isSuccess(kept) && kept.success).toMatchObject({ + environmentId: laptop.environmentId, + resumed: true, + }); + // Launched here already: that exact accepted result still reconciles past the exclusion. + yield* laptopEnv.settings.updateSettings({ + automaticPlacementExcludedEnvironmentIds: [laptop.environmentId], + }); + const replayed = yield* admitHere("local-accepted", acceptedThread); + expect(Result.isSuccess(replayed) && replayed.success).toMatchObject({ + environmentId: laptop.environmentId, + resumed: true, + }); + // Named with an unrelated thread that exists here, the same command proves nothing. + const unrelated = yield* admitHere("local-accepted", parentThreadId); + expect(Result.isFailure(unrelated) && unrelated.failure.reason).toBe("excluded"); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); + +it.effect( + "PR74-R1-F1: only this command's own accepted launch here reconciles past a current exclusion", + () => + Effect.scoped( + Effect.gen(function* () { + const thread = yield* makeBoxThread; + const b = yield* serveBox(thread); + const offers = yield* SubscriptionRef.make< + ReadonlyArray + >([]); + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); + const laptopEnv = yield* makeLaptop(database, offers); + yield* linkTo(laptopEnv.links, b, "full-access"); + yield* seedParent(laptopEnv); + yield* Ref.set(thread.resources, loadOf(0.99)); + const admit = (commandId: string, threadId: ThreadId) => + laptopEnv.remote + .admit( + { sessionId: "web-session", stillAuthorized: Effect.succeed(true) }, + { + commandId: CommandId.make(commandId), + threadId, + projectId: laptopProject.id, + modelSelection: { instanceId: boxClaude.instanceId, model: "claude-sonnet-4-6" }, + runtimeMode: "full-access", + interactionMode: "default", + candidateEnvironmentIds: [laptop.environmentId, box.environmentId], + here: Effect.succeed({ resources: loadOf(0.1), modelAvailable: true }), + }, + ) + .pipe(Effect.result); + // The launch a native client sends here for an admitted operation: `thread.create` under + // the operation's own command id, through the real orchestrator and its receipt. + const launchHere = (commandId: string, threadId: ThreadId) => + laptopEnv.orchestrator + .dispatch({ + type: "thread.create", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make(commandId), + threadId, + projectId: ProjectId.make("project:laptop"), + title: "Placed here", + modelSelection: { instanceId: boxClaude.instanceId, model: "claude-sonnet-4-6" }, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + }) + .pipe(Effect.result); + const pendingThread = ThreadId.make("thread:pending"); + const acceptedThread = ThreadId.make("thread:accepted"); + const typedThread = ThreadId.make("thread:other-type"); + for (const [commandId, threadId] of [ + ["pending", pendingThread], + ["accepted", acceptedThread], + ["rejected", parentThreadId], + ["other-type", typedThread], + ] as const) { + const admitted = yield* admit(commandId, threadId); + expect(Result.isSuccess(admitted) && admitted.success.environmentId).toBe( + laptop.environmentId, + ); + } + // `accepted` is launched here as itself; `rejected`'s launch is refused (its thread exists + // already); `other-type` has only a receipt of another kind under its command id. + expect(Result.isSuccess(yield* launchHere("accepted", acceptedThread))).toBe(true); + expect(Result.isFailure(yield* launchHere("rejected", parentThreadId))).toBe(true); + yield* laptopEnv.threads.bindIntent({ + commandId: CommandId.make("other-type"), + threadId: typedThread, + value: "not a launch", + }); + + yield* laptopEnv.settings.updateSettings({ + automaticPlacementExcludedEnvironmentIds: [laptop.environmentId], + }); + const contact = Effect.all([Ref.get(b.probes), Ref.get(b.bearers)]); + const before = yield* contact; + const reason = (result: Effect.Success>) => + Result.isFailure(result) ? result.failure.reason : "admitted"; + // Never launched, retried naming an unrelated thread that exists here. + expect(reason(yield* admit("pending", parentThreadId))).toBe("excluded"); + // Never launched, retried as admitted. + expect(reason(yield* admit("pending", pendingThread))).toBe("excluded"); + // Its launch here was refused, though its thread exists. + expect(reason(yield* admit("rejected", parentThreadId))).toBe("excluded"); + // A receipt under its id that is not an accepted launch. + expect(reason(yield* admit("other-type", typedThread))).toBe("excluded"); + // Its own accepted launch, but retried for another thread. + expect(reason(yield* admit("accepted", parentThreadId))).toBe("excluded"); + // Its own accepted launch of its own thread: that result reconciles, as bound. + const replayed = yield* admit("accepted", acceptedThread); + expect(Result.isSuccess(replayed) && replayed.success).toMatchObject({ + environmentId: laptop.environmentId, + resumed: true, + }); + // Nothing was asked of the box meanwhile, and nothing was rebound or rescored. + expect(yield* contact).toEqual(before); + yield* laptopEnv.settings.updateSettings({ automaticPlacementExcludedEnvironmentIds: [] }); + const kept = yield* admit("pending", pendingThread); + expect(Result.isSuccess(kept) && kept.success).toMatchObject({ + environmentId: laptop.environmentId, + resumed: true, + }); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); diff --git a/apps/server/src/peer/RemoteDelegation.ts b/apps/server/src/peer/RemoteDelegation.ts index e30b65c590d2..132fc6a9badf 100644 --- a/apps/server/src/peer/RemoteDelegation.ts +++ b/apps/server/src/peer/RemoteDelegation.ts @@ -15,7 +15,9 @@ import { type OrchestratorMcpThreadTimelineItem, type OrchestrationV2RemoteTaskChild, type OrchestrationV2Subagent, - type PeerLinkSummary, + type PeerLink, + OrchestrationV2ThreadPlacementAdmitError, + type ProviderInteractionMode, type ProjectId, type ProviderInstanceId, ProviderDriverKind, @@ -38,6 +40,7 @@ import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Schedule from "effect/Schedule"; +import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import * as Tool from "effect/ai/Tool"; @@ -61,6 +64,43 @@ import * as PeerForwarding from "./PeerForwarding.ts"; import * as PeerLinks from "./PeerLinks.ts"; /** How long one follow of a remote task waits on its thread there before checking again. */ +/** What placement weighs a candidate on, gathered by this server itself. */ +export interface PlacementEvidence { + /** The project here; elsewhere, that machine's project with its repository. */ + readonly projectId: ProjectId; + readonly modelSelection: Pick; + /** The effective mode, sent explicitly wherever the work runs. */ + readonly runtimeMode: RuntimeMode; + /** This environment's own load, and whether it runs the exact model in that mode. */ + readonly here: { + readonly resources: HostResourcesSnapshot | null; + readonly modelAvailable: boolean; + }; +} + +const NativePlacement = Schema.Struct({ + environmentId: Schema.String, + projectId: Schema.String, + reason: Schema.Literal("capacity"), +}); +export interface NativePlacement { + readonly environmentId: EnvironmentId; + readonly projectId: ProjectId; + readonly reason: "capacity"; +} +const NativePlacementJson = Schema.fromJsonString(NativePlacement); +const encodeNativePlacementJson = Schema.encodeSync(NativePlacementJson); +const decodeNativePlacementJson = Schema.decodeEffect(NativePlacementJson); +const encodeNativePlacement = (placement: NativePlacement) => encodeNativePlacementJson(placement); +const decodeNativePlacement = (value: string) => + decodeNativePlacementJson(value).pipe( + Effect.map((placement): NativePlacement => ({ + environmentId: EnvironmentId.make(placement.environmentId), + projectId: placement.projectId as ProjectId, + reason: placement.reason, + })), + ); + const FOLLOW_WAIT_MS = 10 * 60 * 1_000; /** A linked environment that stops answering is retried, backing off to this. */ const FOLLOW_MAX_BACKOFF = Duration.minutes(5); @@ -119,17 +159,36 @@ export class RemoteDelegation extends Context.Service< readonly key: string; /** The thread the binding is recorded against. */ readonly threadId: ThreadId; - readonly projectId: ProjectId; - readonly modelSelection: Pick; - /** The effective mode, sent explicitly wherever the work runs. */ - readonly runtimeMode: RuntimeMode; - /** This environment's own load, and whether it runs the exact model in that mode. */ - readonly here: { - readonly resources: HostResourcesSnapshot | null; - readonly modelAvailable: boolean; - }; - }, + } & PlacementEvidence, ) => Effect.Effect; + /** + * Where a native client's new-thread launch runs: the environment its operation was first + * bound to, or one chosen now from this environment and authenticated links the client may + * launch on, bound with the session that admitted it before anything is sent. Another + * session's operation is refused, never adopted or admitted afresh. + */ + readonly admit: ( + caller: { + readonly sessionId: string; + /** Whether that session is still signed in with operate access; asked before each write. */ + readonly stillAuthorized: Effect.Effect; + }, + input: Omit & { + /** This environment's evidence, read only when nothing is bound yet. */ + readonly here: Effect.Effect< + PlacementEvidence["here"], + OrchestrationV2ThreadPlacementAdmitError + >; + readonly commandId: CommandId; + readonly threadId: ThreadId; + readonly interactionMode: ProviderInteractionMode; + /** Narrows the choice to machines the client may launch on; never widens it. */ + readonly candidateEnvironmentIds: ReadonlyArray; + }, + ) => Effect.Effect< + NativePlacement & { readonly resumed: boolean }, + OrchestrationV2ThreadPlacementAdmitError + >; /** The environment a caller's key was placed in, if it was. */ readonly placed: ( scope: McpInvocationContext.McpInvocationScope, @@ -257,7 +316,7 @@ const make = Effect.gen(function* () { /** The peer's project with this thread's repository, or the one the caller named. */ const resolveRemoteProject = Effect.fn("RemoteDelegation.resolveRemoteProject")(function* ( - scope: McpInvocationContext.McpInvocationScope, + ask: Ask, environmentId: EnvironmentId, parentProjectId: ProjectId, requested: ProjectId | undefined, @@ -282,11 +341,14 @@ const make = Effect.gen(function* () { const matches: Array<{ readonly id: ProjectId; readonly title: string }> = []; let cursor: number | null = 0; while (cursor !== null) { - const listed: Tool.Success = - yield* forwarding.call(scope, ProjectToolkit.tools.t3_project_list, environmentId, { + const listed: Tool.Success = yield* ask( + ProjectToolkit.tools.t3_project_list, + environmentId, + { cursor, limit: 100, - }); + }, + ); matches.push( ...listed.projects.filter((project) => project.repositoryIdentity?.canonicalKey === key), ); @@ -341,43 +403,52 @@ const make = Effect.gen(function* () { return chosen.providerInstanceId; }); + /** One linked environment's answer for placement, asked over whichever caller's forwarding. */ + type Ask = ( + tool: T, + environmentId: EnvironmentId, + params: Tool.Parameters, + ) => Effect.Effect, OrchestratorMcpFailure>; + const askFor = + (scope: McpInvocationContext.McpInvocationScope): Ask => + (tool, environmentId, params) => + forwarding.call(scope, tool, environmentId, params); + const askNative = + (caller: PeerForwarding.NativeCaller): Ask => + (tool, environmentId, params) => + forwarding.callNative(caller, tool, environmentId, params); + /** A linked environment as a placement candidate, from what it answered just now. */ const peerCandidate = Effect.fn("RemoteDelegation.peerCandidate")( - function* ( - scope: McpInvocationContext.McpInvocationScope, - link: PeerLinkSummary, - input: Parameters[1], - ) { - const capabilities = yield* forwarding.call( - scope, - RemoteCapabilitiesTool, - link.environmentId, - { - search: input.modelSelection.model, - }, - ); + function* (ask: Ask, link: PeerLink, input: PlacementEvidence) { + const capabilities = yield* ask(RemoteCapabilitiesTool, link.environmentId, { + search: input.modelSelection.model, + }); const receivedAt = yield* Clock.currentTimeMillis; - const projectAvailable = yield* resolveRemoteProject( - scope, + const projectId = yield* resolveRemoteProject( + ask, link.environmentId, input.projectId, undefined, ).pipe( - Effect.as(true), - Effect.orElseSucceed(() => false), + Effect.asSome, + Effect.orElseSucceed(() => Option.none()), ); return { - environmentId: link.environmentId, - resources: capabilities.hostResources ?? null, - receivedAt, - weight: 1, - operable: true, - projectAvailable, - modelAvailable: runsModel(capabilities, input), - } satisfies ThreadPlacementCandidate; + projectId, + candidate: { + environmentId: link.environmentId, + resources: capabilities.hostResources ?? null, + receivedAt, + weight: 1, + operable: true, + projectAvailable: Option.isSome(projectId), + modelAvailable: runsModel(capabilities, input), + } satisfies ThreadPlacementCandidate, + }; }, // One that does not answer, or refuses this caller, is not a candidate. - (effect, _scope, link) => + (effect, _ask, link) => effect.pipe( Effect.tapError((cause) => Effect.logDebug("remote-delegation.placement-candidate-unavailable", { @@ -389,6 +460,96 @@ const make = Effect.gen(function* () { ), ); + const exclusions = links.placementExclusions.pipe( + Effect.mapError( + () => + new OrchestratorMcpFailure({ + code: "orchestration_error", + message: + "Unable to read which machines automatic placement may use, so nothing was started. Retry with the same clientRequestId.", + }), + ), + ); + + /** + * Linked environments placement may ask, from what is stored here only: an excluded one, or + * one this work could not use, is dropped before any peer is contacted. `restrictTo` narrows + * them to the ones a caller can launch on. + */ + const askableLinks = Effect.fn("RemoteDelegation.askableLinks")(function* (input: { + readonly runtimeMode: RuntimeMode; + readonly excluded: ReadonlySet; + readonly restrictTo?: ReadonlyArray | undefined; + }) { + if (!(yield* links.enabled)) return []; + const usable = (environmentId: EnvironmentId) => + environmentId !== hereId && !input.excluded.has(environmentId); + const stored = + input.restrictTo === undefined + ? yield* links.stored.pipe(Effect.orElseSucceed((): ReadonlyArray => [])) + : (yield* Effect.forEach(input.restrictTo.filter(usable), (environmentId) => + links.get(environmentId).pipe(Effect.orElseSucceed(() => Option.none())), + )).flatMap((link) => (Option.isSome(link) ? [link.value] : [])); + const now = yield* DateTime.now; + return stored.filter( + (link) => + usable(link.environmentId) && + link.access !== "read-only" && + !isRuntimeModeAbove(input.runtimeMode, link.access) && + DateTime.isGreaterThan(link.expiresAt, now), + ); + }); + + /** The shared choice: this environment and the askable links, from their own answers. */ + const choosePlacement = Effect.fn("RemoteDelegation.choosePlacement")(function* ( + ask: Ask, + input: PlacementEvidence & { + readonly excluded: ReadonlySet; + readonly restrictTo?: ReadonlyArray | undefined; + }, + ) { + const askable = yield* askableLinks(input); + const peers = (yield* Effect.forEach(askable, (link) => peerCandidate(ask, link, input), { + concurrency: "unbounded", + })).flatMap((peer) => (Option.isSome(peer) ? [peer.value] : [])); + const now = yield* Clock.currentTimeMillis; + const hereCandidate = + !input.excluded.has(hereId) && + (input.restrictTo === undefined || input.restrictTo.includes(hereId)); + const decision = chooseThreadPlacement( + { originEnvironmentId: hereId }, + [ + ...(hereCandidate + ? [ + { + environmentId: hereId, + resources: input.here.resources, + receivedAt: now, + weight: 1, + operable: true, + projectAvailable: true, + modelAvailable: input.here.modelAvailable, + }, + ] + : []), + ...peers.map((peer) => peer.candidate), + ], + now, + ); + yield* Effect.annotateCurrentSpan({ + "placement.reason": decision.reason, + "placement.candidates": askable.length + (hereCandidate ? 1 : 0), + }); + const projectId = !decision.accepted + ? undefined + : decision.environmentId === hereId + ? input.projectId + : peers + .find((peer) => peer.candidate.environmentId === decision.environmentId) + ?.projectId.pipe(Option.getOrUndefined); + return { decision, projectId }; + }); + const placementBinding = ( scope: McpInvocationContext.McpInvocationScope, input: { readonly operation: string; readonly key: string }, @@ -403,10 +564,27 @@ const make = Effect.gen(function* () { "Unable to record where this task runs, so nothing was started. Retry with the same clientRequestId.", }); + /** A bound environment automatic placement may no longer use: nothing is sent there or anywhere. */ + const boundExcluded = (environmentId: EnvironmentId) => + new OrchestratorMcpFailure({ + code: "target_required", + message: `This request is bound to environment ${environmentId}, which automatic placement may no longer use, so nothing was sent. Reconcile it there, or start new work with a new clientRequestId.`, + }); + const placed: RemoteDelegation["Service"]["placed"] = (scope, input) => - threads - .intentOf(placementBinding(scope, input)) - .pipe(Effect.map(Option.map(EnvironmentId.make)), Effect.mapError(bindingFailure)); + threads.intentOf(placementBinding(scope, input)).pipe( + Effect.map(Option.map(EnvironmentId.make)), + Effect.mapError(bindingFailure), + // Current exclusions hold for every bound destination, this one too. A launch already + // made here replays before placement is consulted (`t3_thread_launch`). + Effect.tap((bound) => + Option.isSome(bound) + ? Effect.flatMap(exclusions, (excluded) => + excluded.has(bound.value) ? Effect.fail(boundExcluded(bound.value)) : Effect.void, + ) + : Effect.void, + ), + ); const place: RemoteDelegation["Service"]["place"] = Effect.fn("RemoteDelegation.place")( function* (scope, input) { @@ -415,41 +593,9 @@ const make = Effect.gen(function* () { // stopped answering: its first attempt may have started the task there. const bound = yield* placed(scope, input); if (Option.isSome(bound)) return bound.value; - - const linked = (yield* links.enabled) - ? yield* links.list.pipe(Effect.orElseSucceed(() => [])) - : []; - const peers = yield* Effect.forEach( - linked.filter( - (link) => - link.environmentId !== hereId && - link.status === "reachable" && - link.access !== "read-only" && - !isRuntimeModeAbove(input.runtimeMode, link.access), - ), - (link) => peerCandidate(scope, link, input), - { concurrency: "unbounded" }, - ); - const now = yield* Clock.currentTimeMillis; - const decision = chooseThreadPlacement( - { originEnvironmentId: hereId }, - [ - { - environmentId: hereId, - resources: input.here.resources, - receivedAt: now, - weight: 1, - operable: true, - projectAvailable: true, - modelAvailable: input.here.modelAvailable, - }, - ...peers.flatMap((peer) => (Option.isSome(peer) ? [peer.value] : [])), - ], - now, - ); - yield* Effect.annotateCurrentSpan({ - "placement.reason": decision.reason, - "placement.candidates": peers.length + 1, + const { decision } = yield* choosePlacement(askFor(scope), { + ...input, + excluded: yield* exclusions, }); if (!decision.accepted) { return yield* new OrchestratorMcpFailure({ @@ -467,6 +613,97 @@ const make = Effect.gen(function* () { }, ); + /** + * Native admission keys hang off this server's own id and the launch's command id. Its + * owner, the session that first admitted it, is bound before anything is chosen. + */ + const nativeBinding = (kind: "placement" | "placement-owner", commandId: CommandId) => + CommandId.make( + `command:native:${encodeURIComponent(hereId)}:${kind}:thread-launch:${encodeURIComponent(commandId)}`, + ); + + const admit: RemoteDelegation["Service"]["admit"] = Effect.fn("RemoteDelegation.admit")( + function* (caller, input) { + const refuse = (reason: OrchestrationV2ThreadPlacementAdmitError["reason"], detail: string) => + new OrchestrationV2ThreadPlacementAdmitError({ + commandId: input.commandId, + reason, + detail, + }); + const unavailableHere = () => + refuse( + "unavailable", + "Unable to read or record where this launch runs, so nothing was bound or started. Send it again.", + ); + const signedOut = () => + refuse( + "unavailable", + "This sign-in can no longer launch threads here, so nothing was bound or started.", + ); + if (!(yield* caller.stillAuthorized)) return yield* signedOut(); + const owner = yield* threads + .bindIntent({ + commandId: nativeBinding("placement-owner", input.commandId), + threadId: input.threadId, + value: caller.sessionId, + }) + .pipe(Effect.mapError(unavailableHere)); + if (owner !== caller.sessionId) { + return yield* refuse( + "other-session", + "This launch was started from another sign-in, so it is not continued or started again from this one. Open it where it was sent from, or start a new draft.", + ); + } + const excluded = yield* links.placementExclusions.pipe(Effect.mapError(unavailableHere)); + const placementId = nativeBinding("placement", input.commandId); + const decode = (value: string) => + decodeNativePlacement(value).pipe(Effect.mapError(unavailableHere)); + // The first admission's answer holds, whatever this one asks: it may already have launched. + const bound = yield* threads.intentOf(placementId).pipe(Effect.mapError(unavailableHere)); + if (Option.isSome(bound)) { + const placement = yield* decode(bound.value); + // Only this operation's own launch, accepted here for its own thread, is reconciled past a + // current exclusion: its `thread.create` receipt. Elsewhere that cannot be known without + // contact, and a thread that merely exists proves nothing. + const acceptedHere = + placement.environmentId === hereId && + (yield* threads.acceptedLaunchOf(input.commandId).pipe( + Effect.map((launched) => Option.isSome(launched) && launched.value === input.threadId), + Effect.mapError(unavailableHere), + )); + if (excluded.has(placement.environmentId) && !acceptedHere) { + return yield* refuse( + "excluded", + `This launch is bound to environment ${placement.environmentId}, which automatic placement may no longer use, so nothing was sent. Reconcile it there, or start a new draft.`, + ); + } + return { ...placement, resumed: true }; + } + const { decision, projectId } = yield* choosePlacement( + askNative({ + limits: { runtimeMode: input.runtimeMode, interactionMode: input.interactionMode }, + }), + { ...input, here: yield* input.here, excluded, restrictTo: input.candidateEnvironmentIds }, + ); + if (!decision.accepted || projectId === undefined) { + return yield* refuse( + "no-eligible-environment", + `No machine you can launch on answered with spare capacity, this repository's project and ${input.modelSelection.instanceId}/${input.modelSelection.model} in ${input.runtimeMode}. Nothing was started.`, + ); + } + if (!(yield* caller.stillAuthorized)) return yield* signedOut(); + const value = encodeNativePlacement({ + environmentId: EnvironmentId.make(decision.environmentId), + projectId, + reason: "capacity", + }); + const held = yield* threads + .bindIntent({ commandId: placementId, threadId: input.threadId, value }) + .pipe(Effect.mapError(unavailableHere)); + return { ...(yield* decode(held)), resumed: held !== value }; + }, + ); + /** * The other side names the thread from its link session and the key. The first launch's * session and answered thread are bound here, and it is sent only under that session; a @@ -548,7 +785,7 @@ const make = Effect.gen(function* () { Effect.orElseSucceed(() => input.environmentId), ); const projectId = yield* resolveRemoteProject( - scope, + askFor(scope), input.environmentId, input.projectId, undefined, @@ -644,7 +881,7 @@ const make = Effect.gen(function* () { ? undefined : yield* resolveInteractionMode(parent.thread.interactionMode, input.interactionMode); const projectId = yield* resolveRemoteProject( - scope, + askFor(scope), target.environmentId, parent.thread.projectId, target.projectId, @@ -1379,6 +1616,7 @@ const make = Effect.gen(function* () { return RemoteDelegation.of({ delegate, place, + admit, placed, launch, followUp, diff --git a/apps/server/src/peer/placementEvidence.ts b/apps/server/src/peer/placementEvidence.ts new file mode 100644 index 000000000000..763f89523683 --- /dev/null +++ b/apps/server/src/peer/placementEvidence.ts @@ -0,0 +1,42 @@ +import { + isProviderAvailable, + type ModelSelection, + type ProjectId, + resolveProjectProviderInstanceEnabled, + type RuntimeMode, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; + +import * as ProviderRegistry from "../provider/ProviderRegistry.ts"; +import * as HostResources from "../resourceTelemetry/HostResources.ts"; +import * as ServerSettings from "../serverSettings.ts"; + +/** + * This environment as a placement candidate: its own load, and whether it runs the exact + * provider instance and model for the project in the requested mode. + */ +export const hereEvidence = Effect.fn("placement.hereEvidence")(function* (input: { + readonly projectId: ProjectId; + readonly modelSelection: Pick; + readonly runtimeMode: RuntimeMode; +}) { + const providers = yield* (yield* ProviderRegistry.ProviderRegistry).getProviders; + const settings = yield* (yield* ServerSettings.ServerSettingsService).getSettings; + const provider = providers.find( + (candidate) => candidate.instanceId === input.modelSelection.instanceId, + ); + const hostResources = yield* Effect.serviceOption(HostResources.HostResources); + return { + resources: Option.isSome(hostResources) ? yield* hostResources.value.read : null, + modelAvailable: + provider !== undefined && + provider.enabled && + isProviderAvailable(provider) && + resolveProjectProviderInstanceEnabled(settings, input.projectId, provider.instanceId) && + provider.models.some((model) => model.slug === input.modelSelection.model) && + (provider.supportedRuntimeModes === undefined || + provider.supportedRuntimeModes.length === 0 || + provider.supportedRuntimeModes.includes(input.runtimeMode)), + }; +}); diff --git a/apps/server/src/relay/AgentAwarenessRelay.test.ts b/apps/server/src/relay/AgentAwarenessRelay.test.ts index 0287c00e5f90..b46c0dde4e86 100644 --- a/apps/server/src/relay/AgentAwarenessRelay.test.ts +++ b/apps/server/src/relay/AgentAwarenessRelay.test.ts @@ -252,6 +252,7 @@ const makeTestRelay = Effect.fnUntraced(function* ( assertProviderInstanceEnabled: unused, isProviderInstanceEnabledForProject: unused, hasReceipt: unused, + acceptedLaunchOf: unused, intentOf: unused, bindIntent: unused, waitForThread: unused, diff --git a/apps/server/src/ws.test.ts b/apps/server/src/ws.test.ts index f8ad4d3229ba..d79969d62a98 100644 --- a/apps/server/src/ws.test.ts +++ b/apps/server/src/ws.test.ts @@ -11,6 +11,9 @@ import { AuthOrchestrationOperateScope, AuthOrchestrationReadScope, AuthSessionId, + CommandId, + ORCHESTRATION_V2_WS_METHODS, + OrchestrationV2ThreadPlacementAdmitError, AuthSourceControlWriteScope, ORCHESTRATION_PROTOCOL_VERSION, ORCHESTRATION_PROTOCOL_QUERY_PARAM, @@ -130,6 +133,7 @@ import * as ServerRuntimeStartup from "./serverRuntimeStartup.ts"; import * as ServerSelfUpdate from "./cloud/selfUpdate.ts"; import * as NpmRelease from "./cloud/npmRelease.ts"; import * as SessionStore from "./auth/SessionStore.ts"; +import * as RemoteDelegation from "./peer/RemoteDelegation.ts"; import * as SourceControlDiscovery from "./sourceControl/SourceControlDiscovery.ts"; import * as SourceControlRepositoryService from "./sourceControl/SourceControlRepositoryService.ts"; import * as SourceControlSignIn from "./sourceControl/SourceControlSignIn.ts"; @@ -2621,3 +2625,219 @@ it.effect("ACP session RPCs list a thread missing its history and repair it on i ), ), ); + +it.effect.each([ + { + label: "an unexpired session with operate access", + expiresInMs: 3_600_000, + listed: true, + admitted: true, + }, + // Listed sessions include an expired one whose socket is still connected. + { + label: "an expired session its socket still holds", + expiresInMs: -60_000, + listed: true, + admitted: false, + }, + { label: "a revoked session", expiresInMs: 3_600_000, listed: false, admitted: false }, +] as const)( + "a native placement admission writes only for $label", + ({ expiresInMs, listed, admitted }) => + Effect.gen(function* () { + const sessionId = AuthSessionId.make("web-session"); + const now = yield* DateTime.now; + const asked: Array<{ readonly sessionId: string; readonly authorized: boolean }> = []; + const exits = yield* Queue.unbounded>(); + const services = Layer.mergeAll( + layerUnusedConfigRpcServices, + Layer.mock(ExternalLauncher.ExternalLauncher)({}), + Layer.mock(ProviderRegistry.ProviderRegistry)({}), + Layer.mock(UsageLimitSources.UsageLimitSources)({}), + Layer.mock(RemoteOpenTargets.RemoteOpenTargets)({}), + Layer.mock(DirectEndpoints.DirectEndpoints)({}), + Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({ namedProjectsRoot: "/unused" }), + Layer.mock(ScratchThreadPromotion.ScratchThreadPromotion)({}), + Layer.mock(ServerEnvironment.ServerEnvironment)({}), + Layer.mock(EnvironmentAuth.EnvironmentAuth)({}), + Layer.mock(ServerRuntimeStartup.ServerRuntimeStartup)({ + enqueueCommand: (effect) => effect, + }), + Layer.mock(SessionStore.SessionStore)({ + cookieName: "admission-test", + legacyCookieName: undefined, + listActive: () => + Effect.succeed( + listed + ? [ + { + sessionId, + subject: "browser", + scopes: [AuthOrchestrationOperateScope], + method: "browser-session-cookie" as const, + client: { deviceType: "desktop" as const }, + issuedAt: now, + expiresAt: DateTime.add(now, { milliseconds: expiresInMs }), + lastConnectedAt: now, + connected: true, + current: true, + }, + ] + : [], + ), + }), + // The delegation itself is covered with real links; here, what the socket hands it. + Layer.mock(RemoteDelegation.RemoteDelegation)({ + admit: (caller, input) => + caller.stillAuthorized.pipe( + Effect.tap((authorized) => + Effect.sync(() => asked.push({ sessionId: caller.sessionId, authorized })), + ), + Effect.flatMap((authorized) => + authorized + ? Effect.succeed({ + environmentId: EnvironmentId.make("environment-box"), + projectId: input.projectId, + reason: "capacity" as const, + resumed: false, + }) + : Effect.fail( + new OrchestrationV2ThreadPlacementAdmitError({ + commandId: input.commandId, + reason: "unavailable", + detail: "signed out", + }), + ), + ), + ), + }), + ); + const server = yield* RpcServer.makeNoSerialization(WsRpcGroup, { + ...WS_RPC_SERVER_OPTIONS, + onFromServer: (response) => + response._tag === "Exit" ? Queue.offer(exits, response.exit) : Effect.void, + }).pipe( + Effect.provide( + layerWsRpc( + { + sessionId, + subject: "browser", + method: "browser-session-cookie", + scopes: [AuthOrchestrationOperateScope], + }, + {}, + yield* ServerBrowser.ServerBrowser, + () => Effect.void, + { secretReferences: true }, + ).pipe( + Layer.provide(services), + Layer.merge(RpcAuthorization.layer([AuthOrchestrationOperateScope])), + ), + ), + ); + yield* server.write(0, { + _tag: "Request", + id: RpcMessage.RequestId("admit"), + tag: ORCHESTRATION_V2_WS_METHODS.admitThreadPlacement, + payload: { + commandId: CommandId.make("web-launch"), + threadId: ThreadId.make("thread:web-launch"), + projectId: ProjectId.make("project:app"), + modelSelection: { + instanceId: ProviderInstanceId.make("claudeAgent"), + model: "claude-sonnet-4-6", + }, + runtimeMode: "full-access", + interactionMode: "default", + candidateEnvironmentIds: [EnvironmentId.make("environment-box")], + }, + headers: Headers.empty, + }); + const exit = yield* Queue.take(exits); + assert.equal(Exit.isSuccess(exit), admitted); + // The socket's own session asks, never one the request names. + assert.deepEqual(asked, [{ sessionId, authorized: admitted }]); + }).pipe( + Effect.provide( + Layer.mergeAll( + Layer.fresh(layerConfigSources), + Layer.mock(PreviewAutomationBroker.PreviewAutomationBroker)({}), + Layer.mock(ServerBrowser.ServerBrowser)({}), + ), + ), + ), +); + +it.effect("a native placement admission needs operate access before anything is asked", () => + Effect.gen(function* () { + let asked = 0; + const exits = yield* Queue.unbounded>(); + const services = Layer.mergeAll( + layerUnusedConfigRpcServices, + Layer.mock(ExternalLauncher.ExternalLauncher)({}), + Layer.mock(ProviderRegistry.ProviderRegistry)({}), + Layer.mock(UsageLimitSources.UsageLimitSources)({}), + Layer.mock(RemoteOpenTargets.RemoteOpenTargets)({}), + Layer.mock(DirectEndpoints.DirectEndpoints)({}), + Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({ namedProjectsRoot: "/unused" }), + Layer.mock(ScratchThreadPromotion.ScratchThreadPromotion)({}), + Layer.mock(ServerEnvironment.ServerEnvironment)({}), + Layer.mock(EnvironmentAuth.EnvironmentAuth)({}), + Layer.mock(RemoteDelegation.RemoteDelegation)({ + admit: () => + Effect.sync(() => { + asked += 1; + }).pipe(Effect.andThen(Effect.die("admitted without operate access"))), + }), + ); + const server = yield* RpcServer.makeNoSerialization(WsRpcGroup, { + ...WS_RPC_SERVER_OPTIONS, + onFromServer: (response) => + response._tag === "Exit" ? Queue.offer(exits, response.exit) : Effect.void, + }).pipe( + Effect.provide( + layerWsRpc( + { + sessionId: AuthSessionId.make("read-only"), + subject: "browser", + method: "browser-session-cookie", + scopes: [AuthAccessReadScope], + }, + {}, + yield* ServerBrowser.ServerBrowser, + () => Effect.void, + { secretReferences: true }, + ).pipe(Layer.provide(services), Layer.merge(RpcAuthorization.layer([AuthAccessReadScope]))), + ), + ); + yield* server.write(0, { + _tag: "Request", + id: RpcMessage.RequestId("admit"), + tag: ORCHESTRATION_V2_WS_METHODS.admitThreadPlacement, + payload: { + commandId: CommandId.make("web-launch"), + threadId: ThreadId.make("thread:web-launch"), + projectId: ProjectId.make("project:app"), + modelSelection: { + instanceId: ProviderInstanceId.make("claudeAgent"), + model: "claude-sonnet-4-6", + }, + runtimeMode: "full-access", + interactionMode: "default", + candidateEnvironmentIds: [], + }, + headers: Headers.empty, + }); + const exit = yield* Queue.take(exits); + assert.equal(Exit.isFailure(exit), true); + assert.equal(asked, 0); + }).pipe( + Effect.provide( + Layer.mergeAll( + Layer.fresh(layerConfigSources), + Layer.mock(PreviewAutomationBroker.PreviewAutomationBroker)({}), + Layer.mock(ServerBrowser.ServerBrowser)({}), + ), + ), + ), +); diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index f9979dcbf7f4..feb2ef6ac65a 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -64,6 +64,9 @@ import { OrchestrationV2GetShellSnapshotError, OrchestrationV2GetThreadProjectionError, OrchestrationV2ThreadLaunchError, + OrchestrationV2ThreadPlacementAdmitError, + type OrchestrationV2ThreadPlacementAdmitInput, + managedRuntimeModeDenial, type OrchestrationProjectShell, type OrchestrationV2ShellSnapshot, type OrchestrationV2ShellStreamItem, @@ -175,6 +178,9 @@ import * as ThreadSearch from "./orchestration-v2/ThreadSearch.ts"; import * as OrchestrationEventStore from "./persistence/OrchestrationEventStore.ts"; import { userFacingDispatchErrorMessage } from "./orchestration-v2/UserFacingErrors.ts"; import * as ProviderRegistry from "./provider/ProviderRegistry.ts"; +import { hereEvidence } from "./peer/placementEvidence.ts"; +import * as RemoteDelegation from "./peer/RemoteDelegation.ts"; +import * as ManagedPolicy from "./policy/ManagedPolicy.ts"; import * as SecretCheck from "./secrets/SecretCheck.ts"; import * as AcpRegistrySessions from "./provider/AcpRegistrySessions.ts"; import * as ProviderInstanceRegistry from "./provider/ProviderInstanceRegistry.ts"; @@ -1580,6 +1586,64 @@ export const layerWsRpc = ( const processDiagnostics = yield* ProcessDiagnostics.ProcessDiagnostics; const triage = yield* TriageService.TriageService; const hostResources = yield* HostResources.HostResources; + const remoteDelegation = yield* Effect.serviceOption(RemoteDelegation.RemoteDelegation); + const managedPolicy = yield* Effect.serviceOption(ManagedPolicy.ManagedPolicy); + const placementContext = yield* Effect.context< + | ProviderRegistry.ProviderRegistry + | ServerSettings.ServerSettingsService + | HostResources.HostResources + >(); + /** + * Where this session's new-thread launch runs, decided and bound here before anything is + * sent. Its session must still be signed in with operate access at each write. + */ + const admitThreadPlacement = Effect.fn("ws.admitThreadPlacement")(function* ( + input: OrchestrationV2ThreadPlacementAdmitInput, + ) { + const unavailableHere = (detail: string) => + new OrchestrationV2ThreadPlacementAdmitError({ + commandId: input.commandId, + reason: "unavailable", + detail, + }); + if (Option.isNone(remoteDelegation)) { + return yield* unavailableHere( + "This server cannot place launches, so nothing was started.", + ); + } + // Read only for a new admission: a bound operation resumes whatever this machine runs now. + const here = hereEvidence(input).pipe( + Effect.provide(placementContext), + Effect.map((evidence) => ({ + ...evidence, + // The launch here would refuse a mode above this machine's managed ceiling. + modelAvailable: + evidence.modelAvailable && + managedRuntimeModeDenial( + Option.isSome(managedPolicy) ? managedPolicy.value.summary : undefined, + input.runtimeMode, + ) === undefined, + })), + Effect.mapError(() => + unavailableHere("Unable to read this machine's providers, so nothing was started."), + ), + ); + // Listed sessions include expired ones a socket still holds; only an unexpired one counts. + const stillAuthorized = Effect.gen(function* () { + const active = yield* sessions.listActive(); + const now = yield* DateTime.now; + return active.some( + (session) => + session.sessionId === currentSessionId && + DateTime.isGreaterThan(session.expiresAt, now) && + session.scopes.includes(AuthOrchestrationOperateScope), + ); + }).pipe(Effect.orElseSucceed(() => false)); + return yield* remoteDelegation.value.admit( + { sessionId: currentSessionId, stillAuthorized }, + { ...input, here }, + ); + }); const processResourceMonitor = yield* ProcessResourceMonitor.ProcessResourceMonitor; const resourceTelemetry = yield* ResourceTelemetry.ResourceTelemetry; const relayClient = yield* RelayClient.RelayClient; @@ -2148,6 +2212,19 @@ export const layerWsRpc = ( ), ), ), + [ORCHESTRATION_V2_WS_METHODS.admitThreadPlacement]: (input) => + startup.enqueueCommand(admitThreadPlacement(input)).pipe( + Effect.mapError((error) => + error._tag === "OrchestrationV2ThreadPlacementAdmitError" + ? error + : new OrchestrationV2ThreadPlacementAdmitError({ + commandId: input.commandId, + reason: "unavailable", + detail: + "This server is not ready to place launches yet, so nothing was started.", + }), + ), + ), [ORCHESTRATION_V2_WS_METHODS.launchThread]: (input) => Effect.annotateCurrentSpan({ "orchestration_v2.command_id": input.commandId, diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 6dc456b412de..ca3459a447e4 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -381,6 +381,7 @@ import { ThreadVisitWatermark } from "./chat/ThreadVisitWatermark"; import { ChatTerminalDrawers } from "./chat/ChatTerminalDrawers"; import { ChatRightPanel } from "./chat/ChatRightPanel"; import { useDraftEnvironmentRouting } from "./chat/useDraftEnvironmentRouting"; +import { attachmentUploadsWaitForPlacement, draftRoutesAutomatically } from "./chat/draftPlacement"; import { branchMismatchBannerItem, extensionStatusBannerItem } from "./chat/ThreadActivityBanners"; import { ThreadNow } from "./chat/ThreadNow"; import type { ThreadStructure } from "@t3tools/client-runtime/state/shell"; @@ -1318,19 +1319,15 @@ export default function ChatView(props: ChatViewProps) { loadBalancingEnabled: settings.loadBalancingEnabled, loadBalancingWeights: settings.loadBalancingWeights, })); - const automaticEnvironment = Boolean( - clientSettingsHydrated && - draftId && - !envLocked && - canAutoBalanceEnvironments && - loadBalancingSettings.loadBalancingEnabled && - draftThread?.environmentSelection !== "manual" && - // A send that may have started there keeps its thread id on that machine. - !draftThread?.sentTo && - (!composerHasAttachments || Boolean(draftThread?.loadBalancedEnvironmentId)) && - (!draftThread?.branch || draftThread.environmentSelection === "auto") && - !draftThread?.worktreePath, - ); + const automaticEnvironment = draftRoutesAutomatically({ + clientSettingsHydrated, + isDraft: Boolean(draftId), + envLocked, + canAutoBalanceEnvironments, + loadBalancingEnabled: loadBalancingSettings.loadBalancingEnabled, + draft: draftThread ?? null, + composerHasAttachments, + }); const autoUpdateEnvironments = automaticEnvironment ? logicalProjectEnvironments.flatMap(({ environmentId }) => { const environment = environmentById.get(environmentId); @@ -3642,6 +3639,13 @@ export default function ChatView(props: ChatViewProps) { composerDraftTarget={composerDraftTarget} environmentId={environmentId} attachmentUploadsCapabilityKnown={attachmentUploadsCapabilityKnown} + attachmentUploadsWaitForPlacement={attachmentUploadsWaitForPlacement( + { + draft: isLocalDraftThread ? (draftThread ?? null) : null, + automatic: automaticEnvironment, + environmentId, + }, + )} supportsAttachmentUploads={supportsAttachmentUploads} supportsQuestionAttachments={supportsQuestionAttachments} maxFileAttachmentBytes={maxFileAttachmentBytes} diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index be3144efb40b..56c3d172c0b8 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -93,6 +93,7 @@ import { makeComposerMentionDragHandlers, } from "./composerMentionDrag"; import { serializeComposerImages } from "./composerImagePersistence"; +import { startComposerAttachmentUploads } from "./draftPlacement"; import { isInsideCollapsedComposerControls, isInsideRestingComposerControlScope, @@ -355,6 +356,8 @@ export interface ChatComposerProps { environmentId: EnvironmentId; canOperateThread: boolean; attachmentUploadsCapabilityKnown: boolean; + /** Automatic work uploads only once its origin bound it here (`attachmentUploadsWaitForPlacement`). */ + attachmentUploadsWaitForPlacement?: boolean; supportsAttachmentUploads: boolean; supportsQuestionAttachments: boolean; maxFileAttachmentBytes: number | null; @@ -491,6 +494,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) environmentId, canOperateThread, attachmentUploadsCapabilityKnown, + attachmentUploadsWaitForPlacement = false, supportsAttachmentUploads, supportsQuestionAttachments, maxFileAttachmentBytes, @@ -844,19 +848,16 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) maxFileAttachmentBytes === null ? [] : composerFiles.filter((file) => file.sizeBytes <= maxFileAttachmentBytes); - const uploadableAttachments = [...composerImages, ...uploadableFiles]; - for (const attachment of uploadableAttachments) { - if (attachment.type === "file" && composerFileNeedsReattach(attachment)) { - continue; - } - startAttachmentUpload({ - environmentId, - image: attachment, - draftTarget: attachmentDraftTarget, - }); - } + startComposerAttachmentUploads({ + environmentId, + attachments: [...composerImages, ...uploadableFiles], + draftTarget: attachmentDraftTarget, + waitForPlacement: attachmentUploadsWaitForPlacement, + start: startAttachmentUpload, + }); }, [ attachmentUploadsCapabilityKnown, + attachmentUploadsWaitForPlacement, attachmentDraftTarget, canOperateThread, composerFiles, diff --git a/apps/web/src/components/chat/draftAttachmentUploads.test.ts b/apps/web/src/components/chat/draftAttachmentUploads.test.ts new file mode 100644 index 000000000000..bb1ad6cab176 --- /dev/null +++ b/apps/web/src/components/chat/draftAttachmentUploads.test.ts @@ -0,0 +1,517 @@ +import { + EnvironmentId, + OrchestrationV2ThreadPlacementAdmitError, + ProjectId, + ProviderInstanceId, +} from "@t3tools/contracts"; +import { scopeProjectRef } from "@t3tools/client-runtime/environment"; +import * as Cause from "effect/Cause"; +import { AsyncResult } from "effect/reactivity"; +import { describe, expect, it, vi } from "vite-plus/test"; +import { createMemoryStorage } from "../../lib/storage"; + +// The network edges of an upload: the session may operate, and each upload cycle is recorded. +const uploads = vi.hoisted(() => ({ + to: [] as Array<{ environmentId: string; name: string }>, + /** Origin admissions asked by the queue and upload cycles, in the order they happened. */ + sequence: [] as Array, +})); +// The origin's own record of the launches it bound, which its admission handler answers again. +const originServer = vi.hoisted(() => ({ + bound: [] as Array<{ + environmentId: string; + commandId: string; + threadId: string; + projectId: string; + destination: string; + destinationProjectId: string; + }>, + /** The queue's own admission requests, as the origin received them. */ + queueAsked: [] as Array<{ + environmentId: string; + commandId: string; + threadId: string; + projectId: string; + candidateEnvironmentIds: ReadonlyArray; + }>, +})); +vi.mock("../../state/session", async (importOriginal) => ({ + ...(await importOriginal()), + readEnvironmentScope: () => true, +})); +vi.mock("@t3tools/client-runtime/state/attachments", async (importOriginal) => ({ + ...(await importOriginal()), + runAttachmentUploadCycle: async (input: { + readonly environmentId: string; + readonly upload: { readonly name: string }; + }) => { + uploads.to.push({ environmentId: input.environmentId, name: input.upload.name }); + uploads.sequence.push(`upload:${input.environmentId}`); + return { status: "uploaded" as const, attachmentId: `upload-${uploads.to.length}` }; + }, +})); +// Both machines connected, on their first established session, as the queue's checks read them. +vi.mock("../../connection/catalog", async () => { + const { AsyncResult: Result, Atom: A } = await import("effect/reactivity"); + const family = (initial: () => unknown) => { + const atoms = new Map(); + return (environmentId: string) => { + if (!atoms.has(environmentId)) atoms.set(environmentId, A.make(initial())); + return atoms.get(environmentId); + }; + }; + return { + environmentCatalog: { + stateAtom: family(() => Result.success({ phase: "connected" })), + sessionGenerationAtom: family(() => 1), + }, + }; +}); +// The queue asks the origin itself, over its admission RPC: answered from what the origin bound. +vi.mock("@t3tools/client-runtime/state/runtime", async (importOriginal) => { + const real = await importOriginal(); + const { AsyncResult: Result } = await import("effect/reactivity"); + const CauseModule = await import("effect/Cause"); + return { + ...real, + runAtomCommand: async (registry: never, command: never, target: never) => { + const { environmentId, input } = target as { + environmentId: string; + input?: { + commandId: string; + threadId: string; + projectId: string; + candidateEnvironmentIds?: ReadonlyArray; + }; + }; + if (input?.candidateEnvironmentIds === undefined) + return real.runAtomCommand(registry, command, target); + const asked = { + environmentId, + commandId: input.commandId, + threadId: input.threadId, + projectId: input.projectId, + candidateEnvironmentIds: [...input.candidateEnvironmentIds], + }; + originServer.queueAsked.push(asked); + uploads.sequence.push(`admit:${environmentId}`); + const bound = originServer.bound.find( + (launch) => + launch.environmentId === asked.environmentId && + launch.commandId === asked.commandId && + launch.threadId === asked.threadId && + launch.projectId === asked.projectId, + ); + if ( + !bound || + asked.candidateEnvironmentIds.length !== 1 || + asked.candidateEnvironmentIds[0] !== bound.destination + ) + return Result.failure(CauseModule.die("not bound at this origin")); + return Result.success({ + environmentId: bound.destination, + projectId: bound.destinationProjectId, + reason: "capacity", + resumed: true, + }); + }, + }; +}); + +/** The origin records the launch it just bound, as its admission handler does. */ +function boundAtOrigin( + target: { + environmentId: string; + input: { commandId: string; threadId: string; projectId: string }; + }, + destination: EnvironmentId, + destinationProjectId: ProjectId, +) { + originServer.bound.push({ + environmentId: target.environmentId, + commandId: target.input.commandId, + threadId: target.input.threadId, + projectId: target.input.projectId, + destination, + destinationProjectId, + }); +} + +const origin = EnvironmentId.make("environment-laptop"); +const box = EnvironmentId.make("environment-box"); +const projectHere = ProjectId.make("project:laptop"); +const projectThere = ProjectId.make("project:box"); +const request = { + modelSelection: { + instanceId: ProviderInstanceId.make("claudeAgent"), + model: "claude-sonnet-4-6", + }, + runtimeMode: "full-access", + interactionMode: "default", + multipleModels: false, +} as const; +const storageFaults = { drops: false }; + +async function openTab() { + return { + store: await import("../../composerDraftStore"), + placement: await import("./draftPlacement"), + queue: await import("../../lib/attachmentUploadQueue"), + }; +} +type Tab = Awaited>; + +async function withStorage(run: () => Promise) { + const memory = createMemoryStorage(); + const storage = { + getItem: (name: string) => memory.getItem(name), + removeItem: (name: string) => memory.removeItem(name), + setItem: (name: string, value: string) => { + if (!storageFaults.drops) memory.setItem(name, value); + }, + }; + vi.stubGlobal("localStorage", storage); + // The image readback reads `window.localStorage`, as in a browser. + vi.stubGlobal("window", Object.assign(new EventTarget(), { localStorage: storage })); + vi.useFakeTimers(); + vi.resetModules(); + uploads.to.length = 0; + uploads.sequence.length = 0; + originServer.bound.length = 0; + originServer.queueAsked.length = 0; + try { + await run(); + } finally { + storageFaults.drops = false; + vi.useRealTimers(); + vi.unstubAllGlobals(); + vi.resetModules(); + } +} + +/** Whether ChatView treats the draft as automatic work, by its own rule, for this state. */ +function routesAutomatically(tab: Tab, draftId: string) { + const id = tab.store.DraftId.make(draftId); + const state = tab.store.useComposerDraftStore.getState(); + const composer = state.getComposerDraft(id); + return tab.placement.draftRoutesAutomatically({ + clientSettingsHydrated: true, + isDraft: true, + envLocked: false, + canAutoBalanceEnvironments: true, + loadBalancingEnabled: true, + draft: state.getDraftSession(id), + composerHasAttachments: (composer?.images.length ?? 0) + (composer?.files.length ?? 0) > 0, + }); +} + +/** The composer's upload effect, as ChatView configures it for this draft now. */ +async function uploadEffect(tab: Tab, draftId: string) { + const id = tab.store.DraftId.make(draftId); + const state = tab.store.useComposerDraftStore.getState(); + const session = state.getDraftSession(id); + const composer = state.getComposerDraft(id)!; + const automatic = routesAutomatically(tab, draftId); + const started = tab.placement.startComposerAttachmentUploads({ + environmentId: session!.environmentId, + attachments: [...composer.images, ...composer.files], + draftTarget: id, + waitForPlacement: tab.placement.attachmentUploadsWaitForPlacement({ + draft: session, + automatic, + environmentId: session!.environmentId, + }), + start: tab.queue.startAttachmentUpload, + }); + await vi.advanceTimersByTimeAsync(0); + // Until the queue's own jobs for these attachments have finished, asked and uploaded or refused. + await tab.queue.awaitAttachmentUploads( + [...composer.images, ...composer.files].map((attachment) => attachment.id), + ); + await vi.advanceTimersByTimeAsync(0); + return started; +} + +async function composeAutoDraft( + tab: Tab, + name: string, + /** The client's own pick, not a binding; null while it is still choosing. */ + picked: EnvironmentId | null = origin, +) { + const id = tab.store.DraftId.make(`draft-${name}`); + const store = tab.store.useComposerDraftStore.getState(); + store.setProjectDraftThreadId(scopeProjectRef(origin, projectHere), id, { + environmentSelection: "auto", + loadBalancedEnvironmentId: picked, + }); + store.setPrompt(id, "Review the report"); + const report = new File(["report bytes"], "report.pdf", { type: "application/pdf" }); + store.addFiles(id, [ + { + type: "file", + id: `file-${name}`, + name: report.name, + mimeType: report.type, + sizeBytes: report.size, + file: report, + }, + ]); + // An image's bytes are kept here, so it survives a reload; a fresh file's are not. + const shot = new File([new Uint8Array([1, 2, 3, 4])], "shot.png", { type: "image/png" }); + const imageId = `shot-${name}`; + store.addImages(id, [ + { + type: "image", + id: imageId, + name: shot.name, + mimeType: shot.type, + sizeBytes: shot.size, + previewUrl: `blob:${imageId}`, + file: shot, + }, + ]); + // As the composer's own effect keeps an image's bytes here, before anything is uploaded. + await store.syncPersistedAttachments(id, [ + { + id: imageId, + name: shot.name, + mimeType: shot.type, + sizeBytes: shot.size, + dataUrl: "data:image/png;base64,AQIDBA==", + }, + ]); + return id; +} + +const place = (tab: Tab, draftId: string, admit: never) => + tab.placement.placeDraftBeforeSend({ + draftId: tab.store.DraftId.make(draftId), + request, + candidateEnvironmentIds: [origin, box], + pinnedHere: false, + admit, + canLaunchOn: () => true, + composerUnchanged: () => true, + }); + +describe("PR74-R1-F4: a fresh automatic draft's attachments", () => { + it("upload nowhere before their placement is bound and stored, then only to that machine", async () => { + await withStorage(async () => { + const tab = await openTab(); + const id = await composeAutoDraft(tab, "auto"); + expect( + tab.store.useComposerDraftStore.getState().getComposerDraft(id)?.persistedAttachments, + ).toHaveLength(1); + // Attached on an automatic draft: the bytes stay here, nothing is uploaded. + expect(await uploadEffect(tab, id)).toBe(0); + expect(uploads.to).toEqual([]); + + // Storage cannot hold the operation: nothing is asked, nothing uploaded. + storageFaults.drops = true; + const unsaved = await place(tab, id, (async () => + AsyncResult.failure(Cause.die("never asked"))) as never); + storageFaults.drops = false; + expect(unsaved.type).toBe("failed"); + expect(await uploadEffect(tab, id)).toBe(0); + + // Its admission's answer is lost: reserved, unbound, still nothing uploaded. + const lost = await place(tab, id, (async () => + AsyncResult.failure(Cause.die("connection lost"))) as never); + expect(lost.type).toBe("failed"); + expect(await uploadEffect(tab, id)).toBe(0); + const reserved = tab.store.useComposerDraftStore + .getState() + .getDraftSession(id)!.placementOperation!; + + vi.clearAllTimers(); + vi.resetModules(); + const reloaded = await openTab(); + expect(await uploadEffect(reloaded, id)).toBe(0); + const asked: Array = []; + const placed = await place(reloaded, id, (async (target: { + environmentId: string; + input: { commandId: string; threadId: string; projectId: string }; + }) => { + asked.push(target.input.commandId); + boundAtOrigin(target, box, projectThere); + return AsyncResult.success({ + environmentId: box, + projectId: projectThere, + reason: "capacity" as const, + resumed: true, + }); + }) as never); + expect(placed).toEqual({ type: "placed", environmentId: box, projectId: projectThere }); + expect(asked).toEqual([reserved.commandId]); + // Bound to B, the draft is still on A: nothing uploads to A. + expect(await uploadEffect(reloaded, id)).toBe(0); + // The routing hook moves it to B; only then do its attachments upload, to B. + reloaded.store.useComposerDraftStore.getState().setDraftThreadContext(id, { + projectRef: scopeProjectRef(box, projectThere), + environmentSelection: "auto", + loadBalancedEnvironmentId: box, + }); + expect(originServer.queueAsked).toEqual([]); + // The image uploads, to B; the file lost its bytes in the reload and needs attaching again. + expect(await uploadEffect(reloaded, id)).toBe(1); + expect(uploads.to).toEqual([{ environmentId: box, name: "shot.png" }]); + // Only after the queue asked the recorded origin again, for the same command and binding. + expect(reserved.admittedOn).toBe(origin); + expect(originServer.queueAsked).toEqual([ + { + environmentId: origin, + commandId: reserved.commandId, + threadId: reloaded.store.useComposerDraftStore.getState().getDraftSession(id)!.threadId, + projectId: reserved.admittedProjectId, + candidateEnvironmentIds: [box], + }, + ]); + expect(uploads.sequence).toEqual([`admit:${origin}`, `upload:${box}`]); + // And the launch there goes out as the operation reserved before the reload. + expect( + reloaded.placement.heldLaunchFor({ + draft: reloaded.store.useComposerDraftStore.getState().getDraftSession(id), + environmentId: box, + projectId: projectThere, + }), + ).toMatchObject({ + type: "reserved", + operation: { + commandId: reserved.commandId, + messageId: reserved.messageId, + createdAt: reserved.createdAt, + admission: { environmentId: box, projectId: projectThere }, + }, + }); + }); + }); + + it("still upload at once on a draft pinned to its machine", async () => { + await withStorage(async () => { + const tab = await openTab(); + const id = await composeAutoDraft(tab, "manual"); + tab.store.useComposerDraftStore.getState().setDraftThreadContext(id, { + environmentSelection: "manual", + loadBalancedEnvironmentId: null, + }); + expect(await uploadEffect(tab, id)).toBe(2); + expect(uploads.to.map(({ environmentId }) => environmentId)).toEqual([origin, origin]); + }); + }); +}); + +describe("PR74-R1-F4: automatic work's uploads wait for its origin", () => { + it("keeps an Auto draft automatic when a fresh attachment arrives before the client's pick", async () => { + await withStorage(async () => { + const tab = await openTab(); + const id = await composeAutoDraft(tab, "unpicked", null); + // Still automatic work by ChatView's own rule: nothing pins it, the bytes stay here. + expect(routesAutomatically(tab, id)).toBe(true); + expect(await uploadEffect(tab, id)).toBe(0); + expect(uploads.to).toEqual([]); + // And its send goes through admission rather than around it. + expect( + tab.placement.placesBeforeSend({ + draft: tab.store.useComposerDraftStore.getState().getDraftSession(id), + automatic: routesAutomatically(tab, id), + pinnedHere: false, + request, + }), + ).toBe(true); + }); + }); + + it("uploads nothing after a reload on its bound machine until its origin reconciles it", async () => { + await withStorage(async () => { + const tab = await openTab(); + const id = await composeAutoDraft(tab, "rebound"); + const bindTo = (async (target: Parameters[0]) => { + boundAtOrigin(target, box, projectThere); + return AsyncResult.success({ + environmentId: box, + projectId: projectThere, + reason: "capacity" as const, + resumed: false, + }); + }) as never; + expect((await place(tab, id, bindTo)).type).toBe("placed"); + // The routing hook moves it to B; the renderer dies before the upload or the launch. + tab.store.useComposerDraftStore.getState().setDraftThreadContext(id, { + projectRef: scopeProjectRef(box, projectThere), + environmentSelection: "auto", + loadBalancedEnvironmentId: box, + }); + await vi.advanceTimersByTimeAsync(300); + vi.clearAllTimers(); + vi.resetModules(); + uploads.to.length = 0; + uploads.sequence.length = 0; + const reloaded = await openTab(); + const session = () => reloaded.store.useComposerDraftStore.getState().getDraftSession(id)!; + expect(session().environmentId).toBe(box); + const reserved = session().placementOperation!; + // On B, bound to B: still nothing uploads before its origin answers this time. + expect(await uploadEffect(reloaded, id)).toBe(0); + // Its origin refuses (replaced sign-in, exclusion) or cannot answer: still nothing. + for (const answer of [ + AsyncResult.failure( + Cause.fail( + new OrchestrationV2ThreadPlacementAdmitError({ + commandId: reserved.commandId as never, + reason: "other-session", + detail: "another sign-in", + }), + ), + ), + AsyncResult.failure( + Cause.fail( + new OrchestrationV2ThreadPlacementAdmitError({ + commandId: reserved.commandId as never, + reason: "excluded", + detail: "excluded", + }), + ), + ), + AsyncResult.failure(Cause.die("connection lost")), + ]) { + expect((await place(reloaded, id, (async () => answer) as never)).type).toBe("failed"); + expect(await uploadEffect(reloaded, id)).toBe(0); + } + expect(uploads.to).toEqual([]); + // Its origin answers with the same binding for the same command: it uploads, only to B. + const asked: Array = []; + const resumed = (async (target: Parameters[0]) => { + asked.push(target.input.commandId); + boundAtOrigin(target, box, projectThere); + return AsyncResult.success({ + environmentId: box, + projectId: projectThere, + reason: "capacity" as const, + resumed: true, + }); + }) as never; + expect(await place(reloaded, id, resumed)).toEqual({ + type: "placed", + environmentId: box, + projectId: projectThere, + }); + expect(asked).toEqual([reserved.commandId]); + expect(originServer.queueAsked).toEqual([]); + expect(await uploadEffect(reloaded, id)).toBe(1); + expect(uploads.to).toEqual([{ environmentId: box, name: "shot.png" }]); + // Only after the queue asked the recorded origin again, for the same command and binding. + expect(reserved.admittedOn).toBe(origin); + expect(originServer.queueAsked).toEqual([ + { + environmentId: origin, + commandId: reserved.commandId, + threadId: session().threadId, + projectId: reserved.admittedProjectId, + candidateEnvironmentIds: [box], + }, + ]); + expect(uploads.sequence).toEqual([`admit:${origin}`, `upload:${box}`]); + expect(session().placementOperation).toEqual(reserved); + }); + }); +}); diff --git a/apps/web/src/components/chat/draftPlacement.test.ts b/apps/web/src/components/chat/draftPlacement.test.ts new file mode 100644 index 000000000000..ec2091426e37 --- /dev/null +++ b/apps/web/src/components/chat/draftPlacement.test.ts @@ -0,0 +1,525 @@ +import { + EnvironmentId, + ProjectId, + ProviderInstanceId, + ThreadId, + type OrchestrationV2ThreadPlacementAdmitInput, + OrchestrationV2ThreadPlacementAdmitError, + type OrchestrationV2ThreadPlacementAdmitResult, +} from "@t3tools/contracts"; +import { scopeProjectRef, scopeThreadRef } from "@t3tools/client-runtime/environment"; +import * as Cause from "effect/Cause"; +import { AsyncResult } from "effect/reactivity"; +import { describe, expect, it, vi } from "vite-plus/test"; +import { createMemoryStorage } from "../../lib/storage"; + +const origin = EnvironmentId.make("environment-laptop"); +const box = EnvironmentId.make("environment-box"); +const projectHere = ProjectId.make("project:laptop"); +const projectThere = ProjectId.make("project:box"); +const request = { + modelSelection: { + instanceId: ProviderInstanceId.make("claudeAgent"), + model: "claude-sonnet-4-6", + }, + runtimeMode: "full-access", + interactionMode: "default", + multipleModels: false, +} as const; +const KEY = "t3code:composer-drafts:v1"; + +const storageFaults = { drops: false }; + +async function openTab() { + return { + store: await import("../../composerDraftStore"), + placement: await import("./draftPlacement"), + }; +} +type Tab = Awaited>; + +async function withStorage(run: (memory: ReturnType) => Promise) { + const memory = createMemoryStorage(); + vi.stubGlobal("localStorage", { + getItem: (name: string) => memory.getItem(name), + removeItem: (name: string) => memory.removeItem(name), + setItem: (name: string, value: string) => { + if (!storageFaults.drops) memory.setItem(name, value); + }, + }); + vi.stubGlobal("window", new EventTarget()); + vi.useFakeTimers(); + vi.resetModules(); + try { + await run(memory); + } finally { + storageFaults.drops = false; + vi.useRealTimers(); + vi.unstubAllGlobals(); + vi.resetModules(); + } +} + +function composeDraft(tab: Tab, name: string) { + const draftId = tab.store.DraftId.make(`draft-${name}`); + const store = tab.store.useComposerDraftStore.getState(); + store.setProjectDraftThreadId(scopeProjectRef(origin, projectHere), draftId, { + environmentSelection: "auto", + loadBalancedEnvironmentId: origin, + }); + store.setPrompt(draftId, "Run the suite"); + return draftId; +} + +/** What storage holds of the draft's placement operation right now. */ +const storedOperation = (memory: ReturnType, draftId: string) => + JSON.parse((memory.getItem(KEY) as string | null) ?? "{}")?.state?.draftThreadsByThreadKey?.[ + draftId + ]?.placementOperation; + +type Admit = (target: { + readonly environmentId: EnvironmentId; + readonly input: OrchestrationV2ThreadPlacementAdmitInput; +}) => Promise>; + +const admittedTo = + ( + seen: Array<{ environmentId: EnvironmentId; commandId: string; stored: unknown }>, + memory: ReturnType, + draftId: string, + ): Admit => + async ({ environmentId, input }) => { + seen.push({ + environmentId, + commandId: input.commandId, + stored: storedOperation(memory, draftId), + }); + return AsyncResult.success({ + environmentId: box, + projectId: projectThere, + reason: "capacity", + resumed: seen.length > 1, + }); + }; + +type PlacementRequest = Parameters[0]["request"]; + +const place = ( + tab: Tab, + draftId: string, + admit: Admit, + options: { + readonly canLaunchOn?: () => boolean; + readonly pinnedHere?: boolean; + readonly request?: PlacementRequest; + } = {}, +) => + tab.placement.placeDraftBeforeSend({ + draftId: tab.store.DraftId.make(draftId), + request: options.request ?? request, + candidateEnvironmentIds: [origin, box], + pinnedHere: options.pinnedHere ?? false, + admit: admit as never, + canLaunchOn: options.canLaunchOn ?? (() => true), + composerUnchanged: () => true, + }); + +/** The origin refuses with one of its typed reasons. */ +const refusing = + (reason: OrchestrationV2ThreadPlacementAdmitError["reason"], asked?: Array): Admit => + async ({ input }) => { + asked?.push(input.commandId); + return AsyncResult.failure( + Cause.fail( + new OrchestrationV2ThreadPlacementAdmitError({ + commandId: input.commandId, + reason, + detail: `refused: ${reason}`, + }), + ), + ); + }; + +describe("placing an automatic draft before its send", () => { + it("stores the operation before asking, and where it was bound before anything moves", async () => { + await withStorage(async (memory) => { + const tab = await openTab(); + const draftId = composeDraft(tab, "ordered"); + const seen: Array<{ environmentId: EnvironmentId; commandId: string; stored: unknown }> = []; + const outcome = await place(tab, draftId, admittedTo(seen, memory, draftId)); + expect(outcome).toEqual({ type: "placed", environmentId: box, projectId: projectThere }); + // The server was asked only once storage held the operation, without a binding yet. + expect(seen).toHaveLength(1); + expect(seen[0]?.environmentId).toBe(origin); + expect(seen[0]?.stored).toMatchObject({ + commandId: seen[0]?.commandId, + admittedOn: origin, + admittedProjectId: projectHere, + }); + expect((seen[0]?.stored as { admission?: unknown } | undefined)?.admission).toBeUndefined(); + // Where it was bound is stored before the draft moves there. + expect(storedOperation(memory, draftId)).toMatchObject({ + commandId: seen[0]?.commandId, + admission: { environmentId: box, projectId: projectThere }, + }); + // Placing moves nothing itself: the routing hook retargets after this answer. + expect( + tab.store.useComposerDraftStore.getState().getDraftSession(tab.store.DraftId.make(draftId)) + ?.environmentId, + ).toBe(origin); + }); + }); + + it("asks the same server for the same operation after a lost answer and a reload", async () => { + await withStorage(async (memory) => { + const tab = await openTab(); + const draftId = composeDraft(tab, "retried"); + const asked: Array<{ environmentId: EnvironmentId; commandId: string }> = []; + // The server bound it, but its answer was lost. + const lost: Admit = async ({ environmentId, input }) => { + asked.push({ environmentId, commandId: input.commandId }); + return AsyncResult.failure(Cause.die("connection lost")); + }; + expect((await place(tab, draftId, lost)).type).toBe("failed"); + const reserved = storedOperation(memory, draftId); + expect(reserved?.admission).toBeUndefined(); + + vi.clearAllTimers(); + vi.resetModules(); + const reloaded = await openTab(); + const seen: Array<{ environmentId: EnvironmentId; commandId: string; stored: unknown }> = []; + expect((await place(reloaded, draftId, admittedTo(seen, memory, draftId))).type).toBe( + "placed", + ); + expect( + [...asked, ...seen].map(({ environmentId, commandId }) => [environmentId, commandId]), + ).toEqual([ + [origin, reserved.commandId], + [origin, reserved.commandId], + ]); + // Bound and stored, a retry still asks that server under this sign-in for the same one. + const again: Array<{ environmentId: EnvironmentId; commandId: string; stored: unknown }> = []; + expect((await place(reloaded, draftId, admittedTo(again, memory, draftId))).type).toBe( + "placed", + ); + expect(again.map(({ environmentId, commandId }) => [environmentId, commandId])).toEqual([ + [origin, reserved.commandId], + ]); + }); + }); + + it("asks nowhere when storage drops the operation, and keeps a known refusal's operation", async () => { + await withStorage(async (memory) => { + const tab = await openTab(); + const draftId = composeDraft(tab, "unsaved"); + const seen: Array<{ environmentId: EnvironmentId; commandId: string; stored: unknown }> = []; + storageFaults.drops = true; + const unsaved = await place(tab, draftId, admittedTo(seen, memory, draftId)); + storageFaults.drops = false; + expect(unsaved.type).toBe("failed"); + expect(seen).toHaveLength(0); + expect( + tab.store.useComposerDraftStore.getState().getDraftSession(tab.store.DraftId.make(draftId)) + ?.placementOperation ?? null, + ).toBeNull(); + + const refused: Admit = async ({ input }) => + AsyncResult.failure( + Cause.fail( + new OrchestrationV2ThreadPlacementAdmitError({ + commandId: input.commandId, + reason: "other-session", + detail: "This launch was started from another sign-in.", + }), + ), + ); + const outcome = await place(tab, draftId, refused); + expect(outcome).toMatchObject({ + type: "failed", + description: "This launch was started from another sign-in.", + }); + // The operation stays, so it is never admitted afresh under a new identity. + expect(storedOperation(memory, draftId)?.commandId).toBeDefined(); + }); + }); + + it("launches nowhere when this client's grant there is gone by the time it is bound", async () => { + await withStorage(async (memory) => { + const tab = await openTab(); + const draftId = composeDraft(tab, "revoked"); + const seen: Array<{ environmentId: EnvironmentId; commandId: string; stored: unknown }> = []; + const outcome = await place(tab, draftId, admittedTo(seen, memory, draftId), { + canLaunchOn: () => false, + }); + expect(outcome.type).toBe("failed"); + // Its binding is kept for a later retry; the draft stays where it was. + expect(storedOperation(memory, draftId)?.admission).toEqual({ + environmentId: box, + projectId: projectThere, + }); + expect( + tab.store.useComposerDraftStore.getState().getDraftSession(tab.store.DraftId.make(draftId)) + ?.environmentId, + ).toBe(origin); + }); + }); + + it.each(["other-session", "excluded", "unavailable"] as const)( + "never follows a stored binding its server now refuses (%s), after a reload", + async (reason) => { + await withStorage(async (memory) => { + const tab = await openTab(); + const draftId = composeDraft(tab, `reconciled-${reason}`); + const seen: Array<{ environmentId: EnvironmentId; commandId: string; stored: unknown }> = + []; + expect((await place(tab, draftId, admittedTo(seen, memory, draftId))).type).toBe("placed"); + const bound = storedOperation(memory, draftId); + vi.clearAllTimers(); + vi.resetModules(); + const reloaded = await openTab(); + // Another sign-in now, the machine excluded since, or this sign-in gone: refused there. + const asked: Array = []; + const outcome = await place(reloaded, draftId, refusing(reason, asked)); + expect(outcome.type).toBe("failed"); + expect(asked).toEqual([bound.commandId]); + // The operation and its binding stay as they were, for reconciling where it runs. + expect(storedOperation(memory, draftId)).toEqual(bound); + expect( + reloaded.store.useComposerDraftStore + .getState() + .getDraftSession(reloaded.store.DraftId.make(draftId))?.environmentId, + ).toBe(origin); + }); + }, + ); + + it("refuses changed work while an operation is open, without asking or replacing it", async () => { + await withStorage(async (memory) => { + const tab = await openTab(); + const draftId = composeDraft(tab, "changed"); + const lost: Admit = async () => AsyncResult.failure(Cause.die("connection lost")); + expect((await place(tab, draftId, lost)).type).toBe("failed"); + const reserved = storedOperation(memory, draftId); + vi.clearAllTimers(); + vi.resetModules(); + const reloaded = await openTab(); + const asked: Array = []; + const changed: ReadonlyArray = [ + { + ...request, + modelSelection: { + instanceId: ProviderInstanceId.make("claudeAgent"), + model: "claude-opus-5-5", + }, + }, + { ...request, runtimeMode: "approval-required" }, + ]; + for (const changedRequest of changed) { + const outcome = await place(reloaded, draftId, refusing("unavailable", asked), { + request: changedRequest, + }); + expect(outcome).toMatchObject({ + type: "failed", + title: "Your last send's placement is still open", + }); + } + expect(asked).toEqual([]); + expect(storedOperation(memory, draftId)).toEqual(reserved); + // Sent unchanged, it is the same operation again. + const seen: Array<{ environmentId: EnvironmentId; commandId: string; stored: unknown }> = []; + expect((await place(reloaded, draftId, admittedTo(seen, memory, draftId))).type).toBe( + "placed", + ); + expect(seen.map(({ commandId }) => commandId)).toEqual([reserved.commandId]); + }); + }); + + it("starts new work only after its server refused the last one without binding it", async () => { + await withStorage(async (memory) => { + const tab = await openTab(); + const draftId = composeDraft(tab, "released"); + const asked: Array = []; + expect((await place(tab, draftId, refusing("no-eligible-environment", asked))).type).toBe( + "failed", + ); + expect(storedOperation(memory, draftId) ?? null).toBeNull(); + const seen: Array<{ environmentId: EnvironmentId; commandId: string; stored: unknown }> = []; + expect((await place(tab, draftId, admittedTo(seen, memory, draftId))).type).toBe("placed"); + expect(seen[0]?.commandId).not.toBe(asked[0]); + }); + }); + + it("keeps attachments on the machine they were added on, rather than moving them", async () => { + await withStorage(async (memory) => { + const tab = await openTab(); + const draftId = composeDraft(tab, "attached"); + const seen: Array<{ environmentId: EnvironmentId; commandId: string; stored: unknown }> = []; + const outcome = await place(tab, draftId, admittedTo(seen, memory, draftId), { + pinnedHere: true, + }); + expect(outcome).toMatchObject({ type: "failed" }); + expect( + tab.store.useComposerDraftStore.getState().getDraftSession(tab.store.DraftId.make(draftId)) + ?.environmentId, + ).toBe(origin); + }); + }); + + it("reconciles a held operation first, whatever automatic routing allows new work", async () => { + await withStorage(async () => { + const tab = await openTab(); + const held = { placementOperation: heldOperation, promotedTo: null }; + const fresh = { placementOperation: null, promotedTo: null }; + const multiple: PlacementRequest = { ...request, multipleModels: true }; + const cases: ReadonlyArray< + readonly [typeof held | typeof fresh, boolean, boolean, PlacementRequest, boolean] + > = [ + [held, false, true, multiple, true], + [held, true, false, request, true], + [fresh, true, false, request, true], + [fresh, false, false, request, false], + [fresh, true, true, request, false], + [fresh, true, false, multiple, false], + ]; + for (const [draft, automatic, pinnedHere, asked, places] of cases) { + expect( + tab.placement.placesBeforeSend({ draft, automatic, pinnedHere, request: asked }), + ).toBe(places); + } + // A send already bound for its launch is not placed again. + expect( + tab.placement.placesBeforeSend({ + draft: { ...held, promotedTo: scopeThreadRef(box, ThreadId.make("thread")) }, + automatic: true, + pinnedHere: false, + request, + }), + ).toBe(false); + }); + }); +}); + +const heldOperation = { + commandId: "command", + messageId: "message", + createdAt: "2026-11-18T12:00:00.000Z", + admittedOn: origin, + admittedProjectId: projectHere, + intent: "{}", +}; + +describe("PR74-R1-F3: a manual pin after a saved admission", () => { + it("stops for reconciliation instead of resuming the old automatic destination", async () => { + await withStorage(async (memory) => { + const tab = await openTab(); + const draftId = composeDraft(tab, "pinned"); + const store = tab.store.useComposerDraftStore.getState(); + const seen: Array<{ environmentId: EnvironmentId; commandId: string; stored: unknown }> = []; + // Saved A→B, but the composer changed during the await: the draft stays on A. + const abandoned = await tab.placement.placeDraftBeforeSend({ + draftId: tab.store.DraftId.make(draftId), + request, + candidateEnvironmentIds: [origin, box], + pinnedHere: false, + admit: admittedTo(seen, memory, draftId) as never, + canLaunchOn: () => true, + composerUnchanged: () => false, + }); + expect(abandoned.type).toBe("abandoned"); + const bound = storedOperation(memory, draftId); + expect(bound.admission).toEqual({ environmentId: box, projectId: projectThere }); + // The user then pins A by hand, as the machine picker does. + store.setDraftThreadContext(tab.store.DraftId.make(draftId), { + projectRef: scopeProjectRef(origin, projectHere), + environmentSelection: "manual", + loadBalancedEnvironmentId: null, + }); + const asked: Array = []; + const outcome = await place(tab, draftId, refusing("unavailable", asked)); + expect(outcome).toMatchObject({ + type: "failed", + title: "Your last send's placement is still open", + }); + expect(asked).toEqual([]); + const after = tab.store.useComposerDraftStore + .getState() + .getDraftSession(tab.store.DraftId.make(draftId)); + expect(after?.environmentSelection).toBe("manual"); + expect(after?.environmentId).toBe(origin); + expect(storedOperation(memory, draftId)).toEqual(bound); + }); + }); +}); + +describe("PR74-R1-F2: a direct annotation send while an operation is held", () => { + it("reconciles the held operation first and launches nothing elsewhere or afresh", async () => { + await withStorage(async (memory) => { + const tab = await openTab(); + const draftId = composeDraft(tab, "annotated"); + const id = tab.store.DraftId.make(draftId); + const seen: Array<{ environmentId: EnvironmentId; commandId: string; stored: unknown }> = []; + // A bound B, but the composer changed while it was asked: the draft stays on A. + await tab.placement.placeDraftBeforeSend({ + draftId: id, + request, + candidateEnvironmentIds: [origin, box], + pinnedHere: false, + admit: admittedTo(seen, memory, draftId) as never, + canLaunchOn: () => true, + composerUnchanged: () => false, + }); + const bound = storedOperation(memory, draftId); + const session = () => tab.store.useComposerDraftStore.getState().getDraftSession(id); + // An annotation send is not new automatic work, but the held operation still goes first. + expect( + tab.placement.placesBeforeSend({ + draft: session(), + automatic: false, + pinnedHere: true, + request, + }), + ).toBe(true); + // Its origin refuses now: nothing is placed, and nothing may launch on A. + const asked: Array = []; + const refused = await place(tab, draftId, refusing("other-session", asked), { + pinnedHere: true, + }); + expect(refused.type).toBe("failed"); + expect(asked).toEqual([bound.commandId]); + // Its origin answers B: the annotation belongs to A's preview, so it does not move. + const kept = await place(tab, draftId, admittedTo(seen, memory, draftId), { + pinnedHere: true, + }); + expect(kept).toMatchObject({ + type: "failed", + title: "Your last send's placement is still open", + }); + // The send itself on A is blocked: no replacement command, no launch. + expect( + tab.placement.heldLaunchFor({ + draft: session(), + environmentId: origin, + projectId: projectHere, + }), + ).toEqual({ type: "blocked" }); + expect(storedOperation(memory, draftId)).toEqual(bound); + // On B, where it was bound, the send goes out as the reserved operation. + tab.store.useComposerDraftStore.getState().setDraftThreadContext(id, { + projectRef: scopeProjectRef(box, projectThere), + environmentSelection: "auto", + loadBalancedEnvironmentId: box, + }); + expect( + tab.placement.heldLaunchFor({ + draft: session(), + environmentId: box, + projectId: projectThere, + }), + ).toEqual({ type: "reserved", operation: bound }); + // A draft holding nothing sends as before. + expect( + tab.placement.heldLaunchFor({ draft: null, environmentId: origin, projectId: projectHere }), + ).toEqual({ type: "none" }); + }); + }); +}); diff --git a/apps/web/src/components/chat/draftPlacement.ts b/apps/web/src/components/chat/draftPlacement.ts new file mode 100644 index 000000000000..af081830eca2 --- /dev/null +++ b/apps/web/src/components/chat/draftPlacement.ts @@ -0,0 +1,345 @@ +import type { + EnvironmentId, + ModelSelection, + OrchestrationV2ThreadPlacementAdmitInput, + OrchestrationV2ThreadPlacementAdmitResult, + ProjectId, + ProviderInteractionMode, + RuntimeMode, +} from "@t3tools/contracts"; +import { CommandId } from "@t3tools/contracts"; +import { + type AtomCommandResult, + squashAtomCommandFailure, +} from "@t3tools/client-runtime/state/runtime"; + +import { + type ComposerFileAttachment, + type ComposerImageAttachment, + type ComposerThreadTarget, + composerFileNeedsReattach, + type DraftId, + type DraftPlacementOperation, + type DraftSessionState, + recordDraftPlacementAdmission, + releaseDraftPlacement, + reserveDraftPlacement, + useComposerDraftStore, +} from "../../composerDraftStore"; +import { newCommandId, newMessageId } from "~/lib/utils"; + +/** What a send asks for, read from the composer at send. */ +export interface DraftPlacementRequest { + readonly modelSelection: ModelSelection; + readonly runtimeMode: RuntimeMode; + readonly interactionMode: ProviderInteractionMode; + /** A multiple-model send prepares its own worktrees here, so it is not placed elsewhere. */ + readonly multipleModels: boolean; +} + +/** + * The work a placement is for, fixed when its operation is reserved: the exact model with its + * options, the modes, the logical project, the workspace and the routing choice. A different one, + * such as a machine pinned by hand since, is new work. + */ +export const draftPlacementIntent = ( + request: DraftPlacementRequest, + draft: Pick< + DraftSessionState, + | "logicalProjectKey" + | "branch" + | "worktreePath" + | "envMode" + | "startFromOrigin" + | "environmentSelection" + >, +) => + JSON.stringify({ + environmentSelection: draft.environmentSelection === "manual" ? "manual" : "auto", + modelSelection: request.modelSelection, + runtimeMode: request.runtimeMode, + interactionMode: request.interactionMode, + multipleModels: request.multipleModels, + logicalProjectKey: draft.logicalProjectKey, + branch: draft.branch, + worktreePath: draft.worktreePath, + envMode: draft.envMode, + startFromOrigin: draft.startFromOrigin, + }); + +/** + * Whether a draft is automatic work: routed by this client's Auto choice, not pinned anywhere. + * Fresh attachments do not pin it; their uploads wait for its placement instead. + */ +export const draftRoutesAutomatically = (input: { + readonly clientSettingsHydrated: boolean; + readonly isDraft: boolean; + readonly envLocked: boolean; + readonly canAutoBalanceEnvironments: boolean; + readonly loadBalancingEnabled: boolean; + readonly draft: Pick< + DraftSessionState, + "environmentSelection" | "sentTo" | "loadBalancedEnvironmentId" | "branch" | "worktreePath" + > | null; + /** Kept for callers; attachments not yet uploaded no longer decide it. */ + readonly composerHasAttachments?: boolean; +}) => { + const { draft } = input; + return Boolean( + input.clientSettingsHydrated && + input.isDraft && + !input.envLocked && + input.canAutoBalanceEnvironments && + input.loadBalancingEnabled && + draft?.environmentSelection !== "manual" && + // A send that may have started there keeps its thread id on that machine. + !draft?.sentTo && + (!draft?.branch || draft.environmentSelection === "auto") && + !draft?.worktreePath, + ); +}; + +/** + * Whether a send goes through placement first: always when its draft holds an operation already + * (reserved, uncertain or bound), whatever automatic routing says now; otherwise only new work + * automatic routing may place. + */ +export const placesBeforeSend = (input: { + readonly draft: Pick | null; + readonly automatic: boolean; + /** Pinned to the machine it is on: attachments uploaded there, or a preview annotation. */ + readonly pinnedHere: boolean; + readonly request: DraftPlacementRequest; +}) => + input.draft !== null && + !input.draft.promotedTo && + (Boolean(input.draft.placementOperation) || + (input.automatic && !input.pinnedHere && !input.request.multipleModels)); + +/** + * What a draft's send may launch under, read at the send itself: nothing held, the operation its + * origin bound to this machine, or blocked, when one is held for another machine or unbound. A + * blocked send makes no effect and never mints a replacement. + */ +export const heldLaunchFor = (input: { + readonly draft: Pick | null; + readonly environmentId: EnvironmentId; + readonly projectId: ProjectId; +}): + | { readonly type: "none" } + | { readonly type: "reserved"; readonly operation: DraftPlacementOperation } + | { readonly type: "blocked" } => { + const operation = input.draft?.placementOperation; + if (!operation) return { type: "none" }; + return operation.admission?.environmentId === input.environmentId && + operation.admission.projectId === input.projectId + ? { type: "reserved", operation } + : { type: "blocked" }; +}; + +/** + * Operations this page has reconciled with their origin and that answered with their stored + * binding. Memory only: a reload forgets it, so a cached binding never authorizes by itself. + */ +const reconciledThisPage = new Set(); +const reconciledKey = ( + commandId: string, + admission: NonNullable, +) => `${commandId}\n${admission.environmentId}\n${admission.projectId}`; + +/** + * Whether a draft's attachments wait to upload: automatic work until its origin has bound it to + * the machine the draft is on and answered so on this page. A client's load-balanced pick, or a + * binding read back from storage alone, is not that answer. + */ +export const attachmentUploadsWaitForPlacement = (input: { + readonly draft: Pick | null; + readonly automatic: boolean; + readonly environmentId: EnvironmentId; +}) => { + const operation = input.draft?.placementOperation; + if (!operation) return input.automatic; + return !( + operation.admission?.environmentId === input.environmentId && + reconciledThisPage.has(reconciledKey(operation.commandId, operation.admission)) + ); +}; + +/** + * Starts the composer's attachment uploads to `environmentId` over this client's own connection + * there, unless they wait for placement. Returns how many it started. + */ +export function startComposerAttachmentUploads(input: { + readonly environmentId: EnvironmentId; + readonly attachments: ReadonlyArray; + readonly draftTarget: ComposerThreadTarget; + readonly waitForPlacement: boolean; + readonly start: (upload: { + readonly environmentId: EnvironmentId; + readonly image: ComposerImageAttachment | ComposerFileAttachment; + readonly draftTarget: ComposerThreadTarget; + }) => void; +}): number { + if (input.waitForPlacement) return 0; + let started = 0; + for (const attachment of input.attachments) { + if (attachment.type === "file" && composerFileNeedsReattach(attachment)) continue; + input.start({ + environmentId: input.environmentId, + image: attachment, + draftTarget: input.draftTarget, + }); + started += 1; + } + return started; +} + +export type DraftPlacementOutcome = + | { + readonly type: "placed"; + readonly environmentId: EnvironmentId; + readonly projectId: ProjectId; + } + | { readonly type: "failed"; readonly title: string; readonly description: string } + /** The draft changed while it was being placed: its send is dropped, its binding kept. */ + | { readonly type: "abandoned" }; + +const unsaved = (title: string): DraftPlacementOutcome => ({ + type: "failed", + title, + description: "Nothing was started. Free up browser storage, then send again.", +}); + +const stillOpen = (description: string): DraftPlacementOutcome => ({ + type: "failed", + title: "Your last send's placement is still open", + description, +}); + +const refusalReason = (error: unknown) => + typeof error === "object" && + error !== null && + "_tag" in error && + error._tag === "OrchestrationV2ThreadPlacementAdmitError" && + "reason" in error + ? String(error.reason) + : undefined; + +/** + * Where a draft's new thread runs: its operation and intent are stored first; the server asked is + * asked again under this sign-in every time, even once bound, and its binding is never rescored. + * Changed work is refused while the operation is open. Nothing is uploaded or launched here. + */ +export async function placeDraftBeforeSend(input: { + readonly draftId: DraftId; + readonly request: DraftPlacementRequest; + /** The machines this client may launch on; they narrow the server's choice. */ + readonly candidateEnvironmentIds: ReadonlyArray; + /** Pinned to the machine it is on: attachments uploaded there, or a preview annotation. */ + readonly pinnedHere: boolean; + readonly admit: (target: { + readonly environmentId: EnvironmentId; + readonly input: OrchestrationV2ThreadPlacementAdmitInput; + }) => Promise>; + /** Whether this client may still launch there, read when it is needed. */ + readonly canLaunchOn: (environmentId: EnvironmentId) => boolean; + /** Whether the composer is still what this send read. */ + readonly composerUnchanged: () => boolean; +}): Promise { + const { draftId } = input; + const session = useComposerDraftStore.getState().getDraftSession(draftId); + if (!session) return { type: "abandoned" }; + const intent = draftPlacementIntent(input.request, session); + const existing = session.placementOperation ?? null; + if (existing) { + if (existing.intent !== intent) + return stillOpen( + "It may already be running with what you sent then. Send that unchanged to finish it, or start a new draft for different work.", + ); + if ( + session.environmentId !== existing.admittedOn && + session.environmentId !== existing.admission?.environmentId + ) + return stillOpen( + "It was placed from another machine than this draft is on now. Switch back to send it, or start a new draft.", + ); + } + // Asked again: until its origin answers this time, its binding authorizes nothing here. + if (existing?.admission) + reconciledThisPage.delete(reconciledKey(existing.commandId, existing.admission)); + const reserved: DraftPlacementOperation = existing ?? { + commandId: newCommandId(), + messageId: newMessageId(), + createdAt: new Date().toISOString(), + admittedOn: session.environmentId, + admittedProjectId: session.projectId, + intent, + }; + if (!existing && !reserveDraftPlacement(draftId, reserved)) + return unsaved("Could not save this draft before sending"); + const result = await input.admit({ + environmentId: reserved.admittedOn, + input: { + commandId: CommandId.make(reserved.commandId), + threadId: session.threadId, + projectId: reserved.admittedProjectId, + modelSelection: input.request.modelSelection, + runtimeMode: input.request.runtimeMode, + interactionMode: input.request.interactionMode, + candidateEnvironmentIds: input.candidateEnvironmentIds, + }, + }); + if (result._tag !== "Success") { + const error = squashAtomCommandFailure(result); + // Refused before anything was bound: the next send is new work, under a new identity. + if (refusalReason(error) === "no-eligible-environment") + releaseDraftPlacement(draftId, reserved.commandId); + return { + type: "failed", + title: "Could not place this draft", + description: + error instanceof Error && error.message.length > 0 + ? error.message + : "Nothing was started. Send it again.", + }; + } + const admission = { + environmentId: result.value.environmentId, + projectId: result.value.projectId, + }; + if (reserved.admission) { + // Its server answers with the binding it holds; anything else is never followed. + if ( + reserved.admission.environmentId !== admission.environmentId || + reserved.admission.projectId !== admission.projectId + ) + return stillOpen( + "Its machine answered with another destination than it was bound to, so nothing was sent.", + ); + } else if (!recordDraftPlacementAdmission(draftId, reserved.commandId, admission)) { + return unsaved("Could not save where this draft runs"); + } + const current = useComposerDraftStore.getState().getDraftSession(draftId); + if ( + !input.composerUnchanged() || + current?.placementOperation?.commandId !== reserved.commandId || + current.environmentId !== session.environmentId || + current.projectId !== session.projectId + ) + return { type: "abandoned" }; + // What is pinned to the machine the draft is on never moves with it. + if (input.pinnedHere && admission.environmentId !== session.environmentId) + return stillOpen( + "It runs on another machine than the attachments or annotation it now holds. Remove them and add them there, or start a new draft.", + ); + // The grant to launch there is this client's own, checked again now that it is needed. + if (!input.canLaunchOn(admission.environmentId)) { + return { + type: "failed", + title: "This machine no longer accepts your launches", + description: "Its sign-in changed while the draft was being placed. Nothing was started.", + }; + } + reconciledThisPage.add(reconciledKey(reserved.commandId, admission)); + return { type: "placed", ...admission }; +} diff --git a/apps/web/src/components/chat/draftUploadReconcile.test.ts b/apps/web/src/components/chat/draftUploadReconcile.test.ts new file mode 100644 index 000000000000..704342138540 --- /dev/null +++ b/apps/web/src/components/chat/draftUploadReconcile.test.ts @@ -0,0 +1,484 @@ +import { EnvironmentId, ProjectId, ProviderInstanceId, ThreadId } from "@t3tools/contracts"; +import { scopeProjectRef } from "@t3tools/client-runtime/environment"; +import { AsyncResult, Atom } from "effect/reactivity"; +import { describe, expect, it, vi } from "vite-plus/test"; +import { createMemoryStorage } from "../../lib/storage"; + +// PR74-R1-F4 (caller disposition D3): every upload of an admitted draft is qualified by its origin +// at the queue itself. One page and one real queue stay alive throughout; only network edges are +// doubles: the origin's admission answer, the upload cycle and persisted-upload verification. +const net = vi.hoisted(() => ({ + origin: { answer: "bound" as "bound" | "other-session" | "excluded" | "lost" }, + admissions: [] as Array<{ environmentId: string; commandId: string }>, + /** Held admission answers, released by the test. */ + hold: null as null | Promise, + cycles: [] as Array<{ environmentId: string; name: string }>, + verifications: [] as Array<{ environmentId: string; attachmentId: string }>, + failNextCycle: false, + /** Machines where this client's operate access was revoked. */ + denied: new Set(), +})); +vi.mock("../../state/session", async (importOriginal) => ({ + ...(await importOriginal()), + readEnvironmentScope: (environmentId: string) => !net.denied.has(environmentId), +})); +vi.mock("@t3tools/client-runtime/state/attachments", async (importOriginal) => ({ + ...(await importOriginal()), + verifyPersistedAttachmentUpload: async (input: { + readonly environmentId: string; + readonly attachmentId: string; + }) => { + net.verifications.push({ + environmentId: input.environmentId, + attachmentId: input.attachmentId, + }); + return { status: "verified" as const }; + }, + runAttachmentUploadCycle: async (input: { + readonly environmentId: string; + readonly upload: { readonly name: string }; + }) => { + net.cycles.push({ environmentId: input.environmentId, name: input.upload.name }); + if (net.failNextCycle) { + net.failNextCycle = false; + return { status: "failed" as const, step: "transfer" as const, error: new Error("lost") }; + } + return { status: "uploaded" as const, attachmentId: `upload-${net.cycles.length}` }; + }, +})); +// Connection state and session generation per machine, which the test changes. +const connection = vi.hoisted(() => ({ + state: new Map(), + generation: new Map(), +})); +vi.mock("../../connection/catalog", async () => { + const { AsyncResult: Result, Atom: A } = await import("effect/reactivity"); + const family = (map: Map, initial: () => unknown) => (environmentId: string) => { + if (!map.has(environmentId)) map.set(environmentId, A.make(initial())); + return map.get(environmentId); + }; + return { + environmentCatalog: { + stateAtom: family(connection.state, () => Result.success({ phase: "connected" })), + sessionGenerationAtom: family(connection.generation, () => 1), + }, + }; +}); +// The origin's admission RPC, answered as `net.origin.answer` says. +vi.mock("@t3tools/client-runtime/state/runtime", async (importOriginal) => { + const real = await importOriginal(); + const { AsyncResult: Result } = await import("effect/reactivity"); + const CauseModule = await import("effect/Cause"); + const { OrchestrationV2ThreadPlacementAdmitError: AdmitError } = + await import("@t3tools/contracts"); + return { + ...real, + runAtomCommand: async (registry: never, command: never, target: never) => { + const admission = ( + target as { input?: { candidateEnvironmentIds?: unknown; commandId?: string } } + ).input; + if (admission?.candidateEnvironmentIds === undefined) + return real.runAtomCommand(registry, command, target); + const { environmentId } = target as { environmentId: string }; + net.admissions.push({ environmentId, commandId: admission.commandId! }); + if (net.hold) await net.hold; + if (net.origin.answer === "lost") return Result.failure(CauseModule.die("connection lost")); + if (net.origin.answer !== "bound") + return Result.failure( + CauseModule.fail( + new AdmitError({ + commandId: admission.commandId as never, + reason: net.origin.answer, + detail: `refused: ${net.origin.answer}`, + }), + ), + ); + return Result.success({ + environmentId: "environment-box", + projectId: "project:box", + reason: "capacity", + resumed: true, + }); + }, + }; +}); + +const origin = EnvironmentId.make("environment-laptop"); +const box = EnvironmentId.make("environment-box"); +const projectHere = ProjectId.make("project:laptop"); +const projectThere = ProjectId.make("project:box"); +const request = { + modelSelection: { + instanceId: ProviderInstanceId.make("claudeAgent"), + model: "claude-sonnet-4-6", + }, + runtimeMode: "full-access", + interactionMode: "default", + multipleModels: false, +} as const; + +async function openPage() { + return { + store: await import("../../composerDraftStore"), + placement: await import("./draftPlacement"), + queue: await import("../../lib/attachmentUploadQueue"), + registry: (await import("../../rpc/atomRegistry")).appAtomRegistry, + catalog: (await import("../../connection/catalog")).environmentCatalog, + }; +} +type Page = Awaited>; + +async function withPage(run: (page: Page) => Promise) { + const memory = createMemoryStorage(); + const storage = { + getItem: (name: string) => memory.getItem(name), + removeItem: (name: string) => memory.removeItem(name), + setItem: (name: string, value: string) => memory.setItem(name, value), + }; + vi.stubGlobal("localStorage", storage); + vi.stubGlobal("window", Object.assign(new EventTarget(), { localStorage: storage })); + vi.useFakeTimers(); + vi.resetModules(); + net.origin.answer = "bound"; + net.admissions.length = 0; + net.cycles.length = 0; + net.verifications.length = 0; + net.hold = null; + net.failNextCycle = false; + net.denied.clear(); + connection.state.clear(); + connection.generation.clear(); + try { + await run(await openPage()); + } finally { + vi.useRealTimers(); + vi.unstubAllGlobals(); + vi.resetModules(); + } +} + +const settle = () => vi.advanceTimersByTimeAsync(0); +/** Waits for the queue's own jobs for these attachments, including a retry they started. */ +const drain = async (page: { + queue: { awaitAttachmentUploads: (ids: string[]) => Promise }; +}) => { + await settle(); + await page.queue.awaitAttachmentUploads(["shot", "later"]); + await settle(); +}; + +function image(id: string) { + const file = new File([new Uint8Array([1, 2, 3, 4])], `${id}.png`, { type: "image/png" }); + return { + type: "image" as const, + id, + name: file.name, + mimeType: file.type, + sizeBytes: file.size, + previewUrl: `blob:${id}`, + file, + }; +} + +/** An automatic draft with an image, admitted on its origin to B and moved there by routing. */ +async function admittedOnBox(page: Page) { + const id = page.store.DraftId.make("draft-held"); + const store = page.store.useComposerDraftStore.getState(); + store.setProjectDraftThreadId(scopeProjectRef(origin, projectHere), id, { + environmentSelection: "auto", + loadBalancedEnvironmentId: null, + }); + store.setPrompt(id, "Review the screenshot"); + store.addImages(id, [image("shot")]); + const placed = await page.placement.placeDraftBeforeSend({ + draftId: id, + request, + candidateEnvironmentIds: [origin, box], + pinnedHere: false, + admit: (async () => + AsyncResult.success({ + environmentId: box, + projectId: projectThere, + reason: "capacity" as const, + resumed: false, + })) as never, + canLaunchOn: () => true, + composerUnchanged: () => true, + }); + expect(placed.type).toBe("placed"); + store.setDraftThreadContext(id, { + projectRef: scopeProjectRef(box, projectThere), + environmentSelection: "auto", + loadBalancedEnvironmentId: box, + }); + return id; +} + +/** The composer's upload effect for the draft as it is now, with ChatView's own Auto rule. */ +async function uploadEffect( + page: Page, + id: ReturnType, + /** False while the origin's answer is held: nothing to wait for yet. */ + wait = true, +) { + const state = page.store.useComposerDraftStore.getState(); + const session = state.getDraftSession(id)!; + const composer = state.getComposerDraft(id)!; + page.placement.startComposerAttachmentUploads({ + environmentId: session.environmentId, + attachments: [...composer.images, ...composer.files], + draftTarget: id, + waitForPlacement: page.placement.attachmentUploadsWaitForPlacement({ + draft: session, + automatic: page.placement.draftRoutesAutomatically({ + clientSettingsHydrated: true, + isDraft: true, + envLocked: false, + canAutoBalanceEnvironments: true, + loadBalancingEnabled: true, + draft: session, + }), + environmentId: session.environmentId, + }), + start: page.queue.startAttachmentUpload, + }); + if (wait) await drain(page); + else await settle(); +} + +/** B drops its connection and comes back, as the queue's own watcher sees it. */ +async function reconnect(page: Page, environmentId: EnvironmentId) { + const atom = connection.state.get(environmentId) as Atom.Writable; + page.registry.set(atom, AsyncResult.success({ phase: "disconnected" })); + await settle(); + page.registry.set(atom, AsyncResult.success({ phase: "connected" })); + await drain(page); +} + +const targetEffects = () => [...net.cycles, ...net.verifications]; + +// The queue's own refusal reasons: a generic failure ("Upload failed") is not one of them. +const STALE = "This draft changed while its placement was confirmed, so nothing was uploaded."; +const UNCONFIRMED = "This draft's placement could not be confirmed, so nothing was uploaded."; +const OWNER_LOST = + "This upload belonged to a placed launch this draft no longer holds, so nothing was uploaded."; +const reasonOf = (page: Page, attachmentId: string) => { + const state = page.queue.readAttachmentUpload(attachmentId); + return state?.status === "failed" ? state.reason : state?.status; +}; + +describe("PR74-R1-F4: an admitted draft's uploads are qualified by its origin each time", () => { + it("retries a failed upload on reconnect only after its origin answers with the same binding", async () => { + await withPage(async (page) => { + const id = await admittedOnBox(page); + // The first upload goes out after its origin confirms the binding, and fails on the way. + net.failNextCycle = true; + await uploadEffect(page, id); + expect(net.cycles).toEqual([{ environmentId: box, name: "shot.png" }]); + expect(page.queue.readAttachmentUpload("shot")?.status).toBe("failed"); + // The origin now refuses (another sign-in, or B excluded) or cannot answer, the page alive. + for (const answer of ["other-session", "excluded", "lost"] as const) { + net.origin.answer = answer; + const before = net.admissions.length; + await reconnect(page, box); + expect(net.admissions.length).toBeGreaterThan(before); + expect(net.cycles).toHaveLength(1); + const reason = answer === "lost" ? UNCONFIRMED : `refused: ${answer}`; + expect(reasonOf(page, "shot")).toBe(reason); + const current = page.store.useComposerDraftStore.getState().getComposerDraft(id)! + .images[0]!; + page.queue.retryAttachmentUpload({ environmentId: box, image: current, draftTarget: id }); + await drain(page); + expect(net.cycles).toHaveLength(1); + expect(reasonOf(page, "shot")).toBe(reason); + } + expect(net.verifications).toEqual([]); + // Answering with the same binding again, the retry goes out to B, as the same operation. + net.origin.answer = "bound"; + await reconnect(page, box); + expect(net.cycles).toEqual([ + { environmentId: box, name: "shot.png" }, + { environmentId: box, name: "shot.png" }, + ]); + expect(net.admissions.every((asked) => asked.environmentId === origin)).toBe(true); + const operation = page.store.useComposerDraftStore + .getState() + .getDraftSession(id)!.placementOperation!; + expect(new Set(net.admissions.map((asked) => asked.commandId))).toEqual( + new Set([operation.commandId]), + ); + }); + }); + + it("sends a newly added attachment nowhere once its origin's answer has changed", async () => { + await withPage(async (page) => { + const id = await admittedOnBox(page); + await uploadEffect(page, id); + expect(net.cycles).toHaveLength(1); + net.origin.answer = "other-session"; + page.store.useComposerDraftStore.getState().addImages(id, [image("later")]); + await uploadEffect(page, id); + expect(net.cycles).toHaveLength(1); + }); + }); + + it.each([ + "the attachment is removed", + "the attachment is replaced", + "the draft moves machine", + "B's session is replaced", + ] as const)("releases nothing when %s while its origin is asked", async (change) => { + await withPage(async (page) => { + const id = await admittedOnBox(page); + let release!: () => void; + net.hold = new Promise((done) => { + release = done; + }); + await uploadEffect(page, id, false); + // Asked, and held there: nothing has gone to B yet. + expect(net.admissions).toHaveLength(1); + expect(targetEffects()).toEqual([]); + const store = page.store.useComposerDraftStore.getState(); + if (change === "the attachment is removed") store.removeImage(id, "shot"); + if (change === "the attachment is replaced") { + store.removeImage(id, "shot"); + store.addImages(id, [image("shot")], { allowDuplicates: true }); + } + if (change === "the draft moves machine") + store.setDraftThreadContext(id, { + projectRef: scopeProjectRef(origin, projectHere), + environmentSelection: "manual", + loadBalancedEnvironmentId: null, + }); + if (change === "B's session is replaced") { + const generation = connection.generation.get(box) as Atom.Writable; + page.registry.set(generation, 2); + } + release(); + await drain(page); + expect(targetEffects()).toEqual([]); + // Refused by the queue's own check, or released with the attachment: never a crash. + expect(reasonOf(page, "shot")).not.toBe("Upload failed"); + }); + }); + + it.each([ + "the origin's connection drops", + "the origin's operate access is revoked", + "the composer's text is edited", + "the same file is attached again", + ] as const)("releases nothing when %s while its origin is asked", async (change) => { + await withPage(async (page) => { + const id = await admittedOnBox(page); + let release!: () => void; + net.hold = new Promise((done) => { + release = done; + }); + await uploadEffect(page, id, false); + expect(net.admissions).toHaveLength(1); + expect(targetEffects()).toEqual([]); + const store = page.store.useComposerDraftStore.getState(); + if (change === "the origin's connection drops") + // Through the catalog itself: the origin's state may not have been read yet. + page.registry.set( + page.catalog.stateAtom(origin) as unknown as Atom.Writable, + AsyncResult.success({ phase: "disconnected" }), + ); + if (change === "the origin's operate access is revoked") net.denied.add(origin); + if (change === "the composer's text is edited") + store.setPrompt(id, "Review the screenshot and the logs"); + if (change === "the same file is attached again") { + const attached = store.getComposerDraft(id)!.images[0]!; + store.removeImage(id, "shot"); + store.addImages(id, [{ ...attached }], { allowDuplicates: true }); + } + release(); + await drain(page); + expect(targetEffects()).toEqual([]); + expect(reasonOf(page, "shot")).toBe(STALE); + }); + }); + + it("verifies nothing when a stored file's marker is replaced while its origin is asked", async () => { + await withPage(async (page) => { + const id = await admittedOnBox(page); + const store = page.store.useComposerDraftStore.getState(); + // A file already uploaded to B, kept here as its marker: no bytes, its upload's id. + const marker = { + type: "file" as const, + id: "report", + name: "report.pdf", + mimeType: "application/pdf", + sizeBytes: 12, + file: null, + uploadedAttachmentId: "upload-report", + uploadEnvironmentId: box, + }; + store.addFiles(id, [marker]); + expect(store.getComposerDraft(id)!.files[0]).toMatchObject({ + file: null, + uploadedAttachmentId: "upload-report", + }); + let release!: () => void; + net.hold = new Promise((done) => { + release = done; + }); + await uploadEffect(page, id, false); + store.removeFile(id, "report"); + store.addFiles(id, [{ ...marker }]); + release(); + await drain(page); + expect(net.verifications).toEqual([]); + expect(reasonOf(page, "report")).toBe(STALE); + }); + }); + + it("keeps a queued retry with its placed launch after the draft lets go of that launch", async () => { + await withPage(async (page) => { + const id = await admittedOnBox(page); + net.failNextCycle = true; + await uploadEffect(page, id); + expect(net.cycles).toHaveLength(1); + // The draft starts over under a new thread: its placed launch is no longer its own. + const store = page.store.useComposerDraftStore.getState(); + page.store.restoreFailedDraftThread( + id, + store.getDraftSession(id)!, + ThreadId.make("thread-new-intent"), + ); + expect(store.getDraftSession(id)?.placementOperation ?? null).toBeNull(); + // B reconnects and the queue retries the failed upload: it is not an ordinary one now. + await reconnect(page, box); + expect(net.cycles).toHaveLength(1); + expect(reasonOf(page, "shot")).toBe(OWNER_LOST); + }); + }); + + it("checks a stored upload at send only after its origin answers with the same binding", async () => { + await withPage(async (page) => { + const id = await admittedOnBox(page); + await uploadEffect(page, id); + expect(page.queue.readAttachmentUpload("shot")?.status).toBe("ready"); + const current = () => + page.store.useComposerDraftStore.getState().getComposerDraft(id)!.images[0]!; + // Its origin now refuses: the stored upload is not even verified there. + net.origin.answer = "excluded"; + await page.queue.refreshExpiredAttachmentUploads({ + environmentId: box, + images: [current()], + draftTarget: id, + }); + expect(net.verifications).toEqual([]); + expect(page.queue.readAttachmentUpload("shot")?.status).toBe("failed"); + }); + }); + + it("sends an unchanged admitted draft's upload to its bound machine after its origin answers", async () => { + await withPage(async (page) => { + const id = await admittedOnBox(page); + await uploadEffect(page, id); + expect(net.cycles).toEqual([{ environmentId: box, name: "shot.png" }]); + expect(net.admissions.map((asked) => asked.environmentId)).toEqual([origin]); + }); + }); +}); diff --git a/apps/web/src/components/chat/useComposerSend.ts b/apps/web/src/components/chat/useComposerSend.ts index ab6988099e26..0df3e650593c 100644 --- a/apps/web/src/components/chat/useComposerSend.ts +++ b/apps/web/src/components/chat/useComposerSend.ts @@ -142,6 +142,7 @@ import { readSentComposer, restoreFailedSendComposer, } from "./chatViewSend"; +import { heldLaunchFor } from "./draftPlacement"; import type { EnvironmentProject, EnvironmentThreadShell, @@ -911,15 +912,40 @@ export function useComposerSend(input: ComposerSendInput) { return; } const retriedLaunch = launchPlan?.type === "retry" ? launchPlan.launch : null; + // A new launch admitted to this machine goes out as the operation reserved for its admission. + // A held placement operation decides what this send may launch, whatever started the send. + const held = draftSessionForSend + ? heldLaunchFor({ + draft: draftSessionForSend, + environmentId, + projectId: activeProject.id, + }) + : ({ type: "none" } as const); + if (held.type === "blocked") { + toastManager.add( + stackedThreadToast({ + type: "warning", + title: "Your last send's placement is still open", + description: + "It was placed on another machine than this draft is on now. Send it again to finish it there, or start a new draft.", + }), + ); + return; + } + const reservedLaunch = + retriedLaunch === null && held.type === "reserved" ? held.operation : null; const messageIdForSend = retriedLaunch ? MessageId.make(retriedLaunch.messageId) - : newMessageId(); - const messageCreatedAt = retriedLaunch?.createdAt ?? new Date().toISOString(); + : reservedLaunch + ? MessageId.make(reservedLaunch.messageId) + : newMessageId(); + const messageCreatedAt = + retriedLaunch?.createdAt ?? reservedLaunch?.createdAt ?? new Date().toISOString(); const launchOperation: PendingDraftLaunch | undefined = launchRequest === null ? undefined : (retriedLaunch ?? { - commandId: newCommandId(), + commandId: reservedLaunch?.commandId ?? newCommandId(), messageId: messageIdForSend, createdAt: messageCreatedAt, request: launchRequest, @@ -1691,10 +1717,22 @@ export function useComposerSend(input: ComposerSendInput) { const onSendLatest = useStableCallback(onSend); const onSendWithRouting = async (...args: Parameters) => { args[0]?.preventDefault(); + const sendContext = composerRef.current?.getSendContext(); + // A direct annotation stays with its preview's machine, but held work is reconciled first. const refresh = - args[3] || sendInFlightRef.current || readPendingEnvironmentChange() !== null + sendInFlightRef.current || + readPendingEnvironmentChange() !== null || + sendContext === undefined ? null - : refreshBeforeSend(); + : refreshBeforeSend( + { + modelSelection: sendContext.selectedModelSelection, + runtimeMode, + interactionMode: DEFAULT_INTERACTION_MODE, + multipleModels: (sendContext.multipleModelSelections?.length ?? 0) > 0, + }, + { directAnnotation: Boolean(args[3]) }, + ); if (refresh === null) return onSend(...args); sendInFlightRef.current = true; const ready = await refresh.finally(() => { diff --git a/apps/web/src/components/chat/useDraftEnvironmentRouting.browser.test.tsx b/apps/web/src/components/chat/useDraftEnvironmentRouting.browser.test.tsx index b4ddb847d082..80489d7263d4 100644 --- a/apps/web/src/components/chat/useDraftEnvironmentRouting.browser.test.tsx +++ b/apps/web/src/components/chat/useDraftEnvironmentRouting.browser.test.tsx @@ -24,11 +24,26 @@ import { DraftId, useComposerDraftStore } from "../../composerDraftStore"; import { useDraftEnvironmentRouting } from "./useDraftEnvironmentRouting"; import { appAtomRegistry, AppAtomRegistryProvider } from "../../rpc/atomRegistry"; import { draftFanoutSelectionsAtom } from "./chatViewSend"; +import type { DraftPlacementOutcome, placeDraftBeforeSend } from "./draftPlacement"; + +type PlaceInput = Parameters[0]; +const sendRequest = { + modelSelection: createModelSelection(ProviderInstanceId.make("codex"), "gpt-6.1-sol"), + runtimeMode: "full-access", + interactionMode: "default", + multipleModels: false, +} as const; const resources = vi.hoisted(() => ({ projects: [] as EnvironmentProject[], pickedEnvironmentId: null as EnvironmentId | null, refreshSelection: vi.fn<(ids: readonly EnvironmentId[]) => Promise>(), + // The server's admission is covered in draftPlacement.test.ts; here, what the hook does with it. + place: vi.fn<(input: PlaceInput) => Promise>(), +})); +vi.mock("./draftPlacement", async (importOriginal) => ({ + ...(await importOriginal()), + placeDraftBeforeSend: resources.place, })); vi.mock("../../hooks/useLoadBalancedEnvironment", () => ({ useLoadBalancedEnvironment: () => ({ @@ -50,6 +65,7 @@ vi.mock("../../hooks/useScratchProject", () => ({ useScratchProject: () => ({}) function setup() { resources.refreshSelection.mockReset(); + resources.place.mockReset(); resources.projects = []; resources.pickedEnvironmentId = null; const a = EnvironmentId.make("machine-a"); @@ -123,7 +139,11 @@ function setup() { projectGroupingSettings: DEFAULT_UNIFIED_SETTINGS, sendInFlightRef: { current: false }, }; - resources.refreshSelection.mockImplementation(async (ids) => (ids.includes(b) ? b : null)); + resources.place.mockImplementation(async () => ({ + type: "placed", + environmentId: b, + projectId, + })); return { a, b, draftId, input, store }; } @@ -164,7 +184,7 @@ function RoutingProbe({ input, onPrepared }: HarnessProps) { {draft?.environmentId}