diff --git a/apps/server/src/mcp/OrchestratorMcpService.test.ts b/apps/server/src/mcp/OrchestratorMcpService.test.ts index f49bcc7c9412f..6a969e21e3abd 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.test.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.test.ts @@ -71,6 +71,7 @@ import * as ScheduledTaskService from "../scheduledTasks/ScheduledTaskService.ts import * as ServerSettings from "../serverSettings.ts"; import * as SecretRequests from "../secrets/SecretRequests.ts"; import * as RemoteDelegation from "../peer/RemoteDelegation.ts"; +import * as HostResources from "../resourceTelemetry/HostResources.ts"; import type { McpInvocationScope } from "./McpInvocationContext.ts"; import { idleThreadProjection, liveThreadShell } from "./McpToolAccess.testkit.ts"; import * as OrchestratorMcpService from "./OrchestratorMcpService.ts"; @@ -1901,6 +1902,13 @@ describe("OrchestratorMcpService provider resolution", () => { }), forkShadow, ]; + const hostSnapshot = { + sampledAt: 1_700_000_000_000, + cpuUtilization: 0.25, + cpuCount: 8, + availableMemoryBytes: 8 * 1024 ** 3, + totalMemoryBytes: 16 * 1024 ** 3, + }; const layerDependencies = Layer.mergeAll( NodeServices.layer, NodeCrypto.layer, @@ -1945,11 +1953,14 @@ describe("OrchestratorMcpService provider resolution", () => { Layer.mock(SecretRequests.SecretRequests)({}), Layer.mock(RemoteDelegation.RemoteDelegation)({}), Layer.mock(ScheduledTaskService.ScheduledTaskService)({}), + Layer.mock(HostResources.HostResources)({ read: Effect.succeed(hostSnapshot) }), ); yield* Effect.gen(function* () { const service = yield* OrchestratorMcpService.OrchestratorMcpService; const capabilities = yield* service.capabilities(scope); + // Linked environments read this for placement. + assert.deepEqual(capabilities.hostResources, hostSnapshot); const byId = new Map( capabilities.providers.map((provider) => [provider.providerInstanceId, provider]), ); diff --git a/apps/server/src/mcp/OrchestratorMcpService.ts b/apps/server/src/mcp/OrchestratorMcpService.ts index 97d578f527b57..b86a564c19f9d 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.ts @@ -136,6 +136,7 @@ import { import { assertTargetWithinLimits, isLiveCallerShell, threadNotFound } from "./threadAccess.ts"; import * as SecretRequests from "../secrets/SecretRequests.ts"; import * as PeerForwarding from "../peer/PeerForwarding.ts"; +import * as HostResources from "../resourceTelemetry/HostResources.ts"; import * as RemoteDelegation from "../peer/RemoteDelegation.ts"; // Return recoverable handles before common MCP client timeouts expire. @@ -1260,6 +1261,7 @@ function timelineItemsWithinBudget( const make = Effect.gen(function* () { const random = yield* SecureRandom.SecureRandom; + const hostResources = yield* Effect.serviceOption(HostResources.HostResources); const threadManagement = yield* ThreadManagementService.ThreadManagementService; const providerRegistry = yield* ProviderRegistry.ProviderRegistry; const providerAdapters = yield* ProviderAdapterRegistry.ProviderAdapterRegistryV2; @@ -2562,7 +2564,9 @@ const make = Effect.gen(function* () { ); const inherited = parent?.thread.modelSelection; const searchTerms = input.search?.toLowerCase().split(/\s+/).filter(Boolean) ?? []; + const resources = Option.isSome(hostResources) ? yield* hostResources.value.read : undefined; return { + ...(resources === undefined ? {} : { hostResources: resources }), parentThreadId: parent?.thread.id ?? null, inheritedProviderInstanceId: inherited?.instanceId ?? null, inheritedModel: inherited?.model ?? null, diff --git a/packages/client-runtime/src/load-balancing.ts b/packages/client-runtime/src/load-balancing.ts index 0b938c6c090e4..84c9c8bd0cb60 100644 --- a/packages/client-runtime/src/load-balancing.ts +++ b/packages/client-runtime/src/load-balancing.ts @@ -1,40 +1 @@ -import type { HostResourcesSnapshot } from "@t3tools/contracts"; - -/** Callers supply only connected machines hosting the project and selected provider. */ -export function chooseLoadBalancedEnvironment( - candidates: ReadonlyArray<{ - environmentId: string; - resources: HostResourcesSnapshot | null; - /** Client receipt time avoids comparing clocks on different machines. */ - receivedAt?: number; - weight: number; - }>, - now: number, -): string | null { - let selected: string | null = null; - let bestScore = 0; - for (const { environmentId, resources, receivedAt, weight } of candidates) { - const sampledAt = receivedAt ?? resources?.sampledAt ?? 0; - if ( - !resources || - !Number.isFinite(weight) || - weight <= 0 || - now - sampledAt > 15_000 || - sampledAt > now + 5_000 || - resources.cpuUtilization === null || - resources.cpuUtilization >= 0.95 || - resources.totalMemoryBytes <= 0 || - resources.cpuCount <= 0 - ) { - continue; - } - const memoryAvailable = resources.availableMemoryBytes / resources.totalMemoryBytes; - if (memoryAvailable <= 0.05) continue; - const score = weight * resources.cpuCount * (1 - resources.cpuUtilization) * memoryAvailable; - if (score > bestScore) { - selected = environmentId; - bestScore = score; - } - } - return selected; -} +export { chooseLoadBalancedEnvironment } from "@t3tools/shared/loadBalancing"; diff --git a/packages/contracts/src/orchestratorMcp.ts b/packages/contracts/src/orchestratorMcp.ts index 3d987a2773ad1..c30c4fea2e785 100644 --- a/packages/contracts/src/orchestratorMcp.ts +++ b/packages/contracts/src/orchestratorMcp.ts @@ -1,4 +1,5 @@ import * as Effect from "effect/Effect"; +import { HostResourcesSnapshot } from "./resourceTelemetry.ts"; import * as Schema from "effect/Schema"; import * as SchemaTransformation from "effect/SchemaTransformation"; @@ -706,6 +707,8 @@ export const OrchestratorMcpCapabilitiesResult = Schema.Struct({ scheduledTasks: Schema.Boolean, maxBatchThreads: Schema.Number, }), + /** This machine's load when answered; placement reads it from linked environments. */ + hostResources: Schema.optional(HostResourcesSnapshot), }); export type OrchestratorMcpCapabilitiesResult = typeof OrchestratorMcpCapabilitiesResult.Type; diff --git a/packages/shared/package.json b/packages/shared/package.json index 3de18958220ef..29b36a3583fe1 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -195,6 +195,14 @@ "types": "./src/remote.ts", "import": "./src/remote.ts" }, + "./loadBalancing": { + "types": "./src/loadBalancing.ts", + "import": "./src/loadBalancing.ts" + }, + "./threadPlacement": { + "types": "./src/threadPlacement.ts", + "import": "./src/threadPlacement.ts" + }, "./relaySigning": { "types": "./src/relaySigning.ts", "import": "./src/relaySigning.ts" diff --git a/packages/shared/src/loadBalancing.ts b/packages/shared/src/loadBalancing.ts new file mode 100644 index 0000000000000..0b938c6c090e4 --- /dev/null +++ b/packages/shared/src/loadBalancing.ts @@ -0,0 +1,40 @@ +import type { HostResourcesSnapshot } from "@t3tools/contracts"; + +/** Callers supply only connected machines hosting the project and selected provider. */ +export function chooseLoadBalancedEnvironment( + candidates: ReadonlyArray<{ + environmentId: string; + resources: HostResourcesSnapshot | null; + /** Client receipt time avoids comparing clocks on different machines. */ + receivedAt?: number; + weight: number; + }>, + now: number, +): string | null { + let selected: string | null = null; + let bestScore = 0; + for (const { environmentId, resources, receivedAt, weight } of candidates) { + const sampledAt = receivedAt ?? resources?.sampledAt ?? 0; + if ( + !resources || + !Number.isFinite(weight) || + weight <= 0 || + now - sampledAt > 15_000 || + sampledAt > now + 5_000 || + resources.cpuUtilization === null || + resources.cpuUtilization >= 0.95 || + resources.totalMemoryBytes <= 0 || + resources.cpuCount <= 0 + ) { + continue; + } + const memoryAvailable = resources.availableMemoryBytes / resources.totalMemoryBytes; + if (memoryAvailable <= 0.05) continue; + const score = weight * resources.cpuCount * (1 - resources.cpuUtilization) * memoryAvailable; + if (score > bestScore) { + selected = environmentId; + bestScore = score; + } + } + return selected; +} diff --git a/packages/shared/src/threadPlacement.test.ts b/packages/shared/src/threadPlacement.test.ts new file mode 100644 index 0000000000000..b2218c21bae88 --- /dev/null +++ b/packages/shared/src/threadPlacement.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, it } from "vite-plus/test"; +import { chooseThreadPlacement, type ThreadPlacementCandidate } from "./threadPlacement.ts"; + +const now = 1_000_000; +const GiB = 1024 ** 3; +const host = ( + environmentId: string, + cpuUtilization: number | null, + overrides: Partial = {}, +): ThreadPlacementCandidate => ({ + environmentId, + resources: { + sampledAt: now, + cpuUtilization, + cpuCount: 8, + availableMemoryBytes: 8 * GiB, + totalMemoryBytes: 16 * GiB, + }, + receivedAt: now, + weight: 50, + operable: true, + projectAvailable: true, + modelAvailable: true, + ...overrides, +}); +const origin = { originEnvironmentId: "here" }; + +describe("chooseThreadPlacement", () => { + it("lets a named target and local state win over capacity", () => { + const candidates = [host("here", 0.9), host("idle", 0.01)]; + expect( + chooseThreadPlacement({ ...origin, pinnedEnvironmentId: "here" }, candidates, now), + ).toEqual({ accepted: true, environmentId: "here", reason: "pinned" }); + expect(chooseThreadPlacement({ ...origin, requiresOrigin: true }, candidates, now)).toEqual({ + accepted: true, + environmentId: "here", + reason: "local-state", + }); + expect( + chooseThreadPlacement( + { ...origin, pinnedEnvironmentId: "idle", requiresOrigin: true }, + candidates, + now, + ), + ).toEqual({ accepted: false, reason: "conflicting-locality" }); + }); + + it("places independent work on the eligible host with the most headroom, in any input order", () => { + const candidates = [host("here", 0.7), host("idle", 0.1), host("busy", 0.5)]; + for (const order of [candidates, [...candidates].reverse()]) { + expect(chooseThreadPlacement(origin, order, now)).toEqual({ + accepted: true, + environmentId: "idle", + reason: "capacity", + }); + } + }); + + it("never picks a host without an operate link, the project, or the requested model", () => { + for (const missing of ["operable", "projectAvailable", "modelAvailable"] as const) { + const decision = chooseThreadPlacement( + origin, + [host("here", 0.8), host("idle", 0.01, { [missing]: false })], + now, + ); + expect(decision).toEqual({ accepted: true, environmentId: "here", reason: "capacity" }); + } + }); + + it("fails explicitly when no host has fresh, unsaturated resources", () => { + const candidates = [ + host("stale", 0.1, { receivedAt: now - 15_001 }), + host("cpu", 0.95), + host("unknown", null), + host("ram", 0.1, { + resources: { + sampledAt: now, + cpuUtilization: 0.1, + cpuCount: 8, + availableMemoryBytes: 0.05 * 16 * GiB, + totalMemoryBytes: 16 * GiB, + }, + }), + host("absent", 0.1, { resources: null }), + ]; + expect(chooseThreadPlacement(origin, candidates, now)).toEqual({ + accepted: false, + reason: "no-eligible-environment", + }); + expect(chooseThreadPlacement(origin, [], now)).toEqual({ + accepted: false, + reason: "no-eligible-environment", + }); + }); + + it("refuses evidence that names one environment twice", () => { + expect(chooseThreadPlacement(origin, [host("a", 0.1), host("a", 0.2)], now)).toEqual({ + accepted: false, + reason: "ambiguous-environment", + }); + }); +}); diff --git a/packages/shared/src/threadPlacement.ts b/packages/shared/src/threadPlacement.ts new file mode 100644 index 0000000000000..6f1c33108ef66 --- /dev/null +++ b/packages/shared/src/threadPlacement.ts @@ -0,0 +1,69 @@ +import type { HostResourcesSnapshot } from "@t3tools/contracts"; +import { chooseLoadBalancedEnvironment } from "./loadBalancing.ts"; + +/** Evidence the deciding server gathered itself; caller-supplied labels are not evidence. */ +export interface ThreadPlacementCandidate { + readonly environmentId: string; + readonly resources: HostResourcesSnapshot | null; + /** When this server received `resources`, so clocks on different machines are never compared. */ + readonly receivedAt: number; + readonly weight: number; + /** An authenticated link that may launch threads there; always true for this environment. */ + readonly operable: boolean; + /** The work's repository has a project there. */ + readonly projectAvailable: boolean; + /** The requested provider, model and modes run there. */ + readonly modelAvailable: boolean; +} + +export interface ThreadPlacementRequest { + readonly originEnvironmentId: string; + /** A target the caller named. It wins; the remote path checks it there. */ + readonly pinnedEnvironmentId?: string | undefined; + /** Workspace, worktree, attachments, referenced context or a dependent parent live here. */ + readonly requiresOrigin?: boolean | undefined; +} + +export type ThreadPlacementDecision = + | { + readonly accepted: true; + readonly environmentId: string; + readonly reason: "pinned" | "local-state" | "capacity"; + } + | { + readonly accepted: false; + readonly reason: "ambiguous-environment" | "conflicting-locality" | "no-eligible-environment"; + }; + +/** + * Where a NEW thread runs. Pure: it has no transport effects. A caller binds an accepted + * remote choice durably before launching, and after an uncertain launch reconciles that + * binding instead of deciding again, so a retry never lands on another host. + */ +export function chooseThreadPlacement( + request: ThreadPlacementRequest, + candidates: ReadonlyArray, + now: number, +): ThreadPlacementDecision { + const ids = new Set(); + for (const { environmentId } of candidates) { + if (ids.has(environmentId)) return { accepted: false, reason: "ambiguous-environment" }; + ids.add(environmentId); + } + const pinned = request.pinnedEnvironmentId; + if (request.requiresOrigin === true) { + if (pinned !== undefined && pinned !== request.originEnvironmentId) + return { accepted: false, reason: "conflicting-locality" }; + return { accepted: true, environmentId: request.originEnvironmentId, reason: "local-state" }; + } + if (pinned !== undefined) return { accepted: true, environmentId: pinned, reason: "pinned" }; + const eligible = candidates + .filter((c) => c.operable && c.projectAvailable && c.modelAvailable) + .sort((a, b) => + a.environmentId < b.environmentId ? -1 : a.environmentId > b.environmentId ? 1 : 0, + ); + const environmentId = chooseLoadBalancedEnvironment(eligible, now); + return environmentId === null + ? { accepted: false, reason: "no-eligible-environment" } + : { accepted: true, environmentId, reason: "capacity" }; +}