From ad825f9a92952ffc8769ec32324fd2b643456389 Mon Sep 17 00:00:00 2001 From: nullStack65 <112618179+nullStack65@users.noreply.github.com> Date: Wed, 18 Nov 2026 04:36:51 +0000 Subject: [PATCH] feat(placement): shared native placement policy and peer load evidence First layer of native thread placement on release/native-0.1.10. No behaviour changes for any existing launch path. - Move chooseLoadBalancedEnvironment (15 s freshness, 95% CPU, 5% available RAM) unchanged from client-runtime to @t3tools/shared so the server and clients use one selector; client-runtime/load-balancing re-exports it, so web is unchanged. - Add chooseThreadPlacement, a pure decision over evidence the deciding server gathered: a named target wins, work that needs local state stays here (a conflicting target fails), and independent work goes to the operable environment with the project and requested model that has the most fresh headroom. Ambiguous evidence and no eligible host fail explicitly. - orchestrator_capabilities now reports this machine's hostResources, an optional field, so a server can read a linked environment's load through the existing peer link. Older peers omit it and are never chosen. Automatic remote placement is not enabled: it first needs the chosen destination bound durably to the caller's request before any remote effect, so a retry after an uncertain launch reuses it. Task: CLO-866 Model: claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- .../src/mcp/OrchestratorMcpService.test.ts | 11 ++ apps/server/src/mcp/OrchestratorMcpService.ts | 4 + packages/client-runtime/src/load-balancing.ts | 41 +------ packages/contracts/src/orchestratorMcp.ts | 3 + packages/shared/package.json | 8 ++ packages/shared/src/loadBalancing.ts | 40 +++++++ packages/shared/src/threadPlacement.test.ts | 102 ++++++++++++++++++ packages/shared/src/threadPlacement.ts | 69 ++++++++++++ 8 files changed, 238 insertions(+), 40 deletions(-) create mode 100644 packages/shared/src/loadBalancing.ts create mode 100644 packages/shared/src/threadPlacement.test.ts create mode 100644 packages/shared/src/threadPlacement.ts diff --git a/apps/server/src/mcp/OrchestratorMcpService.test.ts b/apps/server/src/mcp/OrchestratorMcpService.test.ts index f49bcc7c9412f..6a969e21e3abd 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.test.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.test.ts @@ -71,6 +71,7 @@ import * as ScheduledTaskService from "../scheduledTasks/ScheduledTaskService.ts import * as ServerSettings from "../serverSettings.ts"; import * as SecretRequests from "../secrets/SecretRequests.ts"; import * as RemoteDelegation from "../peer/RemoteDelegation.ts"; +import * as HostResources from "../resourceTelemetry/HostResources.ts"; import type { McpInvocationScope } from "./McpInvocationContext.ts"; import { idleThreadProjection, liveThreadShell } from "./McpToolAccess.testkit.ts"; import * as OrchestratorMcpService from "./OrchestratorMcpService.ts"; @@ -1901,6 +1902,13 @@ describe("OrchestratorMcpService provider resolution", () => { }), forkShadow, ]; + const hostSnapshot = { + sampledAt: 1_700_000_000_000, + cpuUtilization: 0.25, + cpuCount: 8, + availableMemoryBytes: 8 * 1024 ** 3, + totalMemoryBytes: 16 * 1024 ** 3, + }; const layerDependencies = Layer.mergeAll( NodeServices.layer, NodeCrypto.layer, @@ -1945,11 +1953,14 @@ describe("OrchestratorMcpService provider resolution", () => { Layer.mock(SecretRequests.SecretRequests)({}), Layer.mock(RemoteDelegation.RemoteDelegation)({}), Layer.mock(ScheduledTaskService.ScheduledTaskService)({}), + Layer.mock(HostResources.HostResources)({ read: Effect.succeed(hostSnapshot) }), ); yield* Effect.gen(function* () { const service = yield* OrchestratorMcpService.OrchestratorMcpService; const capabilities = yield* service.capabilities(scope); + // Linked environments read this for placement. + assert.deepEqual(capabilities.hostResources, hostSnapshot); const byId = new Map( capabilities.providers.map((provider) => [provider.providerInstanceId, provider]), ); diff --git a/apps/server/src/mcp/OrchestratorMcpService.ts b/apps/server/src/mcp/OrchestratorMcpService.ts index 97d578f527b57..b86a564c19f9d 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.ts @@ -136,6 +136,7 @@ import { import { assertTargetWithinLimits, isLiveCallerShell, threadNotFound } from "./threadAccess.ts"; import * as SecretRequests from "../secrets/SecretRequests.ts"; import * as PeerForwarding from "../peer/PeerForwarding.ts"; +import * as HostResources from "../resourceTelemetry/HostResources.ts"; import * as RemoteDelegation from "../peer/RemoteDelegation.ts"; // Return recoverable handles before common MCP client timeouts expire. @@ -1260,6 +1261,7 @@ function timelineItemsWithinBudget( const make = Effect.gen(function* () { const random = yield* SecureRandom.SecureRandom; + const hostResources = yield* Effect.serviceOption(HostResources.HostResources); const threadManagement = yield* ThreadManagementService.ThreadManagementService; const providerRegistry = yield* ProviderRegistry.ProviderRegistry; const providerAdapters = yield* ProviderAdapterRegistry.ProviderAdapterRegistryV2; @@ -2562,7 +2564,9 @@ const make = Effect.gen(function* () { ); const inherited = parent?.thread.modelSelection; const searchTerms = input.search?.toLowerCase().split(/\s+/).filter(Boolean) ?? []; + const resources = Option.isSome(hostResources) ? yield* hostResources.value.read : undefined; return { + ...(resources === undefined ? {} : { hostResources: resources }), parentThreadId: parent?.thread.id ?? null, inheritedProviderInstanceId: inherited?.instanceId ?? null, inheritedModel: inherited?.model ?? null, diff --git a/packages/client-runtime/src/load-balancing.ts b/packages/client-runtime/src/load-balancing.ts index 0b938c6c090e4..84c9c8bd0cb60 100644 --- a/packages/client-runtime/src/load-balancing.ts +++ b/packages/client-runtime/src/load-balancing.ts @@ -1,40 +1 @@ -import type { HostResourcesSnapshot } from "@t3tools/contracts"; - -/** Callers supply only connected machines hosting the project and selected provider. */ -export function chooseLoadBalancedEnvironment( - candidates: ReadonlyArray<{ - environmentId: string; - resources: HostResourcesSnapshot | null; - /** Client receipt time avoids comparing clocks on different machines. */ - receivedAt?: number; - weight: number; - }>, - now: number, -): string | null { - let selected: string | null = null; - let bestScore = 0; - for (const { environmentId, resources, receivedAt, weight } of candidates) { - const sampledAt = receivedAt ?? resources?.sampledAt ?? 0; - if ( - !resources || - !Number.isFinite(weight) || - weight <= 0 || - now - sampledAt > 15_000 || - sampledAt > now + 5_000 || - resources.cpuUtilization === null || - resources.cpuUtilization >= 0.95 || - resources.totalMemoryBytes <= 0 || - resources.cpuCount <= 0 - ) { - continue; - } - const memoryAvailable = resources.availableMemoryBytes / resources.totalMemoryBytes; - if (memoryAvailable <= 0.05) continue; - const score = weight * resources.cpuCount * (1 - resources.cpuUtilization) * memoryAvailable; - if (score > bestScore) { - selected = environmentId; - bestScore = score; - } - } - return selected; -} +export { chooseLoadBalancedEnvironment } from "@t3tools/shared/loadBalancing"; diff --git a/packages/contracts/src/orchestratorMcp.ts b/packages/contracts/src/orchestratorMcp.ts index 3d987a2773ad1..c30c4fea2e785 100644 --- a/packages/contracts/src/orchestratorMcp.ts +++ b/packages/contracts/src/orchestratorMcp.ts @@ -1,4 +1,5 @@ import * as Effect from "effect/Effect"; +import { HostResourcesSnapshot } from "./resourceTelemetry.ts"; import * as Schema from "effect/Schema"; import * as SchemaTransformation from "effect/SchemaTransformation"; @@ -706,6 +707,8 @@ export const OrchestratorMcpCapabilitiesResult = Schema.Struct({ scheduledTasks: Schema.Boolean, maxBatchThreads: Schema.Number, }), + /** This machine's load when answered; placement reads it from linked environments. */ + hostResources: Schema.optional(HostResourcesSnapshot), }); export type OrchestratorMcpCapabilitiesResult = typeof OrchestratorMcpCapabilitiesResult.Type; diff --git a/packages/shared/package.json b/packages/shared/package.json index 3de18958220ef..29b36a3583fe1 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -195,6 +195,14 @@ "types": "./src/remote.ts", "import": "./src/remote.ts" }, + "./loadBalancing": { + "types": "./src/loadBalancing.ts", + "import": "./src/loadBalancing.ts" + }, + "./threadPlacement": { + "types": "./src/threadPlacement.ts", + "import": "./src/threadPlacement.ts" + }, "./relaySigning": { "types": "./src/relaySigning.ts", "import": "./src/relaySigning.ts" diff --git a/packages/shared/src/loadBalancing.ts b/packages/shared/src/loadBalancing.ts new file mode 100644 index 0000000000000..0b938c6c090e4 --- /dev/null +++ b/packages/shared/src/loadBalancing.ts @@ -0,0 +1,40 @@ +import type { HostResourcesSnapshot } from "@t3tools/contracts"; + +/** Callers supply only connected machines hosting the project and selected provider. */ +export function chooseLoadBalancedEnvironment( + candidates: ReadonlyArray<{ + environmentId: string; + resources: HostResourcesSnapshot | null; + /** Client receipt time avoids comparing clocks on different machines. */ + receivedAt?: number; + weight: number; + }>, + now: number, +): string | null { + let selected: string | null = null; + let bestScore = 0; + for (const { environmentId, resources, receivedAt, weight } of candidates) { + const sampledAt = receivedAt ?? resources?.sampledAt ?? 0; + if ( + !resources || + !Number.isFinite(weight) || + weight <= 0 || + now - sampledAt > 15_000 || + sampledAt > now + 5_000 || + resources.cpuUtilization === null || + resources.cpuUtilization >= 0.95 || + resources.totalMemoryBytes <= 0 || + resources.cpuCount <= 0 + ) { + continue; + } + const memoryAvailable = resources.availableMemoryBytes / resources.totalMemoryBytes; + if (memoryAvailable <= 0.05) continue; + const score = weight * resources.cpuCount * (1 - resources.cpuUtilization) * memoryAvailable; + if (score > bestScore) { + selected = environmentId; + bestScore = score; + } + } + return selected; +} diff --git a/packages/shared/src/threadPlacement.test.ts b/packages/shared/src/threadPlacement.test.ts new file mode 100644 index 0000000000000..b2218c21bae88 --- /dev/null +++ b/packages/shared/src/threadPlacement.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, it } from "vite-plus/test"; +import { chooseThreadPlacement, type ThreadPlacementCandidate } from "./threadPlacement.ts"; + +const now = 1_000_000; +const GiB = 1024 ** 3; +const host = ( + environmentId: string, + cpuUtilization: number | null, + overrides: Partial = {}, +): ThreadPlacementCandidate => ({ + environmentId, + resources: { + sampledAt: now, + cpuUtilization, + cpuCount: 8, + availableMemoryBytes: 8 * GiB, + totalMemoryBytes: 16 * GiB, + }, + receivedAt: now, + weight: 50, + operable: true, + projectAvailable: true, + modelAvailable: true, + ...overrides, +}); +const origin = { originEnvironmentId: "here" }; + +describe("chooseThreadPlacement", () => { + it("lets a named target and local state win over capacity", () => { + const candidates = [host("here", 0.9), host("idle", 0.01)]; + expect( + chooseThreadPlacement({ ...origin, pinnedEnvironmentId: "here" }, candidates, now), + ).toEqual({ accepted: true, environmentId: "here", reason: "pinned" }); + expect(chooseThreadPlacement({ ...origin, requiresOrigin: true }, candidates, now)).toEqual({ + accepted: true, + environmentId: "here", + reason: "local-state", + }); + expect( + chooseThreadPlacement( + { ...origin, pinnedEnvironmentId: "idle", requiresOrigin: true }, + candidates, + now, + ), + ).toEqual({ accepted: false, reason: "conflicting-locality" }); + }); + + it("places independent work on the eligible host with the most headroom, in any input order", () => { + const candidates = [host("here", 0.7), host("idle", 0.1), host("busy", 0.5)]; + for (const order of [candidates, [...candidates].reverse()]) { + expect(chooseThreadPlacement(origin, order, now)).toEqual({ + accepted: true, + environmentId: "idle", + reason: "capacity", + }); + } + }); + + it("never picks a host without an operate link, the project, or the requested model", () => { + for (const missing of ["operable", "projectAvailable", "modelAvailable"] as const) { + const decision = chooseThreadPlacement( + origin, + [host("here", 0.8), host("idle", 0.01, { [missing]: false })], + now, + ); + expect(decision).toEqual({ accepted: true, environmentId: "here", reason: "capacity" }); + } + }); + + it("fails explicitly when no host has fresh, unsaturated resources", () => { + const candidates = [ + host("stale", 0.1, { receivedAt: now - 15_001 }), + host("cpu", 0.95), + host("unknown", null), + host("ram", 0.1, { + resources: { + sampledAt: now, + cpuUtilization: 0.1, + cpuCount: 8, + availableMemoryBytes: 0.05 * 16 * GiB, + totalMemoryBytes: 16 * GiB, + }, + }), + host("absent", 0.1, { resources: null }), + ]; + expect(chooseThreadPlacement(origin, candidates, now)).toEqual({ + accepted: false, + reason: "no-eligible-environment", + }); + expect(chooseThreadPlacement(origin, [], now)).toEqual({ + accepted: false, + reason: "no-eligible-environment", + }); + }); + + it("refuses evidence that names one environment twice", () => { + expect(chooseThreadPlacement(origin, [host("a", 0.1), host("a", 0.2)], now)).toEqual({ + accepted: false, + reason: "ambiguous-environment", + }); + }); +}); diff --git a/packages/shared/src/threadPlacement.ts b/packages/shared/src/threadPlacement.ts new file mode 100644 index 0000000000000..6f1c33108ef66 --- /dev/null +++ b/packages/shared/src/threadPlacement.ts @@ -0,0 +1,69 @@ +import type { HostResourcesSnapshot } from "@t3tools/contracts"; +import { chooseLoadBalancedEnvironment } from "./loadBalancing.ts"; + +/** Evidence the deciding server gathered itself; caller-supplied labels are not evidence. */ +export interface ThreadPlacementCandidate { + readonly environmentId: string; + readonly resources: HostResourcesSnapshot | null; + /** When this server received `resources`, so clocks on different machines are never compared. */ + readonly receivedAt: number; + readonly weight: number; + /** An authenticated link that may launch threads there; always true for this environment. */ + readonly operable: boolean; + /** The work's repository has a project there. */ + readonly projectAvailable: boolean; + /** The requested provider, model and modes run there. */ + readonly modelAvailable: boolean; +} + +export interface ThreadPlacementRequest { + readonly originEnvironmentId: string; + /** A target the caller named. It wins; the remote path checks it there. */ + readonly pinnedEnvironmentId?: string | undefined; + /** Workspace, worktree, attachments, referenced context or a dependent parent live here. */ + readonly requiresOrigin?: boolean | undefined; +} + +export type ThreadPlacementDecision = + | { + readonly accepted: true; + readonly environmentId: string; + readonly reason: "pinned" | "local-state" | "capacity"; + } + | { + readonly accepted: false; + readonly reason: "ambiguous-environment" | "conflicting-locality" | "no-eligible-environment"; + }; + +/** + * Where a NEW thread runs. Pure: it has no transport effects. A caller binds an accepted + * remote choice durably before launching, and after an uncertain launch reconciles that + * binding instead of deciding again, so a retry never lands on another host. + */ +export function chooseThreadPlacement( + request: ThreadPlacementRequest, + candidates: ReadonlyArray, + now: number, +): ThreadPlacementDecision { + const ids = new Set(); + for (const { environmentId } of candidates) { + if (ids.has(environmentId)) return { accepted: false, reason: "ambiguous-environment" }; + ids.add(environmentId); + } + const pinned = request.pinnedEnvironmentId; + if (request.requiresOrigin === true) { + if (pinned !== undefined && pinned !== request.originEnvironmentId) + return { accepted: false, reason: "conflicting-locality" }; + return { accepted: true, environmentId: request.originEnvironmentId, reason: "local-state" }; + } + if (pinned !== undefined) return { accepted: true, environmentId: pinned, reason: "pinned" }; + const eligible = candidates + .filter((c) => c.operable && c.projectAvailable && c.modelAvailable) + .sort((a, b) => + a.environmentId < b.environmentId ? -1 : a.environmentId > b.environmentId ? 1 : 0, + ); + const environmentId = chooseLoadBalancedEnvironment(eligible, now); + return environmentId === null + ? { accepted: false, reason: "no-eligible-environment" } + : { accepted: true, environmentId, reason: "capacity" }; +}