Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions apps/server/src/mcp/OrchestratorMcpService.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ import * as ScheduledTaskService from "../scheduledTasks/ScheduledTaskService.ts
import * as ServerSettings from "../serverSettings.ts";
import * as SecretRequests from "../secrets/SecretRequests.ts";
import * as RemoteDelegation from "../peer/RemoteDelegation.ts";
import * as HostResources from "../resourceTelemetry/HostResources.ts";
import type { McpInvocationScope } from "./McpInvocationContext.ts";
import { idleThreadProjection, liveThreadShell } from "./McpToolAccess.testkit.ts";
import * as OrchestratorMcpService from "./OrchestratorMcpService.ts";
Expand Down Expand Up @@ -1901,6 +1902,13 @@ describe("OrchestratorMcpService provider resolution", () => {
}),
forkShadow,
];
const hostSnapshot = {
sampledAt: 1_700_000_000_000,
cpuUtilization: 0.25,
cpuCount: 8,
availableMemoryBytes: 8 * 1024 ** 3,
totalMemoryBytes: 16 * 1024 ** 3,
};
const layerDependencies = Layer.mergeAll(
NodeServices.layer,
NodeCrypto.layer,
Expand Down Expand Up @@ -1945,11 +1953,14 @@ describe("OrchestratorMcpService provider resolution", () => {
Layer.mock(SecretRequests.SecretRequests)({}),
Layer.mock(RemoteDelegation.RemoteDelegation)({}),
Layer.mock(ScheduledTaskService.ScheduledTaskService)({}),
Layer.mock(HostResources.HostResources)({ read: Effect.succeed(hostSnapshot) }),
);

yield* Effect.gen(function* () {
const service = yield* OrchestratorMcpService.OrchestratorMcpService;
const capabilities = yield* service.capabilities(scope);
// Linked environments read this for placement.
assert.deepEqual(capabilities.hostResources, hostSnapshot);
const byId = new Map(
capabilities.providers.map((provider) => [provider.providerInstanceId, provider]),
);
Expand Down
4 changes: 4 additions & 0 deletions apps/server/src/mcp/OrchestratorMcpService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,7 @@ import {
import { assertTargetWithinLimits, isLiveCallerShell, threadNotFound } from "./threadAccess.ts";
import * as SecretRequests from "../secrets/SecretRequests.ts";
import * as PeerForwarding from "../peer/PeerForwarding.ts";
import * as HostResources from "../resourceTelemetry/HostResources.ts";
import * as RemoteDelegation from "../peer/RemoteDelegation.ts";

// Return recoverable handles before common MCP client timeouts expire.
Expand Down Expand Up @@ -1260,6 +1261,7 @@ function timelineItemsWithinBudget(

const make = Effect.gen(function* () {
const random = yield* SecureRandom.SecureRandom;
const hostResources = yield* Effect.serviceOption(HostResources.HostResources);
const threadManagement = yield* ThreadManagementService.ThreadManagementService;
const providerRegistry = yield* ProviderRegistry.ProviderRegistry;
const providerAdapters = yield* ProviderAdapterRegistry.ProviderAdapterRegistryV2;
Expand Down Expand Up @@ -2562,7 +2564,9 @@ const make = Effect.gen(function* () {
);
const inherited = parent?.thread.modelSelection;
const searchTerms = input.search?.toLowerCase().split(/\s+/).filter(Boolean) ?? [];
const resources = Option.isSome(hostResources) ? yield* hostResources.value.read : undefined;
return {
...(resources === undefined ? {} : { hostResources: resources }),
parentThreadId: parent?.thread.id ?? null,
inheritedProviderInstanceId: inherited?.instanceId ?? null,
inheritedModel: inherited?.model ?? null,
Expand Down
41 changes: 1 addition & 40 deletions packages/client-runtime/src/load-balancing.ts
Original file line number Diff line number Diff line change
@@ -1,40 +1 @@
import type { HostResourcesSnapshot } from "@t3tools/contracts";

/** Callers supply only connected machines hosting the project and selected provider. */
export function chooseLoadBalancedEnvironment(
candidates: ReadonlyArray<{
environmentId: string;
resources: HostResourcesSnapshot | null;
/** Client receipt time avoids comparing clocks on different machines. */
receivedAt?: number;
weight: number;
}>,
now: number,
): string | null {
let selected: string | null = null;
let bestScore = 0;
for (const { environmentId, resources, receivedAt, weight } of candidates) {
const sampledAt = receivedAt ?? resources?.sampledAt ?? 0;
if (
!resources ||
!Number.isFinite(weight) ||
weight <= 0 ||
now - sampledAt > 15_000 ||
sampledAt > now + 5_000 ||
resources.cpuUtilization === null ||
resources.cpuUtilization >= 0.95 ||
resources.totalMemoryBytes <= 0 ||
resources.cpuCount <= 0
) {
continue;
}
const memoryAvailable = resources.availableMemoryBytes / resources.totalMemoryBytes;
if (memoryAvailable <= 0.05) continue;
const score = weight * resources.cpuCount * (1 - resources.cpuUtilization) * memoryAvailable;
if (score > bestScore) {
selected = environmentId;
bestScore = score;
}
}
return selected;
}
export { chooseLoadBalancedEnvironment } from "@t3tools/shared/loadBalancing";
3 changes: 3 additions & 0 deletions packages/contracts/src/orchestratorMcp.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import * as Effect from "effect/Effect";
import { HostResourcesSnapshot } from "./resourceTelemetry.ts";
import * as Schema from "effect/Schema";
import * as SchemaTransformation from "effect/SchemaTransformation";

Expand Down Expand Up @@ -706,6 +707,8 @@ export const OrchestratorMcpCapabilitiesResult = Schema.Struct({
scheduledTasks: Schema.Boolean,
maxBatchThreads: Schema.Number,
}),
/** This machine's load when answered; placement reads it from linked environments. */
hostResources: Schema.optional(HostResourcesSnapshot),
});
export type OrchestratorMcpCapabilitiesResult = typeof OrchestratorMcpCapabilitiesResult.Type;

Expand Down
8 changes: 8 additions & 0 deletions packages/shared/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,14 @@
"types": "./src/remote.ts",
"import": "./src/remote.ts"
},
"./loadBalancing": {
"types": "./src/loadBalancing.ts",
"import": "./src/loadBalancing.ts"
},
"./threadPlacement": {
"types": "./src/threadPlacement.ts",
"import": "./src/threadPlacement.ts"
},
"./relaySigning": {
"types": "./src/relaySigning.ts",
"import": "./src/relaySigning.ts"
Expand Down
40 changes: 40 additions & 0 deletions packages/shared/src/loadBalancing.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
import type { HostResourcesSnapshot } from "@t3tools/contracts";

/** Callers supply only connected machines hosting the project and selected provider. */
export function chooseLoadBalancedEnvironment(
candidates: ReadonlyArray<{
environmentId: string;
resources: HostResourcesSnapshot | null;
/** Client receipt time avoids comparing clocks on different machines. */
receivedAt?: number;
weight: number;
}>,
now: number,
): string | null {
let selected: string | null = null;
let bestScore = 0;
for (const { environmentId, resources, receivedAt, weight } of candidates) {
const sampledAt = receivedAt ?? resources?.sampledAt ?? 0;
if (
!resources ||
!Number.isFinite(weight) ||
weight <= 0 ||
now - sampledAt > 15_000 ||
sampledAt > now + 5_000 ||
resources.cpuUtilization === null ||
resources.cpuUtilization >= 0.95 ||
resources.totalMemoryBytes <= 0 ||
resources.cpuCount <= 0
) {
continue;
}
const memoryAvailable = resources.availableMemoryBytes / resources.totalMemoryBytes;
if (memoryAvailable <= 0.05) continue;
const score = weight * resources.cpuCount * (1 - resources.cpuUtilization) * memoryAvailable;
if (score > bestScore) {
selected = environmentId;
bestScore = score;
}
}
return selected;
}
102 changes: 102 additions & 0 deletions packages/shared/src/threadPlacement.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
import { describe, expect, it } from "vite-plus/test";
import { chooseThreadPlacement, type ThreadPlacementCandidate } from "./threadPlacement.ts";

const now = 1_000_000;
const GiB = 1024 ** 3;
const host = (
environmentId: string,
cpuUtilization: number | null,
overrides: Partial<ThreadPlacementCandidate> = {},
): ThreadPlacementCandidate => ({
environmentId,
resources: {
sampledAt: now,
cpuUtilization,
cpuCount: 8,
availableMemoryBytes: 8 * GiB,
totalMemoryBytes: 16 * GiB,
},
receivedAt: now,
weight: 50,
operable: true,
projectAvailable: true,
modelAvailable: true,
...overrides,
});
const origin = { originEnvironmentId: "here" };

describe("chooseThreadPlacement", () => {
it("lets a named target and local state win over capacity", () => {
const candidates = [host("here", 0.9), host("idle", 0.01)];
expect(
chooseThreadPlacement({ ...origin, pinnedEnvironmentId: "here" }, candidates, now),
).toEqual({ accepted: true, environmentId: "here", reason: "pinned" });
expect(chooseThreadPlacement({ ...origin, requiresOrigin: true }, candidates, now)).toEqual({
accepted: true,
environmentId: "here",
reason: "local-state",
});
expect(
chooseThreadPlacement(
{ ...origin, pinnedEnvironmentId: "idle", requiresOrigin: true },
candidates,
now,
),
).toEqual({ accepted: false, reason: "conflicting-locality" });
});

it("places independent work on the eligible host with the most headroom, in any input order", () => {
const candidates = [host("here", 0.7), host("idle", 0.1), host("busy", 0.5)];
for (const order of [candidates, [...candidates].reverse()]) {
expect(chooseThreadPlacement(origin, order, now)).toEqual({
accepted: true,
environmentId: "idle",
reason: "capacity",
});
}
});

it("never picks a host without an operate link, the project, or the requested model", () => {
for (const missing of ["operable", "projectAvailable", "modelAvailable"] as const) {
const decision = chooseThreadPlacement(
origin,
[host("here", 0.8), host("idle", 0.01, { [missing]: false })],
now,
);
expect(decision).toEqual({ accepted: true, environmentId: "here", reason: "capacity" });
}
});

it("fails explicitly when no host has fresh, unsaturated resources", () => {
const candidates = [
host("stale", 0.1, { receivedAt: now - 15_001 }),
host("cpu", 0.95),
host("unknown", null),
host("ram", 0.1, {
resources: {
sampledAt: now,
cpuUtilization: 0.1,
cpuCount: 8,
availableMemoryBytes: 0.05 * 16 * GiB,
totalMemoryBytes: 16 * GiB,
},
}),
host("absent", 0.1, { resources: null }),
];
expect(chooseThreadPlacement(origin, candidates, now)).toEqual({
accepted: false,
reason: "no-eligible-environment",
});
expect(chooseThreadPlacement(origin, [], now)).toEqual({
accepted: false,
reason: "no-eligible-environment",
});
});

it("refuses evidence that names one environment twice", () => {
expect(chooseThreadPlacement(origin, [host("a", 0.1), host("a", 0.2)], now)).toEqual({
accepted: false,
reason: "ambiguous-environment",
});
});
});
69 changes: 69 additions & 0 deletions packages/shared/src/threadPlacement.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
import type { HostResourcesSnapshot } from "@t3tools/contracts";
import { chooseLoadBalancedEnvironment } from "./loadBalancing.ts";

/** Evidence the deciding server gathered itself; caller-supplied labels are not evidence. */
export interface ThreadPlacementCandidate {
readonly environmentId: string;
readonly resources: HostResourcesSnapshot | null;
/** When this server received `resources`, so clocks on different machines are never compared. */
readonly receivedAt: number;
readonly weight: number;
/** An authenticated link that may launch threads there; always true for this environment. */
readonly operable: boolean;
/** The work's repository has a project there. */
readonly projectAvailable: boolean;
/** The requested provider, model and modes run there. */
readonly modelAvailable: boolean;
}

export interface ThreadPlacementRequest {
readonly originEnvironmentId: string;
/** A target the caller named. It wins; the remote path checks it there. */
readonly pinnedEnvironmentId?: string | undefined;
/** Workspace, worktree, attachments, referenced context or a dependent parent live here. */
readonly requiresOrigin?: boolean | undefined;
}

export type ThreadPlacementDecision =
| {
readonly accepted: true;
readonly environmentId: string;
readonly reason: "pinned" | "local-state" | "capacity";
}
| {
readonly accepted: false;
readonly reason: "ambiguous-environment" | "conflicting-locality" | "no-eligible-environment";
};

/**
* Where a NEW thread runs. Pure: it has no transport effects. A caller binds an accepted
* remote choice durably before launching, and after an uncertain launch reconciles that
* binding instead of deciding again, so a retry never lands on another host.
*/
export function chooseThreadPlacement(
request: ThreadPlacementRequest,
candidates: ReadonlyArray<ThreadPlacementCandidate>,
now: number,
): ThreadPlacementDecision {
const ids = new Set<string>();
for (const { environmentId } of candidates) {
if (ids.has(environmentId)) return { accepted: false, reason: "ambiguous-environment" };
ids.add(environmentId);
}
const pinned = request.pinnedEnvironmentId;
if (request.requiresOrigin === true) {
if (pinned !== undefined && pinned !== request.originEnvironmentId)
return { accepted: false, reason: "conflicting-locality" };
return { accepted: true, environmentId: request.originEnvironmentId, reason: "local-state" };
}
if (pinned !== undefined) return { accepted: true, environmentId: pinned, reason: "pinned" };
const eligible = candidates
.filter((c) => c.operable && c.projectAvailable && c.modelAvailable)
.sort((a, b) =>
a.environmentId < b.environmentId ? -1 : a.environmentId > b.environmentId ? 1 : 0,
);
const environmentId = chooseLoadBalancedEnvironment(eligible, now);
return environmentId === null
? { accepted: false, reason: "no-eligible-environment" }
: { accepted: true, environmentId, reason: "capacity" };
}
Loading