diff --git a/apps/mobile/src/features/settings/ThreadSettingsSections.tsx b/apps/mobile/src/features/settings/ThreadSettingsSections.tsx index 41600619b3a4..bfa2d53dce2e 100644 --- a/apps/mobile/src/features/settings/ThreadSettingsSections.tsx +++ b/apps/mobile/src/features/settings/ThreadSettingsSections.tsx @@ -50,6 +50,13 @@ export function SettingsLabsRouteScreen() { setting="linkedEnvironmentsEnabled" description="Let agents on a machine work on your other machines, and theirs on it. Turn it on for each machine. While off, nothing new starts over a link; running tasks finish." /> + settings.getSettings), + Effect.map((settings) => settings.automaticPlacementByDefault), + Effect.orElseSucceed(() => false), + )); + if (byDefault) { + input = { + ...input, + placement: "auto", + modelSelection: input.modelSelection ?? local.callerModelSelection, + }; + } const key = input.clientRequestId; if (key === undefined && input.placement !== "auto") return undefined; // Work a link sent here was placed on its own side, and never uses this side's links. @@ -201,7 +227,7 @@ const launchPlacedThere = Effect.fn("mcp.launchPlacedThere")(function* ( modelSelection, runtimeMode: local.runtimeMode, }).pipe(Effect.mapError(unavailable)); - const environmentId = yield* delegation.value.place(scope, { + const placing = delegation.value.place(scope, { operation: "thread-launch", key, threadId: local.threadId, @@ -210,6 +236,15 @@ const launchPlacedThere = Effect.fn("mcp.launchPlacedThere")(function* ( runtimeMode: local.runtimeMode, here, }); + // Placement nobody asked for falls back to here, as before, when no machine has room. + const environmentId = byDefault + ? yield* placing.pipe( + Effect.catchIf( + (error) => error.code === "target_required", + () => Effect.succeed(scope.environmentId), + ), + ) + : yield* placing; return environmentId === scope.environmentId ? undefined : yield* sendThere(environmentId, key); }); @@ -330,6 +365,7 @@ export const layer = McpToolAccess.toLayer(ProjectToolkit, { strategy, runtimeMode: defaultRuntimeMode, interactionMode, + callerModelSelection: caller?.modelSelection, }); if (placedThere !== undefined) return placedThere; } diff --git a/apps/server/src/mcp/toolkits/project/tools.ts b/apps/server/src/mcp/toolkits/project/tools.ts index 1a052a1ba75a..2f58e7d9e650 100644 --- a/apps/server/src/mcp/toolkits/project/tools.ts +++ b/apps/server/src/mcp/toolkits/project/tools.ts @@ -162,9 +162,9 @@ const ThreadLaunchTool = Tool.make("t3_thread_launch", { attachments: Schema.optional(Schema.Array(McpAttachmentInput).check(Schema.isMaxLength(8))), clientRequestId: Schema.optional(OrchestratorMcpClientRequestId), placement: Schema.optional( - Schema.Literals(["auto"]).annotate({ + Schema.Literals(["auto", "here"]).annotate({ description: - "'auto', without `environmentId`: here or on a linked environment with spare capacity, this repository's project and your exact `modelSelection`; there, in that project's checkout. Needs `modelSelection` and `clientRequestId`; a retry keeps the environment chosen. Attachments, scratch and worktrees not started from origin stay here.", + "'auto', without `environmentId`: here or on a linked environment with spare capacity, this repository's project and your exact `modelSelection`; there, in that project's checkout. Needs `modelSelection` and `clientRequestId`; a retry keeps the environment chosen. Attachments, scratch and worktrees not started from origin stay here. Omitted with a `clientRequestId`: a worktree started from origin is placed the same way, with your model, when this machine places by default. 'here' keeps it here.", }), ), /** Set by a linked environment launching its agent's delegated task here; ignored otherwise. */ diff --git a/apps/server/src/peer/RemoteDelegation.test.ts b/apps/server/src/peer/RemoteDelegation.test.ts index bebde781edaf..17cd8d7a50bc 100644 --- a/apps/server/src/peer/RemoteDelegation.test.ts +++ b/apps/server/src/peer/RemoteDelegation.test.ts @@ -156,6 +156,7 @@ const loadOf = (cpuUtilization: number): HostResourcesSnapshot => ({ cpuCount: 8, availableMemoryBytes: 8 * GiB, totalMemoryBytes: 16 * GiB, + placementRole: "headless", }); const makeBoxThread = Effect.gen(function* () { @@ -517,6 +518,7 @@ const boxToolkitLayer = (thread: BoxThread) => { ), }); const layerBoxLaunches = Layer.mock(ThreadLaunch.ThreadLaunchService)({ + checkWorktreeBase: () => Effect.void, launch: (input) => Ref.get(thread.runless).pipe( Effect.flatMap((runless) => @@ -892,6 +894,23 @@ const launchPlaced = ( outsideScope, ); +/** A launch that says nothing about placement, so this machine's default decides. */ +const launchUnplaced = ( + laptopEnv: Effect.Success>, + args: Record, +) => + laptopEnv.call( + "t3_thread_launch", + { + projectId: "project:laptop", + title: "Suite", + message: "Run the suite wherever there is room.", + modelSelection: { instanceId: boxClaude.instanceId, model: "claude-sonnet-4-6" }, + ...args, + }, + outsideScope, + ); + /** The laptop's parent thread, mid-turn when its agent delegates. */ const seedParent = ( laptopEnv: Effect.Success>, @@ -1624,6 +1643,73 @@ it.effect("an outside client's auto-placed launch starts at the project's defaul ).pipe(Effect.provide(NodeHttpServer.layerTest)), ); +it.effect("a fresh worktree from origin is placed without asking, unless kept here", () => + Effect.scoped( + Effect.gen(function* () { + const thread = yield* makeBoxThread; + const b = yield* serveBox(thread); + const offers = yield* SubscriptionRef.make< + ReadonlyArray + >([]); + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); + const localLaunches = yield* Ref.make(0); + const a = yield* makeLaptop(database, offers, undefined, Effect.void, { + localLaunch: Ref.update(localLaunches, (count) => count + 1), + }); + yield* linkTo(a.links, b, "full-access"); + const fresh = { type: "worktree", baseRef: "main", startFromOrigin: true }; + // Without a key nothing could find the machine again after a lost answer: it stays here. + yield* launchUnplaced(a, { workspaceStrategy: fresh }); + expect(yield* Ref.get(localLaunches)).toBe(1); + const placed = yield* launchUnplaced(a, { + clientRequestId: "fresh", + workspaceStrategy: fresh, + }); + expect(placed.isError, JSON.stringify(placed.content)).toBe(false); + expect(placed.structuredContent).toMatchObject({ environmentId: box.environmentId }); + expect(yield* Ref.get(localLaunches)).toBe(1); + + // 'here', or the fleet default turned off, keeps it on this machine. + yield* launchPlaced(a, { + placement: "here", + clientRequestId: "kept", + workspaceStrategy: fresh, + }); + expect(yield* Ref.get(localLaunches)).toBe(2); + yield* a.settings.updateSettings({ automaticPlacementByDefault: false }); + yield* launchUnplaced(a, { clientRequestId: "off", workspaceStrategy: fresh }); + expect(yield* Ref.get(localLaunches)).toBe(3); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); + +it.effect("placement nobody asked for runs here when no machine has room", () => + Effect.scoped( + Effect.gen(function* () { + const thread = yield* makeBoxThread; + yield* Ref.set(thread.resources, { ...loadOf(0.99) }); + const b = yield* serveBox(thread); + const offers = yield* SubscriptionRef.make< + ReadonlyArray + >([]); + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); + const localLaunches = yield* Ref.make(0); + const a = yield* makeLaptop(database, offers, undefined, Effect.void, { + localLaunch: Ref.update(localLaunches, (count) => count + 1), + }); + yield* linkTo(a.links, b, "full-access"); + const fresh = { type: "worktree", baseRef: "main", startFromOrigin: true }; + yield* launchUnplaced(a, { clientRequestId: "room", workspaceStrategy: fresh }); + expect(yield* Ref.get(localLaunches)).toBe(1); + expect(yield* Ref.get(thread.launched)).toBeNull(); + // Asked for, it still fails loudly instead. + const asked = yield* launchPlaced(a, { clientRequestId: "full", workspaceStrategy: fresh }); + expect(asked.isError).toBe(true); + expect(yield* Ref.get(localLaunches)).toBe(1); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); + it.effect( "a retry that drops placement or asks for this checkout reaches the thread placed there", () => diff --git a/apps/server/src/process/agentScope.test.ts b/apps/server/src/process/agentScope.test.ts index 49674354b253..a10874b3eed7 100644 --- a/apps/server/src/process/agentScope.test.ts +++ b/apps/server/src/process/agentScope.test.ts @@ -12,12 +12,13 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Logger from "effect/Logger"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; -import { vi } from "vite-plus/test"; +import { afterEach, vi } from "vite-plus/test"; import * as ServerConfig from "../config.ts"; import { makeCodexAppServerSpawnCommand } from "../orchestration-v2/Adapters/CodexAdapterV2.ts"; import * as ProcessRunner from "../processRunner.ts"; import * as ProviderProcesses from "./providerProcesses.ts"; +import { MachinePresence } from "../resourceTelemetry/HostResources.ts"; import { classifyScope, make, ownedAgentSlice, type ScopeState } from "./agentScope.ts"; const GiB = 1024 ** 3; @@ -67,11 +68,13 @@ const withFakeSystemd = ( stopFails = false, leftovers = false, liveOwner = false, + role, }: { readonly oomPolicy?: boolean; readonly stopFails?: boolean; readonly leftovers?: boolean; readonly liveOwner?: boolean; + readonly role?: { current: "headless" | "interactive" }; } = {}, ) => Effect.gen(function* () { @@ -125,7 +128,17 @@ const withFakeSystemd = ( if (leftovers) { units.set(ownedAgentSlice(stateDir), { activeState: "active", result: "success" }); } + const presence = + role === undefined + ? undefined + : MachinePresence.of({ + role: Effect.sync(() => role.current), + read: Effect.sync(() => ({ placementRole: role.current })), + }); const scope = yield* make.pipe( + presence === undefined + ? (effect) => effect + : Effect.provideService(MachinePresence, presence), Effect.provideService(ProviderProcesses.ProviderProcesses, { hasOtherLiveOwner: liveOwner }), Effect.provideService(ProcessRunner.ProcessRunner, runner), Effect.provideService(HostProcess.Platform, "linux"), @@ -195,6 +208,9 @@ const launchCodex = (scope: Effect.Success, threadId: string) => ); describe("AgentScope service", () => { + afterEach(() => { + memory.total = 16 * GiB; + }); it.effect("labels a thread from its newest scope until its next session clears it", () => withFakeSystemd(({ scope, units }) => Effect.gen(function* () { @@ -325,12 +341,46 @@ describe("AgentScope service", () => { "CPUWeight=10", "IOWeight=10", "TasksMax=25%", + "CPUQuota=", ], ]); }), ), ); + it.effect("keeps a person's computer responsive and follows a role change", () => { + memory.total = 8 * GiB; + const role = { current: "interactive" as "headless" | "interactive" }; + return withFakeSystemd( + ({ scope, calls }) => + Effect.gen(function* () { + const limits = () => + calls.filter(([, verb]) => verb === "set-property").map((args) => args.slice(3)); + yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" }); + const cores = NodeOS.cpus().length; + // 8 GB keeps 4 GB for the person before agents slow and 3 GB before a kill. + expect(limits()).toEqual([ + ["app-t3code-agents.slice", `MemoryHigh=${4 * GiB}`, `MemoryMax=${5 * GiB}`], + [ + "app-t3code.slice", + "CPUWeight=1", + "IOWeight=1", + "TasksMax=25%", + `CPUQuota=${Math.max(100, Math.floor(cores * 50))}%`, + ], + ]); + + role.current = "headless"; + yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "b" }); + expect(limits().slice(2)).toEqual([ + ["app-t3code-agents.slice", `MemoryHigh=${4 * GiB}`, `MemoryMax=${6 * GiB}`], + ["app-t3code.slice", "CPUWeight=10", "IOWeight=10", "TasksMax=25%", "CPUQuota="], + ]); + }), + { role }, + ); + }); + it.effect("follows a live memory change at the next launch", () => { memory.total = 16 * GiB; return withFakeSystemd(({ scope, calls, failingVerbs }) => diff --git a/apps/server/src/process/agentScope.ts b/apps/server/src/process/agentScope.ts index 1e52e680b6f8..e5cd473d6889 100644 --- a/apps/server/src/process/agentScope.ts +++ b/apps/server/src/process/agentScope.ts @@ -13,11 +13,13 @@ import { resolveCommandPath } from "@t3tools/shared/shell"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as Semaphore from "effect/Semaphore"; import * as ServerConfig from "../config.ts"; import * as ProcessRunner from "../processRunner.ts"; +import { MachinePresence } from "../resourceTelemetry/HostResources.ts"; import * as ProviderProcesses from "./providerProcesses.ts"; const AGENT_SLICE = "app-t3code-agents.slice"; @@ -27,6 +29,19 @@ const AGENTS_PARENT_SLICE = "app-t3code.slice"; // has those controllers: current systemd delegates cpu, rarely io. TasksMax keeps agents // below logind's 33% per-user task limit, leaving forks for the server. const AGENTS_PARENT_PROPERTIES = ["CPUWeight=10", "IOWeight=10", "TasksMax=25%"]; +// On a person's computer agents also yield to every app and never take more than half +// the cores, so the desktop stays responsive however much work is placed there. +function agentsParentProperties(role: AgentHostRole, cpuCount: number) { + if (role === "headless") return [...AGENTS_PARENT_PROPERTIES, "CPUQuota="]; + return [ + "CPUWeight=1", + "IOWeight=1", + "TasksMax=25%", + `CPUQuota=${Math.max(100, Math.floor(cpuCount * 50))}%`, + ]; +} + +export type AgentHostRole = "headless" | "interactive"; export function ownedAgentSlice(stateDir: string): string { const hash = NodeCrypto.createHash("sha256").update(stateDir).digest("hex").slice(0, 12); @@ -41,7 +56,16 @@ const GiB = 1024 ** 3; * working. Small machines still give agents at least half of RAM before * throttling and three quarters before a kill. */ -function agentSliceMemoryLimits(totalBytes: number) { +export function agentSliceMemoryLimits(totalBytes: number, role: AgentHostRole = "headless") { + if (role === "interactive") { + // A person's computer keeps 4 GB or 40% for them before agents slow, and 3 GB or 30% + // before one is killed. + const high = Math.max(GiB, totalBytes - Math.max(4 * GiB, Math.floor(totalBytes * 0.4))); + return { + high, + max: Math.max(high + GiB / 2, totalBytes - Math.max(3 * GiB, Math.floor(totalBytes * 0.3))), + }; + } return { high: Math.max(totalBytes - 6 * GiB, Math.floor(totalBytes / 2)), max: Math.max(totalBytes - 4 * GiB, Math.floor(totalBytes * 0.75)), @@ -110,6 +134,11 @@ export const make = Effect.gen(function* () { const { stateDir } = yield* ServerConfig.ServerConfig; const { hasOtherLiveOwner } = yield* ProviderProcesses.ProviderProcesses; const ownedSlice = ownedAgentSlice(stateDir); + // Without presence (tests, embedded servers) the machine is treated as headless. + const presence = yield* Effect.serviceOption(MachinePresence); + const hostRole: Effect.Effect = Option.isSome(presence) + ? presence.value.role + : Effect.succeed("headless"); const layerScope = yield* Effect.scope; const resolve = (command: string, env: NodeJS.ProcessEnv) => @@ -202,36 +231,48 @@ export const make = Effect.gen(function* () { }); return false; }); - // VMs and containers can resize RAM live, so each launch re-checks the total - // the limits came from. A failed update is retried by the next launch. + // VMs and containers can resize RAM live, and a machine can become someone's + // computer, so each launch re-checks the total and role the limits came from. + // A failed update is retried by the next launch. let appliedTotal: number | undefined; + let appliedRole: AgentHostRole | undefined; const memoryLimitsLock = yield* Semaphore.make(1); const applyMemoryLimits = memoryLimitsLock.withPermits(1)( Effect.gen(function* () { const total = NodeOS.totalmem(); - if (total === appliedTotal) return; - const limits = agentSliceMemoryLimits(total); + const role = yield* hostRole; + if (total === appliedTotal && role === appliedRole) return; + const limits = agentSliceMemoryLimits(total, role); const applied = yield* setProperties(AGENT_SLICE, [ `MemoryHigh=${limits.high}`, `MemoryMax=${limits.max}`, ]); if (!applied) return; + if (role !== appliedRole) { + const parentApplied = yield* setProperties( + AGENTS_PARENT_SLICE, + agentsParentProperties(role, NodeOS.cpus().length), + ); + if (!parentApplied) return; + } if (appliedTotal !== undefined) { - yield* Effect.logInfo("Agent memory limits follow a memory change", { + yield* Effect.logInfo("Agent limits follow a memory or role change", { slice: AGENT_SLICE, previousTotal: appliedTotal, total, + role, ...limits, }); } appliedTotal = total; + appliedRole = role; }), ); yield* applyMemoryLimits; - yield* setProperties(AGENTS_PARENT_SLICE, AGENTS_PARENT_PROPERTIES); yield* Effect.logInfo("Agent scopes are on", { slice: AGENT_SLICE, - ...agentSliceMemoryLimits(NodeOS.totalmem()), + role: appliedRole, + ...agentSliceMemoryLimits(NodeOS.totalmem(), appliedRole), oomPolicy: supportsOomPolicy, }); return { diff --git a/apps/server/src/resourceTelemetry/HostResources.test.ts b/apps/server/src/resourceTelemetry/HostResources.test.ts index 3e5c636c5d7c..fdf0d064228c 100644 --- a/apps/server/src/resourceTelemetry/HostResources.test.ts +++ b/apps/server/src/resourceTelemetry/HostResources.test.ts @@ -13,12 +13,13 @@ import { vi } from "vite-plus/test"; import * as HostResources from "./HostResources.ts"; // An 8-CPU host; every cpus() call advances each CPU by 100 ms at `host.busy` utilization. -const host = vi.hoisted(() => ({ busy: 0, reads: 0 })); +const host = vi.hoisted(() => ({ busy: 0, reads: 0, load: 0 })); vi.mock("node:os", async (importOriginal) => { const actual = await importOriginal(); return { ...actual, totalmem: () => 4 * 1024 ** 3, + loadavg: () => [host.load, 0, 0], cpus: () => { const elapsed = 100 * host.reads++; const times = { @@ -51,12 +52,28 @@ const unlimitedCgroup = { }; // Each path serves its values in order and repeats the last one. -const linuxHost = (files: Record>) => { +const linuxHost = ( + files: Record>, + directories: Record> = {}, +) => { const reads = new Map(); return HostResources.layer.pipe( Layer.provide( Layer.mergeAll( FileSystem.layerNoop({ + readDirectory: (path) => { + const entries = directories[path]; + return entries + ? Effect.succeed([...entries]) + : Effect.fail( + PlatformError.systemError({ + _tag: "NotFound", + module: "FileSystem", + method: "readDirectory", + pathOrDescriptor: path, + }), + ); + }, readFileString: (path) => { const values = files[path]; if (!values) { @@ -92,6 +109,49 @@ const read = Effect.gen(function* () { }); describe("HostResources.layer", () => { + it.effect("reports load per host CPU and the agents running under this user", () => { + const manager = "/user.slice/user-1000.slice/user@1000.service"; + const server = `${manager}/app.slice/t3.service`; + const agents = `/sys/fs/cgroup${manager}/app.slice/app-t3code.slice/app-t3code-agents.slice`; + return Effect.gen(function* () { + setup(); + host.load = 20; + const snapshot = yield* read; + host.load = 0; + + // 20 runnable on 8 CPUs; two populated scopes from two servers, one finished. + assert.strictEqual(snapshot.loadPerCpu, 2.5); + assert.strictEqual(snapshot.runningAgents, 2); + }).pipe( + Effect.provide( + linuxHost( + { + "/proc/self/cgroup": [`0::${server}\n`], + "/proc/meminfo": unlimitedCgroup["/proc/meminfo"], + [`${agents}/app-t3code-agents-a.slice/t3code-claude-1.scope/cgroup.events`]: [ + "populated 1\nfrozen 0\n", + ], + [`${agents}/app-t3code-agents-a.slice/t3code-codex-2.scope/cgroup.events`]: [ + "populated 0\nfrozen 0\n", + ], + [`${agents}/app-t3code-agents-b.slice/t3code-claude-3.scope/cgroup.events`]: [ + "populated 1\nfrozen 0\n", + ], + }, + { + [agents]: ["app-t3code-agents-a.slice", "app-t3code-agents-b.slice", "cgroup.procs"], + [`${agents}/app-t3code-agents-a.slice`]: [ + "t3code-claude-1.scope", + "t3code-codex-2.scope", + "memory.max", + ], + [`${agents}/app-t3code-agents-b.slice`]: ["t3code-claude-3.scope"], + }, + ), + ), + ); + }); + it.effect("reports the CPU quota and usage of the server's cgroup", () => Effect.gen(function* () { setup(); diff --git a/apps/server/src/resourceTelemetry/HostResources.ts b/apps/server/src/resourceTelemetry/HostResources.ts index 85c985f5a332..7fc5b27dd2ad 100644 --- a/apps/server/src/resourceTelemetry/HostResources.ts +++ b/apps/server/src/resourceTelemetry/HostResources.ts @@ -8,6 +8,7 @@ import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; export class HostResources extends Context.Service< @@ -15,6 +16,19 @@ export class HostResources extends Context.Service< { readonly read: Effect.Effect } >()("t3/resourceTelemetry/HostResources") {} +/** Who this machine is for, joined into each snapshot so placement can protect a person's computer. */ +export class MachinePresence extends Context.Service< + MachinePresence, + { + readonly read: Effect.Effect>; + /** This machine's resolved role, without probing whether its person is active. */ + readonly role: Effect.Effect<"headless" | "interactive">; + } +>()("t3/resourceTelemetry/HostResources/MachinePresence") {} + +// The user manager's slice holding every T3 server's agent scopes (agentScope.ts). +const AGENTS_SLICE_PATH = "app.slice/app-t3code.slice/app-t3code-agents.slice"; + // Linux CPU list syntax, as in Cpus_allowed_list: "0-3,8,10-11". function parseCpuList(list: string): ReadonlySet | null { const cpus = new Set(); @@ -76,6 +90,31 @@ const make = Effect.fn("makeHostResources")(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const readFile = (path: string) => fs.readFileString(path).pipe(Effect.orElseSucceed(() => "")); + const presence = yield* Effect.serviceOption(MachinePresence); + + // Populated agent scopes under this user's manager, from every T3 server on the host. + const countAgents = Effect.fn("HostResources.countAgents")(function* ( + cgroupPath: string | undefined, + ) { + const manager = + cgroupPath === undefined ? undefined : /^(.*\/user@\d+\.service)\//.exec(cgroupPath)?.[1]; + if (manager === undefined) return undefined; + const root = `/sys/fs/cgroup${manager}/${AGENTS_SLICE_PATH}`; + // No agents slice means agent scopes are off here, so neither the count nor caps exist. + const slices = yield* fs.readDirectory(root).pipe(Effect.orElseSucceed(() => null)); + if (slices === null) return undefined; + let running = 0; + for (const slice of slices.filter((name) => name.endsWith(".slice"))) { + const scopes = yield* fs + .readDirectory(`${root}/${slice}`) + .pipe(Effect.orElseSucceed((): ReadonlyArray => [])); + for (const scope of scopes.filter((name) => name.endsWith(".scope"))) { + const events = yield* readFile(`${root}/${slice}/${scope}/cgroup.events`); + if (/^populated 1$/m.test(events)) running++; + } + } + return running; + }); const sample = Effect.fn("HostResources.sample")(function* () { // cgroup v2 only; limits of the server's own cgroup, which is `/` inside a container. @@ -172,12 +211,24 @@ const make = Effect.fn("makeHostResources")(function* () { memoryMax - (memoryCurrent - reclaimable), ); } + // Windows reports no load average. It is per host CPU, since a cgroup quota does not + // shrink the run queue the machine's person feels. + const loadAverage = NodeOS.loadavg()[0]; + const loadPerCpu = + platform === "win32" || loadAverage === undefined || cpu.count <= 0 + ? undefined + : loadAverage / cpu.count; + const runningAgents = platform === "linux" ? yield* countAgents(cgroupPath) : undefined; + const role = Option.isSome(presence) ? yield* presence.value.read : {}; return { sampledAt: DateTime.toEpochMillis(yield* DateTime.now), cpuUtilization, cpuCount, availableMemoryBytes: Math.min(totalMemoryBytes, Math.max(0, availableMemoryBytes)), totalMemoryBytes, + ...(loadPerCpu === undefined ? {} : { loadPerCpu }), + ...(runningAgents === undefined ? {} : { runningAgents }), + ...role, }; }); diff --git a/apps/server/src/resourceTelemetry/MachinePresence.test.ts b/apps/server/src/resourceTelemetry/MachinePresence.test.ts new file mode 100644 index 000000000000..e337d1ca5040 --- /dev/null +++ b/apps/server/src/resourceTelemetry/MachinePresence.test.ts @@ -0,0 +1,127 @@ +import { assert, describe, it } from "@effect/vitest"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import type { MachinePlacementRole } from "@t3tools/contracts"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import { ChildProcessSpawner } from "effect/process"; + +import * as ServerConfig from "../config.ts"; +import * as ProcessRunner from "../processRunner.ts"; +import * as ServerSettings from "../serverSettings.ts"; +import { MachinePresence } from "./HostResources.ts"; +import * as MachinePresenceLive from "./MachinePresence.ts"; + +/** A Linux host whose logind answers `show-user` with `display` and `show-session` with `session`. */ +const linux = (input: { + readonly role: MachinePlacementRole; + readonly display: string; + readonly session?: string; + readonly calls?: Array; + /** logind refusing `show-user`, with this stderr. */ + readonly userError?: string; +}) => + MachinePresenceLive.layer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.succeed(HostProcess.Platform, "linux"), + Layer.succeed( + ProcessRunner.ProcessRunner, + ProcessRunner.ProcessRunner.of({ + run: ({ args }) => + Effect.sync(() => { + input.calls?.push(args[0] ?? ""); + const refused = args[0] === "show-user" && input.userError !== undefined; + return { + stdout: args[0] === "show-user" ? input.display : (input.session ?? ""), + stderr: refused ? (input.userError ?? "") : "", + code: ChildProcessSpawner.ExitCode(refused ? 1 : 0), + timedOut: false, + stdoutTruncated: false, + stderrTruncated: false, + stdoutInvalidUtf8: false, + stderrInvalidUtf8: false, + }; + }), + }), + ), + ServerSettings.layerTest({ machinePlacementRole: input.role }), + ServerConfig.layerTest(process.cwd(), { prefix: "t3-machine-presence-" }).pipe( + Layer.provide(NodeServices.layer), + ), + ), + ), + ); + +const read = Effect.flatMap(MachinePresence, (presence) => presence.read); + +describe("MachinePresence", () => { + it.effect("treats a Linux machine without a graphical login as headless", () => + Effect.gen(function* () { + assert.deepStrictEqual(yield* read, { placementRole: "headless" }); + }).pipe(Effect.provide(linux({ role: "auto", display: "\n" }))), + ); + + it.effect("does not count an SSH or other text login as someone at the machine", () => + Effect.gen(function* () { + for (const session of ["Type=tty\nRemote=yes\nActive=yes\n", "Type=wayland\nRemote=yes\n"]) { + const shown = yield* read.pipe( + Effect.provide(linux({ role: "auto", display: "c41\n", session })), + ); + assert.deepStrictEqual(shown, { placementRole: "headless" }); + } + }), + ); + + it.effect("counts a machine logind cannot describe as someone's, in use", () => + Effect.gen(function* () { + const unknown = yield* read.pipe( + Effect.provide(linux({ role: "auto", display: "", userError: "Failed to connect to bus" })), + ); + assert.deepStrictEqual(unknown, { placementRole: "interactive", humanActive: true }); + const loggedOut = yield* read.pipe( + Effect.provide( + linux({ role: "auto", display: "", userError: "User 1000 is not logged in." }), + ), + ); + assert.deepStrictEqual(loggedOut, { placementRole: "headless" }); + }), + ); + + it.effect("protects a graphical login unless its person is idle, locked or away", () => + Effect.gen(function* () { + const inUse = yield* read.pipe( + Effect.provide( + linux({ + role: "auto", + display: "2\n", + session: "Type=wayland\nRemote=no\nActive=yes\nLockedHint=no\nIdleHint=no\n", + }), + ), + ); + assert.deepStrictEqual(inUse, { placementRole: "interactive", humanActive: true }); + for (const away of ["Active=no", "LockedHint=yes", "IdleHint=yes"]) { + const idle = yield* read.pipe( + Effect.provide(linux({ role: "auto", display: "2\n", session: `Type=x11\n${away}\n` })), + ); + assert.deepStrictEqual(idle, { placementRole: "interactive", humanActive: false }); + } + }), + ); + + it.effect("follows an explicit role without probing for a person on a headless machine", () => + Effect.gen(function* () { + const calls: Array = []; + const headless = yield* read.pipe( + Effect.provide(linux({ role: "headless", display: "2\n", calls })), + ); + assert.deepStrictEqual(headless, { placementRole: "headless" }); + assert.deepStrictEqual(calls, []); + // An interactive machine with nobody logged in graphically has nobody to disturb. + const interactive = yield* read.pipe( + Effect.provide(linux({ role: "interactive", display: "" })), + ); + assert.deepStrictEqual(interactive, { placementRole: "interactive", humanActive: false }); + }), + ); +}); diff --git a/apps/server/src/resourceTelemetry/MachinePresence.ts b/apps/server/src/resourceTelemetry/MachinePresence.ts new file mode 100644 index 000000000000..d5da20b04ca5 --- /dev/null +++ b/apps/server/src/resourceTelemetry/MachinePresence.ts @@ -0,0 +1,110 @@ +// @effect-diagnostics nodeBuiltinImport:off -- the user id comes from the host process. +import * as NodeProcess from "node:process"; + +import type { MachinePlacementRole } from "@t3tools/contracts"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; + +import * as ServerConfig from "../config.ts"; +import * as ProcessRunner from "../processRunner.ts"; +import * as ServerSettings from "../serverSettings.ts"; +import { MachinePresence } from "./HostResources.ts"; + +function parseKeyValues(text: string): Map { + const values = new Map(); + for (const line of text.split("\n")) { + const index = line.indexOf("="); + if (index > 0) values.set(line.slice(0, index), line.slice(index + 1).trim()); + } + return values; +} + +/** The graphical login's state from logind; a session that cannot be read counts as in use. */ +export function linuxSessionInUse(shown: string): boolean { + const values = parseKeyValues(shown); + return !( + values.get("Active") === "no" || + values.get("LockedHint") === "yes" || + values.get("IdleHint") === "yes" + ); +} + +/** A login at this machine's own screen, not a remote or text session. */ +export function isLocalGraphicalSession(shown: string): boolean { + const values = parseKeyValues(shown); + const type = values.get("Type"); + return (type === "x11" || type === "wayland" || type === "mir") && values.get("Remote") !== "yes"; +} + +/** + * What logind says about this user's local graphical login: none, one (with its state), or + * unknown when logind could not be asked. Unknown never means nobody is there. + */ +type GraphicalLogin = + | { readonly kind: "none" } + | { readonly kind: "session"; readonly shown: string } + | { readonly kind: "unknown" }; + +export const make = Effect.gen(function* () { + const platform = yield* HostProcess.Platform; + const runner = yield* ProcessRunner.ProcessRunner; + const settingsService = yield* ServerSettings.ServerSettingsService; + const { mode } = yield* ServerConfig.ServerConfig; + + const run = (args: ReadonlyArray) => + runner.run({ command: "loginctl", args, timeout: "2 seconds" }).pipe(Effect.option); + + // Display can name a remote or text session, such as an SSH login, which nobody sits at. + const graphicalLogin: Effect.Effect = Effect.gen(function* () { + const uid = NodeProcess.getuid?.(); + if (uid === undefined) return { kind: "unknown" } as const; + const user = yield* run(["show-user", String(uid), "-p", "Display", "--value"]); + if (user._tag === "None") return { kind: "unknown" } as const; + if (user.value.code !== 0) + // Without a session or lingering manager logind knows no such user: nobody is logged in. + return /not logged in/i.test(user.value.stderr) + ? ({ kind: "none" } as const) + : ({ kind: "unknown" } as const); + const display = user.value.stdout.trim(); + if (display.length === 0) return { kind: "none" } as const; + const session = yield* run([ + "show-session", + display, + ...["Type", "Remote", "Active", "LockedHint", "IdleHint"].flatMap((key) => ["-p", key]), + ]); + if (session._tag === "None" || session.value.code !== 0) return { kind: "unknown" } as const; + return isLocalGraphicalSession(session.value.stdout) + ? ({ kind: "session", shown: session.value.stdout } as const) + : ({ kind: "none" } as const); + }); + + const role = Effect.gen(function* () { + const settings = yield* settingsService.getSettings.pipe(Effect.option); + const setting: MachinePlacementRole = + settings._tag === "Some" ? settings.value.machinePlacementRole : "auto"; + if (setting !== "auto") return setting; + if (mode === "desktop" || platform !== "linux") return "interactive" as const; + return (yield* graphicalLogin).kind === "none" + ? ("headless" as const) + : ("interactive" as const); + }); + + return MachinePresence.of({ + role, + read: Effect.gen(function* () { + const placementRole = yield* role; + if (placementRole === "headless") return { placementRole }; + // Only Linux can tell, and only Linux caps agents there; elsewhere assume someone is. + if (platform !== "linux") return { placementRole, humanActive: true }; + const login = yield* graphicalLogin; + return { + placementRole, + humanActive: + login.kind === "unknown" || (login.kind === "session" && linuxSessionInUse(login.shown)), + }; + }), + }); +}); + +export const layer = Layer.effect(MachinePresence, make); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 47c4a4c9a20e..efc7b50c8ee5 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -194,6 +194,7 @@ import * as DesktopAppUpdate from "./desktopUpdate/DesktopAppUpdate.ts"; import * as ServiceLauncherClient from "./cloud/serviceLauncherClient.ts"; import * as ProcessDiagnostics from "./diagnostics/ProcessDiagnostics.ts"; import * as HostResources from "./resourceTelemetry/HostResources.ts"; +import * as MachinePresence from "./resourceTelemetry/MachinePresence.ts"; import * as ProcessResourceMonitor from "./diagnostics/ProcessResourceMonitor.ts"; import * as TraceDiagnostics from "./diagnostics/TraceDiagnostics.ts"; import * as TriageService from "./triage/TriageService.ts"; @@ -314,8 +315,14 @@ const layerUsage = UsageService.layer.pipe( Layer.provide(layerServerSettings), ); +// One machine role for placement evidence and the agent slice limits. +const layerMachinePresence = MachinePresence.layer.pipe( + Layer.provide(ProcessRunner.layer), + Layer.provide(layerServerSettings), +); + const layerResourceDiagnostics = Layer.mergeAll( - HostResources.layer, + HostResources.layer.pipe(Layer.provide(layerMachinePresence)), layerResourceTelemetry, ProcessDiagnostics.layer.pipe(Layer.provide(layerResourceTelemetry)), ProcessResourceMonitor.layer.pipe(Layer.provide(layerResourceTelemetry)), @@ -1355,7 +1362,7 @@ const layerMakeServer = Layer.unwrap( Layer.provide(layerApplicationObservability), Layer.provideMerge(FetchHttpClient.layer), Layer.provide(VcsProcess.layer), - Layer.provideMerge(AgentScopeLive.layer), + Layer.provideMerge(AgentScopeLive.layer.pipe(Layer.provide(layerMachinePresence))), Layer.provideMerge(layerPlatformServices), Layer.provide(Layer.succeed(HostProcess.InvokedAs, invokedAs)), ); diff --git a/apps/web/src/components/settings/LabsSettingsPanel.tsx b/apps/web/src/components/settings/LabsSettingsPanel.tsx index 38be7d611518..e64707c469eb 100644 --- a/apps/web/src/components/settings/LabsSettingsPanel.tsx +++ b/apps/web/src/components/settings/LabsSettingsPanel.tsx @@ -5,6 +5,7 @@ import { managedCapabilityDenial } from "@t3tools/contracts"; import { hasCloudPublicConfig } from "~/cloud/publicConfig"; import { useT3ConnectAuthPrompt } from "../clerk/useT3ConnectAuthPrompt"; import { Button } from "../ui/button"; +import { Select, SelectItem, SelectPopup, SelectTrigger, SelectValue } from "../ui/select"; import { Switch } from "../ui/switch"; import { ScopedSwitch } from "./ScopedSwitch"; import { SettingsPageContainer, SettingsRow, SettingsSection } from "./settingsLayout"; @@ -22,6 +23,7 @@ export function LabsSettingsPanel() { {/* Clerk is only mounted when this build has T3 Connect. */} {hasCloudPublicConfig() ? : } + @@ -167,6 +169,64 @@ function LinkedEnvironmentsRow() { ); } +const MACHINE_ROLE_LABELS = { + auto: "Detect", + headless: "Headless", + interactive: "Someone's computer", +} as const; + +function PlacementRow() { + const settings = useScopedSettings(); + const updateSettings = useUpdateScopedSettings(); + return ( + + + + } + > +
+ + updateSettings({ automaticPlacementByDefault: Boolean(checked) }) + } + aria-label="Place new agent threads automatically" + /> + } + /> +
+
+ ); +} + function PromptSuggestionsRow() { const settings = useScopedSettings(); const updateSettings = useUpdateScopedSettings(); diff --git a/apps/web/src/components/settings/settingsSearch.ts b/apps/web/src/components/settings/settingsSearch.ts index 01c86ed15480..d7e327f080b6 100644 --- a/apps/web/src/components/settings/settingsSearch.ts +++ b/apps/web/src/components/settings/settingsSearch.ts @@ -517,6 +517,20 @@ export const SETTINGS_SEARCH_ITEMS = [ scope: "environment", searchTerms: ["beta link another machine vps peer agents work there delegate"], }, + { + id: "machine-placement-role", + title: "Load balancing", + to: "/settings/labs", + scope: "environment", + searchTerms: ["placement load balance headless gui laptop desktop busy machine spread agents"], + }, + { + id: "automatic-placement-default", + title: "Place new agent threads automatically", + to: "/settings/labs", + scope: "environment", + searchTerms: ["placement auto default load balance worktree origin other machine"], + }, { id: "prompt-suggestions-beta", title: "Suggest next prompt", diff --git a/docs/user/remote-access.md b/docs/user/remote-access.md index c963e22b8d6a..e13ea4351abb 100644 --- a/docs/user/remote-access.md +++ b/docs/user/remote-access.md @@ -397,6 +397,29 @@ ends, as for any delegated task. It keeps following the task across restarts. If the other machine revokes the link, or the link is removed here, the task fails with that reason. +### Spread agent work across linked machines + +Agents can let T3 Code pick the machine for independent work: a task with +`placement: "auto"`, or a new thread in a worktree started from origin. In +Closura builds the second needs no request when the agent passes a +`clientRequestId`. Turn **Place new agent threads +automatically** off in **Settings → Labs** to keep those threads on this machine. + +T3 Code picks the machine with the most room, judged by CPU use, load per core, +free memory and how many agents already run there. Headless machines are used +first. A Linux machine marked as someone's computer gets work only when every +headless machine is full and nobody is using it. Agents there are capped at half +its cores and get low CPU and disk priority, and more of its memory is kept free, +so it stays responsive. A Mac or Windows machine, or one on an older release, +never takes placed work from another machine. If no machine has room, work that +started here stays here. + +Each machine decides its own role in **Settings → Labs → Load balancing**. The +**Detect** setting treats the desktop app, a Mac, Windows, or a Linux graphical +login as someone's computer. Set a headless machine that also runs a desktop, +such as a computer-use host, to **Headless** by hand. A linked machine is used +only if its link allows launching threads, not with read-only access. + ## Manage or revoke access On the host, **Settings → Sharing & access** lets authorized administrators create diff --git a/packages/contracts/src/orchestratorMcp.ts b/packages/contracts/src/orchestratorMcp.ts index 4f454f9ad758..47de5847567f 100644 --- a/packages/contracts/src/orchestratorMcp.ts +++ b/packages/contracts/src/orchestratorMcp.ts @@ -184,7 +184,7 @@ export const OrchestratorMcpDelegateTaskInput = Schema.Struct({ placement: Schema.optional( Schema.Literals(["auto"]).annotate({ description: - "'auto': for work that needs none of your checkout's changes. Runs here or on a linked environment with spare capacity, this repository's project and your exact provider and model; there, in that project's checkout. Ignored with target.environmentId or isolation. Reuse clientRequestId on retry to keep the environment chosen.", + "'auto': use for any work that needs none of your checkout's changes. Runs on the least-loaded headless machine, here or linked, with spare capacity, this repository's project and your exact provider and model; there, in that project's checkout. Ignored with target.environmentId or isolation. Reuse clientRequestId on retry to keep the environment chosen.", }), ), mode: Schema.optional( diff --git a/packages/contracts/src/resourceTelemetry.ts b/packages/contracts/src/resourceTelemetry.ts index e827ee1a1235..493dac10ce02 100644 --- a/packages/contracts/src/resourceTelemetry.ts +++ b/packages/contracts/src/resourceTelemetry.ts @@ -12,6 +12,15 @@ export const HostResourcesSnapshot = Schema.Struct({ cpuCount: NonNegativeInt, availableMemoryBytes: NonNegativeInt, totalMemoryBytes: NonNegativeInt, + // Optional so older servers still decode; placement treats a missing value conservatively. + /** One-minute load average per host CPU; work queued past 1 means the machine is saturated. */ + loadPerCpu: Schema.optional(Schema.Number.check(Schema.isGreaterThanOrEqualTo(0))), + /** Agent processes running on this host, across every T3 server there. */ + runningAgents: Schema.optional(NonNegativeInt), + /** Interactive machines belong to a person; placement uses them last and only while idle. */ + placementRole: Schema.optional(Schema.Literals(["headless", "interactive"])), + /** Someone is using this interactive machine now, or it cannot tell. */ + humanActive: Schema.optional(Schema.Boolean), }); export type HostResourcesSnapshot = typeof HostResourcesSnapshot.Type; diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 5e6917eb2819..77e69196e90e 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -1375,6 +1375,9 @@ export const CloudSpeechCredentials = Schema.Struct({ assemblyai: CloudSpeechCredential, }); +export const MachinePlacementRole = Schema.Literals(["auto", "headless", "interactive"]); +export type MachinePlacementRole = typeof MachinePlacementRole.Type; + export const ServerSettings = Schema.Struct({ // Null means unmigrated; empty is authoritative and must not resurrect cleared preferences. favorites: Schema.NullOr(ModelFavorites).pipe(Schema.withDecodingDefault(Effect.succeed(null))), @@ -1507,6 +1510,22 @@ export const ServerSettings = Schema.Struct({ automaticPlacementExcludedEnvironmentIds: Schema.Array(EnvironmentId).pipe( Schema.withDecodingDefault(Effect.succeed([])), ), + /** + * Whether this machine belongs to a person at its keyboard. Automatic placement fills headless + * machines first, uses an interactive one only while its person is idle and every headless one + * is full, and runs agents there under tighter limits. Auto: a desktop app, macOS, Windows or a + * Linux graphical login is interactive; anything else is headless. + */ + machinePlacementRole: MachinePlacementRole.pipe( + Schema.withDecodingDefault(Effect.succeed("auto" as const)), + ), + /** + * Closura: agent-launched threads that need nothing of the caller's checkout are placed + * automatically unless they name an environment. Off launches them here unless they ask. + */ + automaticPlacementByDefault: Schema.Boolean.pipe( + Schema.withDecodingDefault(Effect.succeed(true)), + ), /** Beta: Claude and Codex predict the next prompt after completed turns. Codex follows a change at once; open Claude threads after they restart. */ promptSuggestionsEnabled: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), sidebarAutoSettleKeepPullRequests: Schema.Boolean.pipe( @@ -1940,6 +1959,8 @@ export const ServerSettingsPatch = Schema.Struct({ guestSharingEnabled: Schema.optionalKey(Schema.Boolean), linkedEnvironmentsEnabled: Schema.optionalKey(Schema.Boolean), automaticPlacementExcludedEnvironmentIds: Schema.optionalKey(Schema.Array(EnvironmentId)), + machinePlacementRole: Schema.optionalKey(MachinePlacementRole), + automaticPlacementByDefault: Schema.optionalKey(Schema.Boolean), promptSuggestionsEnabled: Schema.optionalKey(Schema.Boolean), backgroundActivity: Schema.optionalKey( Schema.Struct({ diff --git a/packages/shared/src/loadBalancing.ts b/packages/shared/src/loadBalancing.ts index 0b938c6c090e..1969020c9cb3 100644 --- a/packages/shared/src/loadBalancing.ts +++ b/packages/shared/src/loadBalancing.ts @@ -1,6 +1,49 @@ import type { HostResourcesSnapshot } from "@t3tools/contracts"; -/** Callers supply only connected machines hosting the project and selected provider. */ +const GiB = 1024 ** 3; + +/** + * Headroom a machine must keep after placement. A headless machine is full near its limits; + * an interactive one, a person's computer, is full long before it would feel slow to them. + */ +const LIMITS = { + headless: { cpu: 0.95, loadPerCpu: 1, memoryShare: 0.05, memoryBytes: 1.5 * GiB, agents: null }, + interactive: { cpu: 0.5, loadPerCpu: 0.5, memoryShare: 0.4, memoryBytes: 3 * GiB, agents: 2 }, +} as const; + +function headroom(resources: HostResourcesSnapshot, tier: keyof typeof LIMITS): number | null { + const limits = LIMITS[tier]; + const { cpuUtilization, cpuCount, totalMemoryBytes, availableMemoryBytes } = resources; + if (cpuUtilization === null || totalMemoryBytes <= 0 || cpuCount <= 0) return null; + // A headless machine that cannot report load (Windows) or agents (no systemd) is judged + // without them; someone's computer never is. + const loadPerCpu = resources.loadPerCpu ?? (tier === "headless" ? 0 : null); + const agents = resources.runningAgents ?? (tier === "headless" ? 0 : null); + if (loadPerCpu === null || agents === null) return null; + const memoryAvailable = availableMemoryBytes / totalMemoryBytes; + if ( + cpuUtilization >= limits.cpu || + loadPerCpu >= limits.loadPerCpu || + memoryAvailable <= limits.memoryShare || + availableMemoryBytes < limits.memoryBytes || + (limits.agents !== null && agents >= limits.agents) + ) + return null; + // Run queues past the core count show saturation that a short CPU sample can miss. + const busy = Math.max(cpuUtilization, Math.min(1, loadPerCpu)); + return (cpuCount * (1 - busy) * memoryAvailable) / (1 + agents); +} + +/** + * Callers supply only connected machines hosting the project and selected provider. Headless + * machines take work first; an interactive machine is chosen only when no headless one has room, + * its person is not using it, and it stays well clear of its limits. A machine that does not + * report its role, such as one on an older release, counts as someone's computer in use. + * + * `originEnvironmentId` is where the work starts. When nothing has room it keeps the work if it + * has room by headless limits, as before placement could pick: work started on someone's + * computer is theirs to run there. + */ export function chooseLoadBalancedEnvironment( candidates: ReadonlyArray<{ environmentId: string; @@ -10,9 +53,13 @@ export function chooseLoadBalancedEnvironment( weight: number; }>, now: number, + originEnvironmentId?: string, ): string | null { - let selected: string | null = null; - let bestScore = 0; + const best = { + headless: { id: null as string | null, score: 0 }, + interactive: { id: null as string | null, score: 0 }, + }; + let origin: string | null = null; for (const { environmentId, resources, receivedAt, weight } of candidates) { const sampledAt = receivedAt ?? resources?.sampledAt ?? 0; if ( @@ -20,21 +67,19 @@ export function chooseLoadBalancedEnvironment( !Number.isFinite(weight) || weight <= 0 || now - sampledAt > 15_000 || - sampledAt > now + 5_000 || - resources.cpuUtilization === null || - resources.cpuUtilization >= 0.95 || - resources.totalMemoryBytes <= 0 || - resources.cpuCount <= 0 + sampledAt > now + 5_000 ) { continue; } - const memoryAvailable = resources.availableMemoryBytes / resources.totalMemoryBytes; - if (memoryAvailable <= 0.05) continue; - const score = weight * resources.cpuCount * (1 - resources.cpuUtilization) * memoryAvailable; - if (score > bestScore) { - selected = environmentId; - bestScore = score; - } + if (environmentId === originEnvironmentId && headroom(resources, "headless") !== null) + origin = environmentId; + // A machine that does not say is someone's until it says otherwise, and in use if unsure. + const tier = resources.placementRole === "headless" ? "headless" : "interactive"; + if (tier === "interactive" && resources.humanActive !== false) continue; + const room = headroom(resources, tier); + if (room === null) continue; + const score = weight * room; + if (score > best[tier].score) best[tier] = { id: environmentId, score }; } - return selected; + return best.headless.id ?? best.interactive.id ?? origin; } diff --git a/packages/shared/src/threadPlacement.test.ts b/packages/shared/src/threadPlacement.test.ts index b2218c21bae8..e27aa87a3c03 100644 --- a/packages/shared/src/threadPlacement.test.ts +++ b/packages/shared/src/threadPlacement.test.ts @@ -15,6 +15,9 @@ const host = ( cpuCount: 8, availableMemoryBytes: 8 * GiB, totalMemoryBytes: 16 * GiB, + loadPerCpu: 0, + runningAgents: 0, + placementRole: "headless", }, receivedAt: now, weight: 50, @@ -79,8 +82,10 @@ describe("chooseThreadPlacement", () => { cpuCount: 8, availableMemoryBytes: 0.05 * 16 * GiB, totalMemoryBytes: 16 * GiB, + placementRole: "headless", }, }), + host("queued", 0.1, { resources: { ...host("q", 0.1).resources!, loadPerCpu: 1 } }), host("absent", 0.1, { resources: null }), ]; expect(chooseThreadPlacement(origin, candidates, now)).toEqual({ @@ -93,6 +98,82 @@ describe("chooseThreadPlacement", () => { }); }); + it("fills headless machines first, by run queue and running agents as well as CPU", () => { + const busy = (id: string, extra: object) => + host(id, 0.2, { resources: { ...host(id, 0.2).resources!, ...extra } }); + // A short CPU sample can look idle while 8 runnable tasks per core wait (load 64 on 8 cores). + expect( + chooseThreadPlacement(origin, [busy("asus", { loadPerCpu: 8 }), host("r720", 0.6)], now), + ).toMatchObject({ environmentId: "r720" }); + // Equal machines split by how many agents each already runs. + expect( + chooseThreadPlacement( + origin, + [busy("three", { runningAgents: 3 }), busy("one", { runningAgents: 1 })], + now, + ), + ).toMatchObject({ environmentId: "one" }); + }); + + it("uses a person's computer only when every headless machine is full and they are away", () => { + const laptop = (extra: object) => + host("laptop", 0.05, { + resources: { + ...host("laptop", 0.05).resources!, + placementRole: "interactive", + humanActive: false, + ...extra, + }, + }); + const full = host("r720", 0.96); + expect(chooseThreadPlacement(origin, [laptop({}), host("r720", 0.8)], now)).toMatchObject({ + environmentId: "r720", + }); + expect(chooseThreadPlacement(origin, [laptop({}), full], now)).toMatchObject({ + environmentId: "laptop", + }); + // Present, unsure, an older server that cannot say, or near the person's own headroom: never. + for (const extra of [ + { humanActive: true }, + { humanActive: undefined }, + { placementRole: undefined, humanActive: undefined }, + { loadPerCpu: undefined }, + { cpuUtilization: 0.5 }, + { loadPerCpu: 0.5 }, + { runningAgents: 2 }, + { availableMemoryBytes: 6 * GiB }, + { availableMemoryBytes: 2.9 * GiB, totalMemoryBytes: 4 * GiB }, + ]) { + expect(chooseThreadPlacement(origin, [laptop(extra), full], now)).toEqual({ + accepted: false, + reason: "no-eligible-environment", + }); + } + }); + + it("keeps work on the person's own machine when nothing else has room", () => { + const desk = host("here", 0.3, { + resources: { + ...host("here", 0.3).resources!, + placementRole: "interactive", + humanActive: true, + }, + }); + expect(chooseThreadPlacement(origin, [desk, host("r720", 0.97)], now)).toEqual({ + accepted: true, + environmentId: "here", + reason: "capacity", + }); + // A headless machine with room still wins, and a full origin is not kept. + expect(chooseThreadPlacement(origin, [desk, host("r720", 0.5)], now)).toMatchObject({ + environmentId: "r720", + }); + expect(chooseThreadPlacement(origin, [host("here", 0.97), host("r720", 0.97)], now)).toEqual({ + accepted: false, + reason: "no-eligible-environment", + }); + }); + it("refuses evidence that names one environment twice", () => { expect(chooseThreadPlacement(origin, [host("a", 0.1), host("a", 0.2)], now)).toEqual({ accepted: false, diff --git a/packages/shared/src/threadPlacement.ts b/packages/shared/src/threadPlacement.ts index 6f1c33108ef6..f52a64f7dc43 100644 --- a/packages/shared/src/threadPlacement.ts +++ b/packages/shared/src/threadPlacement.ts @@ -62,7 +62,7 @@ export function chooseThreadPlacement( .sort((a, b) => a.environmentId < b.environmentId ? -1 : a.environmentId > b.environmentId ? 1 : 0, ); - const environmentId = chooseLoadBalancedEnvironment(eligible, now); + const environmentId = chooseLoadBalancedEnvironment(eligible, now, request.originEnvironmentId); return environmentId === null ? { accepted: false, reason: "no-eligible-environment" } : { accepted: true, environmentId, reason: "capacity" };