From 926b6a982199468a316884615cc402524e47a698 Mon Sep 17 00:00:00 2001 From: nullStack65 <112618179+nullStack65@users.noreply.github.com> Date: Wed, 18 Nov 2026 22:09:00 +0000 Subject: [PATCH 1/2] feat(placement): load-aware placement that fills headless machines first Placement now reads load per core, running agents, and whether a machine belongs to a person. Headless machines take work first; someone's computer only when every headless one is full and nobody is using it, and agents there are capped so it stays responsive. Fresh worktrees from origin are placed by default in Closura builds. Model: claude-opus-5-5 Task: implementation Agent: Builder Co-Authored-By: Claude Opus 5.5 --- .../settings/ThreadSettingsSections.tsx | 17 ++- .../src/mcp/toolkits/project/handlers.ts | 37 +++++- apps/server/src/mcp/toolkits/project/tools.ts | 4 +- apps/server/src/peer/RemoteDelegation.test.ts | 76 +++++++++++ apps/server/src/process/agentScope.test.ts | 52 +++++++- apps/server/src/process/agentScope.ts | 57 ++++++-- .../resourceTelemetry/HostResources.test.ts | 64 ++++++++- .../src/resourceTelemetry/HostResources.ts | 50 +++++++ .../resourceTelemetry/MachinePresence.test.ts | 109 +++++++++++++++ .../src/resourceTelemetry/MachinePresence.ts | 125 ++++++++++++++++++ apps/server/src/server.ts | 11 +- .../components/settings/LabsSettingsPanel.tsx | 60 +++++++++ .../src/components/settings/settingsSearch.ts | 14 ++ docs/user/remote-access.md | 21 +++ packages/contracts/src/orchestratorMcp.ts | 2 +- packages/contracts/src/resourceTelemetry.ts | 9 ++ packages/contracts/src/settings.ts | 21 +++ packages/shared/src/loadBalancing.ts | 67 +++++++--- packages/shared/src/threadPlacement.test.ts | 58 ++++++++ 19 files changed, 816 insertions(+), 38 deletions(-) create mode 100644 apps/server/src/resourceTelemetry/MachinePresence.test.ts create mode 100644 apps/server/src/resourceTelemetry/MachinePresence.ts diff --git a/apps/mobile/src/features/settings/ThreadSettingsSections.tsx b/apps/mobile/src/features/settings/ThreadSettingsSections.tsx index 41600619b3a4..bfa2d53dce2e 100644 --- a/apps/mobile/src/features/settings/ThreadSettingsSections.tsx +++ b/apps/mobile/src/features/settings/ThreadSettingsSections.tsx @@ -50,6 +50,13 @@ export function SettingsLabsRouteScreen() { setting="linkedEnvironmentsEnabled" description="Let agents on a machine work on your other machines, and theirs on it. Turn it on for each machine. While off, nothing new starts over a link; running tasks finish." /> + settings.getSettings), + Effect.map((settings) => settings.automaticPlacementByDefault), + Effect.orElseSucceed(() => false), + )); + if (byDefault) { + input = { + ...input, + placement: "auto", + clientRequestId: input.clientRequestId ?? local.threadId, + modelSelection: input.modelSelection ?? local.callerModelSelection, + }; + } const key = input.clientRequestId; if (key === undefined && input.placement !== "auto") return undefined; // Work a link sent here was placed on its own side, and never uses this side's links. @@ -201,7 +226,7 @@ const launchPlacedThere = Effect.fn("mcp.launchPlacedThere")(function* ( modelSelection, runtimeMode: local.runtimeMode, }).pipe(Effect.mapError(unavailable)); - const environmentId = yield* delegation.value.place(scope, { + const placing = delegation.value.place(scope, { operation: "thread-launch", key, threadId: local.threadId, @@ -210,6 +235,15 @@ const launchPlacedThere = Effect.fn("mcp.launchPlacedThere")(function* ( runtimeMode: local.runtimeMode, here, }); + // Placement nobody asked for falls back to here, as before, when no machine has room. + const environmentId = byDefault + ? yield* placing.pipe( + Effect.catchIf( + (error) => error.code === "target_required", + () => Effect.succeed(scope.environmentId), + ), + ) + : yield* placing; return environmentId === scope.environmentId ? undefined : yield* sendThere(environmentId, key); }); @@ -330,6 +364,7 @@ export const layer = McpToolAccess.toLayer(ProjectToolkit, { strategy, runtimeMode: defaultRuntimeMode, interactionMode, + callerModelSelection: caller?.modelSelection, }); if (placedThere !== undefined) return placedThere; } diff --git a/apps/server/src/mcp/toolkits/project/tools.ts b/apps/server/src/mcp/toolkits/project/tools.ts index 1a052a1ba75a..2746bdd4741d 100644 --- a/apps/server/src/mcp/toolkits/project/tools.ts +++ b/apps/server/src/mcp/toolkits/project/tools.ts @@ -162,9 +162,9 @@ const ThreadLaunchTool = Tool.make("t3_thread_launch", { attachments: Schema.optional(Schema.Array(McpAttachmentInput).check(Schema.isMaxLength(8))), clientRequestId: Schema.optional(OrchestratorMcpClientRequestId), placement: Schema.optional( - Schema.Literals(["auto"]).annotate({ + Schema.Literals(["auto", "here"]).annotate({ description: - "'auto', without `environmentId`: here or on a linked environment with spare capacity, this repository's project and your exact `modelSelection`; there, in that project's checkout. Needs `modelSelection` and `clientRequestId`; a retry keeps the environment chosen. Attachments, scratch and worktrees not started from origin stay here.", + "'auto', without `environmentId`: here or on a linked environment with spare capacity, this repository's project and your exact `modelSelection`; there, in that project's checkout. Needs `modelSelection` and `clientRequestId`; a retry keeps the environment chosen. Attachments, scratch and worktrees not started from origin stay here. Omitted: a worktree started from origin is placed the same way, with your model, when this machine places by default. 'here' keeps it here.", }), ), /** Set by a linked environment launching its agent's delegated task here; ignored otherwise. */ diff --git a/apps/server/src/peer/RemoteDelegation.test.ts b/apps/server/src/peer/RemoteDelegation.test.ts index bebde781edaf..f2269ae2622a 100644 --- a/apps/server/src/peer/RemoteDelegation.test.ts +++ b/apps/server/src/peer/RemoteDelegation.test.ts @@ -156,6 +156,7 @@ const loadOf = (cpuUtilization: number): HostResourcesSnapshot => ({ cpuCount: 8, availableMemoryBytes: 8 * GiB, totalMemoryBytes: 16 * GiB, + placementRole: "headless", }); const makeBoxThread = Effect.gen(function* () { @@ -517,6 +518,7 @@ const boxToolkitLayer = (thread: BoxThread) => { ), }); const layerBoxLaunches = Layer.mock(ThreadLaunch.ThreadLaunchService)({ + checkWorktreeBase: () => Effect.void, launch: (input) => Ref.get(thread.runless).pipe( Effect.flatMap((runless) => @@ -892,6 +894,23 @@ const launchPlaced = ( outsideScope, ); +/** A launch that says nothing about placement, so this machine's default decides. */ +const launchUnplaced = ( + laptopEnv: Effect.Success>, + args: Record, +) => + laptopEnv.call( + "t3_thread_launch", + { + projectId: "project:laptop", + title: "Suite", + message: "Run the suite wherever there is room.", + modelSelection: { instanceId: boxClaude.instanceId, model: "claude-sonnet-4-6" }, + ...args, + }, + outsideScope, + ); + /** The laptop's parent thread, mid-turn when its agent delegates. */ const seedParent = ( laptopEnv: Effect.Success>, @@ -1624,6 +1643,63 @@ it.effect("an outside client's auto-placed launch starts at the project's defaul ).pipe(Effect.provide(NodeHttpServer.layerTest)), ); +it.effect("a fresh worktree from origin is placed without asking, unless kept here", () => + Effect.scoped( + Effect.gen(function* () { + const thread = yield* makeBoxThread; + const b = yield* serveBox(thread); + const offers = yield* SubscriptionRef.make< + ReadonlyArray + >([]); + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); + const localLaunches = yield* Ref.make(0); + const a = yield* makeLaptop(database, offers, undefined, Effect.void, { + localLaunch: Ref.update(localLaunches, (count) => count + 1), + }); + yield* linkTo(a.links, b, "full-access"); + const fresh = { type: "worktree", baseRef: "main", startFromOrigin: true }; + const placed = yield* launchUnplaced(a, { workspaceStrategy: fresh }); + expect(placed.isError, JSON.stringify(placed.content)).toBe(false); + expect(placed.structuredContent).toMatchObject({ environmentId: box.environmentId }); + expect(yield* Ref.get(localLaunches)).toBe(0); + + // 'here', or the fleet default turned off, keeps it on this machine. + yield* launchPlaced(a, { placement: "here", workspaceStrategy: fresh }); + expect(yield* Ref.get(localLaunches)).toBe(1); + yield* a.settings.updateSettings({ automaticPlacementByDefault: false }); + yield* launchUnplaced(a, { workspaceStrategy: fresh }); + expect(yield* Ref.get(localLaunches)).toBe(2); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); + +it.effect("placement nobody asked for runs here when no machine has room", () => + Effect.scoped( + Effect.gen(function* () { + const thread = yield* makeBoxThread; + yield* Ref.set(thread.resources, { ...loadOf(0.99) }); + const b = yield* serveBox(thread); + const offers = yield* SubscriptionRef.make< + ReadonlyArray + >([]); + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); + const localLaunches = yield* Ref.make(0); + const a = yield* makeLaptop(database, offers, undefined, Effect.void, { + localLaunch: Ref.update(localLaunches, (count) => count + 1), + }); + yield* linkTo(a.links, b, "full-access"); + const fresh = { type: "worktree", baseRef: "main", startFromOrigin: true }; + yield* launchUnplaced(a, { workspaceStrategy: fresh }); + expect(yield* Ref.get(localLaunches)).toBe(1); + expect(yield* Ref.get(thread.launched)).toBeNull(); + // Asked for, it still fails loudly instead. + const asked = yield* launchPlaced(a, { clientRequestId: "full", workspaceStrategy: fresh }); + expect(asked.isError).toBe(true); + expect(yield* Ref.get(localLaunches)).toBe(1); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); + it.effect( "a retry that drops placement or asks for this checkout reaches the thread placed there", () => diff --git a/apps/server/src/process/agentScope.test.ts b/apps/server/src/process/agentScope.test.ts index 49674354b253..a10874b3eed7 100644 --- a/apps/server/src/process/agentScope.test.ts +++ b/apps/server/src/process/agentScope.test.ts @@ -12,12 +12,13 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Logger from "effect/Logger"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; -import { vi } from "vite-plus/test"; +import { afterEach, vi } from "vite-plus/test"; import * as ServerConfig from "../config.ts"; import { makeCodexAppServerSpawnCommand } from "../orchestration-v2/Adapters/CodexAdapterV2.ts"; import * as ProcessRunner from "../processRunner.ts"; import * as ProviderProcesses from "./providerProcesses.ts"; +import { MachinePresence } from "../resourceTelemetry/HostResources.ts"; import { classifyScope, make, ownedAgentSlice, type ScopeState } from "./agentScope.ts"; const GiB = 1024 ** 3; @@ -67,11 +68,13 @@ const withFakeSystemd = ( stopFails = false, leftovers = false, liveOwner = false, + role, }: { readonly oomPolicy?: boolean; readonly stopFails?: boolean; readonly leftovers?: boolean; readonly liveOwner?: boolean; + readonly role?: { current: "headless" | "interactive" }; } = {}, ) => Effect.gen(function* () { @@ -125,7 +128,17 @@ const withFakeSystemd = ( if (leftovers) { units.set(ownedAgentSlice(stateDir), { activeState: "active", result: "success" }); } + const presence = + role === undefined + ? undefined + : MachinePresence.of({ + role: Effect.sync(() => role.current), + read: Effect.sync(() => ({ placementRole: role.current })), + }); const scope = yield* make.pipe( + presence === undefined + ? (effect) => effect + : Effect.provideService(MachinePresence, presence), Effect.provideService(ProviderProcesses.ProviderProcesses, { hasOtherLiveOwner: liveOwner }), Effect.provideService(ProcessRunner.ProcessRunner, runner), Effect.provideService(HostProcess.Platform, "linux"), @@ -195,6 +208,9 @@ const launchCodex = (scope: Effect.Success, threadId: string) => ); describe("AgentScope service", () => { + afterEach(() => { + memory.total = 16 * GiB; + }); it.effect("labels a thread from its newest scope until its next session clears it", () => withFakeSystemd(({ scope, units }) => Effect.gen(function* () { @@ -325,12 +341,46 @@ describe("AgentScope service", () => { "CPUWeight=10", "IOWeight=10", "TasksMax=25%", + "CPUQuota=", ], ]); }), ), ); + it.effect("keeps a person's computer responsive and follows a role change", () => { + memory.total = 8 * GiB; + const role = { current: "interactive" as "headless" | "interactive" }; + return withFakeSystemd( + ({ scope, calls }) => + Effect.gen(function* () { + const limits = () => + calls.filter(([, verb]) => verb === "set-property").map((args) => args.slice(3)); + yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" }); + const cores = NodeOS.cpus().length; + // 8 GB keeps 4 GB for the person before agents slow and 3 GB before a kill. + expect(limits()).toEqual([ + ["app-t3code-agents.slice", `MemoryHigh=${4 * GiB}`, `MemoryMax=${5 * GiB}`], + [ + "app-t3code.slice", + "CPUWeight=1", + "IOWeight=1", + "TasksMax=25%", + `CPUQuota=${Math.max(100, Math.floor(cores * 50))}%`, + ], + ]); + + role.current = "headless"; + yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "b" }); + expect(limits().slice(2)).toEqual([ + ["app-t3code-agents.slice", `MemoryHigh=${4 * GiB}`, `MemoryMax=${6 * GiB}`], + ["app-t3code.slice", "CPUWeight=10", "IOWeight=10", "TasksMax=25%", "CPUQuota="], + ]); + }), + { role }, + ); + }); + it.effect("follows a live memory change at the next launch", () => { memory.total = 16 * GiB; return withFakeSystemd(({ scope, calls, failingVerbs }) => diff --git a/apps/server/src/process/agentScope.ts b/apps/server/src/process/agentScope.ts index 1e52e680b6f8..e5cd473d6889 100644 --- a/apps/server/src/process/agentScope.ts +++ b/apps/server/src/process/agentScope.ts @@ -13,11 +13,13 @@ import { resolveCommandPath } from "@t3tools/shared/shell"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as Semaphore from "effect/Semaphore"; import * as ServerConfig from "../config.ts"; import * as ProcessRunner from "../processRunner.ts"; +import { MachinePresence } from "../resourceTelemetry/HostResources.ts"; import * as ProviderProcesses from "./providerProcesses.ts"; const AGENT_SLICE = "app-t3code-agents.slice"; @@ -27,6 +29,19 @@ const AGENTS_PARENT_SLICE = "app-t3code.slice"; // has those controllers: current systemd delegates cpu, rarely io. TasksMax keeps agents // below logind's 33% per-user task limit, leaving forks for the server. const AGENTS_PARENT_PROPERTIES = ["CPUWeight=10", "IOWeight=10", "TasksMax=25%"]; +// On a person's computer agents also yield to every app and never take more than half +// the cores, so the desktop stays responsive however much work is placed there. +function agentsParentProperties(role: AgentHostRole, cpuCount: number) { + if (role === "headless") return [...AGENTS_PARENT_PROPERTIES, "CPUQuota="]; + return [ + "CPUWeight=1", + "IOWeight=1", + "TasksMax=25%", + `CPUQuota=${Math.max(100, Math.floor(cpuCount * 50))}%`, + ]; +} + +export type AgentHostRole = "headless" | "interactive"; export function ownedAgentSlice(stateDir: string): string { const hash = NodeCrypto.createHash("sha256").update(stateDir).digest("hex").slice(0, 12); @@ -41,7 +56,16 @@ const GiB = 1024 ** 3; * working. Small machines still give agents at least half of RAM before * throttling and three quarters before a kill. */ -function agentSliceMemoryLimits(totalBytes: number) { +export function agentSliceMemoryLimits(totalBytes: number, role: AgentHostRole = "headless") { + if (role === "interactive") { + // A person's computer keeps 4 GB or 40% for them before agents slow, and 3 GB or 30% + // before one is killed. + const high = Math.max(GiB, totalBytes - Math.max(4 * GiB, Math.floor(totalBytes * 0.4))); + return { + high, + max: Math.max(high + GiB / 2, totalBytes - Math.max(3 * GiB, Math.floor(totalBytes * 0.3))), + }; + } return { high: Math.max(totalBytes - 6 * GiB, Math.floor(totalBytes / 2)), max: Math.max(totalBytes - 4 * GiB, Math.floor(totalBytes * 0.75)), @@ -110,6 +134,11 @@ export const make = Effect.gen(function* () { const { stateDir } = yield* ServerConfig.ServerConfig; const { hasOtherLiveOwner } = yield* ProviderProcesses.ProviderProcesses; const ownedSlice = ownedAgentSlice(stateDir); + // Without presence (tests, embedded servers) the machine is treated as headless. + const presence = yield* Effect.serviceOption(MachinePresence); + const hostRole: Effect.Effect = Option.isSome(presence) + ? presence.value.role + : Effect.succeed("headless"); const layerScope = yield* Effect.scope; const resolve = (command: string, env: NodeJS.ProcessEnv) => @@ -202,36 +231,48 @@ export const make = Effect.gen(function* () { }); return false; }); - // VMs and containers can resize RAM live, so each launch re-checks the total - // the limits came from. A failed update is retried by the next launch. + // VMs and containers can resize RAM live, and a machine can become someone's + // computer, so each launch re-checks the total and role the limits came from. + // A failed update is retried by the next launch. let appliedTotal: number | undefined; + let appliedRole: AgentHostRole | undefined; const memoryLimitsLock = yield* Semaphore.make(1); const applyMemoryLimits = memoryLimitsLock.withPermits(1)( Effect.gen(function* () { const total = NodeOS.totalmem(); - if (total === appliedTotal) return; - const limits = agentSliceMemoryLimits(total); + const role = yield* hostRole; + if (total === appliedTotal && role === appliedRole) return; + const limits = agentSliceMemoryLimits(total, role); const applied = yield* setProperties(AGENT_SLICE, [ `MemoryHigh=${limits.high}`, `MemoryMax=${limits.max}`, ]); if (!applied) return; + if (role !== appliedRole) { + const parentApplied = yield* setProperties( + AGENTS_PARENT_SLICE, + agentsParentProperties(role, NodeOS.cpus().length), + ); + if (!parentApplied) return; + } if (appliedTotal !== undefined) { - yield* Effect.logInfo("Agent memory limits follow a memory change", { + yield* Effect.logInfo("Agent limits follow a memory or role change", { slice: AGENT_SLICE, previousTotal: appliedTotal, total, + role, ...limits, }); } appliedTotal = total; + appliedRole = role; }), ); yield* applyMemoryLimits; - yield* setProperties(AGENTS_PARENT_SLICE, AGENTS_PARENT_PROPERTIES); yield* Effect.logInfo("Agent scopes are on", { slice: AGENT_SLICE, - ...agentSliceMemoryLimits(NodeOS.totalmem()), + role: appliedRole, + ...agentSliceMemoryLimits(NodeOS.totalmem(), appliedRole), oomPolicy: supportsOomPolicy, }); return { diff --git a/apps/server/src/resourceTelemetry/HostResources.test.ts b/apps/server/src/resourceTelemetry/HostResources.test.ts index 3e5c636c5d7c..fdf0d064228c 100644 --- a/apps/server/src/resourceTelemetry/HostResources.test.ts +++ b/apps/server/src/resourceTelemetry/HostResources.test.ts @@ -13,12 +13,13 @@ import { vi } from "vite-plus/test"; import * as HostResources from "./HostResources.ts"; // An 8-CPU host; every cpus() call advances each CPU by 100 ms at `host.busy` utilization. -const host = vi.hoisted(() => ({ busy: 0, reads: 0 })); +const host = vi.hoisted(() => ({ busy: 0, reads: 0, load: 0 })); vi.mock("node:os", async (importOriginal) => { const actual = await importOriginal(); return { ...actual, totalmem: () => 4 * 1024 ** 3, + loadavg: () => [host.load, 0, 0], cpus: () => { const elapsed = 100 * host.reads++; const times = { @@ -51,12 +52,28 @@ const unlimitedCgroup = { }; // Each path serves its values in order and repeats the last one. -const linuxHost = (files: Record>) => { +const linuxHost = ( + files: Record>, + directories: Record> = {}, +) => { const reads = new Map(); return HostResources.layer.pipe( Layer.provide( Layer.mergeAll( FileSystem.layerNoop({ + readDirectory: (path) => { + const entries = directories[path]; + return entries + ? Effect.succeed([...entries]) + : Effect.fail( + PlatformError.systemError({ + _tag: "NotFound", + module: "FileSystem", + method: "readDirectory", + pathOrDescriptor: path, + }), + ); + }, readFileString: (path) => { const values = files[path]; if (!values) { @@ -92,6 +109,49 @@ const read = Effect.gen(function* () { }); describe("HostResources.layer", () => { + it.effect("reports load per host CPU and the agents running under this user", () => { + const manager = "/user.slice/user-1000.slice/user@1000.service"; + const server = `${manager}/app.slice/t3.service`; + const agents = `/sys/fs/cgroup${manager}/app.slice/app-t3code.slice/app-t3code-agents.slice`; + return Effect.gen(function* () { + setup(); + host.load = 20; + const snapshot = yield* read; + host.load = 0; + + // 20 runnable on 8 CPUs; two populated scopes from two servers, one finished. + assert.strictEqual(snapshot.loadPerCpu, 2.5); + assert.strictEqual(snapshot.runningAgents, 2); + }).pipe( + Effect.provide( + linuxHost( + { + "/proc/self/cgroup": [`0::${server}\n`], + "/proc/meminfo": unlimitedCgroup["/proc/meminfo"], + [`${agents}/app-t3code-agents-a.slice/t3code-claude-1.scope/cgroup.events`]: [ + "populated 1\nfrozen 0\n", + ], + [`${agents}/app-t3code-agents-a.slice/t3code-codex-2.scope/cgroup.events`]: [ + "populated 0\nfrozen 0\n", + ], + [`${agents}/app-t3code-agents-b.slice/t3code-claude-3.scope/cgroup.events`]: [ + "populated 1\nfrozen 0\n", + ], + }, + { + [agents]: ["app-t3code-agents-a.slice", "app-t3code-agents-b.slice", "cgroup.procs"], + [`${agents}/app-t3code-agents-a.slice`]: [ + "t3code-claude-1.scope", + "t3code-codex-2.scope", + "memory.max", + ], + [`${agents}/app-t3code-agents-b.slice`]: ["t3code-claude-3.scope"], + }, + ), + ), + ); + }); + it.effect("reports the CPU quota and usage of the server's cgroup", () => Effect.gen(function* () { setup(); diff --git a/apps/server/src/resourceTelemetry/HostResources.ts b/apps/server/src/resourceTelemetry/HostResources.ts index 85c985f5a332..b1e0445f7ae6 100644 --- a/apps/server/src/resourceTelemetry/HostResources.ts +++ b/apps/server/src/resourceTelemetry/HostResources.ts @@ -8,6 +8,7 @@ import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; export class HostResources extends Context.Service< @@ -15,6 +16,19 @@ export class HostResources extends Context.Service< { readonly read: Effect.Effect } >()("t3/resourceTelemetry/HostResources") {} +/** Who this machine is for, joined into each snapshot so placement can protect a person's computer. */ +export class MachinePresence extends Context.Service< + MachinePresence, + { + readonly read: Effect.Effect>; + /** This machine's resolved role, without probing whether its person is active. */ + readonly role: Effect.Effect<"headless" | "interactive">; + } +>()("t3/resourceTelemetry/HostResources/MachinePresence") {} + +// The user manager's slice holding every T3 server's agent scopes (agentScope.ts). +const AGENTS_SLICE_PATH = "app.slice/app-t3code.slice/app-t3code-agents.slice"; + // Linux CPU list syntax, as in Cpus_allowed_list: "0-3,8,10-11". function parseCpuList(list: string): ReadonlySet | null { const cpus = new Set(); @@ -76,6 +90,30 @@ const make = Effect.fn("makeHostResources")(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const readFile = (path: string) => fs.readFileString(path).pipe(Effect.orElseSucceed(() => "")); + const presence = yield* Effect.serviceOption(MachinePresence); + + // Populated agent scopes under this user's manager, from every T3 server on the host. + const countAgents = Effect.fn("HostResources.countAgents")(function* ( + cgroupPath: string | undefined, + ) { + const manager = + cgroupPath === undefined ? undefined : /^(.*\/user@\d+\.service)\//.exec(cgroupPath)?.[1]; + if (manager === undefined) return undefined; + const root = `/sys/fs/cgroup${manager}/${AGENTS_SLICE_PATH}`; + const slices = yield* fs.readDirectory(root).pipe(Effect.orElseSucceed(() => null)); + if (slices === null) return 0; + let running = 0; + for (const slice of slices.filter((name) => name.endsWith(".slice"))) { + const scopes = yield* fs + .readDirectory(`${root}/${slice}`) + .pipe(Effect.orElseSucceed((): ReadonlyArray => [])); + for (const scope of scopes.filter((name) => name.endsWith(".scope"))) { + const events = yield* readFile(`${root}/${slice}/${scope}/cgroup.events`); + if (/^populated 1$/m.test(events)) running++; + } + } + return running; + }); const sample = Effect.fn("HostResources.sample")(function* () { // cgroup v2 only; limits of the server's own cgroup, which is `/` inside a container. @@ -172,12 +210,24 @@ const make = Effect.fn("makeHostResources")(function* () { memoryMax - (memoryCurrent - reclaimable), ); } + // Windows reports no load average. It is per host CPU, since a cgroup quota does not + // shrink the run queue the machine's person feels. + const loadAverage = NodeOS.loadavg()[0]; + const loadPerCpu = + platform === "win32" || loadAverage === undefined || cpu.count <= 0 + ? undefined + : loadAverage / cpu.count; + const runningAgents = platform === "linux" ? yield* countAgents(cgroupPath) : undefined; + const role = Option.isSome(presence) ? yield* presence.value.read : {}; return { sampledAt: DateTime.toEpochMillis(yield* DateTime.now), cpuUtilization, cpuCount, availableMemoryBytes: Math.min(totalMemoryBytes, Math.max(0, availableMemoryBytes)), totalMemoryBytes, + ...(loadPerCpu === undefined ? {} : { loadPerCpu }), + ...(runningAgents === undefined ? {} : { runningAgents }), + ...role, }; }); diff --git a/apps/server/src/resourceTelemetry/MachinePresence.test.ts b/apps/server/src/resourceTelemetry/MachinePresence.test.ts new file mode 100644 index 000000000000..32631492fb97 --- /dev/null +++ b/apps/server/src/resourceTelemetry/MachinePresence.test.ts @@ -0,0 +1,109 @@ +import { assert, describe, it } from "@effect/vitest"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import type { MachinePlacementRole } from "@t3tools/contracts"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import { ChildProcessSpawner } from "effect/process"; + +import * as ServerConfig from "../config.ts"; +import * as ProcessRunner from "../processRunner.ts"; +import * as ServerSettings from "../serverSettings.ts"; +import { MachinePresence } from "./HostResources.ts"; +import * as MachinePresenceLive from "./MachinePresence.ts"; + +/** A Linux host whose logind answers `show-user` with `display` and `show-session` with `session`. */ +const linux = (input: { + readonly role: MachinePlacementRole; + readonly display: string; + readonly session?: string; + readonly calls?: Array; +}) => + MachinePresenceLive.layer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.succeed(HostProcess.Platform, "linux"), + Layer.succeed( + ProcessRunner.ProcessRunner, + ProcessRunner.ProcessRunner.of({ + run: ({ args }) => + Effect.sync(() => { + input.calls?.push(args[0] ?? ""); + return { + stdout: args[0] === "show-user" ? input.display : (input.session ?? ""), + stderr: "", + code: ChildProcessSpawner.ExitCode(0), + timedOut: false, + stdoutTruncated: false, + stderrTruncated: false, + stdoutInvalidUtf8: false, + stderrInvalidUtf8: false, + }; + }), + }), + ), + ServerSettings.layerTest({ machinePlacementRole: input.role }), + ServerConfig.layerTest(process.cwd(), { prefix: "t3-machine-presence-" }).pipe( + Layer.provide(NodeServices.layer), + ), + ), + ), + ); + +const read = Effect.flatMap(MachinePresence, (presence) => presence.read); + +describe("MachinePresence", () => { + it.effect("treats a Linux machine without a graphical login as headless", () => + Effect.gen(function* () { + assert.deepStrictEqual(yield* read, { placementRole: "headless" }); + }).pipe(Effect.provide(linux({ role: "auto", display: "\n" }))), + ); + + it.effect("does not count an SSH or other text login as someone at the machine", () => + Effect.gen(function* () { + for (const session of ["Type=tty\nRemote=yes\nActive=yes\n", "Type=wayland\nRemote=yes\n"]) { + const shown = yield* read.pipe( + Effect.provide(linux({ role: "auto", display: "c41\n", session })), + ); + assert.deepStrictEqual(shown, { placementRole: "headless" }); + } + }), + ); + + it.effect("protects a graphical login unless its person is idle, locked or away", () => + Effect.gen(function* () { + const inUse = yield* read.pipe( + Effect.provide( + linux({ + role: "auto", + display: "2\n", + session: "Type=wayland\nRemote=no\nActive=yes\nLockedHint=no\nIdleHint=no\n", + }), + ), + ); + assert.deepStrictEqual(inUse, { placementRole: "interactive", humanActive: true }); + for (const away of ["Active=no", "LockedHint=yes", "IdleHint=yes"]) { + const idle = yield* read.pipe( + Effect.provide(linux({ role: "auto", display: "2\n", session: `Type=x11\n${away}\n` })), + ); + assert.deepStrictEqual(idle, { placementRole: "interactive", humanActive: false }); + } + }), + ); + + it.effect("follows an explicit role without probing for a person on a headless machine", () => + Effect.gen(function* () { + const calls: Array = []; + const headless = yield* read.pipe( + Effect.provide(linux({ role: "headless", display: "2\n", calls })), + ); + assert.deepStrictEqual(headless, { placementRole: "headless" }); + assert.deepStrictEqual(calls, []); + // An interactive machine with nobody logged in graphically has nobody to disturb. + const interactive = yield* read.pipe( + Effect.provide(linux({ role: "interactive", display: "" })), + ); + assert.deepStrictEqual(interactive, { placementRole: "interactive", humanActive: false }); + }), + ); +}); diff --git a/apps/server/src/resourceTelemetry/MachinePresence.ts b/apps/server/src/resourceTelemetry/MachinePresence.ts new file mode 100644 index 000000000000..5d2080e03477 --- /dev/null +++ b/apps/server/src/resourceTelemetry/MachinePresence.ts @@ -0,0 +1,125 @@ +// @effect-diagnostics nodeBuiltinImport:off -- the user id comes from the host process. +import * as NodeProcess from "node:process"; + +import type { MachinePlacementRole } from "@t3tools/contracts"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; + +import * as ServerConfig from "../config.ts"; +import * as ProcessRunner from "../processRunner.ts"; +import * as ServerSettings from "../serverSettings.ts"; +import { MachinePresence } from "./HostResources.ts"; + +/** A person who has not touched a Mac's keyboard or pointer for this long is away. */ +const DARWIN_IDLE_NANOS = 10 * 60 * 1e9; + +function parseKeyValues(text: string): Map { + const values = new Map(); + for (const line of text.split("\n")) { + const index = line.indexOf("="); + if (index > 0) values.set(line.slice(0, index), line.slice(index + 1).trim()); + } + return values; +} + +/** The graphical login's state from logind; a session that cannot be read counts as in use. */ +export function linuxSessionInUse(shown: string): boolean { + const values = parseKeyValues(shown); + return !( + values.get("Active") === "no" || + values.get("LockedHint") === "yes" || + values.get("IdleHint") === "yes" + ); +} + +/** A login at this machine's own screen, not a remote or text session. */ +export function isLocalGraphicalSession(shown: string): boolean { + const values = parseKeyValues(shown); + const type = values.get("Type"); + return (type === "x11" || type === "wayland" || type === "mir") && values.get("Remote") !== "yes"; +} + +/** HIDIdleTime from `ioreg -c IOHIDSystem`; undefined when it cannot be read. */ +export function darwinIdleNanos(output: string): number | undefined { + const value = /"HIDIdleTime"\s*=\s*(\d+)/.exec(output)?.[1]; + return value === undefined ? undefined : Number(value); +} + +export const make = Effect.gen(function* () { + const platform = yield* HostProcess.Platform; + const runner = yield* ProcessRunner.ProcessRunner; + const settingsService = yield* ServerSettings.ServerSettingsService; + const { mode } = yield* ServerConfig.ServerConfig; + + const run = (command: string, args: ReadonlyArray) => + runner.run({ command, args, timeout: "2 seconds" }).pipe( + Effect.map((result) => (result.code === 0 ? result.stdout : undefined)), + Effect.orElseSucceed(() => undefined), + ); + + // The logind state of this user's local graphical login, or undefined when there is none. + // Display can name a remote or text session, such as an SSH login, which nobody sits at. + const graphicalSession = Effect.gen(function* () { + const uid = NodeProcess.getuid?.(); + if (uid === undefined) return undefined; + const display = (yield* run("loginctl", [ + "show-user", + String(uid), + "-p", + "Display", + "--value", + ]))?.trim(); + if (display === undefined || display.length === 0) return undefined; + const shown = yield* run("loginctl", [ + "show-session", + display, + ...["Type", "Remote", "Active", "LockedHint", "IdleHint"].flatMap((key) => ["-p", key]), + ]); + // A session logind names but cannot describe is treated as someone's. + if (shown === undefined) return ""; + return isLocalGraphicalSession(shown) ? shown : undefined; + }); + + const resolveRole = Effect.fn("MachinePresence.resolveRole")(function* ( + setting: MachinePlacementRole, + ) { + if (setting !== "auto") return setting; + if (mode === "desktop" || platform !== "linux") return "interactive" as const; + return (yield* graphicalSession) === undefined + ? ("headless" as const) + : ("interactive" as const); + }); + + const humanActive = Effect.gen(function* () { + if (platform === "darwin") { + const idle = darwinIdleNanos( + (yield* run("/usr/sbin/ioreg", ["-c", "IOHIDSystem", "-d", "4", "-k", "HIDIdleTime"])) ?? + "", + ); + return idle === undefined || idle < DARWIN_IDLE_NANOS; + } + if (platform !== "linux") return true; + const session = yield* graphicalSession; + return session !== undefined && linuxSessionInUse(session); + }); + + const role = Effect.gen(function* () { + const settings = yield* settingsService.getSettings.pipe(Effect.option); + return yield* resolveRole( + settings._tag === "Some" ? settings.value.machinePlacementRole : "auto", + ); + }); + + return MachinePresence.of({ + role, + read: Effect.gen(function* () { + const placementRole = yield* role; + return placementRole === "headless" + ? { placementRole } + : { placementRole, humanActive: yield* humanActive }; + }), + }); +}); + +export const layer = Layer.effect(MachinePresence, make); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 47c4a4c9a20e..efc7b50c8ee5 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -194,6 +194,7 @@ import * as DesktopAppUpdate from "./desktopUpdate/DesktopAppUpdate.ts"; import * as ServiceLauncherClient from "./cloud/serviceLauncherClient.ts"; import * as ProcessDiagnostics from "./diagnostics/ProcessDiagnostics.ts"; import * as HostResources from "./resourceTelemetry/HostResources.ts"; +import * as MachinePresence from "./resourceTelemetry/MachinePresence.ts"; import * as ProcessResourceMonitor from "./diagnostics/ProcessResourceMonitor.ts"; import * as TraceDiagnostics from "./diagnostics/TraceDiagnostics.ts"; import * as TriageService from "./triage/TriageService.ts"; @@ -314,8 +315,14 @@ const layerUsage = UsageService.layer.pipe( Layer.provide(layerServerSettings), ); +// One machine role for placement evidence and the agent slice limits. +const layerMachinePresence = MachinePresence.layer.pipe( + Layer.provide(ProcessRunner.layer), + Layer.provide(layerServerSettings), +); + const layerResourceDiagnostics = Layer.mergeAll( - HostResources.layer, + HostResources.layer.pipe(Layer.provide(layerMachinePresence)), layerResourceTelemetry, ProcessDiagnostics.layer.pipe(Layer.provide(layerResourceTelemetry)), ProcessResourceMonitor.layer.pipe(Layer.provide(layerResourceTelemetry)), @@ -1355,7 +1362,7 @@ const layerMakeServer = Layer.unwrap( Layer.provide(layerApplicationObservability), Layer.provideMerge(FetchHttpClient.layer), Layer.provide(VcsProcess.layer), - Layer.provideMerge(AgentScopeLive.layer), + Layer.provideMerge(AgentScopeLive.layer.pipe(Layer.provide(layerMachinePresence))), Layer.provideMerge(layerPlatformServices), Layer.provide(Layer.succeed(HostProcess.InvokedAs, invokedAs)), ); diff --git a/apps/web/src/components/settings/LabsSettingsPanel.tsx b/apps/web/src/components/settings/LabsSettingsPanel.tsx index 38be7d611518..e64707c469eb 100644 --- a/apps/web/src/components/settings/LabsSettingsPanel.tsx +++ b/apps/web/src/components/settings/LabsSettingsPanel.tsx @@ -5,6 +5,7 @@ import { managedCapabilityDenial } from "@t3tools/contracts"; import { hasCloudPublicConfig } from "~/cloud/publicConfig"; import { useT3ConnectAuthPrompt } from "../clerk/useT3ConnectAuthPrompt"; import { Button } from "../ui/button"; +import { Select, SelectItem, SelectPopup, SelectTrigger, SelectValue } from "../ui/select"; import { Switch } from "../ui/switch"; import { ScopedSwitch } from "./ScopedSwitch"; import { SettingsPageContainer, SettingsRow, SettingsSection } from "./settingsLayout"; @@ -22,6 +23,7 @@ export function LabsSettingsPanel() { {/* Clerk is only mounted when this build has T3 Connect. */} {hasCloudPublicConfig() ? : } + @@ -167,6 +169,64 @@ function LinkedEnvironmentsRow() { ); } +const MACHINE_ROLE_LABELS = { + auto: "Detect", + headless: "Headless", + interactive: "Someone's computer", +} as const; + +function PlacementRow() { + const settings = useScopedSettings(); + const updateSettings = useUpdateScopedSettings(); + return ( + + + + } + > +
+ + updateSettings({ automaticPlacementByDefault: Boolean(checked) }) + } + aria-label="Place new agent threads automatically" + /> + } + /> +
+
+ ); +} + function PromptSuggestionsRow() { const settings = useScopedSettings(); const updateSettings = useUpdateScopedSettings(); diff --git a/apps/web/src/components/settings/settingsSearch.ts b/apps/web/src/components/settings/settingsSearch.ts index 01c86ed15480..d7e327f080b6 100644 --- a/apps/web/src/components/settings/settingsSearch.ts +++ b/apps/web/src/components/settings/settingsSearch.ts @@ -517,6 +517,20 @@ export const SETTINGS_SEARCH_ITEMS = [ scope: "environment", searchTerms: ["beta link another machine vps peer agents work there delegate"], }, + { + id: "machine-placement-role", + title: "Load balancing", + to: "/settings/labs", + scope: "environment", + searchTerms: ["placement load balance headless gui laptop desktop busy machine spread agents"], + }, + { + id: "automatic-placement-default", + title: "Place new agent threads automatically", + to: "/settings/labs", + scope: "environment", + searchTerms: ["placement auto default load balance worktree origin other machine"], + }, { id: "prompt-suggestions-beta", title: "Suggest next prompt", diff --git a/docs/user/remote-access.md b/docs/user/remote-access.md index c963e22b8d6a..269ed2c402d6 100644 --- a/docs/user/remote-access.md +++ b/docs/user/remote-access.md @@ -397,6 +397,27 @@ ends, as for any delegated task. It keeps following the task across restarts. If the other machine revokes the link, or the link is removed here, the task fails with that reason. +### Spread agent work across linked machines + +Agents can let T3 Code pick the machine for independent work: a task with +`placement: "auto"`, or a new thread in a worktree started from origin. In +Closura builds the second needs no request. Turn **Place new agent threads +automatically** off in **Settings → Labs** to keep those threads on this machine. + +T3 Code picks the machine with the most room, judged by CPU use, load per core, +free memory and how many agents already run there. Headless machines are used +first. A machine marked as someone's computer gets work only when every +headless machine is full and nobody is using it. Agents there are capped at half +its cores and get low CPU and disk priority, and more of its memory is kept free, +so it stays responsive. If no machine has room, work the agent did not +explicitly place starts here. + +Each machine decides its own role in **Settings → Labs → Load balancing**. The +**Detect** setting treats the desktop app, a Mac, Windows, or a Linux graphical +login as someone's computer. Set a headless machine that also runs a desktop, +such as a computer-use host, to **Headless** by hand. A linked machine is used +only if its link allows launching threads, not with read-only access. + ## Manage or revoke access On the host, **Settings → Sharing & access** lets authorized administrators create diff --git a/packages/contracts/src/orchestratorMcp.ts b/packages/contracts/src/orchestratorMcp.ts index 4f454f9ad758..47de5847567f 100644 --- a/packages/contracts/src/orchestratorMcp.ts +++ b/packages/contracts/src/orchestratorMcp.ts @@ -184,7 +184,7 @@ export const OrchestratorMcpDelegateTaskInput = Schema.Struct({ placement: Schema.optional( Schema.Literals(["auto"]).annotate({ description: - "'auto': for work that needs none of your checkout's changes. Runs here or on a linked environment with spare capacity, this repository's project and your exact provider and model; there, in that project's checkout. Ignored with target.environmentId or isolation. Reuse clientRequestId on retry to keep the environment chosen.", + "'auto': use for any work that needs none of your checkout's changes. Runs on the least-loaded headless machine, here or linked, with spare capacity, this repository's project and your exact provider and model; there, in that project's checkout. Ignored with target.environmentId or isolation. Reuse clientRequestId on retry to keep the environment chosen.", }), ), mode: Schema.optional( diff --git a/packages/contracts/src/resourceTelemetry.ts b/packages/contracts/src/resourceTelemetry.ts index e827ee1a1235..493dac10ce02 100644 --- a/packages/contracts/src/resourceTelemetry.ts +++ b/packages/contracts/src/resourceTelemetry.ts @@ -12,6 +12,15 @@ export const HostResourcesSnapshot = Schema.Struct({ cpuCount: NonNegativeInt, availableMemoryBytes: NonNegativeInt, totalMemoryBytes: NonNegativeInt, + // Optional so older servers still decode; placement treats a missing value conservatively. + /** One-minute load average per host CPU; work queued past 1 means the machine is saturated. */ + loadPerCpu: Schema.optional(Schema.Number.check(Schema.isGreaterThanOrEqualTo(0))), + /** Agent processes running on this host, across every T3 server there. */ + runningAgents: Schema.optional(NonNegativeInt), + /** Interactive machines belong to a person; placement uses them last and only while idle. */ + placementRole: Schema.optional(Schema.Literals(["headless", "interactive"])), + /** Someone is using this interactive machine now, or it cannot tell. */ + humanActive: Schema.optional(Schema.Boolean), }); export type HostResourcesSnapshot = typeof HostResourcesSnapshot.Type; diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 5e6917eb2819..77e69196e90e 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -1375,6 +1375,9 @@ export const CloudSpeechCredentials = Schema.Struct({ assemblyai: CloudSpeechCredential, }); +export const MachinePlacementRole = Schema.Literals(["auto", "headless", "interactive"]); +export type MachinePlacementRole = typeof MachinePlacementRole.Type; + export const ServerSettings = Schema.Struct({ // Null means unmigrated; empty is authoritative and must not resurrect cleared preferences. favorites: Schema.NullOr(ModelFavorites).pipe(Schema.withDecodingDefault(Effect.succeed(null))), @@ -1507,6 +1510,22 @@ export const ServerSettings = Schema.Struct({ automaticPlacementExcludedEnvironmentIds: Schema.Array(EnvironmentId).pipe( Schema.withDecodingDefault(Effect.succeed([])), ), + /** + * Whether this machine belongs to a person at its keyboard. Automatic placement fills headless + * machines first, uses an interactive one only while its person is idle and every headless one + * is full, and runs agents there under tighter limits. Auto: a desktop app, macOS, Windows or a + * Linux graphical login is interactive; anything else is headless. + */ + machinePlacementRole: MachinePlacementRole.pipe( + Schema.withDecodingDefault(Effect.succeed("auto" as const)), + ), + /** + * Closura: agent-launched threads that need nothing of the caller's checkout are placed + * automatically unless they name an environment. Off launches them here unless they ask. + */ + automaticPlacementByDefault: Schema.Boolean.pipe( + Schema.withDecodingDefault(Effect.succeed(true)), + ), /** Beta: Claude and Codex predict the next prompt after completed turns. Codex follows a change at once; open Claude threads after they restart. */ promptSuggestionsEnabled: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), sidebarAutoSettleKeepPullRequests: Schema.Boolean.pipe( @@ -1940,6 +1959,8 @@ export const ServerSettingsPatch = Schema.Struct({ guestSharingEnabled: Schema.optionalKey(Schema.Boolean), linkedEnvironmentsEnabled: Schema.optionalKey(Schema.Boolean), automaticPlacementExcludedEnvironmentIds: Schema.optionalKey(Schema.Array(EnvironmentId)), + machinePlacementRole: Schema.optionalKey(MachinePlacementRole), + automaticPlacementByDefault: Schema.optionalKey(Schema.Boolean), promptSuggestionsEnabled: Schema.optionalKey(Schema.Boolean), backgroundActivity: Schema.optionalKey( Schema.Struct({ diff --git a/packages/shared/src/loadBalancing.ts b/packages/shared/src/loadBalancing.ts index 0b938c6c090e..5a1e3de2ad33 100644 --- a/packages/shared/src/loadBalancing.ts +++ b/packages/shared/src/loadBalancing.ts @@ -1,6 +1,43 @@ import type { HostResourcesSnapshot } from "@t3tools/contracts"; -/** Callers supply only connected machines hosting the project and selected provider. */ +const GiB = 1024 ** 3; + +/** + * Headroom a machine must keep after placement. A headless machine is full near its limits; + * an interactive one, a person's computer, is full long before it would feel slow to them. + */ +const LIMITS = { + headless: { cpu: 0.95, loadPerCpu: 1, memoryShare: 0.05, memoryBytes: 1.5 * GiB, agents: null }, + interactive: { cpu: 0.5, loadPerCpu: 0.5, memoryShare: 0.4, memoryBytes: 3 * GiB, agents: 2 }, +} as const; + +function headroom(resources: HostResourcesSnapshot, tier: keyof typeof LIMITS): number | null { + const limits = LIMITS[tier]; + const { cpuUtilization, cpuCount, totalMemoryBytes, availableMemoryBytes } = resources; + if (cpuUtilization === null || totalMemoryBytes <= 0 || cpuCount <= 0) return null; + // An older server sends no load or agent count; only a headless one is trusted without them. + const loadPerCpu = resources.loadPerCpu ?? (tier === "headless" ? 0 : null); + const agents = resources.runningAgents ?? (tier === "headless" ? 0 : null); + if (loadPerCpu === null || agents === null) return null; + const memoryAvailable = availableMemoryBytes / totalMemoryBytes; + if ( + cpuUtilization >= limits.cpu || + loadPerCpu >= limits.loadPerCpu || + memoryAvailable <= limits.memoryShare || + availableMemoryBytes < limits.memoryBytes || + (limits.agents !== null && agents >= limits.agents) + ) + return null; + // Run queues past the core count show saturation that a short CPU sample can miss. + const busy = Math.max(cpuUtilization, Math.min(1, loadPerCpu)); + return (cpuCount * (1 - busy) * memoryAvailable) / (1 + agents); +} + +/** + * Callers supply only connected machines hosting the project and selected provider. Headless + * machines take work first; an interactive machine is chosen only when no headless one has room, + * its person is not using it, and it stays well clear of its limits. + */ export function chooseLoadBalancedEnvironment( candidates: ReadonlyArray<{ environmentId: string; @@ -11,8 +48,10 @@ export function chooseLoadBalancedEnvironment( }>, now: number, ): string | null { - let selected: string | null = null; - let bestScore = 0; + const best = { + headless: { id: null as string | null, score: 0 }, + interactive: { id: null as string | null, score: 0 }, + }; for (const { environmentId, resources, receivedAt, weight } of candidates) { const sampledAt = receivedAt ?? resources?.sampledAt ?? 0; if ( @@ -20,21 +59,17 @@ export function chooseLoadBalancedEnvironment( !Number.isFinite(weight) || weight <= 0 || now - sampledAt > 15_000 || - sampledAt > now + 5_000 || - resources.cpuUtilization === null || - resources.cpuUtilization >= 0.95 || - resources.totalMemoryBytes <= 0 || - resources.cpuCount <= 0 + sampledAt > now + 5_000 ) { continue; } - const memoryAvailable = resources.availableMemoryBytes / resources.totalMemoryBytes; - if (memoryAvailable <= 0.05) continue; - const score = weight * resources.cpuCount * (1 - resources.cpuUtilization) * memoryAvailable; - if (score > bestScore) { - selected = environmentId; - bestScore = score; - } + // A machine that does not say is someone's until it says otherwise, and in use if unsure. + const tier = resources.placementRole === "headless" ? "headless" : "interactive"; + if (tier === "interactive" && resources.humanActive !== false) continue; + const room = headroom(resources, tier); + if (room === null) continue; + const score = weight * room; + if (score > best[tier].score) best[tier] = { id: environmentId, score }; } - return selected; + return best.headless.id ?? best.interactive.id; } diff --git a/packages/shared/src/threadPlacement.test.ts b/packages/shared/src/threadPlacement.test.ts index b2218c21bae8..a7e67b91accb 100644 --- a/packages/shared/src/threadPlacement.test.ts +++ b/packages/shared/src/threadPlacement.test.ts @@ -15,6 +15,9 @@ const host = ( cpuCount: 8, availableMemoryBytes: 8 * GiB, totalMemoryBytes: 16 * GiB, + loadPerCpu: 0, + runningAgents: 0, + placementRole: "headless", }, receivedAt: now, weight: 50, @@ -79,8 +82,10 @@ describe("chooseThreadPlacement", () => { cpuCount: 8, availableMemoryBytes: 0.05 * 16 * GiB, totalMemoryBytes: 16 * GiB, + placementRole: "headless", }, }), + host("queued", 0.1, { resources: { ...host("q", 0.1).resources!, loadPerCpu: 1 } }), host("absent", 0.1, { resources: null }), ]; expect(chooseThreadPlacement(origin, candidates, now)).toEqual({ @@ -93,6 +98,59 @@ describe("chooseThreadPlacement", () => { }); }); + it("fills headless machines first, by run queue and running agents as well as CPU", () => { + const busy = (id: string, extra: object) => + host(id, 0.2, { resources: { ...host(id, 0.2).resources!, ...extra } }); + // A short CPU sample can look idle while 8 runnable tasks per core wait (load 64 on 8 cores). + expect( + chooseThreadPlacement(origin, [busy("asus", { loadPerCpu: 8 }), host("r720", 0.6)], now), + ).toMatchObject({ environmentId: "r720" }); + // Equal machines split by how many agents each already runs. + expect( + chooseThreadPlacement( + origin, + [busy("three", { runningAgents: 3 }), busy("one", { runningAgents: 1 })], + now, + ), + ).toMatchObject({ environmentId: "one" }); + }); + + it("uses a person's computer only when every headless machine is full and they are away", () => { + const laptop = (extra: object) => + host("laptop", 0.05, { + resources: { + ...host("laptop", 0.05).resources!, + placementRole: "interactive", + humanActive: false, + ...extra, + }, + }); + const full = host("r720", 0.96); + expect(chooseThreadPlacement(origin, [laptop({}), host("r720", 0.8)], now)).toMatchObject({ + environmentId: "r720", + }); + expect(chooseThreadPlacement(origin, [laptop({}), full], now)).toMatchObject({ + environmentId: "laptop", + }); + // Present, unsure, an older server that cannot say, or near the person's own headroom: never. + for (const extra of [ + { humanActive: true }, + { humanActive: undefined }, + { placementRole: undefined, humanActive: undefined }, + { loadPerCpu: undefined }, + { cpuUtilization: 0.5 }, + { loadPerCpu: 0.5 }, + { runningAgents: 2 }, + { availableMemoryBytes: 6 * GiB }, + { availableMemoryBytes: 2.9 * GiB, totalMemoryBytes: 4 * GiB }, + ]) { + expect(chooseThreadPlacement(origin, [laptop(extra), full], now)).toEqual({ + accepted: false, + reason: "no-eligible-environment", + }); + } + }); + it("refuses evidence that names one environment twice", () => { expect(chooseThreadPlacement(origin, [host("a", 0.1), host("a", 0.2)], now)).toEqual({ accepted: false, From 12c8580deef4d6735f41218979a6657c4e094907 Mon Sep 17 00:00:00 2001 From: nullStack65 <112618179+nullStack65@users.noreply.github.com> Date: Wed, 18 Nov 2026 22:24:03 +0000 Subject: [PATCH 2/2] fix(placement): fail closed on unknown presence; keep work on its own machine From review: logind errors no longer read as headless, a machine without agent scopes reports no agent count, default placement needs the caller's clientRequestId so a retry finds the chosen machine, and work started on someone's computer stays there when nothing else has room. Model: claude-opus-5-5 Task: fix Agent: Builder Co-Authored-By: Claude Opus 5.5 --- .../src/mcp/toolkits/project/handlers.ts | 5 +- apps/server/src/mcp/toolkits/project/tools.ts | 2 +- apps/server/src/peer/RemoteDelegation.test.ts | 24 +++-- .../src/resourceTelemetry/HostResources.ts | 3 +- .../resourceTelemetry/MachinePresence.test.ts | 22 ++++- .../src/resourceTelemetry/MachinePresence.ts | 95 ++++++++----------- docs/user/remote-access.md | 10 +- packages/shared/src/loadBalancing.ts | 16 +++- packages/shared/src/threadPlacement.test.ts | 23 +++++ packages/shared/src/threadPlacement.ts | 2 +- 10 files changed, 126 insertions(+), 76 deletions(-) diff --git a/apps/server/src/mcp/toolkits/project/handlers.ts b/apps/server/src/mcp/toolkits/project/handlers.ts index b56daee099b6..b2f5ee193634 100644 --- a/apps/server/src/mcp/toolkits/project/handlers.ts +++ b/apps/server/src/mcp/toolkits/project/handlers.ts @@ -134,9 +134,11 @@ const launchPlacedThere = Effect.fn("mcp.launchPlacedThere")(function* ( }, ) { // A fresh worktree from origin needs nothing of this checkout, so the fleet places it - // by default; anything else, or an explicit 'here', stays unless it asks. + // by default; anything else, or an explicit 'here', stays unless it asks. Only with the + // caller's key: a retry after a lost answer must reach the machine first chosen. const byDefault = input.placement === undefined && + input.clientRequestId !== undefined && input.environmentId === undefined && local.strategy?.type === "worktree" && local.strategy.startFromOrigin === true && @@ -151,7 +153,6 @@ const launchPlacedThere = Effect.fn("mcp.launchPlacedThere")(function* ( input = { ...input, placement: "auto", - clientRequestId: input.clientRequestId ?? local.threadId, modelSelection: input.modelSelection ?? local.callerModelSelection, }; } diff --git a/apps/server/src/mcp/toolkits/project/tools.ts b/apps/server/src/mcp/toolkits/project/tools.ts index 2746bdd4741d..2f58e7d9e650 100644 --- a/apps/server/src/mcp/toolkits/project/tools.ts +++ b/apps/server/src/mcp/toolkits/project/tools.ts @@ -164,7 +164,7 @@ const ThreadLaunchTool = Tool.make("t3_thread_launch", { placement: Schema.optional( Schema.Literals(["auto", "here"]).annotate({ description: - "'auto', without `environmentId`: here or on a linked environment with spare capacity, this repository's project and your exact `modelSelection`; there, in that project's checkout. Needs `modelSelection` and `clientRequestId`; a retry keeps the environment chosen. Attachments, scratch and worktrees not started from origin stay here. Omitted: a worktree started from origin is placed the same way, with your model, when this machine places by default. 'here' keeps it here.", + "'auto', without `environmentId`: here or on a linked environment with spare capacity, this repository's project and your exact `modelSelection`; there, in that project's checkout. Needs `modelSelection` and `clientRequestId`; a retry keeps the environment chosen. Attachments, scratch and worktrees not started from origin stay here. Omitted with a `clientRequestId`: a worktree started from origin is placed the same way, with your model, when this machine places by default. 'here' keeps it here.", }), ), /** Set by a linked environment launching its agent's delegated task here; ignored otherwise. */ diff --git a/apps/server/src/peer/RemoteDelegation.test.ts b/apps/server/src/peer/RemoteDelegation.test.ts index f2269ae2622a..17cd8d7a50bc 100644 --- a/apps/server/src/peer/RemoteDelegation.test.ts +++ b/apps/server/src/peer/RemoteDelegation.test.ts @@ -1658,17 +1658,27 @@ it.effect("a fresh worktree from origin is placed without asking, unless kept he }); yield* linkTo(a.links, b, "full-access"); const fresh = { type: "worktree", baseRef: "main", startFromOrigin: true }; - const placed = yield* launchUnplaced(a, { workspaceStrategy: fresh }); + // Without a key nothing could find the machine again after a lost answer: it stays here. + yield* launchUnplaced(a, { workspaceStrategy: fresh }); + expect(yield* Ref.get(localLaunches)).toBe(1); + const placed = yield* launchUnplaced(a, { + clientRequestId: "fresh", + workspaceStrategy: fresh, + }); expect(placed.isError, JSON.stringify(placed.content)).toBe(false); expect(placed.structuredContent).toMatchObject({ environmentId: box.environmentId }); - expect(yield* Ref.get(localLaunches)).toBe(0); + expect(yield* Ref.get(localLaunches)).toBe(1); // 'here', or the fleet default turned off, keeps it on this machine. - yield* launchPlaced(a, { placement: "here", workspaceStrategy: fresh }); - expect(yield* Ref.get(localLaunches)).toBe(1); - yield* a.settings.updateSettings({ automaticPlacementByDefault: false }); - yield* launchUnplaced(a, { workspaceStrategy: fresh }); + yield* launchPlaced(a, { + placement: "here", + clientRequestId: "kept", + workspaceStrategy: fresh, + }); expect(yield* Ref.get(localLaunches)).toBe(2); + yield* a.settings.updateSettings({ automaticPlacementByDefault: false }); + yield* launchUnplaced(a, { clientRequestId: "off", workspaceStrategy: fresh }); + expect(yield* Ref.get(localLaunches)).toBe(3); }), ).pipe(Effect.provide(NodeHttpServer.layerTest)), ); @@ -1689,7 +1699,7 @@ it.effect("placement nobody asked for runs here when no machine has room", () => }); yield* linkTo(a.links, b, "full-access"); const fresh = { type: "worktree", baseRef: "main", startFromOrigin: true }; - yield* launchUnplaced(a, { workspaceStrategy: fresh }); + yield* launchUnplaced(a, { clientRequestId: "room", workspaceStrategy: fresh }); expect(yield* Ref.get(localLaunches)).toBe(1); expect(yield* Ref.get(thread.launched)).toBeNull(); // Asked for, it still fails loudly instead. diff --git a/apps/server/src/resourceTelemetry/HostResources.ts b/apps/server/src/resourceTelemetry/HostResources.ts index b1e0445f7ae6..7fc5b27dd2ad 100644 --- a/apps/server/src/resourceTelemetry/HostResources.ts +++ b/apps/server/src/resourceTelemetry/HostResources.ts @@ -100,8 +100,9 @@ const make = Effect.fn("makeHostResources")(function* () { cgroupPath === undefined ? undefined : /^(.*\/user@\d+\.service)\//.exec(cgroupPath)?.[1]; if (manager === undefined) return undefined; const root = `/sys/fs/cgroup${manager}/${AGENTS_SLICE_PATH}`; + // No agents slice means agent scopes are off here, so neither the count nor caps exist. const slices = yield* fs.readDirectory(root).pipe(Effect.orElseSucceed(() => null)); - if (slices === null) return 0; + if (slices === null) return undefined; let running = 0; for (const slice of slices.filter((name) => name.endsWith(".slice"))) { const scopes = yield* fs diff --git a/apps/server/src/resourceTelemetry/MachinePresence.test.ts b/apps/server/src/resourceTelemetry/MachinePresence.test.ts index 32631492fb97..e337d1ca5040 100644 --- a/apps/server/src/resourceTelemetry/MachinePresence.test.ts +++ b/apps/server/src/resourceTelemetry/MachinePresence.test.ts @@ -18,6 +18,8 @@ const linux = (input: { readonly display: string; readonly session?: string; readonly calls?: Array; + /** logind refusing `show-user`, with this stderr. */ + readonly userError?: string; }) => MachinePresenceLive.layer.pipe( Layer.provide( @@ -29,10 +31,11 @@ const linux = (input: { run: ({ args }) => Effect.sync(() => { input.calls?.push(args[0] ?? ""); + const refused = args[0] === "show-user" && input.userError !== undefined; return { stdout: args[0] === "show-user" ? input.display : (input.session ?? ""), - stderr: "", - code: ChildProcessSpawner.ExitCode(0), + stderr: refused ? (input.userError ?? "") : "", + code: ChildProcessSpawner.ExitCode(refused ? 1 : 0), timedOut: false, stdoutTruncated: false, stderrTruncated: false, @@ -70,6 +73,21 @@ describe("MachinePresence", () => { }), ); + it.effect("counts a machine logind cannot describe as someone's, in use", () => + Effect.gen(function* () { + const unknown = yield* read.pipe( + Effect.provide(linux({ role: "auto", display: "", userError: "Failed to connect to bus" })), + ); + assert.deepStrictEqual(unknown, { placementRole: "interactive", humanActive: true }); + const loggedOut = yield* read.pipe( + Effect.provide( + linux({ role: "auto", display: "", userError: "User 1000 is not logged in." }), + ), + ); + assert.deepStrictEqual(loggedOut, { placementRole: "headless" }); + }), + ); + it.effect("protects a graphical login unless its person is idle, locked or away", () => Effect.gen(function* () { const inUse = yield* read.pipe( diff --git a/apps/server/src/resourceTelemetry/MachinePresence.ts b/apps/server/src/resourceTelemetry/MachinePresence.ts index 5d2080e03477..d5da20b04ca5 100644 --- a/apps/server/src/resourceTelemetry/MachinePresence.ts +++ b/apps/server/src/resourceTelemetry/MachinePresence.ts @@ -11,9 +11,6 @@ import * as ProcessRunner from "../processRunner.ts"; import * as ServerSettings from "../serverSettings.ts"; import { MachinePresence } from "./HostResources.ts"; -/** A person who has not touched a Mac's keyboard or pointer for this long is away. */ -const DARWIN_IDLE_NANOS = 10 * 60 * 1e9; - function parseKeyValues(text: string): Map { const values = new Map(); for (const line of text.split("\n")) { @@ -40,11 +37,14 @@ export function isLocalGraphicalSession(shown: string): boolean { return (type === "x11" || type === "wayland" || type === "mir") && values.get("Remote") !== "yes"; } -/** HIDIdleTime from `ioreg -c IOHIDSystem`; undefined when it cannot be read. */ -export function darwinIdleNanos(output: string): number | undefined { - const value = /"HIDIdleTime"\s*=\s*(\d+)/.exec(output)?.[1]; - return value === undefined ? undefined : Number(value); -} +/** + * What logind says about this user's local graphical login: none, one (with its state), or + * unknown when logind could not be asked. Unknown never means nobody is there. + */ +type GraphicalLogin = + | { readonly kind: "none" } + | { readonly kind: "session"; readonly shown: string } + | { readonly kind: "unknown" }; export const make = Effect.gen(function* () { const platform = yield* HostProcess.Platform; @@ -52,72 +52,57 @@ export const make = Effect.gen(function* () { const settingsService = yield* ServerSettings.ServerSettingsService; const { mode } = yield* ServerConfig.ServerConfig; - const run = (command: string, args: ReadonlyArray) => - runner.run({ command, args, timeout: "2 seconds" }).pipe( - Effect.map((result) => (result.code === 0 ? result.stdout : undefined)), - Effect.orElseSucceed(() => undefined), - ); + const run = (args: ReadonlyArray) => + runner.run({ command: "loginctl", args, timeout: "2 seconds" }).pipe(Effect.option); - // The logind state of this user's local graphical login, or undefined when there is none. // Display can name a remote or text session, such as an SSH login, which nobody sits at. - const graphicalSession = Effect.gen(function* () { + const graphicalLogin: Effect.Effect = Effect.gen(function* () { const uid = NodeProcess.getuid?.(); - if (uid === undefined) return undefined; - const display = (yield* run("loginctl", [ - "show-user", - String(uid), - "-p", - "Display", - "--value", - ]))?.trim(); - if (display === undefined || display.length === 0) return undefined; - const shown = yield* run("loginctl", [ + if (uid === undefined) return { kind: "unknown" } as const; + const user = yield* run(["show-user", String(uid), "-p", "Display", "--value"]); + if (user._tag === "None") return { kind: "unknown" } as const; + if (user.value.code !== 0) + // Without a session or lingering manager logind knows no such user: nobody is logged in. + return /not logged in/i.test(user.value.stderr) + ? ({ kind: "none" } as const) + : ({ kind: "unknown" } as const); + const display = user.value.stdout.trim(); + if (display.length === 0) return { kind: "none" } as const; + const session = yield* run([ "show-session", display, ...["Type", "Remote", "Active", "LockedHint", "IdleHint"].flatMap((key) => ["-p", key]), ]); - // A session logind names but cannot describe is treated as someone's. - if (shown === undefined) return ""; - return isLocalGraphicalSession(shown) ? shown : undefined; + if (session._tag === "None" || session.value.code !== 0) return { kind: "unknown" } as const; + return isLocalGraphicalSession(session.value.stdout) + ? ({ kind: "session", shown: session.value.stdout } as const) + : ({ kind: "none" } as const); }); - const resolveRole = Effect.fn("MachinePresence.resolveRole")(function* ( - setting: MachinePlacementRole, - ) { + const role = Effect.gen(function* () { + const settings = yield* settingsService.getSettings.pipe(Effect.option); + const setting: MachinePlacementRole = + settings._tag === "Some" ? settings.value.machinePlacementRole : "auto"; if (setting !== "auto") return setting; if (mode === "desktop" || platform !== "linux") return "interactive" as const; - return (yield* graphicalSession) === undefined + return (yield* graphicalLogin).kind === "none" ? ("headless" as const) : ("interactive" as const); }); - const humanActive = Effect.gen(function* () { - if (platform === "darwin") { - const idle = darwinIdleNanos( - (yield* run("/usr/sbin/ioreg", ["-c", "IOHIDSystem", "-d", "4", "-k", "HIDIdleTime"])) ?? - "", - ); - return idle === undefined || idle < DARWIN_IDLE_NANOS; - } - if (platform !== "linux") return true; - const session = yield* graphicalSession; - return session !== undefined && linuxSessionInUse(session); - }); - - const role = Effect.gen(function* () { - const settings = yield* settingsService.getSettings.pipe(Effect.option); - return yield* resolveRole( - settings._tag === "Some" ? settings.value.machinePlacementRole : "auto", - ); - }); - return MachinePresence.of({ role, read: Effect.gen(function* () { const placementRole = yield* role; - return placementRole === "headless" - ? { placementRole } - : { placementRole, humanActive: yield* humanActive }; + if (placementRole === "headless") return { placementRole }; + // Only Linux can tell, and only Linux caps agents there; elsewhere assume someone is. + if (platform !== "linux") return { placementRole, humanActive: true }; + const login = yield* graphicalLogin; + return { + placementRole, + humanActive: + login.kind === "unknown" || (login.kind === "session" && linuxSessionInUse(login.shown)), + }; }), }); }); diff --git a/docs/user/remote-access.md b/docs/user/remote-access.md index 269ed2c402d6..e13ea4351abb 100644 --- a/docs/user/remote-access.md +++ b/docs/user/remote-access.md @@ -401,16 +401,18 @@ fails with that reason. Agents can let T3 Code pick the machine for independent work: a task with `placement: "auto"`, or a new thread in a worktree started from origin. In -Closura builds the second needs no request. Turn **Place new agent threads +Closura builds the second needs no request when the agent passes a +`clientRequestId`. Turn **Place new agent threads automatically** off in **Settings → Labs** to keep those threads on this machine. T3 Code picks the machine with the most room, judged by CPU use, load per core, free memory and how many agents already run there. Headless machines are used -first. A machine marked as someone's computer gets work only when every +first. A Linux machine marked as someone's computer gets work only when every headless machine is full and nobody is using it. Agents there are capped at half its cores and get low CPU and disk priority, and more of its memory is kept free, -so it stays responsive. If no machine has room, work the agent did not -explicitly place starts here. +so it stays responsive. A Mac or Windows machine, or one on an older release, +never takes placed work from another machine. If no machine has room, work that +started here stays here. Each machine decides its own role in **Settings → Labs → Load balancing**. The **Detect** setting treats the desktop app, a Mac, Windows, or a Linux graphical diff --git a/packages/shared/src/loadBalancing.ts b/packages/shared/src/loadBalancing.ts index 5a1e3de2ad33..1969020c9cb3 100644 --- a/packages/shared/src/loadBalancing.ts +++ b/packages/shared/src/loadBalancing.ts @@ -15,7 +15,8 @@ function headroom(resources: HostResourcesSnapshot, tier: keyof typeof LIMITS): const limits = LIMITS[tier]; const { cpuUtilization, cpuCount, totalMemoryBytes, availableMemoryBytes } = resources; if (cpuUtilization === null || totalMemoryBytes <= 0 || cpuCount <= 0) return null; - // An older server sends no load or agent count; only a headless one is trusted without them. + // A headless machine that cannot report load (Windows) or agents (no systemd) is judged + // without them; someone's computer never is. const loadPerCpu = resources.loadPerCpu ?? (tier === "headless" ? 0 : null); const agents = resources.runningAgents ?? (tier === "headless" ? 0 : null); if (loadPerCpu === null || agents === null) return null; @@ -36,7 +37,12 @@ function headroom(resources: HostResourcesSnapshot, tier: keyof typeof LIMITS): /** * Callers supply only connected machines hosting the project and selected provider. Headless * machines take work first; an interactive machine is chosen only when no headless one has room, - * its person is not using it, and it stays well clear of its limits. + * its person is not using it, and it stays well clear of its limits. A machine that does not + * report its role, such as one on an older release, counts as someone's computer in use. + * + * `originEnvironmentId` is where the work starts. When nothing has room it keeps the work if it + * has room by headless limits, as before placement could pick: work started on someone's + * computer is theirs to run there. */ export function chooseLoadBalancedEnvironment( candidates: ReadonlyArray<{ @@ -47,11 +53,13 @@ export function chooseLoadBalancedEnvironment( weight: number; }>, now: number, + originEnvironmentId?: string, ): string | null { const best = { headless: { id: null as string | null, score: 0 }, interactive: { id: null as string | null, score: 0 }, }; + let origin: string | null = null; for (const { environmentId, resources, receivedAt, weight } of candidates) { const sampledAt = receivedAt ?? resources?.sampledAt ?? 0; if ( @@ -63,6 +71,8 @@ export function chooseLoadBalancedEnvironment( ) { continue; } + if (environmentId === originEnvironmentId && headroom(resources, "headless") !== null) + origin = environmentId; // A machine that does not say is someone's until it says otherwise, and in use if unsure. const tier = resources.placementRole === "headless" ? "headless" : "interactive"; if (tier === "interactive" && resources.humanActive !== false) continue; @@ -71,5 +81,5 @@ export function chooseLoadBalancedEnvironment( const score = weight * room; if (score > best[tier].score) best[tier] = { id: environmentId, score }; } - return best.headless.id ?? best.interactive.id; + return best.headless.id ?? best.interactive.id ?? origin; } diff --git a/packages/shared/src/threadPlacement.test.ts b/packages/shared/src/threadPlacement.test.ts index a7e67b91accb..e27aa87a3c03 100644 --- a/packages/shared/src/threadPlacement.test.ts +++ b/packages/shared/src/threadPlacement.test.ts @@ -151,6 +151,29 @@ describe("chooseThreadPlacement", () => { } }); + it("keeps work on the person's own machine when nothing else has room", () => { + const desk = host("here", 0.3, { + resources: { + ...host("here", 0.3).resources!, + placementRole: "interactive", + humanActive: true, + }, + }); + expect(chooseThreadPlacement(origin, [desk, host("r720", 0.97)], now)).toEqual({ + accepted: true, + environmentId: "here", + reason: "capacity", + }); + // A headless machine with room still wins, and a full origin is not kept. + expect(chooseThreadPlacement(origin, [desk, host("r720", 0.5)], now)).toMatchObject({ + environmentId: "r720", + }); + expect(chooseThreadPlacement(origin, [host("here", 0.97), host("r720", 0.97)], now)).toEqual({ + accepted: false, + reason: "no-eligible-environment", + }); + }); + it("refuses evidence that names one environment twice", () => { expect(chooseThreadPlacement(origin, [host("a", 0.1), host("a", 0.2)], now)).toEqual({ accepted: false, diff --git a/packages/shared/src/threadPlacement.ts b/packages/shared/src/threadPlacement.ts index 6f1c33108ef6..f52a64f7dc43 100644 --- a/packages/shared/src/threadPlacement.ts +++ b/packages/shared/src/threadPlacement.ts @@ -62,7 +62,7 @@ export function chooseThreadPlacement( .sort((a, b) => a.environmentId < b.environmentId ? -1 : a.environmentId > b.environmentId ? 1 : 0, ); - const environmentId = chooseLoadBalancedEnvironment(eligible, now); + const environmentId = chooseLoadBalancedEnvironment(eligible, now, request.originEnvironmentId); return environmentId === null ? { accepted: false, reason: "no-eligible-environment" } : { accepted: true, environmentId, reason: "capacity" };