diff --git a/apps/server/src/mcp/OrchestratorMcpService.test.ts b/apps/server/src/mcp/OrchestratorMcpService.test.ts index 97907e2d50b2c..217b1127d1c07 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.test.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.test.ts @@ -4489,8 +4489,11 @@ describe("OrchestratorMcpService thread lookup", () => { shells: ReadonlyArray, sent: Ref.Ref, overrides: Partial = {}, + launch: Partial = {}, ) => - layerOrchestratorService.pipe( + OrchestratorMcpService.layer.pipe( + Layer.provide(Layer.mock(ThreadLaunchService.ThreadLaunchService)(launch)), + Layer.provide(Layer.mock(GitVcsDriver.GitVcsDriver)({})), Layer.provide( Layer.mergeAll( NodeServices.layer, @@ -4638,6 +4641,63 @@ describe("OrchestratorMcpService thread lookup", () => { }), ); + it.effect("retries a failed worktree setup when a send is held behind it", () => + Effect.gen(function* () { + const target = liveThreadShell(ThreadId.make("thread:held")); + const failedRunId = RunId.make("run:setup-failed"); + const retried: Array = []; + const sendWith = (worktreePath: string | null) => + Effect.gen(function* () { + const sent = yield* Ref.make(0); + yield* Effect.gen(function* () { + const mcp = yield* OrchestratorMcpService.OrchestratorMcpService; + yield* mcp.sendToThread(client, { threadId: target.id, message: "hello" }); + }).pipe( + Effect.provide( + service( + [target], + sent, + { + getProjectThreadRecords: () => Effect.succeed(idleThreadProjection(target)), + getThreadRecords: () => + Effect.succeed({ + thread: { ...target, worktreePath }, + runs: [ + { + id: failedRunId, + status: "failed", + startedAt: null, + workspacePreparation: { type: "worktree" }, + }, + ], + } as never), + sendToThread: () => + Effect.succeed({ + run: { id: RunId.make("run:held"), status: "queued", queueHeld: true }, + delivery: "queued", + } as never), + }, + { + retryPreparation: (input) => { + retried.push(input); + return Effect.succeed({} as never); + }, + }, + ), + ), + ); + }); + + yield* sendWith(null); + assert.deepStrictEqual( + retried.map((input) => [input.threadId, input.runId]), + [[target.id, failedRunId]], + ); + yield* sendWith("/repo/worktree"); + assert.equal(retried.length, 1); + }), + ); + it.effect("refuses waiting without orchestration capability before reading the target", () => Effect.gen(function* () { const sent = yield* Ref.make(0); diff --git a/apps/server/src/mcp/OrchestratorMcpService.ts b/apps/server/src/mcp/OrchestratorMcpService.ts index 800e8cbb3ba9d..961ce2c28789a 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.ts @@ -72,7 +72,7 @@ import { } from "@t3tools/contracts"; import { getProviderOptionCurrentValue, withLegacyUltracodeEffort } from "@t3tools/shared/model"; import { backgroundWorkHoldsCompletion } from "@t3tools/shared/orchestrationV2PendingBackgroundWork"; -import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError"; +import { latestExecutedRun, runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError"; import { resolveProjectSettings } from "@t3tools/shared/projectSettings"; import { formatThreadLink } from "@t3tools/shared/threadLinks"; import { @@ -3610,6 +3610,27 @@ const make = Effect.gen(function* () { ), ), ); + if (result.run.status === "queued" && result.run.queueHeld === true) { + yield* threadManagement.getThreadRecords(input.threadId, ["runs"]).pipe( + Effect.flatMap(({ thread, runs }) => { + const failed = latestExecutedRun(runs); + return thread.worktreePath === null && + failed?.status === "failed" && + failed.workspacePreparation?.type === "worktree" + ? threadLaunch.retryPreparation({ + commandId: CommandId.make( + `${stableCommandId({ scope, requestKey: key, operation: "thread-send" })}:retry-preparation`, + ), + threadId: input.threadId, + runId: failed.id, + }) + : Effect.void; + }), + Effect.catchCause((cause) => + Effect.logWarning("thread-send could not retry the failed worktree setup", { cause }), + ), + ); + } return { threadId: input.threadId, link: threadLinkFor(scope, input.threadId, target.thread.title), diff --git a/apps/server/src/mcp/toolkits/project/handlers.ts b/apps/server/src/mcp/toolkits/project/handlers.ts index 6bd5b14690d74..716f0b09c8d51 100644 --- a/apps/server/src/mcp/toolkits/project/handlers.ts +++ b/apps/server/src/mcp/toolkits/project/handlers.ts @@ -3,6 +3,7 @@ import { agentLaunchInboxRouting, withLegacyPeerInboxReason, CommandId, + DEFAULT_AUTOMATIC_PLACEMENT_HOSTS, type EnvironmentId, MessageId, type ModelSelection, @@ -35,6 +36,8 @@ import * as ProviderRegistry from "../../../provider/ProviderRegistry.ts"; import * as ServerSettings from "../../../serverSettings.ts"; import * as DateTime from "effect/DateTime"; import { resolveNewThreadModelSelection } from "@t3tools/shared/usageLimits"; +import { withinHostCap } from "@t3tools/shared/loadBalancing"; +import { placementHostLimits } from "@t3tools/shared/threadPlacement"; import { resolveProjectSettings } from "@t3tools/shared/projectSettings"; import { resolveRuntimeMode } from "../../delegatedTaskModes.ts"; import * as McpInvocationContext from "../../McpInvocationContext.ts"; @@ -238,8 +241,21 @@ const launchPlacedThere = Effect.fn("mcp.launchPlacedThere")(function* ( runtimeMode: local.runtimeMode, here, }); - // Placement nobody asked for falls back to here, as before, when no machine has room. - const environmentId = byDefault + // Placement nobody asked for falls back to here when no machine has room, within here's limit. + const hereMayKeep = + byDefault && + withinHostCap( + here.resources, + placementHostLimits( + yield* ServerSettings.ServerSettingsService.pipe( + Effect.flatMap((settings) => settings.getSettings), + Effect.map((settings) => settings.automaticPlacementHosts), + Effect.orElseSucceed(() => DEFAULT_AUTOMATIC_PLACEMENT_HOSTS), + ), + scope.environmentId, + ).cap, + ); + const environmentId = hereMayKeep ? yield* placing.pipe( Effect.catchIf( (error) => error.code === "target_required", diff --git a/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts b/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts index d8b7dae693632..1151c4e953d6e 100644 --- a/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts @@ -604,6 +604,33 @@ it.effect("maintains delegated candidate indexes across legacy rows and payload }).pipe(Effect.provide(layerTest)), ); +it.effect("counts threads with live or unheld queued runs as active lanes", () => + Effect.gen(function* () { + const projections = yield* ProjectionStore.ProjectionStoreV2; + const now = yield* DateTime.now; + for (const name of ["running", "waiting", "queued", "held", "completed", "deleted", "both"]) { + const thread = yield* createThread(`lanes-${name}`, { + deletedAt: name === "deleted" ? now : null, + }); + if (name === "queued" || name === "held") { + yield* createRun(thread, "queued", { + startedAt: null, + ...(name === "held" ? { queueHeld: true } : {}), + }); + } else if (name === "both") { + yield* createRun(thread, "running"); + yield* createRun(thread, "queued", { ordinal: 2, startedAt: null }); + } else { + yield* createRun( + thread, + name === "waiting" ? "waiting" : name === "completed" ? "completed" : "running", + ); + } + } + assert.strictEqual(yield* projections.countActiveRunThreads(), 4); + }).pipe(Effect.provide(layerTest)), +); + it.effect("recovers terminal subagent results until their cross-thread transfer exists", () => Effect.gen(function* () { const projections = yield* ProjectionStore.ProjectionStoreV2; diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 4d3229a7e9889..8402be3e50058 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -610,6 +610,7 @@ export interface ProjectionStoreV2Shape { >; /** Threads that are not deleted, archived ones included. */ readonly countThreads: () => Effect.Effect; + readonly countActiveRunThreads: () => Effect.Effect; /** Every active thread, in shell order. */ readonly listActiveShellIndex: () => Effect.Effect< ReadonlyArray, @@ -9136,6 +9137,18 @@ export const layer: Layer.Layer = return rows[0]?.count ?? 0; }, Effect.mapError(shellSnapshotReadError)); + const countActiveRunThreads = Effect.fn("ProjectionStore.countActiveRunThreads")(function* () { + const rows = yield* sql<{ readonly count: number }>` + SELECT COUNT(DISTINCT run.thread_id) AS count + FROM orchestration_v2_projection_runs AS run + JOIN orchestration_v2_projection_threads AS thread ON thread.thread_id = run.thread_id + WHERE thread.deleted_at IS NULL + AND (run.status IN ('preparing', 'starting', 'running', 'waiting') + OR (run.status = 'queued' AND json_extract(run.payload_json, '$.queueHeld') IS NOT 1)) + `; + return rows[0]?.count ?? 0; + }, Effect.mapError(shellSnapshotReadError)); + const listActiveShellIndex = Effect.fn("ProjectionStore.listActiveShellIndex")(function* () { const rows = yield* sql<{ readonly thread_id: string; @@ -9518,6 +9531,7 @@ export const layer: Layer.Layer = getPullRequestSnapshot: readOnReader(getPullRequestSnapshot), listActiveSubagentThreadIds: readOnReader(listActiveSubagentThreadIds), countThreads: readOnReader(countThreads), + countActiveRunThreads: readOnReader(countActiveRunThreads), listActiveShellIndex: readOnReader(listActiveShellIndex), listActiveThreads: readOnReader(listActiveThreads), readShellSnapshot: readOnReader(readShellSnapshot), diff --git a/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts b/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts index 02aa457ca849b..88b72fdc54269 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts @@ -217,6 +217,7 @@ it.effect( getShellSnapshot: () => Effect.die("unused getShellSnapshot"), listActiveSubagentThreadIds: () => Effect.die("unused listActiveSubagentThreadIds"), countThreads: () => Effect.die("unused countThreads"), + countActiveRunThreads: () => Effect.die("unused countActiveRunThreads"), listActiveShellIndex: () => Effect.die("unused listActiveShellIndex"), listActiveThreads: () => Effect.die("unused listActiveThreads"), listOlderSettledThreads: () => Effect.die("unused listOlderSettledThreads"), diff --git a/apps/server/src/orchestration-v2/ThreadLaunchService.ts b/apps/server/src/orchestration-v2/ThreadLaunchService.ts index c0045f7aa8058..58a31b4493a30 100644 --- a/apps/server/src/orchestration-v2/ThreadLaunchService.ts +++ b/apps/server/src/orchestration-v2/ThreadLaunchService.ts @@ -725,6 +725,7 @@ const make = Effect.gen(function* () { path: null, }, { + reuseEmptyBranch: true, progress: { onWorktreeClaimed: (path) => Effect.sync(() => { diff --git a/apps/server/src/peer/PeerLinks.ts b/apps/server/src/peer/PeerLinks.ts index f79a18257a828..1aba75d7c6dd9 100644 --- a/apps/server/src/peer/PeerLinks.ts +++ b/apps/server/src/peer/PeerLinks.ts @@ -1,5 +1,6 @@ import { type AppConnectionReport, + type AutomaticPlacementHost, AuthMcpClientAccess, AuthMcpRegistrationError, AuthMcpTokenError, @@ -126,6 +127,7 @@ export class PeerLinks extends Context.Service< * fail, so placement contacts nobody rather than everyone. */ readonly placementExclusions: Effect.Effect, PeerLinkError>; + readonly placementHosts: Effect.Effect, PeerLinkError>; /** The stored links, read here only: no peer is asked whether it answers. */ readonly stored: Effect.Effect, PeerLinkError>; /** @@ -281,6 +283,10 @@ const make = Effect.gen(function* () { Effect.map((settings) => settings.linkedEnvironmentsEnabled), Effect.orElseSucceed(() => false), ); + const placementHosts = serverSettings.getSettings.pipe( + Effect.map((settings) => settings.automaticPlacementHosts), + Effect.catch(failStorage), + ); const placementExclusions = serverSettings.getSettings.pipe( Effect.map( (settings): ReadonlySet => @@ -853,6 +859,7 @@ const make = Effect.gen(function* () { environments, stored, placementExclusions, + placementHosts, session, get, renewed, diff --git a/apps/server/src/peer/RemoteDelegation.test.ts b/apps/server/src/peer/RemoteDelegation.test.ts index 17cd8d7a50bc0..d4ff90e280d4d 100644 --- a/apps/server/src/peer/RemoteDelegation.test.ts +++ b/apps/server/src/peer/RemoteDelegation.test.ts @@ -1710,6 +1710,112 @@ it.effect("placement nobody asked for runs here when no machine has room", () => ).pipe(Effect.provide(NodeHttpServer.layerTest)), ); +it.effect( + "a host at its lane cap refuses, counts its own fresh placements, and a laptop capped at no lanes keeps nothing", + () => + Effect.scoped( + Effect.gen(function* () { + const thread = yield* makeBoxThread; + yield* Ref.set(thread.resources, { ...loadOf(0.1), runningAgents: 4 }); + const b = yield* serveBox(thread); + const offers = yield* SubscriptionRef.make< + ReadonlyArray + >([]); + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); + const localLaunches = yield* Ref.make(0); + const a = yield* makeLaptop(database, offers, undefined, Effect.void, { + localLaunch: Ref.update(localLaunches, (count) => count + 1), + }); + yield* linkTo(a.links, b, "full-access"); + yield* a.settings.updateSettings({ + automaticPlacementHosts: [ + { environmentId: box.environmentId, maxLanes: 4, minAvailableMemoryGiB: 4 }, + { environmentId: laptop.environmentId, maxLanes: 0, minAvailableMemoryGiB: 4 }, + ], + }); + const fresh = { type: "worktree", baseRef: "main", startFromOrigin: true }; + const capped = yield* launchUnplaced(a, { + clientRequestId: "capped", + workspaceStrategy: fresh, + }); + expect(capped.isError).toBe(true); + expect(JSON.stringify(capped.content)).toContain("no-eligible-environment"); + expect(yield* Ref.get(localLaunches)).toBe(0); + expect(yield* Ref.get(thread.launched)).toBeNull(); + + yield* Ref.set(thread.resources, { ...loadOf(0.1), runningAgents: 3 }); + const placed = yield* launchUnplaced(a, { + clientRequestId: "room", + workspaceStrategy: fresh, + }); + expect(placed.isError, JSON.stringify(placed.content)).toBe(false); + expect(placed.structuredContent).toMatchObject({ environmentId: box.environmentId }); + expect(yield* Ref.get(localLaunches)).toBe(0); + + const behind = yield* launchUnplaced(a, { + clientRequestId: "same-sample", + workspaceStrategy: fresh, + }); + expect(behind.isError).toBe(true); + expect(JSON.stringify(behind.content)).toContain("no-eligible-environment"); + expect(yield* Ref.get(localLaunches)).toBe(0); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); + +it.effect( + "sequential launches fill every active lane once the host reports them, and launches on one sample cannot share the last", + () => + Effect.scoped( + Effect.gen(function* () { + const thread = yield* makeBoxThread; + const b = yield* serveBox(thread); + const offers = yield* SubscriptionRef.make< + ReadonlyArray + >([]); + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); + const a = yield* makeLaptop(database, offers, undefined, Effect.void, { + localLaunch: Effect.void, + }); + yield* linkTo(a.links, b, "full-access"); + yield* a.settings.updateSettings({ + automaticPlacementHosts: [ + { environmentId: box.environmentId, maxLanes: 4, minAvailableMemoryGiB: 4 }, + { environmentId: laptop.environmentId, maxLanes: 0, minAvailableMemoryGiB: 4 }, + ], + }); + const fresh = { type: "worktree", baseRef: "main", startFromOrigin: true }; + for (const active of [0, 1, 2, 3]) { + yield* Ref.set(thread.resources, { + ...loadOf(0.1), + runningAgents: 18, + activeRuns: active, + }); + const placed = yield* launchUnplaced(a, { + clientRequestId: `lane-${active}`, + workspaceStrategy: fresh, + }); + expect(placed.isError, JSON.stringify(placed.content)).toBe(false); + expect(placed.structuredContent).toMatchObject({ environmentId: box.environmentId }); + } + const behind = yield* launchUnplaced(a, { + clientRequestId: "lane-same-sample", + workspaceStrategy: fresh, + }); + expect(behind.isError).toBe(true); + expect(JSON.stringify(behind.content)).toContain("no-eligible-environment"); + + yield* Ref.set(thread.resources, { ...loadOf(0.1), runningAgents: 18, activeRuns: 4 }); + const full = yield* launchUnplaced(a, { + clientRequestId: "lane-full", + workspaceStrategy: fresh, + }); + expect(full.isError).toBe(true); + expect(JSON.stringify(full.content)).toContain("no-eligible-environment"); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); + it.effect( "a retry that drops placement or asks for this checkout reaches the thread placed there", () => diff --git a/apps/server/src/peer/RemoteDelegation.ts b/apps/server/src/peer/RemoteDelegation.ts index ed4ea992a9749..6441b6a8b4c2c 100644 --- a/apps/server/src/peer/RemoteDelegation.ts +++ b/apps/server/src/peer/RemoteDelegation.ts @@ -1,5 +1,6 @@ import { CommandId, + DEFAULT_AUTOMATIC_PLACEMENT_HOSTS, EnvironmentId, isRuntimeModeAbove, type ModelSelection, @@ -29,6 +30,7 @@ import { } from "@t3tools/contracts"; import { chooseThreadPlacement, + placementHostLimits, type ThreadPlacementCandidate, } from "@t3tools/shared/threadPlacement"; import * as Cause from "effect/Cause"; @@ -501,6 +503,37 @@ const make = Effect.gen(function* () { ); }); + const recentPlacements = new Map< + string, + ReadonlyArray<{ readonly at: number; readonly lanes: number }> + >(); + const recentAt = (environmentId: string, now: number) => + (recentPlacements.get(environmentId) ?? []).filter((entry) => now - entry.at < 60_000); + const reportedLanes = (resources: HostResourcesSnapshot | null) => + resources === null ? undefined : (resources.activeRuns ?? resources.runningAgents); + const withRecentPlacements = < + C extends { readonly environmentId: string; readonly resources: HostResourcesSnapshot | null }, + >( + candidate: C, + now: number, + ): C => { + const resources = candidate.resources; + const reported = reportedLanes(resources); + if (resources === null || reported === undefined) return candidate; + const recent = recentAt(candidate.environmentId, now); + let expected = reported; + for (let index = recent.length - 1; index >= 0; index--) + expected = Math.max(expected, recent[index]!.lanes + recent.length - index); + if (expected === reported) return candidate; + return { + ...candidate, + resources: + resources.activeRuns !== undefined + ? { ...resources, activeRuns: expected } + : { ...resources, runningAgents: expected }, + }; + }; + /** The shared choice: this environment and the askable links, from their own answers. */ const choosePlacement = Effect.fn("RemoteDelegation.choosePlacement")(function* ( ask: Ask, @@ -510,6 +543,9 @@ const make = Effect.gen(function* () { }, ) { const askable = yield* askableLinks(input); + const hosts = yield* links.placementHosts.pipe( + Effect.orElseSucceed(() => DEFAULT_AUTOMATIC_PLACEMENT_HOSTS), + ); const peers = (yield* Effect.forEach(askable, (link) => peerCandidate(ask, link, input), { concurrency: "unbounded", })).flatMap((peer) => (Option.isSome(peer) ? [peer.value] : [])); @@ -517,26 +553,45 @@ const make = Effect.gen(function* () { const hereCandidate = !input.excluded.has(hereId) && (input.restrictTo === undefined || input.restrictTo.includes(hereId)); - const decision = chooseThreadPlacement( - { originEnvironmentId: hereId }, - [ - ...(hereCandidate - ? [ - { - environmentId: hereId, - resources: input.here.resources, - receivedAt: now, - weight: 1, - operable: true, - projectAvailable: true, - modelAvailable: input.here.modelAvailable, - }, - ] - : []), - ...peers.map((peer) => peer.candidate), - ], - now, - ); + const candidates = [ + ...(hereCandidate + ? [ + { + environmentId: hereId, + resources: input.here.resources, + receivedAt: now, + weight: 1, + operable: true, + projectAvailable: true, + modelAvailable: input.here.modelAvailable, + ...placementHostLimits(hosts, hereId), + }, + ] + : []), + ...peers.map((peer) => ({ + ...peer.candidate, + ...placementHostLimits(hosts, peer.candidate.environmentId), + })), + ]; + const decision = yield* Effect.sync(() => { + const chosen = chooseThreadPlacement( + { originEnvironmentId: hereId }, + candidates.map((candidate) => withRecentPlacements(candidate, now)), + now, + ); + const lanes = chosen.accepted + ? reportedLanes( + candidates.find((candidate) => candidate.environmentId === chosen.environmentId) + ?.resources ?? null, + ) + : undefined; + if (chosen.accepted && lanes !== undefined) + recentPlacements.set(chosen.environmentId, [ + ...recentAt(chosen.environmentId, now), + { at: now, lanes }, + ]); + return chosen; + }); yield* Effect.annotateCurrentSpan({ "placement.reason": decision.reason, "placement.candidates": askable.length + (hereCandidate ? 1 : 0), @@ -601,7 +656,7 @@ const make = Effect.gen(function* () { if (!decision.accepted) { return yield* new OrchestratorMcpFailure({ code: "target_required", - message: `No environment can take this task now (${decision.reason}): none answered with spare capacity, this repository's project and ${input.modelSelection.instanceId}/${input.modelSelection.model}. Nothing was started. Name an environment, or omit placement to run it here.`, + message: `No environment can take this task now (${decision.reason}): every machine automatic placement may use is at its lane or memory limit, or lacks this repository's project or ${input.modelSelection.instanceId}/${input.modelSelection.model}. Nothing was started. Retry later; name an environment only for work bound to that machine.`, }); } return yield* threads diff --git a/apps/server/src/resourceTelemetry/HostResources.test.ts b/apps/server/src/resourceTelemetry/HostResources.test.ts index fdf0d064228cf..8823dd0017c8e 100644 --- a/apps/server/src/resourceTelemetry/HostResources.test.ts +++ b/apps/server/src/resourceTelemetry/HostResources.test.ts @@ -55,6 +55,7 @@ const unlimitedCgroup = { const linuxHost = ( files: Record>, directories: Record> = {}, + activeRuns?: number, ) => { const reads = new Map(); return HostResources.layer.pipe( @@ -92,6 +93,10 @@ const linuxHost = ( }, }), Layer.succeed(HostProcess.Platform, "linux"), + Layer.succeed( + HostResources.ActiveRuns, + HostResources.ActiveRuns.of({ count: Effect.succeed(activeRuns) }), + ), Layer.succeed( ChildProcessSpawner.ChildProcessSpawner, ChildProcessSpawner.make(() => Effect.die("HostResources spawned on linux")), @@ -109,48 +114,53 @@ const read = Effect.gen(function* () { }); describe("HostResources.layer", () => { - it.effect("reports load per host CPU and the agents running under this user", () => { - const manager = "/user.slice/user-1000.slice/user@1000.service"; - const server = `${manager}/app.slice/t3.service`; - const agents = `/sys/fs/cgroup${manager}/app.slice/app-t3code.slice/app-t3code-agents.slice`; - return Effect.gen(function* () { - setup(); - host.load = 20; - const snapshot = yield* read; - host.load = 0; + it.effect( + "reports load per host CPU, the agents running under this user and this server's active runs", + () => { + const manager = "/user.slice/user-1000.slice/user@1000.service"; + const server = `${manager}/app.slice/t3.service`; + const agents = `/sys/fs/cgroup${manager}/app.slice/app-t3code.slice/app-t3code-agents.slice`; + return Effect.gen(function* () { + setup(); + host.load = 20; + const snapshot = yield* read; + host.load = 0; - // 20 runnable on 8 CPUs; two populated scopes from two servers, one finished. - assert.strictEqual(snapshot.loadPerCpu, 2.5); - assert.strictEqual(snapshot.runningAgents, 2); - }).pipe( - Effect.provide( - linuxHost( - { - "/proc/self/cgroup": [`0::${server}\n`], - "/proc/meminfo": unlimitedCgroup["/proc/meminfo"], - [`${agents}/app-t3code-agents-a.slice/t3code-claude-1.scope/cgroup.events`]: [ - "populated 1\nfrozen 0\n", - ], - [`${agents}/app-t3code-agents-a.slice/t3code-codex-2.scope/cgroup.events`]: [ - "populated 0\nfrozen 0\n", - ], - [`${agents}/app-t3code-agents-b.slice/t3code-claude-3.scope/cgroup.events`]: [ - "populated 1\nfrozen 0\n", - ], - }, - { - [agents]: ["app-t3code-agents-a.slice", "app-t3code-agents-b.slice", "cgroup.procs"], - [`${agents}/app-t3code-agents-a.slice`]: [ - "t3code-claude-1.scope", - "t3code-codex-2.scope", - "memory.max", - ], - [`${agents}/app-t3code-agents-b.slice`]: ["t3code-claude-3.scope"], - }, + // 20 runnable on 8 CPUs; two populated scopes from two servers, one finished. + assert.strictEqual(snapshot.loadPerCpu, 2.5); + assert.strictEqual(snapshot.runningAgents, 2); + assert.strictEqual(snapshot.activeRuns, 1); + }).pipe( + Effect.provide( + linuxHost( + { + "/proc/self/cgroup": [`0::${server}\n`], + "/proc/meminfo": unlimitedCgroup["/proc/meminfo"], + [`${agents}/app-t3code-agents-a.slice/t3code-claude-1.scope/cgroup.events`]: [ + "populated 1\nfrozen 0\n", + ], + [`${agents}/app-t3code-agents-a.slice/t3code-codex-2.scope/cgroup.events`]: [ + "populated 0\nfrozen 0\n", + ], + [`${agents}/app-t3code-agents-b.slice/t3code-claude-3.scope/cgroup.events`]: [ + "populated 1\nfrozen 0\n", + ], + }, + { + [agents]: ["app-t3code-agents-a.slice", "app-t3code-agents-b.slice", "cgroup.procs"], + [`${agents}/app-t3code-agents-a.slice`]: [ + "t3code-claude-1.scope", + "t3code-codex-2.scope", + "memory.max", + ], + [`${agents}/app-t3code-agents-b.slice`]: ["t3code-claude-3.scope"], + }, + 1, + ), ), - ), - ); - }); + ); + }, + ); it.effect("reports the CPU quota and usage of the server's cgroup", () => Effect.gen(function* () { diff --git a/apps/server/src/resourceTelemetry/HostResources.ts b/apps/server/src/resourceTelemetry/HostResources.ts index 7fc5b27dd2ad3..465e2a2ce9fdc 100644 --- a/apps/server/src/resourceTelemetry/HostResources.ts +++ b/apps/server/src/resourceTelemetry/HostResources.ts @@ -26,6 +26,11 @@ export class MachinePresence extends Context.Service< } >()("t3/resourceTelemetry/HostResources/MachinePresence") {} +export class ActiveRuns extends Context.Service< + ActiveRuns, + { readonly count: Effect.Effect } +>()("t3/resourceTelemetry/HostResources/ActiveRuns") {} + // The user manager's slice holding every T3 server's agent scopes (agentScope.ts). const AGENTS_SLICE_PATH = "app.slice/app-t3code.slice/app-t3code-agents.slice"; @@ -91,6 +96,7 @@ const make = Effect.fn("makeHostResources")(function* () { const readFile = (path: string) => fs.readFileString(path).pipe(Effect.orElseSucceed(() => "")); const presence = yield* Effect.serviceOption(MachinePresence); + const activeRuns = yield* Effect.serviceOption(ActiveRuns); // Populated agent scopes under this user's manager, from every T3 server on the host. const countAgents = Effect.fn("HostResources.countAgents")(function* ( @@ -220,6 +226,7 @@ const make = Effect.fn("makeHostResources")(function* () { : loadAverage / cpu.count; const runningAgents = platform === "linux" ? yield* countAgents(cgroupPath) : undefined; const role = Option.isSome(presence) ? yield* presence.value.read : {}; + const runs = Option.isSome(activeRuns) ? yield* activeRuns.value.count : undefined; return { sampledAt: DateTime.toEpochMillis(yield* DateTime.now), cpuUtilization, @@ -228,6 +235,7 @@ const make = Effect.fn("makeHostResources")(function* () { totalMemoryBytes, ...(loadPerCpu === undefined ? {} : { loadPerCpu }), ...(runningAgents === undefined ? {} : { runningAgents }), + ...(runs === undefined ? {} : { activeRuns: runs }), ...role, }; }); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 1a7895b0cd954..35ad3c6f35e83 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -326,8 +326,18 @@ const layerMachinePresence = MachinePresence.layer.pipe( Layer.provide(layerServerSettings), ); +const layerActiveRuns = Layer.effect( + HostResources.ActiveRuns, + Effect.gen(function* () { + const store = yield* ProjectionStoreV2.ProjectionStoreV2; + return HostResources.ActiveRuns.of({ + count: store.countActiveRunThreads().pipe(Effect.orElseSucceed(() => undefined)), + }); + }), +).pipe(Layer.provide(ProjectionStoreV2.layer), Layer.provide(SqlitePersistence.layerConfig)); + const layerResourceDiagnostics = Layer.mergeAll( - HostResources.layer.pipe(Layer.provide(layerMachinePresence)), + HostResources.layer.pipe(Layer.provide(Layer.merge(layerMachinePresence, layerActiveRuns))), layerResourceTelemetry, ProcessDiagnostics.layer.pipe(Layer.provide(layerResourceTelemetry)), ProcessResourceMonitor.layer.pipe(Layer.provide(layerResourceTelemetry)), diff --git a/apps/server/src/vcs/GitVcsDriver.ts b/apps/server/src/vcs/GitVcsDriver.ts index d0da1b3d09b3d..89a39e9a1be81 100644 --- a/apps/server/src/vcs/GitVcsDriver.ts +++ b/apps/server/src/vcs/GitVcsDriver.ts @@ -154,6 +154,7 @@ export interface CreateWorktreeOptions { readonly worktreesDirectory?: string; /** Clone the checkout from the macOS worktree checkout cache. */ readonly fastCheckout?: boolean; + readonly reuseEmptyBranch?: boolean; } export interface GitCommitProgress { diff --git a/apps/server/src/vcs/GitVcsDriverCore.test.ts b/apps/server/src/vcs/GitVcsDriverCore.test.ts index efaf9826408cd..c00bdd648a368 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.test.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.test.ts @@ -6532,6 +6532,91 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect( + "reuses an existing branch with no commits of its own only when asked, and refuses one with work", + () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(cwd); + yield* git(cwd, ["branch", "stale"]); + yield* git(cwd, ["branch", "worked"]); + yield* writeTextFile(cwd, "next.txt", "next\n"); + yield* git(cwd, ["add", "next.txt"]); + yield* git(cwd, ["commit", "-m", "next"]); + const tip = (yield* git(cwd, ["rev-parse", "HEAD"])).trim(); + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.GitVcsDriver; + const reuse = { reuseEmptyBranch: true }; + const unasked = yield* driver + .createWorktree({ + cwd, + path: path.join(yield* makeTmpDir("git-worktrees-"), "unasked"), + refName: initialBranch, + newRefName: "stale", + }) + .pipe(Effect.flip); + assert.equal(unasked.reason, "branch_already_exists"); + const stale = (yield* git(cwd, ["rev-parse", "refs/heads/stale"])).trim(); + const occupied = yield* makeTmpDir("git-worktrees-"); + yield* writeTextFile(occupied, "taken.txt", "taken\n"); + yield* driver + .createWorktree( + { cwd, path: occupied, refName: initialBranch, newRefName: "stale" }, + reuse, + ) + .pipe(Effect.flip); + assert.equal((yield* git(cwd, ["rev-parse", "refs/heads/stale"])).trim(), stale); + const reused = yield* driver.createWorktree( + { + cwd, + path: path.join(yield* makeTmpDir("git-worktrees-"), "stale"), + refName: initialBranch, + newRefName: "stale", + }, + reuse, + ); + assert.equal((yield* git(reused.worktree.path, ["rev-parse", "HEAD"])).trim(), tip); + assert.equal( + (yield* git(reused.worktree.path, ["branch", "--show-current"])).trim(), + "stale", + ); + + const again = yield* driver + .createWorktree( + { + cwd, + path: path.join(yield* makeTmpDir("git-worktrees-"), "stale-again"), + refName: initialBranch, + newRefName: "stale", + }, + reuse, + ) + .pipe(Effect.flip); + assert.equal(again.reason, "branch_already_exists"); + + const worktreePath = path.join(yield* makeTmpDir("git-worktrees-"), "worked"); + yield* git(cwd, ["worktree", "add", "--", worktreePath, "worked"]); + yield* writeTextFile(worktreePath, "mine.txt", "mine\n"); + yield* git(worktreePath, ["add", "mine.txt"]); + yield* git(worktreePath, ["commit", "-m", "mine"]); + const worked = (yield* git(cwd, ["rev-parse", "refs/heads/worked"])).trim(); + yield* git(cwd, ["worktree", "remove", "--", worktreePath]); + const refused = yield* driver + .createWorktree( + { + cwd, + path: path.join(yield* makeTmpDir("git-worktrees-"), "worked-again"), + refName: initialBranch, + newRefName: "worked", + }, + reuse, + ) + .pipe(Effect.flip); + assert.equal(refused.reason, "branch_already_exists"); + assert.equal((yield* git(cwd, ["rev-parse", "refs/heads/worked"])).trim(), worked); + }), + ); + it.effect("explains a new branch that collides with an existing branch namespace", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); diff --git a/apps/server/src/vcs/GitVcsDriverCore.ts b/apps/server/src/vcs/GitVcsDriverCore.ts index 89bb1fa039ddd..12acef995c6af 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.ts @@ -4593,13 +4593,14 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* operation: string, input: GitVcsDriver.GitCreateBranchInput, fallbackErrorDetail: string, + force = false, ) { // `git branch` does not read a lone - as the previous checkout. const startPoint = input.startPoint === "-" ? "@{-1}" : input.startPoint; yield* executeGitWithStableDiagnostics( operation, input.cwd, - ["branch", "--no-track", "--", input.branch, startPoint], + ["branch", ...(force ? ["--force"] : []), "--no-track", "--", input.branch, startPoint], { fallbackErrorDetail }, ); const startRef = yield* executeGit( @@ -5156,12 +5157,54 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ? `t3-create-${yield* random.uuidV4()}` : undefined; + let reusedTip: string | null = null; if (newRefName) { + const branch = { cwd: input.cwd, branch: newRefName, startPoint: input.refName }; // Kept apart from the checkout so the shared-config lock only covers the tracking write. yield* createBranchAt( "GitVcsDriver.createWorktree.createBranch", - { cwd: input.cwd, branch: newRefName, startPoint: input.refName }, + branch, "git branch failed", + ).pipe( + Effect.catchIf( + (error) => options?.reuseEmptyBranch === true && error.reason === "branch_already_exists", + (error) => + Effect.gen(function* () { + const checkedOut = (yield* executeWorktreeList( + "GitVcsDriver.createWorktree.branchCheckedOut", + input.cwd, + [], + )).stdout + .split("\0") + .includes(`branch refs/heads/${newRefName}`); + const keep = + checkedOut || + (yield* executeGit( + "GitVcsDriver.createWorktree.branchHasNoUniqueCommits", + input.cwd, + [ + "merge-base", + "--is-ancestor", + `refs/heads/${newRefName}`, + input.refName === "-" ? "@{-1}" : input.refName, + ], + { allowNonZeroExit: true }, + )).exitCode !== 0; + if (keep) return yield* error; + const tip = (yield* executeGit( + "GitVcsDriver.createWorktree.reusedBranchTip", + input.cwd, + ["rev-parse", "--verify", `refs/heads/${newRefName}`], + )).stdout.trim(); + yield* createBranchAt( + "GitVcsDriver.createWorktree.reuseBranch", + branch, + "git branch failed", + true, + ); + reusedTip = tip; + }), + ), ); } const checkoutRef = newRefName ?? input.refName; @@ -5259,7 +5302,16 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* : Effect.void ).pipe( Effect.andThen( - deleteLocalBranch({ cwd: input.cwd, refName: newRefName, force: true }), + reusedTip === null + ? deleteLocalBranch({ cwd: input.cwd, refName: newRefName, force: true }) + : executeGit("GitVcsDriver.createWorktree.restoreReusedBranch", input.cwd, [ + "branch", + "--force", + "--no-track", + "--", + newRefName, + reusedTip, + ]), ), Effect.catch((cause) => Effect.logWarning("could not clean up a failed worktree or its branch", { cause }), diff --git a/packages/contracts/src/resourceTelemetry.ts b/packages/contracts/src/resourceTelemetry.ts index 493dac10ce028..5a7047d868ec7 100644 --- a/packages/contracts/src/resourceTelemetry.ts +++ b/packages/contracts/src/resourceTelemetry.ts @@ -17,6 +17,7 @@ export const HostResourcesSnapshot = Schema.Struct({ loadPerCpu: Schema.optional(Schema.Number.check(Schema.isGreaterThanOrEqualTo(0))), /** Agent processes running on this host, across every T3 server there. */ runningAgents: Schema.optional(NonNegativeInt), + activeRuns: Schema.optional(NonNegativeInt), /** Interactive machines belong to a person; placement uses them last and only while idle. */ placementRole: Schema.optional(Schema.Literals(["headless", "interactive"])), /** Someone is using this interactive machine now, or it cannot tell. */ diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 8e08976ff6b7a..3a1e77f3a8c3f 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -16,6 +16,7 @@ import * as SchemaTransformation from "effect/SchemaTransformation"; import { EnvironmentId, ForwardCompatibleNullable, + NonNegativeInt, ForwardCompatibleOptional, OmittedWhenNull, ProjectId, @@ -1429,6 +1430,38 @@ export const CloudSpeechCredentials = Schema.Struct({ export const MachinePlacementRole = Schema.Literals(["auto", "headless", "interactive"]); export type MachinePlacementRole = typeof MachinePlacementRole.Type; +export const AutomaticPlacementHost = Schema.Struct({ + environmentId: EnvironmentId, + maxLanes: NonNegativeInt, + minAvailableMemoryGiB: Schema.Number.check(Schema.isGreaterThanOrEqualTo(0)).pipe( + Schema.withDecodingDefault(Effect.succeed(4)), + ), +}); +export type AutomaticPlacementHost = typeof AutomaticPlacementHost.Type; + +export const DEFAULT_AUTOMATIC_PLACEMENT_HOSTS: ReadonlyArray = [ + { + environmentId: EnvironmentId.make("5cd73d78-08c9-47d4-8381-a53e008340d4"), + maxLanes: 4, + minAvailableMemoryGiB: 4, + }, + { + environmentId: EnvironmentId.make("26cc7088-9c9e-4e60-87dc-51af42d30424"), + maxLanes: 1, + minAvailableMemoryGiB: 4, + }, + { + environmentId: EnvironmentId.make("47f1a903-7121-4886-8b2f-7c8f50174b61"), + maxLanes: 1, + minAvailableMemoryGiB: 4, + }, + { + environmentId: EnvironmentId.make("2f360b2e-0386-498a-b9cd-62999f3525ba"), + maxLanes: 0, + minAvailableMemoryGiB: 4, + }, +]; + export const ServerSettings = Schema.Struct({ // Null means unmigrated; empty is authoritative and must not resurrect cleared preferences. favorites: Schema.NullOr(ModelFavorites).pipe(Schema.withDecodingDefault(Effect.succeed(null))), @@ -1584,6 +1617,15 @@ export const ServerSettings = Schema.Struct({ automaticPlacementByDefault: Schema.Boolean.pipe( Schema.withDecodingDefault(Effect.succeed(true)), ), + /** + * Closura: the fleet order and limits for automatic placement. R720 first with at most four + * lanes, ASUS one heavy lane, Jeff's Omarchy desktop light work, Kameron's laptop none: work + * runs there only when it is bound to that laptop. A host past its limit refuses, so the + * dispatch router overflows to the cloud. Unlisted hosts follow the listed ones, uncapped. + */ + automaticPlacementHosts: Schema.Array(AutomaticPlacementHost).pipe( + Schema.withDecodingDefault(Effect.succeed(DEFAULT_AUTOMATIC_PLACEMENT_HOSTS)), + ), /** * The watcher thread. It hears a run that ran another model than selected, hit a provider * safeguard fallback or a usage limit, and each lane that ended, promised to continue with @@ -2054,6 +2096,7 @@ export const ServerSettingsPatch = Schema.Struct({ automaticPlacementExcludedEnvironmentIds: Schema.optionalKey(Schema.Array(EnvironmentId)), machinePlacementRole: Schema.optionalKey(MachinePlacementRole), automaticPlacementByDefault: Schema.optionalKey(Schema.Boolean), + automaticPlacementHosts: Schema.optionalKey(Schema.Array(AutomaticPlacementHost)), modelAlertWatcher: Schema.optionalKey(Schema.NullOr(ModelAlertWatcher)), modelAlertWebhookUrl: Schema.optionalKey(TrimmedString), watcherStuckMinutes: Schema.optionalKey( diff --git a/packages/shared/src/loadBalancing.ts b/packages/shared/src/loadBalancing.ts index 1969020c9cb3b..9eb66e0565b12 100644 --- a/packages/shared/src/loadBalancing.ts +++ b/packages/shared/src/loadBalancing.ts @@ -11,7 +11,30 @@ const LIMITS = { interactive: { cpu: 0.5, loadPerCpu: 0.5, memoryShare: 0.4, memoryBytes: 3 * GiB, agents: 2 }, } as const; -function headroom(resources: HostResourcesSnapshot, tier: keyof typeof LIMITS): number | null { +export interface PlacementHostCap { + readonly maxLanes: number; + readonly minAvailableMemoryBytes: number; +} + +export function withinHostCap( + resources: HostResourcesSnapshot | null, + cap: PlacementHostCap | undefined, +): boolean { + if (cap === undefined) return true; + if (resources === null) return false; + const lanes = resources.activeRuns ?? resources.runningAgents; + return ( + lanes !== undefined && + lanes < cap.maxLanes && + resources.availableMemoryBytes >= cap.minAvailableMemoryBytes + ); +} + +function headroom( + resources: HostResourcesSnapshot, + tier: keyof typeof LIMITS, + cap: PlacementHostCap | undefined, +): number | null { const limits = LIMITS[tier]; const { cpuUtilization, cpuCount, totalMemoryBytes, availableMemoryBytes } = resources; if (cpuUtilization === null || totalMemoryBytes <= 0 || cpuCount <= 0) return null; @@ -29,6 +52,7 @@ function headroom(resources: HostResourcesSnapshot, tier: keyof typeof LIMITS): (limits.agents !== null && agents >= limits.agents) ) return null; + if (!withinHostCap(resources, cap)) return null; // Run queues past the core count show saturation that a short CPU sample can miss. const busy = Math.max(cpuUtilization, Math.min(1, loadPerCpu)); return (cpuCount * (1 - busy) * memoryAvailable) / (1 + agents); @@ -40,9 +64,9 @@ function headroom(resources: HostResourcesSnapshot, tier: keyof typeof LIMITS): * its person is not using it, and it stays well clear of its limits. A machine that does not * report its role, such as one on an older release, counts as someone's computer in use. * - * `originEnvironmentId` is where the work starts. When nothing has room it keeps the work if it - * has room by headless limits, as before placement could pick: work started on someone's - * computer is theirs to run there. + * Within a tier the lowest `rank` with room wins, then the most room. A `cap` refuses a machine + * at its lane count or memory floor, and one that cannot report its lanes. When nothing has room + * the answer is null, never the machine the work started on. */ export function chooseLoadBalancedEnvironment( candidates: ReadonlyArray<{ @@ -51,16 +75,16 @@ export function chooseLoadBalancedEnvironment( /** Client receipt time avoids comparing clocks on different machines. */ receivedAt?: number; weight: number; + rank?: number | undefined; + cap?: PlacementHostCap | undefined; }>, now: number, - originEnvironmentId?: string, ): string | null { const best = { - headless: { id: null as string | null, score: 0 }, - interactive: { id: null as string | null, score: 0 }, + headless: { id: null as string | null, rank: Infinity, score: 0 }, + interactive: { id: null as string | null, rank: Infinity, score: 0 }, }; - let origin: string | null = null; - for (const { environmentId, resources, receivedAt, weight } of candidates) { + for (const { environmentId, resources, receivedAt, weight, rank, cap } of candidates) { const sampledAt = receivedAt ?? resources?.sampledAt ?? 0; if ( !resources || @@ -71,15 +95,16 @@ export function chooseLoadBalancedEnvironment( ) { continue; } - if (environmentId === originEnvironmentId && headroom(resources, "headless") !== null) - origin = environmentId; // A machine that does not say is someone's until it says otherwise, and in use if unsure. const tier = resources.placementRole === "headless" ? "headless" : "interactive"; if (tier === "interactive" && resources.humanActive !== false) continue; - const room = headroom(resources, tier); + const room = headroom(resources, tier, cap); if (room === null) continue; const score = weight * room; - if (score > best[tier].score) best[tier] = { id: environmentId, score }; + const order = rank ?? Infinity; + const current = best[tier]; + if (order < current.rank || (order === current.rank && score > current.score)) + best[tier] = { id: environmentId, rank: order, score }; } - return best.headless.id ?? best.interactive.id ?? origin; + return best.headless.id ?? best.interactive.id; } diff --git a/packages/shared/src/threadPlacement.test.ts b/packages/shared/src/threadPlacement.test.ts index e27aa87a3c038..993adc74b8efd 100644 --- a/packages/shared/src/threadPlacement.test.ts +++ b/packages/shared/src/threadPlacement.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from "vite-plus/test"; -import { chooseThreadPlacement, type ThreadPlacementCandidate } from "./threadPlacement.ts"; +import { + chooseThreadPlacement, + placementHostLimits, + type ThreadPlacementCandidate, +} from "./threadPlacement.ts"; const now = 1_000_000; const GiB = 1024 ** 3; @@ -151,7 +155,7 @@ describe("chooseThreadPlacement", () => { } }); - it("keeps work on the person's own machine when nothing else has room", () => { + it("never keeps work on the person's own machine when nothing else has room", () => { const desk = host("here", 0.3, { resources: { ...host("here", 0.3).resources!, @@ -160,11 +164,9 @@ describe("chooseThreadPlacement", () => { }, }); expect(chooseThreadPlacement(origin, [desk, host("r720", 0.97)], now)).toEqual({ - accepted: true, - environmentId: "here", - reason: "capacity", + accepted: false, + reason: "no-eligible-environment", }); - // A headless machine with room still wins, and a full origin is not kept. expect(chooseThreadPlacement(origin, [desk, host("r720", 0.5)], now)).toMatchObject({ environmentId: "r720", }); @@ -174,6 +176,118 @@ describe("chooseThreadPlacement", () => { }); }); + describe("with the fleet host order and caps", () => { + const fleet = [ + { environmentId: "r720", maxLanes: 4, minAvailableMemoryGiB: 4 }, + { environmentId: "asus", maxLanes: 1, minAvailableMemoryGiB: 4 }, + { environmentId: "omarchy", maxLanes: 1, minAvailableMemoryGiB: 4 }, + { environmentId: "laptop", maxLanes: 0, minAvailableMemoryGiB: 4 }, + ]; + const machine = ( + environmentId: string, + cpuUtilization: number, + extra: Partial> = {}, + ) => + host(environmentId, cpuUtilization, { + resources: { ...host(environmentId, cpuUtilization).resources!, ...extra }, + ...placementHostLimits(fleet, environmentId), + }); + const away = { placementRole: "interactive", humanActive: false } as const; + const laptopOrigin = { originEnvironmentId: "laptop" }; + + it("sends a lane started from the laptop to R720, then ASUS, then the Omarchy desktop", () => { + const laptop = machine("laptop", 0.01, away); + expect( + chooseThreadPlacement( + laptopOrigin, + [laptop, machine("omarchy", 0.01, away), machine("asus", 0.01), machine("r720", 0.6)], + now, + ), + ).toEqual({ accepted: true, environmentId: "r720", reason: "capacity" }); + expect( + chooseThreadPlacement( + laptopOrigin, + [ + laptop, + machine("omarchy", 0.01, away), + machine("asus", 0.6), + machine("r720", 0.1, { runningAgents: 4 }), + ], + now, + ), + ).toMatchObject({ environmentId: "asus" }); + expect( + chooseThreadPlacement( + laptopOrigin, + [ + laptop, + machine("omarchy", 0.01, away), + machine("asus", 0.1, { runningAgents: 1 }), + machine("r720", 0.1, { runningAgents: 4 }), + ], + now, + ), + ).toMatchObject({ environmentId: "omarchy" }); + }); + + it("refuses with no-eligible-environment when every host is at its cap, never the laptop", () => { + expect( + chooseThreadPlacement( + laptopOrigin, + [ + machine("laptop", 0.01, away), + machine("omarchy", 0.01, { ...away, runningAgents: 1 }), + machine("asus", 0.1, { runningAgents: 1 }), + machine("r720", 0.1, { runningAgents: 4 }), + ], + now, + ), + ).toEqual({ accepted: false, reason: "no-eligible-environment" }); + }); + + it("caps R720 at four active lanes and a memory floor, counts idle scopes only without a run count, and refuses a host that cannot count its lanes", () => { + for (const [extra, expected] of [ + [{ runningAgents: 3 }, "r720"], + [{ runningAgents: 4 }, null], + [{ runningAgents: 18 }, null], + [{ runningAgents: 0, availableMemoryBytes: 3.9 * GiB }, null], + [{ runningAgents: undefined }, null], + [{ runningAgents: 18, activeRuns: 3 }, "r720"], + [{ runningAgents: 0, activeRuns: 4 }, null], + [{ runningAgents: undefined, activeRuns: 0 }, "r720"], + ] as const) { + const decision = chooseThreadPlacement(origin, [machine("r720", 0.1, extra)], now); + expect(decision.accepted ? decision.environmentId : null).toBe(expected); + } + }); + + it("tries unlisted hosts after listed ones, without a cap", () => { + expect( + chooseThreadPlacement(origin, [machine("spare", 0.01), machine("asus", 0.5)], now), + ).toMatchObject({ environmentId: "asus" }); + expect( + chooseThreadPlacement( + origin, + [ + machine("spare", 0.5, { runningAgents: 30 }), + machine("asus", 0.1, { runningAgents: 1 }), + ], + now, + ), + ).toMatchObject({ environmentId: "spare" }); + }); + + it("still runs laptop-bound work on the laptop", () => { + expect( + chooseThreadPlacement( + { ...laptopOrigin, requiresOrigin: true }, + [machine("laptop", 0.9), machine("r720", 0.1)], + now, + ), + ).toEqual({ accepted: true, environmentId: "laptop", reason: "local-state" }); + }); + }); + it("refuses evidence that names one environment twice", () => { expect(chooseThreadPlacement(origin, [host("a", 0.1), host("a", 0.2)], now)).toEqual({ accepted: false, diff --git a/packages/shared/src/threadPlacement.ts b/packages/shared/src/threadPlacement.ts index f52a64f7dc43d..8b103a9826cad 100644 --- a/packages/shared/src/threadPlacement.ts +++ b/packages/shared/src/threadPlacement.ts @@ -1,5 +1,7 @@ import type { HostResourcesSnapshot } from "@t3tools/contracts"; -import { chooseLoadBalancedEnvironment } from "./loadBalancing.ts"; +import { chooseLoadBalancedEnvironment, type PlacementHostCap } from "./loadBalancing.ts"; + +const GiB = 1024 ** 3; /** Evidence the deciding server gathered itself; caller-supplied labels are not evidence. */ export interface ThreadPlacementCandidate { @@ -14,6 +16,8 @@ export interface ThreadPlacementCandidate { readonly projectAvailable: boolean; /** The requested provider, model and modes run there. */ readonly modelAvailable: boolean; + readonly rank?: number | undefined; + readonly cap?: PlacementHostCap | undefined; } export interface ThreadPlacementRequest { @@ -62,8 +66,25 @@ export function chooseThreadPlacement( .sort((a, b) => a.environmentId < b.environmentId ? -1 : a.environmentId > b.environmentId ? 1 : 0, ); - const environmentId = chooseLoadBalancedEnvironment(eligible, now, request.originEnvironmentId); + const environmentId = chooseLoadBalancedEnvironment(eligible, now); return environmentId === null ? { accepted: false, reason: "no-eligible-environment" } : { accepted: true, environmentId, reason: "capacity" }; } + +export function placementHostLimits( + hosts: ReadonlyArray<{ + readonly environmentId: string; + readonly maxLanes: number; + readonly minAvailableMemoryGiB: number; + }>, + environmentId: string, +): { readonly rank?: number; readonly cap?: PlacementHostCap } { + const rank = hosts.findIndex((host) => host.environmentId === environmentId); + const host = hosts[rank]; + if (host === undefined) return {}; + return { + rank, + cap: { maxLanes: host.maxLanes, minAvailableMemoryBytes: host.minAvailableMemoryGiB * GiB }, + }; +}