From 2dac3a713e128a708a520cf6b265c711cdd515e9 Mon Sep 17 00:00:00 2001 From: nullStack65 <112618179+nullStack65@users.noreply.github.com> Date: Fri, 20 Nov 2026 09:40:16 +0000 Subject: [PATCH 1/4] feat(placement): fleet host order and lane caps; reuse empty branches; retry held worktree setup Automatic placement now tries hosts in a configured order (R720, ASUS, omarchy, then the laptop) and refuses a host at its lane cap or memory floor. It never falls back to the machine the work started on, so a full fleet answers no-eligible-environment and cloud overflow can take it. The refusal no longer tells callers to name an environment, which is how R720 was pinned past its limits. Worktree setup reuses an existing branch with no commits of its own instead of failing with branch_already_exists. A send held behind a failed worktree setup now retries that setup. Co-Authored-By: Claude Fable 5.2 --- .../src/mcp/OrchestratorMcpService.test.ts | 62 ++++++++- apps/server/src/mcp/OrchestratorMcpService.ts | 23 +++- .../src/mcp/toolkits/project/handlers.ts | 20 ++- apps/server/src/peer/PeerLinks.ts | 7 + apps/server/src/peer/RemoteDelegation.test.ts | 43 ++++++ apps/server/src/peer/RemoteDelegation.ts | 13 +- apps/server/src/vcs/GitVcsDriverCore.test.ts | 56 ++++++++ apps/server/src/vcs/GitVcsDriverCore.ts | 40 +++++- packages/contracts/src/settings.ts | 41 ++++++ packages/shared/src/loadBalancing.ts | 53 ++++++-- packages/shared/src/threadPlacement.test.ts | 123 +++++++++++++++++- packages/shared/src/threadPlacement.ts | 25 +++- 12 files changed, 476 insertions(+), 30 deletions(-) diff --git a/apps/server/src/mcp/OrchestratorMcpService.test.ts b/apps/server/src/mcp/OrchestratorMcpService.test.ts index 97907e2d50b2c..217b1127d1c07 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.test.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.test.ts @@ -4489,8 +4489,11 @@ describe("OrchestratorMcpService thread lookup", () => { shells: ReadonlyArray, sent: Ref.Ref, overrides: Partial = {}, + launch: Partial = {}, ) => - layerOrchestratorService.pipe( + OrchestratorMcpService.layer.pipe( + Layer.provide(Layer.mock(ThreadLaunchService.ThreadLaunchService)(launch)), + Layer.provide(Layer.mock(GitVcsDriver.GitVcsDriver)({})), Layer.provide( Layer.mergeAll( NodeServices.layer, @@ -4638,6 +4641,63 @@ describe("OrchestratorMcpService thread lookup", () => { }), ); + it.effect("retries a failed worktree setup when a send is held behind it", () => + Effect.gen(function* () { + const target = liveThreadShell(ThreadId.make("thread:held")); + const failedRunId = RunId.make("run:setup-failed"); + const retried: Array = []; + const sendWith = (worktreePath: string | null) => + Effect.gen(function* () { + const sent = yield* Ref.make(0); + yield* Effect.gen(function* () { + const mcp = yield* OrchestratorMcpService.OrchestratorMcpService; + yield* mcp.sendToThread(client, { threadId: target.id, message: "hello" }); + }).pipe( + Effect.provide( + service( + [target], + sent, + { + getProjectThreadRecords: () => Effect.succeed(idleThreadProjection(target)), + getThreadRecords: () => + Effect.succeed({ + thread: { ...target, worktreePath }, + runs: [ + { + id: failedRunId, + status: "failed", + startedAt: null, + workspacePreparation: { type: "worktree" }, + }, + ], + } as never), + sendToThread: () => + Effect.succeed({ + run: { id: RunId.make("run:held"), status: "queued", queueHeld: true }, + delivery: "queued", + } as never), + }, + { + retryPreparation: (input) => { + retried.push(input); + return Effect.succeed({} as never); + }, + }, + ), + ), + ); + }); + + yield* sendWith(null); + assert.deepStrictEqual( + retried.map((input) => [input.threadId, input.runId]), + [[target.id, failedRunId]], + ); + yield* sendWith("/repo/worktree"); + assert.equal(retried.length, 1); + }), + ); + it.effect("refuses waiting without orchestration capability before reading the target", () => Effect.gen(function* () { const sent = yield* Ref.make(0); diff --git a/apps/server/src/mcp/OrchestratorMcpService.ts b/apps/server/src/mcp/OrchestratorMcpService.ts index 800e8cbb3ba9d..961ce2c28789a 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.ts @@ -72,7 +72,7 @@ import { } from "@t3tools/contracts"; import { getProviderOptionCurrentValue, withLegacyUltracodeEffort } from "@t3tools/shared/model"; import { backgroundWorkHoldsCompletion } from "@t3tools/shared/orchestrationV2PendingBackgroundWork"; -import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError"; +import { latestExecutedRun, runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError"; import { resolveProjectSettings } from "@t3tools/shared/projectSettings"; import { formatThreadLink } from "@t3tools/shared/threadLinks"; import { @@ -3610,6 +3610,27 @@ const make = Effect.gen(function* () { ), ), ); + if (result.run.status === "queued" && result.run.queueHeld === true) { + yield* threadManagement.getThreadRecords(input.threadId, ["runs"]).pipe( + Effect.flatMap(({ thread, runs }) => { + const failed = latestExecutedRun(runs); + return thread.worktreePath === null && + failed?.status === "failed" && + failed.workspacePreparation?.type === "worktree" + ? threadLaunch.retryPreparation({ + commandId: CommandId.make( + `${stableCommandId({ scope, requestKey: key, operation: "thread-send" })}:retry-preparation`, + ), + threadId: input.threadId, + runId: failed.id, + }) + : Effect.void; + }), + Effect.catchCause((cause) => + Effect.logWarning("thread-send could not retry the failed worktree setup", { cause }), + ), + ); + } return { threadId: input.threadId, link: threadLinkFor(scope, input.threadId, target.thread.title), diff --git a/apps/server/src/mcp/toolkits/project/handlers.ts b/apps/server/src/mcp/toolkits/project/handlers.ts index 6bd5b14690d74..716f0b09c8d51 100644 --- a/apps/server/src/mcp/toolkits/project/handlers.ts +++ b/apps/server/src/mcp/toolkits/project/handlers.ts @@ -3,6 +3,7 @@ import { agentLaunchInboxRouting, withLegacyPeerInboxReason, CommandId, + DEFAULT_AUTOMATIC_PLACEMENT_HOSTS, type EnvironmentId, MessageId, type ModelSelection, @@ -35,6 +36,8 @@ import * as ProviderRegistry from "../../../provider/ProviderRegistry.ts"; import * as ServerSettings from "../../../serverSettings.ts"; import * as DateTime from "effect/DateTime"; import { resolveNewThreadModelSelection } from "@t3tools/shared/usageLimits"; +import { withinHostCap } from "@t3tools/shared/loadBalancing"; +import { placementHostLimits } from "@t3tools/shared/threadPlacement"; import { resolveProjectSettings } from "@t3tools/shared/projectSettings"; import { resolveRuntimeMode } from "../../delegatedTaskModes.ts"; import * as McpInvocationContext from "../../McpInvocationContext.ts"; @@ -238,8 +241,21 @@ const launchPlacedThere = Effect.fn("mcp.launchPlacedThere")(function* ( runtimeMode: local.runtimeMode, here, }); - // Placement nobody asked for falls back to here, as before, when no machine has room. - const environmentId = byDefault + // Placement nobody asked for falls back to here when no machine has room, within here's limit. + const hereMayKeep = + byDefault && + withinHostCap( + here.resources, + placementHostLimits( + yield* ServerSettings.ServerSettingsService.pipe( + Effect.flatMap((settings) => settings.getSettings), + Effect.map((settings) => settings.automaticPlacementHosts), + Effect.orElseSucceed(() => DEFAULT_AUTOMATIC_PLACEMENT_HOSTS), + ), + scope.environmentId, + ).cap, + ); + const environmentId = hereMayKeep ? yield* placing.pipe( Effect.catchIf( (error) => error.code === "target_required", diff --git a/apps/server/src/peer/PeerLinks.ts b/apps/server/src/peer/PeerLinks.ts index f79a18257a828..1aba75d7c6dd9 100644 --- a/apps/server/src/peer/PeerLinks.ts +++ b/apps/server/src/peer/PeerLinks.ts @@ -1,5 +1,6 @@ import { type AppConnectionReport, + type AutomaticPlacementHost, AuthMcpClientAccess, AuthMcpRegistrationError, AuthMcpTokenError, @@ -126,6 +127,7 @@ export class PeerLinks extends Context.Service< * fail, so placement contacts nobody rather than everyone. */ readonly placementExclusions: Effect.Effect, PeerLinkError>; + readonly placementHosts: Effect.Effect, PeerLinkError>; /** The stored links, read here only: no peer is asked whether it answers. */ readonly stored: Effect.Effect, PeerLinkError>; /** @@ -281,6 +283,10 @@ const make = Effect.gen(function* () { Effect.map((settings) => settings.linkedEnvironmentsEnabled), Effect.orElseSucceed(() => false), ); + const placementHosts = serverSettings.getSettings.pipe( + Effect.map((settings) => settings.automaticPlacementHosts), + Effect.catch(failStorage), + ); const placementExclusions = serverSettings.getSettings.pipe( Effect.map( (settings): ReadonlySet => @@ -853,6 +859,7 @@ const make = Effect.gen(function* () { environments, stored, placementExclusions, + placementHosts, session, get, renewed, diff --git a/apps/server/src/peer/RemoteDelegation.test.ts b/apps/server/src/peer/RemoteDelegation.test.ts index 17cd8d7a50bc0..a8561b75eef97 100644 --- a/apps/server/src/peer/RemoteDelegation.test.ts +++ b/apps/server/src/peer/RemoteDelegation.test.ts @@ -1710,6 +1710,49 @@ it.effect("placement nobody asked for runs here when no machine has room", () => ).pipe(Effect.provide(NodeHttpServer.layerTest)), ); +it.effect("a host at its lane cap refuses, and a laptop capped at no lanes keeps nothing", () => + Effect.scoped( + Effect.gen(function* () { + const thread = yield* makeBoxThread; + yield* Ref.set(thread.resources, { ...loadOf(0.1), runningAgents: 4 }); + const b = yield* serveBox(thread); + const offers = yield* SubscriptionRef.make< + ReadonlyArray + >([]); + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); + const localLaunches = yield* Ref.make(0); + const a = yield* makeLaptop(database, offers, undefined, Effect.void, { + localLaunch: Ref.update(localLaunches, (count) => count + 1), + }); + yield* linkTo(a.links, b, "full-access"); + yield* a.settings.updateSettings({ + automaticPlacementHosts: [ + { environmentId: box.environmentId, maxLanes: 4, minAvailableMemoryGiB: 4 }, + { environmentId: laptop.environmentId, maxLanes: 0, minAvailableMemoryGiB: 4 }, + ], + }); + const fresh = { type: "worktree", baseRef: "main", startFromOrigin: true }; + const capped = yield* launchUnplaced(a, { + clientRequestId: "capped", + workspaceStrategy: fresh, + }); + expect(capped.isError).toBe(true); + expect(JSON.stringify(capped.content)).toContain("no-eligible-environment"); + expect(yield* Ref.get(localLaunches)).toBe(0); + expect(yield* Ref.get(thread.launched)).toBeNull(); + + yield* Ref.set(thread.resources, { ...loadOf(0.1), runningAgents: 3 }); + const placed = yield* launchUnplaced(a, { + clientRequestId: "room", + workspaceStrategy: fresh, + }); + expect(placed.isError, JSON.stringify(placed.content)).toBe(false); + expect(placed.structuredContent).toMatchObject({ environmentId: box.environmentId }); + expect(yield* Ref.get(localLaunches)).toBe(0); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); + it.effect( "a retry that drops placement or asks for this checkout reaches the thread placed there", () => diff --git a/apps/server/src/peer/RemoteDelegation.ts b/apps/server/src/peer/RemoteDelegation.ts index ed4ea992a9749..1c8b8f5c5777c 100644 --- a/apps/server/src/peer/RemoteDelegation.ts +++ b/apps/server/src/peer/RemoteDelegation.ts @@ -1,5 +1,6 @@ import { CommandId, + DEFAULT_AUTOMATIC_PLACEMENT_HOSTS, EnvironmentId, isRuntimeModeAbove, type ModelSelection, @@ -29,6 +30,7 @@ import { } from "@t3tools/contracts"; import { chooseThreadPlacement, + placementHostLimits, type ThreadPlacementCandidate, } from "@t3tools/shared/threadPlacement"; import * as Cause from "effect/Cause"; @@ -510,6 +512,9 @@ const make = Effect.gen(function* () { }, ) { const askable = yield* askableLinks(input); + const hosts = yield* links.placementHosts.pipe( + Effect.orElseSucceed(() => DEFAULT_AUTOMATIC_PLACEMENT_HOSTS), + ); const peers = (yield* Effect.forEach(askable, (link) => peerCandidate(ask, link, input), { concurrency: "unbounded", })).flatMap((peer) => (Option.isSome(peer) ? [peer.value] : [])); @@ -530,10 +535,14 @@ const make = Effect.gen(function* () { operable: true, projectAvailable: true, modelAvailable: input.here.modelAvailable, + ...placementHostLimits(hosts, hereId), }, ] : []), - ...peers.map((peer) => peer.candidate), + ...peers.map((peer) => ({ + ...peer.candidate, + ...placementHostLimits(hosts, peer.candidate.environmentId), + })), ], now, ); @@ -601,7 +610,7 @@ const make = Effect.gen(function* () { if (!decision.accepted) { return yield* new OrchestratorMcpFailure({ code: "target_required", - message: `No environment can take this task now (${decision.reason}): none answered with spare capacity, this repository's project and ${input.modelSelection.instanceId}/${input.modelSelection.model}. Nothing was started. Name an environment, or omit placement to run it here.`, + message: `No environment can take this task now (${decision.reason}): every machine automatic placement may use is at its lane or memory limit, or lacks this repository's project or ${input.modelSelection.instanceId}/${input.modelSelection.model}. Nothing was started. Retry later or send it to cloud overflow; name an environment only for work bound to that machine.`, }); } return yield* threads diff --git a/apps/server/src/vcs/GitVcsDriverCore.test.ts b/apps/server/src/vcs/GitVcsDriverCore.test.ts index efaf9826408cd..e593cbcd7589b 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.test.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.test.ts @@ -6532,6 +6532,62 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect( + "reuses an existing branch with no commits of its own, and refuses one with work", + () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(cwd); + yield* git(cwd, ["branch", "stale"]); + yield* git(cwd, ["branch", "worked"]); + yield* writeTextFile(cwd, "next.txt", "next\n"); + yield* git(cwd, ["add", "next.txt"]); + yield* git(cwd, ["commit", "-m", "next"]); + const tip = (yield* git(cwd, ["rev-parse", "HEAD"])).trim(); + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.GitVcsDriver; + const reused = yield* driver.createWorktree({ + cwd, + path: path.join(yield* makeTmpDir("git-worktrees-"), "stale"), + refName: initialBranch, + newRefName: "stale", + }); + assert.equal((yield* git(reused.worktree.path, ["rev-parse", "HEAD"])).trim(), tip); + assert.equal( + (yield* git(reused.worktree.path, ["branch", "--show-current"])).trim(), + "stale", + ); + + const again = yield* driver + .createWorktree({ + cwd, + path: path.join(yield* makeTmpDir("git-worktrees-"), "stale-again"), + refName: initialBranch, + newRefName: "stale", + }) + .pipe(Effect.flip); + assert.equal(again.reason, "branch_already_exists"); + + const worktreePath = path.join(yield* makeTmpDir("git-worktrees-"), "worked"); + yield* git(cwd, ["worktree", "add", "--", worktreePath, "worked"]); + yield* writeTextFile(worktreePath, "mine.txt", "mine\n"); + yield* git(worktreePath, ["add", "mine.txt"]); + yield* git(worktreePath, ["commit", "-m", "mine"]); + const worked = (yield* git(cwd, ["rev-parse", "refs/heads/worked"])).trim(); + yield* git(cwd, ["worktree", "remove", "--", worktreePath]); + const refused = yield* driver + .createWorktree({ + cwd, + path: path.join(yield* makeTmpDir("git-worktrees-"), "worked-again"), + refName: initialBranch, + newRefName: "worked", + }) + .pipe(Effect.flip); + assert.equal(refused.reason, "branch_already_exists"); + assert.equal((yield* git(cwd, ["rev-parse", "refs/heads/worked"])).trim(), worked); + }), + ); + it.effect("explains a new branch that collides with an existing branch namespace", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); diff --git a/apps/server/src/vcs/GitVcsDriverCore.ts b/apps/server/src/vcs/GitVcsDriverCore.ts index 89bb1fa039ddd..497c36be2354b 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.ts @@ -4593,13 +4593,14 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* operation: string, input: GitVcsDriver.GitCreateBranchInput, fallbackErrorDetail: string, + force = false, ) { // `git branch` does not read a lone - as the previous checkout. const startPoint = input.startPoint === "-" ? "@{-1}" : input.startPoint; yield* executeGitWithStableDiagnostics( operation, input.cwd, - ["branch", "--no-track", "--", input.branch, startPoint], + ["branch", ...(force ? ["--force"] : []), "--no-track", "--", input.branch, startPoint], { fallbackErrorDetail }, ); const startRef = yield* executeGit( @@ -5157,11 +5158,46 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* : undefined; if (newRefName) { + const branch = { cwd: input.cwd, branch: newRefName, startPoint: input.refName }; // Kept apart from the checkout so the shared-config lock only covers the tracking write. yield* createBranchAt( "GitVcsDriver.createWorktree.createBranch", - { cwd: input.cwd, branch: newRefName, startPoint: input.refName }, + branch, "git branch failed", + ).pipe( + Effect.catchIf( + (error) => error.reason === "branch_already_exists", + (error) => + Effect.gen(function* () { + const checkedOut = (yield* executeWorktreeList( + "GitVcsDriver.createWorktree.branchCheckedOut", + input.cwd, + [], + )).stdout + .split("\0") + .includes(`branch refs/heads/${newRefName}`); + const keep = + checkedOut || + (yield* executeGit( + "GitVcsDriver.createWorktree.branchHasNoUniqueCommits", + input.cwd, + [ + "merge-base", + "--is-ancestor", + `refs/heads/${newRefName}`, + input.refName === "-" ? "@{-1}" : input.refName, + ], + { allowNonZeroExit: true }, + )).exitCode !== 0; + if (keep) return yield* error; + yield* createBranchAt( + "GitVcsDriver.createWorktree.reuseBranch", + branch, + "git branch failed", + true, + ); + }), + ), ); } const checkoutRef = newRefName ?? input.refName; diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 8e08976ff6b7a..0a81383ec12c6 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -16,6 +16,7 @@ import * as SchemaTransformation from "effect/SchemaTransformation"; import { EnvironmentId, ForwardCompatibleNullable, + NonNegativeInt, ForwardCompatibleOptional, OmittedWhenNull, ProjectId, @@ -1429,6 +1430,36 @@ export const CloudSpeechCredentials = Schema.Struct({ export const MachinePlacementRole = Schema.Literals(["auto", "headless", "interactive"]); export type MachinePlacementRole = typeof MachinePlacementRole.Type; +export const AutomaticPlacementHost = Schema.Struct({ + environmentId: EnvironmentId, + maxLanes: NonNegativeInt, + minAvailableMemoryGiB: Schema.Number.check(Schema.isGreaterThanOrEqualTo(0)), +}); +export type AutomaticPlacementHost = typeof AutomaticPlacementHost.Type; + +export const DEFAULT_AUTOMATIC_PLACEMENT_HOSTS: ReadonlyArray = [ + { + environmentId: EnvironmentId.make("5cd73d78-08c9-47d4-8381-a53e008340d4"), + maxLanes: 4, + minAvailableMemoryGiB: 4, + }, + { + environmentId: EnvironmentId.make("26cc7088-9c9e-4e60-87dc-51af42d30424"), + maxLanes: 1, + minAvailableMemoryGiB: 4, + }, + { + environmentId: EnvironmentId.make("47f1a903-7121-4886-8b2f-7c8f50174b61"), + maxLanes: 1, + minAvailableMemoryGiB: 4, + }, + { + environmentId: EnvironmentId.make("2f360b2e-0386-498a-b9cd-62999f3525ba"), + maxLanes: 0, + minAvailableMemoryGiB: 4, + }, +]; + export const ServerSettings = Schema.Struct({ // Null means unmigrated; empty is authoritative and must not resurrect cleared preferences. favorites: Schema.NullOr(ModelFavorites).pipe(Schema.withDecodingDefault(Effect.succeed(null))), @@ -1584,6 +1615,15 @@ export const ServerSettings = Schema.Struct({ automaticPlacementByDefault: Schema.Boolean.pipe( Schema.withDecodingDefault(Effect.succeed(true)), ), + /** + * Closura: the fleet order and limits for automatic placement. R720 first with at most four + * lanes, ASUS one heavy lane, Jeff's Omarchy desktop light work, Kameron's laptop none: work + * runs there only when it is bound to that laptop. A host past its limit refuses, so the + * dispatch router overflows to the cloud. Unlisted hosts follow the listed ones, uncapped. + */ + automaticPlacementHosts: Schema.Array(AutomaticPlacementHost).pipe( + Schema.withDecodingDefault(Effect.succeed(DEFAULT_AUTOMATIC_PLACEMENT_HOSTS)), + ), /** * The watcher thread. It hears a run that ran another model than selected, hit a provider * safeguard fallback or a usage limit, and each lane that ended, promised to continue with @@ -2054,6 +2094,7 @@ export const ServerSettingsPatch = Schema.Struct({ automaticPlacementExcludedEnvironmentIds: Schema.optionalKey(Schema.Array(EnvironmentId)), machinePlacementRole: Schema.optionalKey(MachinePlacementRole), automaticPlacementByDefault: Schema.optionalKey(Schema.Boolean), + automaticPlacementHosts: Schema.optionalKey(Schema.Array(AutomaticPlacementHost)), modelAlertWatcher: Schema.optionalKey(Schema.NullOr(ModelAlertWatcher)), modelAlertWebhookUrl: Schema.optionalKey(TrimmedString), watcherStuckMinutes: Schema.optionalKey( diff --git a/packages/shared/src/loadBalancing.ts b/packages/shared/src/loadBalancing.ts index 1969020c9cb3b..15cda778e21b6 100644 --- a/packages/shared/src/loadBalancing.ts +++ b/packages/shared/src/loadBalancing.ts @@ -11,7 +11,30 @@ const LIMITS = { interactive: { cpu: 0.5, loadPerCpu: 0.5, memoryShare: 0.4, memoryBytes: 3 * GiB, agents: 2 }, } as const; -function headroom(resources: HostResourcesSnapshot, tier: keyof typeof LIMITS): number | null { +export interface PlacementHostCap { + readonly maxLanes: number; + readonly minAvailableMemoryBytes: number; +} + +export function withinHostCap( + resources: HostResourcesSnapshot | null, + cap: PlacementHostCap | undefined, +): boolean { + if (cap === undefined) return true; + if (resources === null) return false; + const lanes = resources.runningAgents; + return ( + lanes !== undefined && + lanes < cap.maxLanes && + resources.availableMemoryBytes >= cap.minAvailableMemoryBytes + ); +} + +function headroom( + resources: HostResourcesSnapshot, + tier: keyof typeof LIMITS, + cap: PlacementHostCap | undefined, +): number | null { const limits = LIMITS[tier]; const { cpuUtilization, cpuCount, totalMemoryBytes, availableMemoryBytes } = resources; if (cpuUtilization === null || totalMemoryBytes <= 0 || cpuCount <= 0) return null; @@ -29,6 +52,7 @@ function headroom(resources: HostResourcesSnapshot, tier: keyof typeof LIMITS): (limits.agents !== null && agents >= limits.agents) ) return null; + if (!withinHostCap(resources, cap)) return null; // Run queues past the core count show saturation that a short CPU sample can miss. const busy = Math.max(cpuUtilization, Math.min(1, loadPerCpu)); return (cpuCount * (1 - busy) * memoryAvailable) / (1 + agents); @@ -40,9 +64,9 @@ function headroom(resources: HostResourcesSnapshot, tier: keyof typeof LIMITS): * its person is not using it, and it stays well clear of its limits. A machine that does not * report its role, such as one on an older release, counts as someone's computer in use. * - * `originEnvironmentId` is where the work starts. When nothing has room it keeps the work if it - * has room by headless limits, as before placement could pick: work started on someone's - * computer is theirs to run there. + * Within a tier the lowest `rank` with room wins, then the most room. A `cap` refuses a machine + * at its lane count or memory floor, and one that cannot report its lanes. When nothing has room + * the answer is null, never the machine the work started on. */ export function chooseLoadBalancedEnvironment( candidates: ReadonlyArray<{ @@ -51,16 +75,16 @@ export function chooseLoadBalancedEnvironment( /** Client receipt time avoids comparing clocks on different machines. */ receivedAt?: number; weight: number; + rank?: number | undefined; + cap?: PlacementHostCap | undefined; }>, now: number, - originEnvironmentId?: string, ): string | null { const best = { - headless: { id: null as string | null, score: 0 }, - interactive: { id: null as string | null, score: 0 }, + headless: { id: null as string | null, rank: Infinity, score: 0 }, + interactive: { id: null as string | null, rank: Infinity, score: 0 }, }; - let origin: string | null = null; - for (const { environmentId, resources, receivedAt, weight } of candidates) { + for (const { environmentId, resources, receivedAt, weight, rank, cap } of candidates) { const sampledAt = receivedAt ?? resources?.sampledAt ?? 0; if ( !resources || @@ -71,15 +95,16 @@ export function chooseLoadBalancedEnvironment( ) { continue; } - if (environmentId === originEnvironmentId && headroom(resources, "headless") !== null) - origin = environmentId; // A machine that does not say is someone's until it says otherwise, and in use if unsure. const tier = resources.placementRole === "headless" ? "headless" : "interactive"; if (tier === "interactive" && resources.humanActive !== false) continue; - const room = headroom(resources, tier); + const room = headroom(resources, tier, cap); if (room === null) continue; const score = weight * room; - if (score > best[tier].score) best[tier] = { id: environmentId, score }; + const order = rank ?? Infinity; + const current = best[tier]; + if (order < current.rank || (order === current.rank && score > current.score)) + best[tier] = { id: environmentId, rank: order, score }; } - return best.headless.id ?? best.interactive.id ?? origin; + return best.headless.id ?? best.interactive.id; } diff --git a/packages/shared/src/threadPlacement.test.ts b/packages/shared/src/threadPlacement.test.ts index e27aa87a3c038..56f7e8d435ffd 100644 --- a/packages/shared/src/threadPlacement.test.ts +++ b/packages/shared/src/threadPlacement.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from "vite-plus/test"; -import { chooseThreadPlacement, type ThreadPlacementCandidate } from "./threadPlacement.ts"; +import { + chooseThreadPlacement, + placementHostLimits, + type ThreadPlacementCandidate, +} from "./threadPlacement.ts"; const now = 1_000_000; const GiB = 1024 ** 3; @@ -151,7 +155,7 @@ describe("chooseThreadPlacement", () => { } }); - it("keeps work on the person's own machine when nothing else has room", () => { + it("never keeps work on the person's own machine when nothing else has room", () => { const desk = host("here", 0.3, { resources: { ...host("here", 0.3).resources!, @@ -160,11 +164,9 @@ describe("chooseThreadPlacement", () => { }, }); expect(chooseThreadPlacement(origin, [desk, host("r720", 0.97)], now)).toEqual({ - accepted: true, - environmentId: "here", - reason: "capacity", + accepted: false, + reason: "no-eligible-environment", }); - // A headless machine with room still wins, and a full origin is not kept. expect(chooseThreadPlacement(origin, [desk, host("r720", 0.5)], now)).toMatchObject({ environmentId: "r720", }); @@ -174,6 +176,115 @@ describe("chooseThreadPlacement", () => { }); }); + describe("with the fleet host order and caps", () => { + const fleet = [ + { environmentId: "r720", maxLanes: 4, minAvailableMemoryGiB: 4 }, + { environmentId: "asus", maxLanes: 1, minAvailableMemoryGiB: 4 }, + { environmentId: "omarchy", maxLanes: 1, minAvailableMemoryGiB: 4 }, + { environmentId: "laptop", maxLanes: 0, minAvailableMemoryGiB: 4 }, + ]; + const machine = ( + environmentId: string, + cpuUtilization: number, + extra: Partial> = {}, + ) => + host(environmentId, cpuUtilization, { + resources: { ...host(environmentId, cpuUtilization).resources!, ...extra }, + ...placementHostLimits(fleet, environmentId), + }); + const away = { placementRole: "interactive", humanActive: false } as const; + const laptopOrigin = { originEnvironmentId: "laptop" }; + + it("sends a lane started from the laptop to R720, then ASUS, then the Omarchy desktop", () => { + const laptop = machine("laptop", 0.01, away); + expect( + chooseThreadPlacement( + laptopOrigin, + [laptop, machine("omarchy", 0.01, away), machine("asus", 0.01), machine("r720", 0.6)], + now, + ), + ).toEqual({ accepted: true, environmentId: "r720", reason: "capacity" }); + expect( + chooseThreadPlacement( + laptopOrigin, + [ + laptop, + machine("omarchy", 0.01, away), + machine("asus", 0.6), + machine("r720", 0.1, { runningAgents: 4 }), + ], + now, + ), + ).toMatchObject({ environmentId: "asus" }); + expect( + chooseThreadPlacement( + laptopOrigin, + [ + laptop, + machine("omarchy", 0.01, away), + machine("asus", 0.1, { runningAgents: 1 }), + machine("r720", 0.1, { runningAgents: 4 }), + ], + now, + ), + ).toMatchObject({ environmentId: "omarchy" }); + }); + + it("refuses with no-eligible-environment when every host is at its cap, never the laptop", () => { + expect( + chooseThreadPlacement( + laptopOrigin, + [ + machine("laptop", 0.01, away), + machine("omarchy", 0.01, { ...away, runningAgents: 1 }), + machine("asus", 0.1, { runningAgents: 1 }), + machine("r720", 0.1, { runningAgents: 4 }), + ], + now, + ), + ).toEqual({ accepted: false, reason: "no-eligible-environment" }); + }); + + it("caps R720 at four lanes and a memory floor, and refuses a host that cannot count its lanes", () => { + for (const [extra, expected] of [ + [{ runningAgents: 3 }, "r720"], + [{ runningAgents: 4 }, null], + [{ runningAgents: 18 }, null], + [{ runningAgents: 0, availableMemoryBytes: 3.9 * GiB }, null], + [{ runningAgents: undefined }, null], + ] as const) { + const decision = chooseThreadPlacement(origin, [machine("r720", 0.1, extra)], now); + expect(decision.accepted ? decision.environmentId : null).toBe(expected); + } + }); + + it("tries unlisted hosts after listed ones, without a cap", () => { + expect( + chooseThreadPlacement(origin, [machine("spare", 0.01), machine("asus", 0.5)], now), + ).toMatchObject({ environmentId: "asus" }); + expect( + chooseThreadPlacement( + origin, + [ + machine("spare", 0.5, { runningAgents: 30 }), + machine("asus", 0.1, { runningAgents: 1 }), + ], + now, + ), + ).toMatchObject({ environmentId: "spare" }); + }); + + it("still runs laptop-bound work on the laptop", () => { + expect( + chooseThreadPlacement( + { ...laptopOrigin, requiresOrigin: true }, + [machine("laptop", 0.9), machine("r720", 0.1)], + now, + ), + ).toEqual({ accepted: true, environmentId: "laptop", reason: "local-state" }); + }); + }); + it("refuses evidence that names one environment twice", () => { expect(chooseThreadPlacement(origin, [host("a", 0.1), host("a", 0.2)], now)).toEqual({ accepted: false, diff --git a/packages/shared/src/threadPlacement.ts b/packages/shared/src/threadPlacement.ts index f52a64f7dc43d..8b103a9826cad 100644 --- a/packages/shared/src/threadPlacement.ts +++ b/packages/shared/src/threadPlacement.ts @@ -1,5 +1,7 @@ import type { HostResourcesSnapshot } from "@t3tools/contracts"; -import { chooseLoadBalancedEnvironment } from "./loadBalancing.ts"; +import { chooseLoadBalancedEnvironment, type PlacementHostCap } from "./loadBalancing.ts"; + +const GiB = 1024 ** 3; /** Evidence the deciding server gathered itself; caller-supplied labels are not evidence. */ export interface ThreadPlacementCandidate { @@ -14,6 +16,8 @@ export interface ThreadPlacementCandidate { readonly projectAvailable: boolean; /** The requested provider, model and modes run there. */ readonly modelAvailable: boolean; + readonly rank?: number | undefined; + readonly cap?: PlacementHostCap | undefined; } export interface ThreadPlacementRequest { @@ -62,8 +66,25 @@ export function chooseThreadPlacement( .sort((a, b) => a.environmentId < b.environmentId ? -1 : a.environmentId > b.environmentId ? 1 : 0, ); - const environmentId = chooseLoadBalancedEnvironment(eligible, now, request.originEnvironmentId); + const environmentId = chooseLoadBalancedEnvironment(eligible, now); return environmentId === null ? { accepted: false, reason: "no-eligible-environment" } : { accepted: true, environmentId, reason: "capacity" }; } + +export function placementHostLimits( + hosts: ReadonlyArray<{ + readonly environmentId: string; + readonly maxLanes: number; + readonly minAvailableMemoryGiB: number; + }>, + environmentId: string, +): { readonly rank?: number; readonly cap?: PlacementHostCap } { + const rank = hosts.findIndex((host) => host.environmentId === environmentId); + const host = hosts[rank]; + if (host === undefined) return {}; + return { + rank, + cap: { maxLanes: host.maxLanes, minAvailableMemoryBytes: host.minAvailableMemoryGiB * GiB }, + }; +} From 2861a327b03c469b0c8ce999c98e9014e30f8bc3 Mon Sep 17 00:00:00 2001 From: nullStack65 <112618179+nullStack65@users.noreply.github.com> Date: Fri, 20 Nov 2026 10:11:19 +0000 Subject: [PATCH 2/4] fix(placement): review round fixes for host caps and branch reuse Branch reuse now needs the caller to ask for it, and only thread launch does, so the worktree handoff tool and worktree restore keep refusing an existing branch. A failed worktree add after a reuse puts the branch back on its old tip instead of deleting it. Placement counts its own accepted choices from the last minute against a host's lanes, so launches fanned out against one resource sample cannot all land on the same host. The refusal no longer names a cloud overflow route this server does not have, and a host entry without a memory floor decodes with 4 GiB instead of discarding the settings. Co-Authored-By: Claude Fable 5.2 --- .../orchestration-v2/ThreadLaunchService.ts | 1 + apps/server/src/peer/RemoteDelegation.test.ts | 90 ++++++++++--------- apps/server/src/peer/RemoteDelegation.ts | 72 +++++++++------ apps/server/src/vcs/GitVcsDriver.ts | 1 + apps/server/src/vcs/GitVcsDriverCore.test.ts | 67 ++++++++++---- apps/server/src/vcs/GitVcsDriverCore.ts | 20 ++++- packages/contracts/src/settings.ts | 4 +- 7 files changed, 168 insertions(+), 87 deletions(-) diff --git a/apps/server/src/orchestration-v2/ThreadLaunchService.ts b/apps/server/src/orchestration-v2/ThreadLaunchService.ts index c0045f7aa8058..58a31b4493a30 100644 --- a/apps/server/src/orchestration-v2/ThreadLaunchService.ts +++ b/apps/server/src/orchestration-v2/ThreadLaunchService.ts @@ -725,6 +725,7 @@ const make = Effect.gen(function* () { path: null, }, { + reuseEmptyBranch: true, progress: { onWorktreeClaimed: (path) => Effect.sync(() => { diff --git a/apps/server/src/peer/RemoteDelegation.test.ts b/apps/server/src/peer/RemoteDelegation.test.ts index a8561b75eef97..e5ed7745f157c 100644 --- a/apps/server/src/peer/RemoteDelegation.test.ts +++ b/apps/server/src/peer/RemoteDelegation.test.ts @@ -1710,47 +1710,57 @@ it.effect("placement nobody asked for runs here when no machine has room", () => ).pipe(Effect.provide(NodeHttpServer.layerTest)), ); -it.effect("a host at its lane cap refuses, and a laptop capped at no lanes keeps nothing", () => - Effect.scoped( - Effect.gen(function* () { - const thread = yield* makeBoxThread; - yield* Ref.set(thread.resources, { ...loadOf(0.1), runningAgents: 4 }); - const b = yield* serveBox(thread); - const offers = yield* SubscriptionRef.make< - ReadonlyArray - >([]); - const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); - const localLaunches = yield* Ref.make(0); - const a = yield* makeLaptop(database, offers, undefined, Effect.void, { - localLaunch: Ref.update(localLaunches, (count) => count + 1), - }); - yield* linkTo(a.links, b, "full-access"); - yield* a.settings.updateSettings({ - automaticPlacementHosts: [ - { environmentId: box.environmentId, maxLanes: 4, minAvailableMemoryGiB: 4 }, - { environmentId: laptop.environmentId, maxLanes: 0, minAvailableMemoryGiB: 4 }, - ], - }); - const fresh = { type: "worktree", baseRef: "main", startFromOrigin: true }; - const capped = yield* launchUnplaced(a, { - clientRequestId: "capped", - workspaceStrategy: fresh, - }); - expect(capped.isError).toBe(true); - expect(JSON.stringify(capped.content)).toContain("no-eligible-environment"); - expect(yield* Ref.get(localLaunches)).toBe(0); - expect(yield* Ref.get(thread.launched)).toBeNull(); +it.effect( + "a host at its lane cap refuses, counts its own fresh placements, and a laptop capped at no lanes keeps nothing", + () => + Effect.scoped( + Effect.gen(function* () { + const thread = yield* makeBoxThread; + yield* Ref.set(thread.resources, { ...loadOf(0.1), runningAgents: 4 }); + const b = yield* serveBox(thread); + const offers = yield* SubscriptionRef.make< + ReadonlyArray + >([]); + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); + const localLaunches = yield* Ref.make(0); + const a = yield* makeLaptop(database, offers, undefined, Effect.void, { + localLaunch: Ref.update(localLaunches, (count) => count + 1), + }); + yield* linkTo(a.links, b, "full-access"); + yield* a.settings.updateSettings({ + automaticPlacementHosts: [ + { environmentId: box.environmentId, maxLanes: 4, minAvailableMemoryGiB: 4 }, + { environmentId: laptop.environmentId, maxLanes: 0, minAvailableMemoryGiB: 4 }, + ], + }); + const fresh = { type: "worktree", baseRef: "main", startFromOrigin: true }; + const capped = yield* launchUnplaced(a, { + clientRequestId: "capped", + workspaceStrategy: fresh, + }); + expect(capped.isError).toBe(true); + expect(JSON.stringify(capped.content)).toContain("no-eligible-environment"); + expect(yield* Ref.get(localLaunches)).toBe(0); + expect(yield* Ref.get(thread.launched)).toBeNull(); - yield* Ref.set(thread.resources, { ...loadOf(0.1), runningAgents: 3 }); - const placed = yield* launchUnplaced(a, { - clientRequestId: "room", - workspaceStrategy: fresh, - }); - expect(placed.isError, JSON.stringify(placed.content)).toBe(false); - expect(placed.structuredContent).toMatchObject({ environmentId: box.environmentId }); - expect(yield* Ref.get(localLaunches)).toBe(0); - }), - ).pipe(Effect.provide(NodeHttpServer.layerTest)), + yield* Ref.set(thread.resources, { ...loadOf(0.1), runningAgents: 3 }); + const placed = yield* launchUnplaced(a, { + clientRequestId: "room", + workspaceStrategy: fresh, + }); + expect(placed.isError, JSON.stringify(placed.content)).toBe(false); + expect(placed.structuredContent).toMatchObject({ environmentId: box.environmentId }); + expect(yield* Ref.get(localLaunches)).toBe(0); + + const behind = yield* launchUnplaced(a, { + clientRequestId: "same-sample", + workspaceStrategy: fresh, + }); + expect(behind.isError).toBe(true); + expect(JSON.stringify(behind.content)).toContain("no-eligible-environment"); + expect(yield* Ref.get(localLaunches)).toBe(0); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), ); it.effect( diff --git a/apps/server/src/peer/RemoteDelegation.ts b/apps/server/src/peer/RemoteDelegation.ts index 1c8b8f5c5777c..6d14267643965 100644 --- a/apps/server/src/peer/RemoteDelegation.ts +++ b/apps/server/src/peer/RemoteDelegation.ts @@ -503,6 +503,22 @@ const make = Effect.gen(function* () { ); }); + const recentPlacements = new Map>(); + const recentAt = (environmentId: string, now: number) => + (recentPlacements.get(environmentId) ?? []).filter((at) => now - at < 60_000); + const withRecentPlacements = < + C extends { readonly environmentId: string; readonly resources: HostResourcesSnapshot | null }, + >( + candidate: C, + now: number, + ): C => { + const recent = recentAt(candidate.environmentId, now).length; + const lanes = candidate.resources?.runningAgents; + return recent === 0 || candidate.resources === null || lanes === undefined + ? candidate + : { ...candidate, resources: { ...candidate.resources, runningAgents: lanes + recent } }; + }; + /** The shared choice: this environment and the askable links, from their own answers. */ const choosePlacement = Effect.fn("RemoteDelegation.choosePlacement")(function* ( ask: Ask, @@ -522,30 +538,36 @@ const make = Effect.gen(function* () { const hereCandidate = !input.excluded.has(hereId) && (input.restrictTo === undefined || input.restrictTo.includes(hereId)); - const decision = chooseThreadPlacement( - { originEnvironmentId: hereId }, - [ - ...(hereCandidate - ? [ - { - environmentId: hereId, - resources: input.here.resources, - receivedAt: now, - weight: 1, - operable: true, - projectAvailable: true, - modelAvailable: input.here.modelAvailable, - ...placementHostLimits(hosts, hereId), - }, - ] - : []), - ...peers.map((peer) => ({ - ...peer.candidate, - ...placementHostLimits(hosts, peer.candidate.environmentId), - })), - ], - now, - ); + const candidates = [ + ...(hereCandidate + ? [ + { + environmentId: hereId, + resources: input.here.resources, + receivedAt: now, + weight: 1, + operable: true, + projectAvailable: true, + modelAvailable: input.here.modelAvailable, + ...placementHostLimits(hosts, hereId), + }, + ] + : []), + ...peers.map((peer) => ({ + ...peer.candidate, + ...placementHostLimits(hosts, peer.candidate.environmentId), + })), + ]; + const decision = yield* Effect.sync(() => { + const chosen = chooseThreadPlacement( + { originEnvironmentId: hereId }, + candidates.map((candidate) => withRecentPlacements(candidate, now)), + now, + ); + if (chosen.accepted) + recentPlacements.set(chosen.environmentId, [...recentAt(chosen.environmentId, now), now]); + return chosen; + }); yield* Effect.annotateCurrentSpan({ "placement.reason": decision.reason, "placement.candidates": askable.length + (hereCandidate ? 1 : 0), @@ -610,7 +632,7 @@ const make = Effect.gen(function* () { if (!decision.accepted) { return yield* new OrchestratorMcpFailure({ code: "target_required", - message: `No environment can take this task now (${decision.reason}): every machine automatic placement may use is at its lane or memory limit, or lacks this repository's project or ${input.modelSelection.instanceId}/${input.modelSelection.model}. Nothing was started. Retry later or send it to cloud overflow; name an environment only for work bound to that machine.`, + message: `No environment can take this task now (${decision.reason}): every machine automatic placement may use is at its lane or memory limit, or lacks this repository's project or ${input.modelSelection.instanceId}/${input.modelSelection.model}. Nothing was started. Retry later; name an environment only for work bound to that machine.`, }); } return yield* threads diff --git a/apps/server/src/vcs/GitVcsDriver.ts b/apps/server/src/vcs/GitVcsDriver.ts index d0da1b3d09b3d..89a39e9a1be81 100644 --- a/apps/server/src/vcs/GitVcsDriver.ts +++ b/apps/server/src/vcs/GitVcsDriver.ts @@ -154,6 +154,7 @@ export interface CreateWorktreeOptions { readonly worktreesDirectory?: string; /** Clone the checkout from the macOS worktree checkout cache. */ readonly fastCheckout?: boolean; + readonly reuseEmptyBranch?: boolean; } export interface GitCommitProgress { diff --git a/apps/server/src/vcs/GitVcsDriverCore.test.ts b/apps/server/src/vcs/GitVcsDriverCore.test.ts index e593cbcd7589b..c00bdd648a368 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.test.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.test.ts @@ -6533,7 +6533,7 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { ); it.effect( - "reuses an existing branch with no commits of its own, and refuses one with work", + "reuses an existing branch with no commits of its own only when asked, and refuses one with work", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); @@ -6546,12 +6546,35 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const tip = (yield* git(cwd, ["rev-parse", "HEAD"])).trim(); const path = yield* Path.Path; const driver = yield* GitVcsDriver.GitVcsDriver; - const reused = yield* driver.createWorktree({ - cwd, - path: path.join(yield* makeTmpDir("git-worktrees-"), "stale"), - refName: initialBranch, - newRefName: "stale", - }); + const reuse = { reuseEmptyBranch: true }; + const unasked = yield* driver + .createWorktree({ + cwd, + path: path.join(yield* makeTmpDir("git-worktrees-"), "unasked"), + refName: initialBranch, + newRefName: "stale", + }) + .pipe(Effect.flip); + assert.equal(unasked.reason, "branch_already_exists"); + const stale = (yield* git(cwd, ["rev-parse", "refs/heads/stale"])).trim(); + const occupied = yield* makeTmpDir("git-worktrees-"); + yield* writeTextFile(occupied, "taken.txt", "taken\n"); + yield* driver + .createWorktree( + { cwd, path: occupied, refName: initialBranch, newRefName: "stale" }, + reuse, + ) + .pipe(Effect.flip); + assert.equal((yield* git(cwd, ["rev-parse", "refs/heads/stale"])).trim(), stale); + const reused = yield* driver.createWorktree( + { + cwd, + path: path.join(yield* makeTmpDir("git-worktrees-"), "stale"), + refName: initialBranch, + newRefName: "stale", + }, + reuse, + ); assert.equal((yield* git(reused.worktree.path, ["rev-parse", "HEAD"])).trim(), tip); assert.equal( (yield* git(reused.worktree.path, ["branch", "--show-current"])).trim(), @@ -6559,12 +6582,15 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { ); const again = yield* driver - .createWorktree({ - cwd, - path: path.join(yield* makeTmpDir("git-worktrees-"), "stale-again"), - refName: initialBranch, - newRefName: "stale", - }) + .createWorktree( + { + cwd, + path: path.join(yield* makeTmpDir("git-worktrees-"), "stale-again"), + refName: initialBranch, + newRefName: "stale", + }, + reuse, + ) .pipe(Effect.flip); assert.equal(again.reason, "branch_already_exists"); @@ -6576,12 +6602,15 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const worked = (yield* git(cwd, ["rev-parse", "refs/heads/worked"])).trim(); yield* git(cwd, ["worktree", "remove", "--", worktreePath]); const refused = yield* driver - .createWorktree({ - cwd, - path: path.join(yield* makeTmpDir("git-worktrees-"), "worked-again"), - refName: initialBranch, - newRefName: "worked", - }) + .createWorktree( + { + cwd, + path: path.join(yield* makeTmpDir("git-worktrees-"), "worked-again"), + refName: initialBranch, + newRefName: "worked", + }, + reuse, + ) .pipe(Effect.flip); assert.equal(refused.reason, "branch_already_exists"); assert.equal((yield* git(cwd, ["rev-parse", "refs/heads/worked"])).trim(), worked); diff --git a/apps/server/src/vcs/GitVcsDriverCore.ts b/apps/server/src/vcs/GitVcsDriverCore.ts index 497c36be2354b..12acef995c6af 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.ts @@ -5157,6 +5157,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ? `t3-create-${yield* random.uuidV4()}` : undefined; + let reusedTip: string | null = null; if (newRefName) { const branch = { cwd: input.cwd, branch: newRefName, startPoint: input.refName }; // Kept apart from the checkout so the shared-config lock only covers the tracking write. @@ -5166,7 +5167,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* "git branch failed", ).pipe( Effect.catchIf( - (error) => error.reason === "branch_already_exists", + (error) => options?.reuseEmptyBranch === true && error.reason === "branch_already_exists", (error) => Effect.gen(function* () { const checkedOut = (yield* executeWorktreeList( @@ -5190,12 +5191,18 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* { allowNonZeroExit: true }, )).exitCode !== 0; if (keep) return yield* error; + const tip = (yield* executeGit( + "GitVcsDriver.createWorktree.reusedBranchTip", + input.cwd, + ["rev-parse", "--verify", `refs/heads/${newRefName}`], + )).stdout.trim(); yield* createBranchAt( "GitVcsDriver.createWorktree.reuseBranch", branch, "git branch failed", true, ); + reusedTip = tip; }), ), ); @@ -5295,7 +5302,16 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* : Effect.void ).pipe( Effect.andThen( - deleteLocalBranch({ cwd: input.cwd, refName: newRefName, force: true }), + reusedTip === null + ? deleteLocalBranch({ cwd: input.cwd, refName: newRefName, force: true }) + : executeGit("GitVcsDriver.createWorktree.restoreReusedBranch", input.cwd, [ + "branch", + "--force", + "--no-track", + "--", + newRefName, + reusedTip, + ]), ), Effect.catch((cause) => Effect.logWarning("could not clean up a failed worktree or its branch", { cause }), diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 0a81383ec12c6..3a1e77f3a8c3f 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -1433,7 +1433,9 @@ export type MachinePlacementRole = typeof MachinePlacementRole.Type; export const AutomaticPlacementHost = Schema.Struct({ environmentId: EnvironmentId, maxLanes: NonNegativeInt, - minAvailableMemoryGiB: Schema.Number.check(Schema.isGreaterThanOrEqualTo(0)), + minAvailableMemoryGiB: Schema.Number.check(Schema.isGreaterThanOrEqualTo(0)).pipe( + Schema.withDecodingDefault(Effect.succeed(4)), + ), }); export type AutomaticPlacementHost = typeof AutomaticPlacementHost.Type; From aa474b05ec11f2db0db8d54149a53bb0af456ab4 Mon Sep 17 00:00:00 2001 From: nullStack65 <112618179+nullStack65@users.noreply.github.com> Date: Fri, 20 Nov 2026 10:27:50 +0000 Subject: [PATCH 3/4] fix(placement): cap lanes by this server's active runs, not idle agent scopes Idle provider sessions keep an agent scope populated for 30 minutes to 4 hours, so the scope count held host lane caps closed with little running. Snapshots now carry activeRuns: threads with a preparing, starting, running, waiting or unheld queued run. Caps and the recent-placement reservation use it, falling back to the scope count for peers that do not report it yet. Co-Authored-By: Claude Fable 5.2 --- .../ProjectionRecovery.test.ts | 27 ++++++ .../src/orchestration-v2/ProjectionStore.ts | 14 +++ .../ProviderTurnControlService.test.ts | 1 + apps/server/src/peer/RemoteDelegation.test.ts | 15 ++++ apps/server/src/peer/RemoteDelegation.ts | 15 +++- .../resourceTelemetry/HostResources.test.ts | 90 ++++++++++--------- .../src/resourceTelemetry/HostResources.ts | 8 ++ apps/server/src/server.ts | 12 ++- packages/contracts/src/resourceTelemetry.ts | 1 + packages/shared/src/loadBalancing.ts | 2 +- packages/shared/src/threadPlacement.test.ts | 5 +- 11 files changed, 144 insertions(+), 46 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts b/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts index d8b7dae693632..1151c4e953d6e 100644 --- a/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts @@ -604,6 +604,33 @@ it.effect("maintains delegated candidate indexes across legacy rows and payload }).pipe(Effect.provide(layerTest)), ); +it.effect("counts threads with live or unheld queued runs as active lanes", () => + Effect.gen(function* () { + const projections = yield* ProjectionStore.ProjectionStoreV2; + const now = yield* DateTime.now; + for (const name of ["running", "waiting", "queued", "held", "completed", "deleted", "both"]) { + const thread = yield* createThread(`lanes-${name}`, { + deletedAt: name === "deleted" ? now : null, + }); + if (name === "queued" || name === "held") { + yield* createRun(thread, "queued", { + startedAt: null, + ...(name === "held" ? { queueHeld: true } : {}), + }); + } else if (name === "both") { + yield* createRun(thread, "running"); + yield* createRun(thread, "queued", { ordinal: 2, startedAt: null }); + } else { + yield* createRun( + thread, + name === "waiting" ? "waiting" : name === "completed" ? "completed" : "running", + ); + } + } + assert.strictEqual(yield* projections.countActiveRunThreads(), 4); + }).pipe(Effect.provide(layerTest)), +); + it.effect("recovers terminal subagent results until their cross-thread transfer exists", () => Effect.gen(function* () { const projections = yield* ProjectionStore.ProjectionStoreV2; diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 4d3229a7e9889..8402be3e50058 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -610,6 +610,7 @@ export interface ProjectionStoreV2Shape { >; /** Threads that are not deleted, archived ones included. */ readonly countThreads: () => Effect.Effect; + readonly countActiveRunThreads: () => Effect.Effect; /** Every active thread, in shell order. */ readonly listActiveShellIndex: () => Effect.Effect< ReadonlyArray, @@ -9136,6 +9137,18 @@ export const layer: Layer.Layer = return rows[0]?.count ?? 0; }, Effect.mapError(shellSnapshotReadError)); + const countActiveRunThreads = Effect.fn("ProjectionStore.countActiveRunThreads")(function* () { + const rows = yield* sql<{ readonly count: number }>` + SELECT COUNT(DISTINCT run.thread_id) AS count + FROM orchestration_v2_projection_runs AS run + JOIN orchestration_v2_projection_threads AS thread ON thread.thread_id = run.thread_id + WHERE thread.deleted_at IS NULL + AND (run.status IN ('preparing', 'starting', 'running', 'waiting') + OR (run.status = 'queued' AND json_extract(run.payload_json, '$.queueHeld') IS NOT 1)) + `; + return rows[0]?.count ?? 0; + }, Effect.mapError(shellSnapshotReadError)); + const listActiveShellIndex = Effect.fn("ProjectionStore.listActiveShellIndex")(function* () { const rows = yield* sql<{ readonly thread_id: string; @@ -9518,6 +9531,7 @@ export const layer: Layer.Layer = getPullRequestSnapshot: readOnReader(getPullRequestSnapshot), listActiveSubagentThreadIds: readOnReader(listActiveSubagentThreadIds), countThreads: readOnReader(countThreads), + countActiveRunThreads: readOnReader(countActiveRunThreads), listActiveShellIndex: readOnReader(listActiveShellIndex), listActiveThreads: readOnReader(listActiveThreads), readShellSnapshot: readOnReader(readShellSnapshot), diff --git a/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts b/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts index 02aa457ca849b..88b72fdc54269 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts @@ -217,6 +217,7 @@ it.effect( getShellSnapshot: () => Effect.die("unused getShellSnapshot"), listActiveSubagentThreadIds: () => Effect.die("unused listActiveSubagentThreadIds"), countThreads: () => Effect.die("unused countThreads"), + countActiveRunThreads: () => Effect.die("unused countActiveRunThreads"), listActiveShellIndex: () => Effect.die("unused listActiveShellIndex"), listActiveThreads: () => Effect.die("unused listActiveThreads"), listOlderSettledThreads: () => Effect.die("unused listOlderSettledThreads"), diff --git a/apps/server/src/peer/RemoteDelegation.test.ts b/apps/server/src/peer/RemoteDelegation.test.ts index e5ed7745f157c..b132610a9adb3 100644 --- a/apps/server/src/peer/RemoteDelegation.test.ts +++ b/apps/server/src/peer/RemoteDelegation.test.ts @@ -1759,6 +1759,21 @@ it.effect( expect(behind.isError).toBe(true); expect(JSON.stringify(behind.content)).toContain("no-eligible-environment"); expect(yield* Ref.get(localLaunches)).toBe(0); + + yield* Ref.set(thread.resources, { ...loadOf(0.1), runningAgents: 18, activeRuns: 2 }); + const idleScopes = yield* launchUnplaced(a, { + clientRequestId: "idle-scopes", + workspaceStrategy: fresh, + }); + expect(idleScopes.isError, JSON.stringify(idleScopes.content)).toBe(false); + expect(idleScopes.structuredContent).toMatchObject({ environmentId: box.environmentId }); + + const full = yield* launchUnplaced(a, { + clientRequestId: "active-full", + workspaceStrategy: fresh, + }); + expect(full.isError).toBe(true); + expect(JSON.stringify(full.content)).toContain("no-eligible-environment"); }), ).pipe(Effect.provide(NodeHttpServer.layerTest)), ); diff --git a/apps/server/src/peer/RemoteDelegation.ts b/apps/server/src/peer/RemoteDelegation.ts index 6d14267643965..4b37200275529 100644 --- a/apps/server/src/peer/RemoteDelegation.ts +++ b/apps/server/src/peer/RemoteDelegation.ts @@ -513,10 +513,19 @@ const make = Effect.gen(function* () { now: number, ): C => { const recent = recentAt(candidate.environmentId, now).length; - const lanes = candidate.resources?.runningAgents; - return recent === 0 || candidate.resources === null || lanes === undefined + const resources = candidate.resources; + if (recent === 0 || resources === null) return candidate; + if (resources.activeRuns !== undefined) + return { + ...candidate, + resources: { ...resources, activeRuns: resources.activeRuns + recent }, + }; + return resources.runningAgents === undefined ? candidate - : { ...candidate, resources: { ...candidate.resources, runningAgents: lanes + recent } }; + : { + ...candidate, + resources: { ...resources, runningAgents: resources.runningAgents + recent }, + }; }; /** The shared choice: this environment and the askable links, from their own answers. */ diff --git a/apps/server/src/resourceTelemetry/HostResources.test.ts b/apps/server/src/resourceTelemetry/HostResources.test.ts index fdf0d064228cf..8823dd0017c8e 100644 --- a/apps/server/src/resourceTelemetry/HostResources.test.ts +++ b/apps/server/src/resourceTelemetry/HostResources.test.ts @@ -55,6 +55,7 @@ const unlimitedCgroup = { const linuxHost = ( files: Record>, directories: Record> = {}, + activeRuns?: number, ) => { const reads = new Map(); return HostResources.layer.pipe( @@ -92,6 +93,10 @@ const linuxHost = ( }, }), Layer.succeed(HostProcess.Platform, "linux"), + Layer.succeed( + HostResources.ActiveRuns, + HostResources.ActiveRuns.of({ count: Effect.succeed(activeRuns) }), + ), Layer.succeed( ChildProcessSpawner.ChildProcessSpawner, ChildProcessSpawner.make(() => Effect.die("HostResources spawned on linux")), @@ -109,48 +114,53 @@ const read = Effect.gen(function* () { }); describe("HostResources.layer", () => { - it.effect("reports load per host CPU and the agents running under this user", () => { - const manager = "/user.slice/user-1000.slice/user@1000.service"; - const server = `${manager}/app.slice/t3.service`; - const agents = `/sys/fs/cgroup${manager}/app.slice/app-t3code.slice/app-t3code-agents.slice`; - return Effect.gen(function* () { - setup(); - host.load = 20; - const snapshot = yield* read; - host.load = 0; + it.effect( + "reports load per host CPU, the agents running under this user and this server's active runs", + () => { + const manager = "/user.slice/user-1000.slice/user@1000.service"; + const server = `${manager}/app.slice/t3.service`; + const agents = `/sys/fs/cgroup${manager}/app.slice/app-t3code.slice/app-t3code-agents.slice`; + return Effect.gen(function* () { + setup(); + host.load = 20; + const snapshot = yield* read; + host.load = 0; - // 20 runnable on 8 CPUs; two populated scopes from two servers, one finished. - assert.strictEqual(snapshot.loadPerCpu, 2.5); - assert.strictEqual(snapshot.runningAgents, 2); - }).pipe( - Effect.provide( - linuxHost( - { - "/proc/self/cgroup": [`0::${server}\n`], - "/proc/meminfo": unlimitedCgroup["/proc/meminfo"], - [`${agents}/app-t3code-agents-a.slice/t3code-claude-1.scope/cgroup.events`]: [ - "populated 1\nfrozen 0\n", - ], - [`${agents}/app-t3code-agents-a.slice/t3code-codex-2.scope/cgroup.events`]: [ - "populated 0\nfrozen 0\n", - ], - [`${agents}/app-t3code-agents-b.slice/t3code-claude-3.scope/cgroup.events`]: [ - "populated 1\nfrozen 0\n", - ], - }, - { - [agents]: ["app-t3code-agents-a.slice", "app-t3code-agents-b.slice", "cgroup.procs"], - [`${agents}/app-t3code-agents-a.slice`]: [ - "t3code-claude-1.scope", - "t3code-codex-2.scope", - "memory.max", - ], - [`${agents}/app-t3code-agents-b.slice`]: ["t3code-claude-3.scope"], - }, + // 20 runnable on 8 CPUs; two populated scopes from two servers, one finished. + assert.strictEqual(snapshot.loadPerCpu, 2.5); + assert.strictEqual(snapshot.runningAgents, 2); + assert.strictEqual(snapshot.activeRuns, 1); + }).pipe( + Effect.provide( + linuxHost( + { + "/proc/self/cgroup": [`0::${server}\n`], + "/proc/meminfo": unlimitedCgroup["/proc/meminfo"], + [`${agents}/app-t3code-agents-a.slice/t3code-claude-1.scope/cgroup.events`]: [ + "populated 1\nfrozen 0\n", + ], + [`${agents}/app-t3code-agents-a.slice/t3code-codex-2.scope/cgroup.events`]: [ + "populated 0\nfrozen 0\n", + ], + [`${agents}/app-t3code-agents-b.slice/t3code-claude-3.scope/cgroup.events`]: [ + "populated 1\nfrozen 0\n", + ], + }, + { + [agents]: ["app-t3code-agents-a.slice", "app-t3code-agents-b.slice", "cgroup.procs"], + [`${agents}/app-t3code-agents-a.slice`]: [ + "t3code-claude-1.scope", + "t3code-codex-2.scope", + "memory.max", + ], + [`${agents}/app-t3code-agents-b.slice`]: ["t3code-claude-3.scope"], + }, + 1, + ), ), - ), - ); - }); + ); + }, + ); it.effect("reports the CPU quota and usage of the server's cgroup", () => Effect.gen(function* () { diff --git a/apps/server/src/resourceTelemetry/HostResources.ts b/apps/server/src/resourceTelemetry/HostResources.ts index 7fc5b27dd2ad3..465e2a2ce9fdc 100644 --- a/apps/server/src/resourceTelemetry/HostResources.ts +++ b/apps/server/src/resourceTelemetry/HostResources.ts @@ -26,6 +26,11 @@ export class MachinePresence extends Context.Service< } >()("t3/resourceTelemetry/HostResources/MachinePresence") {} +export class ActiveRuns extends Context.Service< + ActiveRuns, + { readonly count: Effect.Effect } +>()("t3/resourceTelemetry/HostResources/ActiveRuns") {} + // The user manager's slice holding every T3 server's agent scopes (agentScope.ts). const AGENTS_SLICE_PATH = "app.slice/app-t3code.slice/app-t3code-agents.slice"; @@ -91,6 +96,7 @@ const make = Effect.fn("makeHostResources")(function* () { const readFile = (path: string) => fs.readFileString(path).pipe(Effect.orElseSucceed(() => "")); const presence = yield* Effect.serviceOption(MachinePresence); + const activeRuns = yield* Effect.serviceOption(ActiveRuns); // Populated agent scopes under this user's manager, from every T3 server on the host. const countAgents = Effect.fn("HostResources.countAgents")(function* ( @@ -220,6 +226,7 @@ const make = Effect.fn("makeHostResources")(function* () { : loadAverage / cpu.count; const runningAgents = platform === "linux" ? yield* countAgents(cgroupPath) : undefined; const role = Option.isSome(presence) ? yield* presence.value.read : {}; + const runs = Option.isSome(activeRuns) ? yield* activeRuns.value.count : undefined; return { sampledAt: DateTime.toEpochMillis(yield* DateTime.now), cpuUtilization, @@ -228,6 +235,7 @@ const make = Effect.fn("makeHostResources")(function* () { totalMemoryBytes, ...(loadPerCpu === undefined ? {} : { loadPerCpu }), ...(runningAgents === undefined ? {} : { runningAgents }), + ...(runs === undefined ? {} : { activeRuns: runs }), ...role, }; }); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 1a7895b0cd954..35ad3c6f35e83 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -326,8 +326,18 @@ const layerMachinePresence = MachinePresence.layer.pipe( Layer.provide(layerServerSettings), ); +const layerActiveRuns = Layer.effect( + HostResources.ActiveRuns, + Effect.gen(function* () { + const store = yield* ProjectionStoreV2.ProjectionStoreV2; + return HostResources.ActiveRuns.of({ + count: store.countActiveRunThreads().pipe(Effect.orElseSucceed(() => undefined)), + }); + }), +).pipe(Layer.provide(ProjectionStoreV2.layer), Layer.provide(SqlitePersistence.layerConfig)); + const layerResourceDiagnostics = Layer.mergeAll( - HostResources.layer.pipe(Layer.provide(layerMachinePresence)), + HostResources.layer.pipe(Layer.provide(Layer.merge(layerMachinePresence, layerActiveRuns))), layerResourceTelemetry, ProcessDiagnostics.layer.pipe(Layer.provide(layerResourceTelemetry)), ProcessResourceMonitor.layer.pipe(Layer.provide(layerResourceTelemetry)), diff --git a/packages/contracts/src/resourceTelemetry.ts b/packages/contracts/src/resourceTelemetry.ts index 493dac10ce028..5a7047d868ec7 100644 --- a/packages/contracts/src/resourceTelemetry.ts +++ b/packages/contracts/src/resourceTelemetry.ts @@ -17,6 +17,7 @@ export const HostResourcesSnapshot = Schema.Struct({ loadPerCpu: Schema.optional(Schema.Number.check(Schema.isGreaterThanOrEqualTo(0))), /** Agent processes running on this host, across every T3 server there. */ runningAgents: Schema.optional(NonNegativeInt), + activeRuns: Schema.optional(NonNegativeInt), /** Interactive machines belong to a person; placement uses them last and only while idle. */ placementRole: Schema.optional(Schema.Literals(["headless", "interactive"])), /** Someone is using this interactive machine now, or it cannot tell. */ diff --git a/packages/shared/src/loadBalancing.ts b/packages/shared/src/loadBalancing.ts index 15cda778e21b6..9eb66e0565b12 100644 --- a/packages/shared/src/loadBalancing.ts +++ b/packages/shared/src/loadBalancing.ts @@ -22,7 +22,7 @@ export function withinHostCap( ): boolean { if (cap === undefined) return true; if (resources === null) return false; - const lanes = resources.runningAgents; + const lanes = resources.activeRuns ?? resources.runningAgents; return ( lanes !== undefined && lanes < cap.maxLanes && diff --git a/packages/shared/src/threadPlacement.test.ts b/packages/shared/src/threadPlacement.test.ts index 56f7e8d435ffd..993adc74b8efd 100644 --- a/packages/shared/src/threadPlacement.test.ts +++ b/packages/shared/src/threadPlacement.test.ts @@ -245,13 +245,16 @@ describe("chooseThreadPlacement", () => { ).toEqual({ accepted: false, reason: "no-eligible-environment" }); }); - it("caps R720 at four lanes and a memory floor, and refuses a host that cannot count its lanes", () => { + it("caps R720 at four active lanes and a memory floor, counts idle scopes only without a run count, and refuses a host that cannot count its lanes", () => { for (const [extra, expected] of [ [{ runningAgents: 3 }, "r720"], [{ runningAgents: 4 }, null], [{ runningAgents: 18 }, null], [{ runningAgents: 0, availableMemoryBytes: 3.9 * GiB }, null], [{ runningAgents: undefined }, null], + [{ runningAgents: 18, activeRuns: 3 }, "r720"], + [{ runningAgents: 0, activeRuns: 4 }, null], + [{ runningAgents: undefined, activeRuns: 0 }, "r720"], ] as const) { const decision = chooseThreadPlacement(origin, [machine("r720", 0.1, extra)], now); expect(decision.accepted ? decision.environmentId : null).toBe(expected); From b44997b7701ea702e2acc0e86aa09a6df1e13e65 Mon Sep 17 00:00:00 2001 From: nullStack65 <112618179+nullStack65@users.noreply.github.com> Date: Fri, 20 Nov 2026 10:38:49 +0000 Subject: [PATCH 4/4] fix(placement): stop counting a reservation once its host reports the run A placement reserved a lane for a flat 60 seconds on top of the host's fresh count, so sequential launches were refused at about half the cap. Each reservation now records the lane count the host reported when it was placed, and a host counts the larger of its reported lanes and any reservation's lanes plus the reservations made since. Co-Authored-By: Claude Fable 5.2 --- apps/server/src/peer/RemoteDelegation.test.ts | 50 ++++++++++++++++--- apps/server/src/peer/RemoteDelegation.ts | 49 +++++++++++------- 2 files changed, 76 insertions(+), 23 deletions(-) diff --git a/apps/server/src/peer/RemoteDelegation.test.ts b/apps/server/src/peer/RemoteDelegation.test.ts index b132610a9adb3..d4ff90e280d4d 100644 --- a/apps/server/src/peer/RemoteDelegation.test.ts +++ b/apps/server/src/peer/RemoteDelegation.test.ts @@ -1759,17 +1759,55 @@ it.effect( expect(behind.isError).toBe(true); expect(JSON.stringify(behind.content)).toContain("no-eligible-environment"); expect(yield* Ref.get(localLaunches)).toBe(0); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); - yield* Ref.set(thread.resources, { ...loadOf(0.1), runningAgents: 18, activeRuns: 2 }); - const idleScopes = yield* launchUnplaced(a, { - clientRequestId: "idle-scopes", +it.effect( + "sequential launches fill every active lane once the host reports them, and launches on one sample cannot share the last", + () => + Effect.scoped( + Effect.gen(function* () { + const thread = yield* makeBoxThread; + const b = yield* serveBox(thread); + const offers = yield* SubscriptionRef.make< + ReadonlyArray + >([]); + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); + const a = yield* makeLaptop(database, offers, undefined, Effect.void, { + localLaunch: Effect.void, + }); + yield* linkTo(a.links, b, "full-access"); + yield* a.settings.updateSettings({ + automaticPlacementHosts: [ + { environmentId: box.environmentId, maxLanes: 4, minAvailableMemoryGiB: 4 }, + { environmentId: laptop.environmentId, maxLanes: 0, minAvailableMemoryGiB: 4 }, + ], + }); + const fresh = { type: "worktree", baseRef: "main", startFromOrigin: true }; + for (const active of [0, 1, 2, 3]) { + yield* Ref.set(thread.resources, { + ...loadOf(0.1), + runningAgents: 18, + activeRuns: active, + }); + const placed = yield* launchUnplaced(a, { + clientRequestId: `lane-${active}`, + workspaceStrategy: fresh, + }); + expect(placed.isError, JSON.stringify(placed.content)).toBe(false); + expect(placed.structuredContent).toMatchObject({ environmentId: box.environmentId }); + } + const behind = yield* launchUnplaced(a, { + clientRequestId: "lane-same-sample", workspaceStrategy: fresh, }); - expect(idleScopes.isError, JSON.stringify(idleScopes.content)).toBe(false); - expect(idleScopes.structuredContent).toMatchObject({ environmentId: box.environmentId }); + expect(behind.isError).toBe(true); + expect(JSON.stringify(behind.content)).toContain("no-eligible-environment"); + yield* Ref.set(thread.resources, { ...loadOf(0.1), runningAgents: 18, activeRuns: 4 }); const full = yield* launchUnplaced(a, { - clientRequestId: "active-full", + clientRequestId: "lane-full", workspaceStrategy: fresh, }); expect(full.isError).toBe(true); diff --git a/apps/server/src/peer/RemoteDelegation.ts b/apps/server/src/peer/RemoteDelegation.ts index 4b37200275529..6441b6a8b4c2c 100644 --- a/apps/server/src/peer/RemoteDelegation.ts +++ b/apps/server/src/peer/RemoteDelegation.ts @@ -503,29 +503,35 @@ const make = Effect.gen(function* () { ); }); - const recentPlacements = new Map>(); + const recentPlacements = new Map< + string, + ReadonlyArray<{ readonly at: number; readonly lanes: number }> + >(); const recentAt = (environmentId: string, now: number) => - (recentPlacements.get(environmentId) ?? []).filter((at) => now - at < 60_000); + (recentPlacements.get(environmentId) ?? []).filter((entry) => now - entry.at < 60_000); + const reportedLanes = (resources: HostResourcesSnapshot | null) => + resources === null ? undefined : (resources.activeRuns ?? resources.runningAgents); const withRecentPlacements = < C extends { readonly environmentId: string; readonly resources: HostResourcesSnapshot | null }, >( candidate: C, now: number, ): C => { - const recent = recentAt(candidate.environmentId, now).length; const resources = candidate.resources; - if (recent === 0 || resources === null) return candidate; - if (resources.activeRuns !== undefined) - return { - ...candidate, - resources: { ...resources, activeRuns: resources.activeRuns + recent }, - }; - return resources.runningAgents === undefined - ? candidate - : { - ...candidate, - resources: { ...resources, runningAgents: resources.runningAgents + recent }, - }; + const reported = reportedLanes(resources); + if (resources === null || reported === undefined) return candidate; + const recent = recentAt(candidate.environmentId, now); + let expected = reported; + for (let index = recent.length - 1; index >= 0; index--) + expected = Math.max(expected, recent[index]!.lanes + recent.length - index); + if (expected === reported) return candidate; + return { + ...candidate, + resources: + resources.activeRuns !== undefined + ? { ...resources, activeRuns: expected } + : { ...resources, runningAgents: expected }, + }; }; /** The shared choice: this environment and the askable links, from their own answers. */ @@ -573,8 +579,17 @@ const make = Effect.gen(function* () { candidates.map((candidate) => withRecentPlacements(candidate, now)), now, ); - if (chosen.accepted) - recentPlacements.set(chosen.environmentId, [...recentAt(chosen.environmentId, now), now]); + const lanes = chosen.accepted + ? reportedLanes( + candidates.find((candidate) => candidate.environmentId === chosen.environmentId) + ?.resources ?? null, + ) + : undefined; + if (chosen.accepted && lanes !== undefined) + recentPlacements.set(chosen.environmentId, [ + ...recentAt(chosen.environmentId, now), + { at: now, lanes }, + ]); return chosen; }); yield* Effect.annotateCurrentSpan({