Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 61 additions & 1 deletion apps/server/src/mcp/OrchestratorMcpService.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4489,8 +4489,11 @@ describe("OrchestratorMcpService thread lookup", () => {
shells: ReadonlyArray<OrchestrationV2ThreadShell>,
sent: Ref.Ref<number>,
overrides: Partial<ThreadManagementService.ThreadManagementService["Service"]> = {},
launch: Partial<ThreadLaunchService.ThreadLaunchService["Service"]> = {},
) =>
layerOrchestratorService.pipe(
OrchestratorMcpService.layer.pipe(
Layer.provide(Layer.mock(ThreadLaunchService.ThreadLaunchService)(launch)),
Layer.provide(Layer.mock(GitVcsDriver.GitVcsDriver)({})),
Layer.provide(
Layer.mergeAll(
NodeServices.layer,
Expand Down Expand Up @@ -4638,6 +4641,63 @@ describe("OrchestratorMcpService thread lookup", () => {
}),
);

it.effect("retries a failed worktree setup when a send is held behind it", () =>
Effect.gen(function* () {
const target = liveThreadShell(ThreadId.make("thread:held"));
const failedRunId = RunId.make("run:setup-failed");
const retried: Array<ThreadLaunchService.ThreadLaunchPreparationInput> = [];
const sendWith = (worktreePath: string | null) =>
Effect.gen(function* () {
const sent = yield* Ref.make(0);
yield* Effect.gen(function* () {
const mcp = yield* OrchestratorMcpService.OrchestratorMcpService;
yield* mcp.sendToThread(client, { threadId: target.id, message: "hello" });
}).pipe(
Effect.provide(
service(
[target],
sent,
{
getProjectThreadRecords: () => Effect.succeed(idleThreadProjection(target)),
getThreadRecords: () =>
Effect.succeed({
thread: { ...target, worktreePath },
runs: [
{
id: failedRunId,
status: "failed",
startedAt: null,
workspacePreparation: { type: "worktree" },
},
],
} as never),
sendToThread: () =>
Effect.succeed({
run: { id: RunId.make("run:held"), status: "queued", queueHeld: true },
delivery: "queued",
} as never),
},
{
retryPreparation: (input) => {
retried.push(input);
return Effect.succeed({} as never);
},
},
),
),
);
});

yield* sendWith(null);
assert.deepStrictEqual(
retried.map((input) => [input.threadId, input.runId]),
[[target.id, failedRunId]],
);
yield* sendWith("/repo/worktree");
assert.equal(retried.length, 1);
}),
);

it.effect("refuses waiting without orchestration capability before reading the target", () =>
Effect.gen(function* () {
const sent = yield* Ref.make(0);
Expand Down
23 changes: 22 additions & 1 deletion apps/server/src/mcp/OrchestratorMcpService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ import {
} from "@t3tools/contracts";
import { getProviderOptionCurrentValue, withLegacyUltracodeEffort } from "@t3tools/shared/model";
import { backgroundWorkHoldsCompletion } from "@t3tools/shared/orchestrationV2PendingBackgroundWork";
import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError";
import { latestExecutedRun, runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError";
import { resolveProjectSettings } from "@t3tools/shared/projectSettings";
import { formatThreadLink } from "@t3tools/shared/threadLinks";
import {
Expand Down Expand Up @@ -3610,6 +3610,27 @@ const make = Effect.gen(function* () {
),
),
);
if (result.run.status === "queued" && result.run.queueHeld === true) {
yield* threadManagement.getThreadRecords(input.threadId, ["runs"]).pipe(
Effect.flatMap(({ thread, runs }) => {
const failed = latestExecutedRun(runs);
return thread.worktreePath === null &&
failed?.status === "failed" &&
failed.workspacePreparation?.type === "worktree"
? threadLaunch.retryPreparation({
commandId: CommandId.make(
`${stableCommandId({ scope, requestKey: key, operation: "thread-send" })}:retry-preparation`,
),
threadId: input.threadId,
runId: failed.id,
})
: Effect.void;
}),
Effect.catchCause((cause) =>
Effect.logWarning("thread-send could not retry the failed worktree setup", { cause }),
),
);
}
return {
threadId: input.threadId,
link: threadLinkFor(scope, input.threadId, target.thread.title),
Expand Down
20 changes: 18 additions & 2 deletions apps/server/src/mcp/toolkits/project/handlers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import {
agentLaunchInboxRouting,
withLegacyPeerInboxReason,
CommandId,
DEFAULT_AUTOMATIC_PLACEMENT_HOSTS,
type EnvironmentId,
MessageId,
type ModelSelection,
Expand Down Expand Up @@ -35,6 +36,8 @@ import * as ProviderRegistry from "../../../provider/ProviderRegistry.ts";
import * as ServerSettings from "../../../serverSettings.ts";
import * as DateTime from "effect/DateTime";
import { resolveNewThreadModelSelection } from "@t3tools/shared/usageLimits";
import { withinHostCap } from "@t3tools/shared/loadBalancing";
import { placementHostLimits } from "@t3tools/shared/threadPlacement";
import { resolveProjectSettings } from "@t3tools/shared/projectSettings";
import { resolveRuntimeMode } from "../../delegatedTaskModes.ts";
import * as McpInvocationContext from "../../McpInvocationContext.ts";
Expand Down Expand Up @@ -238,8 +241,21 @@ const launchPlacedThere = Effect.fn("mcp.launchPlacedThere")(function* (
runtimeMode: local.runtimeMode,
here,
});
// Placement nobody asked for falls back to here, as before, when no machine has room.
const environmentId = byDefault
// Placement nobody asked for falls back to here when no machine has room, within here's limit.
const hereMayKeep =
byDefault &&
withinHostCap(
here.resources,
placementHostLimits(
yield* ServerSettings.ServerSettingsService.pipe(
Effect.flatMap((settings) => settings.getSettings),
Effect.map((settings) => settings.automaticPlacementHosts),
Effect.orElseSucceed(() => DEFAULT_AUTOMATIC_PLACEMENT_HOSTS),
),
scope.environmentId,
).cap,
);
const environmentId = hereMayKeep
? yield* placing.pipe(
Effect.catchIf(
(error) => error.code === "target_required",
Expand Down
27 changes: 27 additions & 0 deletions apps/server/src/orchestration-v2/ProjectionRecovery.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -604,6 +604,33 @@ it.effect("maintains delegated candidate indexes across legacy rows and payload
}).pipe(Effect.provide(layerTest)),
);

it.effect("counts threads with live or unheld queued runs as active lanes", () =>
Effect.gen(function* () {
const projections = yield* ProjectionStore.ProjectionStoreV2;
const now = yield* DateTime.now;
for (const name of ["running", "waiting", "queued", "held", "completed", "deleted", "both"]) {
const thread = yield* createThread(`lanes-${name}`, {
deletedAt: name === "deleted" ? now : null,
});
if (name === "queued" || name === "held") {
yield* createRun(thread, "queued", {
startedAt: null,
...(name === "held" ? { queueHeld: true } : {}),
});
} else if (name === "both") {
yield* createRun(thread, "running");
yield* createRun(thread, "queued", { ordinal: 2, startedAt: null });
} else {
yield* createRun(
thread,
name === "waiting" ? "waiting" : name === "completed" ? "completed" : "running",
);
}
}
assert.strictEqual(yield* projections.countActiveRunThreads(), 4);
}).pipe(Effect.provide(layerTest)),
);

it.effect("recovers terminal subagent results until their cross-thread transfer exists", () =>
Effect.gen(function* () {
const projections = yield* ProjectionStore.ProjectionStoreV2;
Expand Down
14 changes: 14 additions & 0 deletions apps/server/src/orchestration-v2/ProjectionStore.ts
Original file line number Diff line number Diff line change
Expand Up @@ -610,6 +610,7 @@ export interface ProjectionStoreV2Shape {
>;
/** Threads that are not deleted, archived ones included. */
readonly countThreads: () => Effect.Effect<number, ProjectionStoreV2Error>;
readonly countActiveRunThreads: () => Effect.Effect<number, ProjectionStoreV2Error>;
/** Every active thread, in shell order. */
readonly listActiveShellIndex: () => Effect.Effect<
ReadonlyArray<ActiveShellIndexEntry>,
Expand Down Expand Up @@ -9136,6 +9137,18 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
return rows[0]?.count ?? 0;
}, Effect.mapError(shellSnapshotReadError));

const countActiveRunThreads = Effect.fn("ProjectionStore.countActiveRunThreads")(function* () {
const rows = yield* sql<{ readonly count: number }>`
SELECT COUNT(DISTINCT run.thread_id) AS count
FROM orchestration_v2_projection_runs AS run
JOIN orchestration_v2_projection_threads AS thread ON thread.thread_id = run.thread_id
WHERE thread.deleted_at IS NULL
AND (run.status IN ('preparing', 'starting', 'running', 'waiting')
OR (run.status = 'queued' AND json_extract(run.payload_json, '$.queueHeld') IS NOT 1))
`;
return rows[0]?.count ?? 0;
}, Effect.mapError(shellSnapshotReadError));

const listActiveShellIndex = Effect.fn("ProjectionStore.listActiveShellIndex")(function* () {
const rows = yield* sql<{
readonly thread_id: string;
Expand Down Expand Up @@ -9518,6 +9531,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
getPullRequestSnapshot: readOnReader(getPullRequestSnapshot),
listActiveSubagentThreadIds: readOnReader(listActiveSubagentThreadIds),
countThreads: readOnReader(countThreads),
countActiveRunThreads: readOnReader(countActiveRunThreads),
listActiveShellIndex: readOnReader(listActiveShellIndex),
listActiveThreads: readOnReader(listActiveThreads),
readShellSnapshot: readOnReader(readShellSnapshot),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,7 @@ it.effect(
getShellSnapshot: () => Effect.die("unused getShellSnapshot"),
listActiveSubagentThreadIds: () => Effect.die("unused listActiveSubagentThreadIds"),
countThreads: () => Effect.die("unused countThreads"),
countActiveRunThreads: () => Effect.die("unused countActiveRunThreads"),
listActiveShellIndex: () => Effect.die("unused listActiveShellIndex"),
listActiveThreads: () => Effect.die("unused listActiveThreads"),
listOlderSettledThreads: () => Effect.die("unused listOlderSettledThreads"),
Expand Down
1 change: 1 addition & 0 deletions apps/server/src/orchestration-v2/ThreadLaunchService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -725,6 +725,7 @@ const make = Effect.gen(function* () {
path: null,
},
{
reuseEmptyBranch: true,
progress: {
onWorktreeClaimed: (path) =>
Effect.sync(() => {
Expand Down
7 changes: 7 additions & 0 deletions apps/server/src/peer/PeerLinks.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import {
type AppConnectionReport,
type AutomaticPlacementHost,
AuthMcpClientAccess,
AuthMcpRegistrationError,
AuthMcpTokenError,
Expand Down Expand Up @@ -126,6 +127,7 @@ export class PeerLinks extends Context.Service<
* fail, so placement contacts nobody rather than everyone.
*/
readonly placementExclusions: Effect.Effect<ReadonlySet<EnvironmentId>, PeerLinkError>;
readonly placementHosts: Effect.Effect<ReadonlyArray<AutomaticPlacementHost>, PeerLinkError>;
/** The stored links, read here only: no peer is asked whether it answers. */
readonly stored: Effect.Effect<ReadonlyArray<PeerLink>, PeerLinkError>;
/**
Expand Down Expand Up @@ -281,6 +283,10 @@ const make = Effect.gen(function* () {
Effect.map((settings) => settings.linkedEnvironmentsEnabled),
Effect.orElseSucceed(() => false),
);
const placementHosts = serverSettings.getSettings.pipe(
Effect.map((settings) => settings.automaticPlacementHosts),
Effect.catch(failStorage),
);
const placementExclusions = serverSettings.getSettings.pipe(
Effect.map(
(settings): ReadonlySet<EnvironmentId> =>
Expand Down Expand Up @@ -853,6 +859,7 @@ const make = Effect.gen(function* () {
environments,
stored,
placementExclusions,
placementHosts,
session,
get,
renewed,
Expand Down
106 changes: 106 additions & 0 deletions apps/server/src/peer/RemoteDelegation.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1710,6 +1710,112 @@ it.effect("placement nobody asked for runs here when no machine has room", () =>
).pipe(Effect.provide(NodeHttpServer.layerTest)),
);

it.effect(
"a host at its lane cap refuses, counts its own fresh placements, and a laptop capped at no lanes keeps nothing",
() =>
Effect.scoped(
Effect.gen(function* () {
const thread = yield* makeBoxThread;
yield* Ref.set(thread.resources, { ...loadOf(0.1), runningAgents: 4 });
const b = yield* serveBox(thread);
const offers = yield* SubscriptionRef.make<
ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest>
>([]);
const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build);
const localLaunches = yield* Ref.make(0);
const a = yield* makeLaptop(database, offers, undefined, Effect.void, {
localLaunch: Ref.update(localLaunches, (count) => count + 1),
});
yield* linkTo(a.links, b, "full-access");
yield* a.settings.updateSettings({
automaticPlacementHosts: [
{ environmentId: box.environmentId, maxLanes: 4, minAvailableMemoryGiB: 4 },
{ environmentId: laptop.environmentId, maxLanes: 0, minAvailableMemoryGiB: 4 },
],
});
const fresh = { type: "worktree", baseRef: "main", startFromOrigin: true };
const capped = yield* launchUnplaced(a, {
clientRequestId: "capped",
workspaceStrategy: fresh,
});
expect(capped.isError).toBe(true);
expect(JSON.stringify(capped.content)).toContain("no-eligible-environment");
expect(yield* Ref.get(localLaunches)).toBe(0);
expect(yield* Ref.get(thread.launched)).toBeNull();

yield* Ref.set(thread.resources, { ...loadOf(0.1), runningAgents: 3 });
const placed = yield* launchUnplaced(a, {
clientRequestId: "room",
workspaceStrategy: fresh,
});
expect(placed.isError, JSON.stringify(placed.content)).toBe(false);
expect(placed.structuredContent).toMatchObject({ environmentId: box.environmentId });
expect(yield* Ref.get(localLaunches)).toBe(0);

const behind = yield* launchUnplaced(a, {
clientRequestId: "same-sample",
workspaceStrategy: fresh,
});
expect(behind.isError).toBe(true);
expect(JSON.stringify(behind.content)).toContain("no-eligible-environment");
expect(yield* Ref.get(localLaunches)).toBe(0);
}),
).pipe(Effect.provide(NodeHttpServer.layerTest)),
);

it.effect(
"sequential launches fill every active lane once the host reports them, and launches on one sample cannot share the last",
() =>
Effect.scoped(
Effect.gen(function* () {
const thread = yield* makeBoxThread;
const b = yield* serveBox(thread);
const offers = yield* SubscriptionRef.make<
ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest>
>([]);
const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build);
const a = yield* makeLaptop(database, offers, undefined, Effect.void, {
localLaunch: Effect.void,
});
yield* linkTo(a.links, b, "full-access");
yield* a.settings.updateSettings({
automaticPlacementHosts: [
{ environmentId: box.environmentId, maxLanes: 4, minAvailableMemoryGiB: 4 },
{ environmentId: laptop.environmentId, maxLanes: 0, minAvailableMemoryGiB: 4 },
],
});
const fresh = { type: "worktree", baseRef: "main", startFromOrigin: true };
for (const active of [0, 1, 2, 3]) {
yield* Ref.set(thread.resources, {
...loadOf(0.1),
runningAgents: 18,
activeRuns: active,
});
const placed = yield* launchUnplaced(a, {
clientRequestId: `lane-${active}`,
workspaceStrategy: fresh,
});
expect(placed.isError, JSON.stringify(placed.content)).toBe(false);
expect(placed.structuredContent).toMatchObject({ environmentId: box.environmentId });
}
const behind = yield* launchUnplaced(a, {
clientRequestId: "lane-same-sample",
workspaceStrategy: fresh,
});
expect(behind.isError).toBe(true);
expect(JSON.stringify(behind.content)).toContain("no-eligible-environment");

yield* Ref.set(thread.resources, { ...loadOf(0.1), runningAgents: 18, activeRuns: 4 });
const full = yield* launchUnplaced(a, {
clientRequestId: "lane-full",
workspaceStrategy: fresh,
});
expect(full.isError).toBe(true);
expect(JSON.stringify(full.content)).toContain("no-eligible-environment");
}),
).pipe(Effect.provide(NodeHttpServer.layerTest)),
);

it.effect(
"a retry that drops placement or asks for this checkout reaches the thread placed there",
() =>
Expand Down
Loading
Loading