Skip to content

Commit 8b6ac19

Browse files
fix(server): share provider update locks, bound peer calls, mark lost child work interrupted
- Build one ProviderMaintenanceRunner for the server and hand it to every WebSocket connection, so two clients share the provider update locks. - Time out a forwarded peer tool call after two minutes, so a half-open linked environment cannot hang task_cancel or task_status. - When a provider process is released with awaited background work still on its roster, record that work on the run as restart recovery does, so a delegated child settles as interrupted instead of completed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 18cf8f2 commit 8b6ac19

9 files changed

Lines changed: 255 additions & 25 deletions

‎apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts‎

Lines changed: 2 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError";
21
import { resolveProjectSettings } from "@t3tools/shared/projectSettings";
32
import {
43
CommandId,
@@ -40,6 +39,7 @@ import {
4039
awaitingProviderThreadId,
4140
cancelledRosterTaskWork,
4241
cancelledTurnItemWork,
42+
latestStartedRun,
4343
mergeRestartCancelledBackgroundWork,
4444
neverStarted,
4545
} from "./RestartBackgroundNote.ts";
@@ -173,23 +173,6 @@ const cancelledItem = (
173173
updatedAt: now,
174174
};
175175

176-
function latestStartedRun(
177-
projection: ProjectionStore.ProjectionRuntimeRecoveryState,
178-
providerThreadId: ProviderThreadId,
179-
) {
180-
return projection.runs.reduce<OrchestrationV2ThreadProjection["runs"][number] | undefined>(
181-
(latest, run) =>
182-
run.providerThreadId === providerThreadId &&
183-
run.status !== "queued" &&
184-
run.status !== "rolled_back" &&
185-
!neverStarted(run) &&
186-
(latest === undefined || runRanAfter(run, latest))
187-
? run
188-
: latest,
189-
undefined,
190-
);
191-
}
192-
193176
export const make = Effect.gen(function* () {
194177
const settings = yield* ServerSettings.ServerSettingsService;
195178
const projections = yield* ProjectionStore.ProjectionStoreV2;
@@ -976,7 +959,7 @@ export const make = Effect.gen(function* () {
976959
});
977960
}
978961
for (const [providerThreadId, work] of cancelledBackgroundWork) {
979-
const noteRun = latestStartedRun(projection, providerThreadId);
962+
const noteRun = latestStartedRun(projection.runs, providerThreadId);
980963
if (noteRun === undefined) continue;
981964
// Use a separate event; stale run snapshots can overwrite intervening lifecycle changes.
982965
events.push({

‎apps/server/src/orchestration-v2/ProviderSessionManager.test.ts‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ import { assert, it } from "@effect/vitest";
55
import { AgentScope } from "@t3tools/shared/AgentScope";
66
import {
77
EnvironmentId,
8+
EventId,
9+
MessageId,
810
type ModelSelection,
911
type OrchestrationV2AppThread,
1012
type OrchestrationV2DomainEvent,
@@ -60,6 +62,7 @@ import * as ProviderInstanceRegistry from "../provider/ProviderInstanceRegistry.
6062
import * as GitVcsDriver from "../vcs/GitVcsDriver.ts";
6163
import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
6264
import * as ProviderSessionManager from "./ProviderSessionManager.ts";
65+
import { runOutcome } from "./RestartBackgroundNote.ts";
6366
import * as CommandOutputHub from "./CommandOutputHub.ts";
6467
import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
6568
import {
@@ -3669,6 +3672,69 @@ it.effect.each(["a runtime whose tasks can outlive it", "server shutdown"] as co
36693672
}),
36703673
);
36713674

3675+
// regression: a delegated child whose process ended under awaited background work settled as completed
3676+
it.effect("ProviderSessionManagerV2 records the background work a released process lost", () =>
3677+
Effect.gen(function* () {
3678+
const state = yield* Ref.make(emptyState);
3679+
const effect = Effect.gen(function* () {
3680+
const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
3681+
const eventSink = yield* EventSink.EventSinkV2;
3682+
const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
3683+
const { threadId, providerSessionId, providerThreads, writeProviderThread } =
3684+
yield* openSessionWithBackgroundCommand("provider-session-manager-lost-awaited-work");
3685+
const now = yield* DateTime.now;
3686+
const runId = RunId.make("run:child-awaiting-reviewer");
3687+
yield* eventSink.write({
3688+
events: [
3689+
{
3690+
id: EventId.make("event:child-run"),
3691+
type: "run.updated",
3692+
threadId,
3693+
runId,
3694+
occurredAt: now,
3695+
payload: {
3696+
id: runId,
3697+
threadId,
3698+
ordinal: 1,
3699+
providerInstanceId: modelSelection.instanceId,
3700+
modelSelection,
3701+
providerThreadId: providerThreads[0]!.id,
3702+
userMessageId: MessageId.make("message:child-prompt"),
3703+
rootNodeId: null,
3704+
activeAttemptId: null,
3705+
status: "completed",
3706+
requestedAt: now,
3707+
startedAt: now,
3708+
completedAt: now,
3709+
checkpointId: null,
3710+
contextHandoffId: null,
3711+
},
3712+
},
3713+
],
3714+
});
3715+
yield* writeProviderThread({
3716+
...providerThreads[0]!,
3717+
pendingBackgroundTasks: [
3718+
...providerThreads[0]!.pendingBackgroundTasks!,
3719+
{ taskId: "reviewer", kind: "subagent", description: "Review the diff" },
3720+
],
3721+
});
3722+
3723+
yield* manager.close(providerSessionId);
3724+
3725+
const { runs } = yield* projectionStore.getThreadRecords(threadId, ["runs"]);
3726+
const run = runs.find((candidate) => candidate.id === runId)!;
3727+
assert.sameMembers(
3728+
(run.restartCancelledBackgroundWork ?? []).map((work) => work.id),
3729+
["dev-server", "reviewer"],
3730+
);
3731+
assert.equal(runOutcome(run), "interrupted");
3732+
});
3733+
3734+
yield* effect.pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 60_000 })));
3735+
}),
3736+
);
3737+
36723738
// regression: #19355 settled and idle-released threads stayed Waiting
36733739
it.effect("ProviderSessionManagerV2 keeps provider updates that land while release clears", () =>
36743740
Effect.gen(function* () {

‎apps/server/src/orchestration-v2/ProviderSessionManager.ts‎

Lines changed: 41 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,11 @@ import {
6565
} from "@t3tools/provider-core/server/subagentProjection";
6666
import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts";
6767
import * as ProjectionStore from "./ProjectionStore.ts";
68+
import {
69+
cancelledRosterTaskWork,
70+
latestStartedRun,
71+
mergeRestartCancelledBackgroundWork,
72+
} from "./RestartBackgroundNote.ts";
6873

6974
const DEFAULT_IDLE_TIMEOUT_MS = 30 * 60 * 1000;
7075
const DEFAULT_MAX_IDLE_PIN_MS = 4 * 60 * 60 * 1000;
@@ -967,14 +972,30 @@ export const layerWithOptions = (
967972
}: ReleasedProviderThread) {
968973
if (taskIds.size === 0) return;
969974
for (let attempt = 0; attempt < 3; attempt++) {
970-
const projection = yield* projectionStore.getThreadRecords(threadId, ["providerThreads"]);
975+
const projection = yield* projectionStore.getThreadRecords(threadId, [
976+
"providerThreads",
977+
"runs",
978+
]);
971979
const providerThread = projection.providerThreads.find(
972980
(candidate) => candidate.id === providerThreadId,
973981
);
974982
const tasks = providerThread?.pendingBackgroundTasks ?? [];
975983
const remaining = tasks.filter((task) => !taskIds.has(task.taskId));
976984
if (providerThread === undefined || remaining.length === tasks.length) return;
977985
const now = yield* DateTime.now;
986+
// Record the lost work as restart recovery does, so a child that was waiting on it
987+
// settles as interrupted rather than completed.
988+
const lost =
989+
providerThread.ownerNodeId === null
990+
? tasks
991+
.filter((task) => taskIds.has(task.taskId))
992+
.filter(
993+
(task) => task.parentTaskId === undefined || !taskIds.has(task.parentTaskId),
994+
)
995+
.map(cancelledRosterTaskWork)
996+
: [];
997+
const noteRun =
998+
lost.length === 0 ? undefined : latestStartedRun(projection.runs, providerThreadId);
978999
const { committed } = yield* eventSink.writeIfProviderThreadUnchanged({
9791000
expected: providerThread,
9801001
events: [
@@ -991,6 +1012,25 @@ export const layerWithOptions = (
9911012
updatedAt: now,
9921013
},
9931014
},
1015+
...(noteRun === undefined
1016+
? []
1017+
: [
1018+
{
1019+
id: yield* idAllocator.allocate.event({ threadId, providerSessionId }),
1020+
type: "run.background-work-cancelled" as const,
1021+
threadId,
1022+
runId: noteRun.id,
1023+
providerInstanceId: noteRun.providerInstanceId,
1024+
occurredAt: now,
1025+
payload: {
1026+
runId: noteRun.id,
1027+
restartCancelledBackgroundWork: mergeRestartCancelledBackgroundWork(
1028+
noteRun.restartCancelledBackgroundWork ?? [],
1029+
lost,
1030+
),
1031+
},
1032+
},
1033+
]),
9941034
],
9951035
});
9961036
if (committed) return;

‎apps/server/src/orchestration-v2/RestartBackgroundNote.ts‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,24 @@ export function neverStarted(
185185
return run.startedAt === null && run.restartContinuationOfRunId === undefined;
186186
}
187187

188+
/** The run that lost a provider thread's background work when its process ended. */
189+
export function latestStartedRun<Run extends OrchestrationV2Run>(
190+
runs: ReadonlyArray<Run>,
191+
providerThreadId: OrchestrationV2ProviderThread["id"],
192+
): Run | undefined {
193+
return runs.reduce<Run | undefined>(
194+
(latest, run) =>
195+
run.providerThreadId === providerThreadId &&
196+
run.status !== "queued" &&
197+
run.status !== "rolled_back" &&
198+
!neverStarted(run) &&
199+
(latest === undefined || runRanAfter(run, latest))
200+
? run
201+
: latest,
202+
undefined,
203+
);
204+
}
205+
188206
/** Restart-cancelled unstarted runs supersede nothing; user-stopped unstarted runs do. */
189207
export function cancelledBeforeStart(
190208
run: Pick<OrchestrationV2Run, "status" | "startedAt" | "restartContinuationOfRunId">,

‎apps/server/src/peer/PeerMcpClient.test.ts‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -398,7 +398,7 @@ it.effect("renews a missing peer session once and reuses its replacement", () =>
398398
}),
399399
);
400400

401-
it.effect("bounds the session handshake, not a tool call that waits longer", () =>
401+
it.effect("bounds the handshake tightly and a tool call past any wait there", () =>
402402
Effect.gen(function* () {
403403
const arrived = yield* Queue.unbounded<number>();
404404
const answer = yield* Deferred.make<void>();
@@ -408,8 +408,8 @@ it.effect("bounds the session handshake, not a tool call that waits longer", ()
408408
const index = requests++;
409409
yield* Queue.offer(arrived, index);
410410
// The first initialize, then the next initialized notification, never answer;
411-
// the third session's tool call answers late.
412-
if (index === 0 || index === 2) return yield* Effect.never;
411+
// the third session's tool call answers late, and the one after it never does.
412+
if (index === 0 || index === 2 || index === 6) return yield* Effect.never;
413413
if (index === 5) yield* Deferred.await(answer);
414414
return HttpClientResponse.fromWeb(
415415
request,
@@ -437,5 +437,13 @@ it.effect("bounds the session handshake, not a tool call that waits longer", ()
437437
yield* TestClock.adjust(Duration.minutes(1));
438438
yield* Deferred.succeed(answer, undefined);
439439
expect(yield* Fiber.join(waiting)).toEqual({ modes: "approval-required/plan" });
440+
441+
const hung = yield* probe.pipe(Effect.flip, Effect.forkScoped);
442+
yield* Queue.takeN(arrived, 1);
443+
yield* TestClock.adjust(Duration.minutes(2));
444+
expect(yield* Fiber.join(hung)).toMatchObject({
445+
code: "orchestration_error",
446+
message: "The linked environment stopped answering.",
447+
});
440448
}),
441449
);

‎apps/server/src/peer/PeerMcpClient.ts‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import {
77
type RuntimeMode,
88
} from "@t3tools/contracts";
99
import * as Context from "effect/Context";
10+
import * as Duration from "effect/Duration";
1011
import * as Effect from "effect/Effect";
1112
import * as Layer from "effect/Layer";
1213
import * as Ref from "effect/Ref";
@@ -22,6 +23,12 @@ import * as PeerLinks from "./PeerLinks.ts";
2223
/** The only MCP protocol version a T3 Code `/mcp` registers. */
2324
const PROTOCOL_VERSION = "2025-06-18";
2425

26+
/**
27+
* A forwarded call's ceiling once the session is open. Waits there stop at 45 s, so only a
28+
* half-open peer runs this long; without it, a cancel or status read on that link never returns.
29+
*/
30+
const FORWARDED_CALL_TIMEOUT = Duration.minutes(2);
31+
2532
/** A title can be any length; the header carrying it stays well under HTTP header limits. */
2633
const SENDER_TITLE_MAX_LENGTH = 200;
2734

@@ -162,7 +169,7 @@ const make = Effect.gen(function* () {
162169
);
163170
return sessionId;
164171
},
165-
// Only the handshake is bounded: a forwarded tool call may wait for minutes.
172+
// The handshake is bounded tightly; a forwarded tool call may legitimately wait longer.
166173
(effect, url) => Effect.timeout(effect, PeerLinks.probeTimeout(url)),
167174
);
168175

@@ -300,6 +307,7 @@ const make = Effect.gen(function* () {
300307
},
301308
(effect, input) =>
302309
effect.pipe(
310+
Effect.timeout(FORWARDED_CALL_TIMEOUT),
303311
Effect.tapError((error) =>
304312
links.recordOutcome(input.environmentId, { error: error.message }),
305313
),

‎apps/server/src/server.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,7 @@ import * as AntigravityInstallation from "./provider/AntigravityInstallation.ts"
120120
import * as ProviderInstallation from "./provider/providerInstallation.ts";
121121
import * as CodexInstallation from "./provider/CodexInstallation.ts";
122122
import * as ProviderInstallationGate from "./provider/ProviderInstallationGate.ts";
123+
import * as ProviderMaintenanceRunner from "./provider/providerMaintenanceRunner.ts";
123124
import * as ProviderInstanceRegistry from "./provider/ProviderInstanceRegistry.ts";
124125
import * as ProviderAdapterRegistry from "./orchestration-v2/ProviderAdapterRegistry.ts";
125126
import * as ProviderRegistry from "./provider/ProviderRegistry.ts";
@@ -912,6 +913,8 @@ const layerMakeRoutes = Layer.mergeAll(
912913
Layer.provide(PeerLinkRequests.layer),
913914
// Share the service so HTTP/WS clients single-flight reads and invalidate each other's cached patches.
914915
Layer.provide(layerPullRequestService),
916+
// Provider updates hold their locks here, shared by every WebSocket client.
917+
Layer.provide(ProviderMaintenanceRunner.layer),
915918
// The same activity instance RunExecutionService reports to, by layer memoization.
916919
Layer.provide(
917920
ServerBrowser.layer.pipe(

0 commit comments

Comments
 (0)