Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 2 additions & 19 deletions apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError";
import { resolveProjectSettings } from "@t3tools/shared/projectSettings";
import {
CommandId,
Expand Down Expand Up @@ -40,6 +39,7 @@ import {
awaitingProviderThreadId,
cancelledRosterTaskWork,
cancelledTurnItemWork,
latestStartedRun,
mergeRestartCancelledBackgroundWork,
neverStarted,
} from "./RestartBackgroundNote.ts";
Expand Down Expand Up @@ -173,23 +173,6 @@ const cancelledItem = (
updatedAt: now,
};

function latestStartedRun(
projection: ProjectionStore.ProjectionRuntimeRecoveryState,
providerThreadId: ProviderThreadId,
) {
return projection.runs.reduce<OrchestrationV2ThreadProjection["runs"][number] | undefined>(
(latest, run) =>
run.providerThreadId === providerThreadId &&
run.status !== "queued" &&
run.status !== "rolled_back" &&
!neverStarted(run) &&
(latest === undefined || runRanAfter(run, latest))
? run
: latest,
undefined,
);
}

export const make = Effect.gen(function* () {
const settings = yield* ServerSettings.ServerSettingsService;
const projections = yield* ProjectionStore.ProjectionStoreV2;
Expand Down Expand Up @@ -976,7 +959,7 @@ export const make = Effect.gen(function* () {
});
}
for (const [providerThreadId, work] of cancelledBackgroundWork) {
const noteRun = latestStartedRun(projection, providerThreadId);
const noteRun = latestStartedRun(projection.runs, providerThreadId);
if (noteRun === undefined) continue;
// Use a separate event; stale run snapshots can overwrite intervening lifecycle changes.
events.push({
Expand Down
66 changes: 66 additions & 0 deletions apps/server/src/orchestration-v2/ProviderSessionManager.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ import { assert, it } from "@effect/vitest";
import { AgentScope } from "@t3tools/shared/AgentScope";
import {
EnvironmentId,
EventId,
MessageId,
type ModelSelection,
type OrchestrationV2AppThread,
type OrchestrationV2DomainEvent,
Expand Down Expand Up @@ -60,6 +62,7 @@ import * as ProviderInstanceRegistry from "../provider/ProviderInstanceRegistry.
import * as GitVcsDriver from "../vcs/GitVcsDriver.ts";
import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
import * as ProviderSessionManager from "./ProviderSessionManager.ts";
import { runOutcome } from "./RestartBackgroundNote.ts";
import * as CommandOutputHub from "./CommandOutputHub.ts";
import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
import {
Expand Down Expand Up @@ -3669,6 +3672,69 @@ it.effect.each(["a runtime whose tasks can outlive it", "server shutdown"] as co
}),
);

// regression: a delegated child whose process ended under awaited background work settled as completed
it.effect("ProviderSessionManagerV2 records the background work a released process lost", () =>
Effect.gen(function* () {
const state = yield* Ref.make(emptyState);
const effect = Effect.gen(function* () {
const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
const eventSink = yield* EventSink.EventSinkV2;
const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
const { threadId, providerSessionId, providerThreads, writeProviderThread } =
yield* openSessionWithBackgroundCommand("provider-session-manager-lost-awaited-work");
const now = yield* DateTime.now;
const runId = RunId.make("run:child-awaiting-reviewer");
yield* eventSink.write({
events: [
{
id: EventId.make("event:child-run"),
type: "run.updated",
threadId,
runId,
occurredAt: now,
payload: {
id: runId,
threadId,
ordinal: 1,
providerInstanceId: modelSelection.instanceId,
modelSelection,
providerThreadId: providerThreads[0]!.id,
userMessageId: MessageId.make("message:child-prompt"),
rootNodeId: null,
activeAttemptId: null,
status: "completed",
requestedAt: now,
startedAt: now,
completedAt: now,
checkpointId: null,
contextHandoffId: null,
},
},
],
});
yield* writeProviderThread({
...providerThreads[0]!,
pendingBackgroundTasks: [
...providerThreads[0]!.pendingBackgroundTasks!,
{ taskId: "reviewer", kind: "subagent", description: "Review the diff" },
],
});

yield* manager.close(providerSessionId);

const { runs } = yield* projectionStore.getThreadRecords(threadId, ["runs"]);
const run = runs.find((candidate) => candidate.id === runId)!;
assert.sameMembers(
(run.restartCancelledBackgroundWork ?? []).map((work) => work.id),
["dev-server", "reviewer"],
);
assert.equal(runOutcome(run), "interrupted");
});

yield* effect.pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 60_000 })));
}),
);

// regression: #19355 settled and idle-released threads stayed Waiting
it.effect("ProviderSessionManagerV2 keeps provider updates that land while release clears", () =>
Effect.gen(function* () {
Expand Down
42 changes: 41 additions & 1 deletion apps/server/src/orchestration-v2/ProviderSessionManager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,11 @@ import {
} from "@t3tools/provider-core/server/subagentProjection";
import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts";
import * as ProjectionStore from "./ProjectionStore.ts";
import {
cancelledRosterTaskWork,
latestStartedRun,
mergeRestartCancelledBackgroundWork,
} from "./RestartBackgroundNote.ts";

const DEFAULT_IDLE_TIMEOUT_MS = 30 * 60 * 1000;
const DEFAULT_MAX_IDLE_PIN_MS = 4 * 60 * 60 * 1000;
Expand Down Expand Up @@ -967,14 +972,30 @@ export const layerWithOptions = (
}: ReleasedProviderThread) {
if (taskIds.size === 0) return;
for (let attempt = 0; attempt < 3; attempt++) {
const projection = yield* projectionStore.getThreadRecords(threadId, ["providerThreads"]);
const projection = yield* projectionStore.getThreadRecords(threadId, [
"providerThreads",
"runs",
]);
const providerThread = projection.providerThreads.find(
(candidate) => candidate.id === providerThreadId,
);
const tasks = providerThread?.pendingBackgroundTasks ?? [];
const remaining = tasks.filter((task) => !taskIds.has(task.taskId));
if (providerThread === undefined || remaining.length === tasks.length) return;
const now = yield* DateTime.now;
// Record the lost work as restart recovery does, so a child that was waiting on it
// settles as interrupted rather than completed.
const lost =
providerThread.ownerNodeId === null
? tasks
.filter((task) => taskIds.has(task.taskId))
.filter(
(task) => task.parentTaskId === undefined || !taskIds.has(task.parentTaskId),
)
.map(cancelledRosterTaskWork)
: [];
const noteRun =
lost.length === 0 ? undefined : latestStartedRun(projection.runs, providerThreadId);
const { committed } = yield* eventSink.writeIfProviderThreadUnchanged({
expected: providerThread,
events: [
Expand All @@ -991,6 +1012,25 @@ export const layerWithOptions = (
updatedAt: now,
},
},
...(noteRun === undefined
? []
: [
{
id: yield* idAllocator.allocate.event({ threadId, providerSessionId }),
type: "run.background-work-cancelled" as const,
threadId,
runId: noteRun.id,
providerInstanceId: noteRun.providerInstanceId,
occurredAt: now,
payload: {
runId: noteRun.id,
restartCancelledBackgroundWork: mergeRestartCancelledBackgroundWork(
noteRun.restartCancelledBackgroundWork ?? [],
lost,
),
},
},
]),
],
});
if (committed) return;
Expand Down
18 changes: 18 additions & 0 deletions apps/server/src/orchestration-v2/RestartBackgroundNote.ts
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,24 @@ export function neverStarted(
return run.startedAt === null && run.restartContinuationOfRunId === undefined;
}

/** The run that lost a provider thread's background work when its process ended. */
export function latestStartedRun<Run extends OrchestrationV2Run>(
runs: ReadonlyArray<Run>,
providerThreadId: OrchestrationV2ProviderThread["id"],
): Run | undefined {
return runs.reduce<Run | undefined>(
(latest, run) =>
run.providerThreadId === providerThreadId &&
run.status !== "queued" &&
run.status !== "rolled_back" &&
!neverStarted(run) &&
(latest === undefined || runRanAfter(run, latest))
? run
: latest,
undefined,
);
}

/** Restart-cancelled unstarted runs supersede nothing; user-stopped unstarted runs do. */
export function cancelledBeforeStart(
run: Pick<OrchestrationV2Run, "status" | "startedAt" | "restartContinuationOfRunId">,
Expand Down
14 changes: 11 additions & 3 deletions apps/server/src/peer/PeerMcpClient.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -398,7 +398,7 @@ it.effect("renews a missing peer session once and reuses its replacement", () =>
}),
);

it.effect("bounds the session handshake, not a tool call that waits longer", () =>
it.effect("bounds the handshake tightly and a tool call past any wait there", () =>
Effect.gen(function* () {
const arrived = yield* Queue.unbounded<number>();
const answer = yield* Deferred.make<void>();
Expand All @@ -408,8 +408,8 @@ it.effect("bounds the session handshake, not a tool call that waits longer", ()
const index = requests++;
yield* Queue.offer(arrived, index);
// The first initialize, then the next initialized notification, never answer;
// the third session's tool call answers late.
if (index === 0 || index === 2) return yield* Effect.never;
// the third session's tool call answers late, and the one after it never does.
if (index === 0 || index === 2 || index === 6) return yield* Effect.never;
if (index === 5) yield* Deferred.await(answer);
return HttpClientResponse.fromWeb(
request,
Expand Down Expand Up @@ -437,5 +437,13 @@ it.effect("bounds the session handshake, not a tool call that waits longer", ()
yield* TestClock.adjust(Duration.minutes(1));
yield* Deferred.succeed(answer, undefined);
expect(yield* Fiber.join(waiting)).toEqual({ modes: "approval-required/plan" });

const hung = yield* probe.pipe(Effect.flip, Effect.forkScoped);
yield* Queue.takeN(arrived, 1);
yield* TestClock.adjust(Duration.minutes(2));
expect(yield* Fiber.join(hung)).toMatchObject({
code: "orchestration_error",
message: "The linked environment stopped answering.",
});
}),
);
10 changes: 9 additions & 1 deletion apps/server/src/peer/PeerMcpClient.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import {
type RuntimeMode,
} from "@t3tools/contracts";
import * as Context from "effect/Context";
import * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
import * as Layer from "effect/Layer";
import * as Ref from "effect/Ref";
Expand All @@ -22,6 +23,12 @@ import * as PeerLinks from "./PeerLinks.ts";
/** The only MCP protocol version a T3 Code `/mcp` registers. */
const PROTOCOL_VERSION = "2025-06-18";

/**
* A forwarded call's ceiling once the session is open. Waits there stop at 45 s, so only a
* half-open peer runs this long; without it, a cancel or status read on that link never returns.
*/
const FORWARDED_CALL_TIMEOUT = Duration.minutes(2);

/** A title can be any length; the header carrying it stays well under HTTP header limits. */
const SENDER_TITLE_MAX_LENGTH = 200;

Expand Down Expand Up @@ -162,7 +169,7 @@ const make = Effect.gen(function* () {
);
return sessionId;
},
// Only the handshake is bounded: a forwarded tool call may wait for minutes.
// The handshake is bounded tightly; a forwarded tool call may legitimately wait longer.
(effect, url) => Effect.timeout(effect, PeerLinks.probeTimeout(url)),
);

Expand Down Expand Up @@ -300,6 +307,7 @@ const make = Effect.gen(function* () {
},
(effect, input) =>
effect.pipe(
Effect.timeout(FORWARDED_CALL_TIMEOUT),
Effect.tapError((error) =>
links.recordOutcome(input.environmentId, { error: error.message }),
),
Expand Down
3 changes: 3 additions & 0 deletions apps/server/src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,7 @@ import * as AntigravityInstallation from "./provider/AntigravityInstallation.ts"
import * as ProviderInstallation from "./provider/providerInstallation.ts";
import * as CodexInstallation from "./provider/CodexInstallation.ts";
import * as ProviderInstallationGate from "./provider/ProviderInstallationGate.ts";
import * as ProviderMaintenanceRunner from "./provider/providerMaintenanceRunner.ts";
import * as ProviderInstanceRegistry from "./provider/ProviderInstanceRegistry.ts";
import * as ProviderAdapterRegistry from "./orchestration-v2/ProviderAdapterRegistry.ts";
import * as ProviderRegistry from "./provider/ProviderRegistry.ts";
Expand Down Expand Up @@ -912,6 +913,8 @@ const layerMakeRoutes = Layer.mergeAll(
Layer.provide(PeerLinkRequests.layer),
// Share the service so HTTP/WS clients single-flight reads and invalidate each other's cached patches.
Layer.provide(layerPullRequestService),
// Provider updates hold their locks here, shared by every WebSocket client.
Layer.provide(ProviderMaintenanceRunner.layer),
// The same activity instance RunExecutionService reports to, by layer memoization.
Layer.provide(
ServerBrowser.layer.pipe(
Expand Down
Loading
Loading