Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion FORK.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@ adds the feature, and remove it in the PR that drops the feature.
`apps/server/src/cloud/npmRelease.ts`.
- #14 Background work an agent waits on keeps its thread working until the result reaches the
agent; settling cannot detach that work, wake dispatches retry, and work lost with a provider
session or Claude process is reported: `ClaudeAdapterV2.ts`
session or Claude process is reported. After a provider crash, upstream's provider-exit
continuation (pingdotgg/t3code#40170) wakes the agent instead: `ClaudeAdapterV2.ts`
(`deliveringBackgroundTasksByNativeThread`, `finalizeActiveTurnAfterQueryExit`),
`Orchestrator.ts` (`automaticSettlementBlocker`), `ThreadManagementService.ts`
(`settleAfterRun`), `ProviderSessionManager.ts`, `ProviderContinuationService.ts`,
Expand Down
45 changes: 35 additions & 10 deletions apps/server/src/orchestration-v2/ProviderSessionManager.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3701,20 +3701,21 @@ it.effect.each(["detach", "stream failure", "provider stop", "early close"] as c
);

it.effect.each([
{ release: "crash", wakes: true },
{ release: "idle", wakes: true },
{ release: "crash", wakes: false },
{ release: "settle", wakes: false },
] as const)(
"ProviderSessionManagerV2 records a waited-on job its session took down: $release",
({ release, wakes }) =>
Effect.gen(function* () {
const state = yield* Ref.make(emptyState);
const offered: Array<ProviderContinuationRequests.ProviderContinuationRequest> = [];
const woken = yield* Deferred.make<void>();
const effect = Effect.gen(function* () {
const eventSink = yield* EventSink.EventSinkV2;
const idAllocator = yield* IdAllocator.IdAllocatorV2;
const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
const outbox = yield* EffectOutbox.EffectOutboxV2;
const now = yield* DateTime.now;
const threadId = yield* idAllocator.allocate.thread({
fixtureName: `provider-session-manager-lost-job-${release}`,
Expand Down Expand Up @@ -3776,9 +3777,27 @@ it.effect.each([
modelSelection,
runtimePolicy,
});
const deploy = {
kind: "shell",
label: "Wait for deploy (id deploy)",
id: "deploy",
awaited: true,
} as const;
if (release === "crash") {
const afterSequence = yield* eventSink.latestSequence({ threadId });
yield* runtime.events.pipe(Stream.runDrain, Effect.ignore, Effect.forkScoped);
yield* Deferred.await(woken);
// The provider-exit note lands after the roster note, so any wake from it has run.
yield* eventSink.stream({ threadId, afterSequence }).pipe(
Stream.filter(
({ event }) =>
event.type === "run.background-work-cancelled" &&
event.payload.restartCancelledBackgroundWork.length === 2,
),
Stream.take(1),
Stream.runDrain,
);
} else if (release === "idle") {
yield* manager.release({ providerSessionId, reason: "idle_timeout" });
} else {
yield* manager.detach({ providerSessionId, threadId, detail: "Thread settled." });
}
Expand All @@ -3790,16 +3809,25 @@ it.effect.each([
?.pendingBackgroundTasks,
[],
);
// Only the job the agent waited on is reported; the dev server was expected to stop.
// The provider-exit note also lists the dev server, but only awaited work wakes the agent.
assert.deepStrictEqual(
projection.runs.find((run) => run.id === runId)?.restartCancelledBackgroundWork,
[{ kind: "shell", label: "Wait for deploy (id deploy)", id: "deploy", awaited: true }],
release === "crash"
? [deploy, { kind: "shell", label: "vp run dev (id dev-server)", id: "dev-server" }]
: [deploy],
);
// A crash wakes the agent to check on the deploy; a settle the user asked for does not.
// An idle release wakes the agent here; a crash wakes it once, through the provider-exit
// continuation; a settle the user asked for wakes nothing.
assert.deepStrictEqual(
offered.map((request) => [request.threadId, request.delivery]),
wakes ? [[threadId, "message_text"]] : [],
);
assert.deepStrictEqual(
Option.getOrUndefined(yield* outbox.get(`effect:restart-continuation:${runId}`))?.request,
release === "crash"
? { type: "provider-runtime.continue", sourceRunId: runId, cause: "provider_exit" }
: undefined,
);
});

yield* effect.pipe(
Expand All @@ -3810,10 +3838,7 @@ it.effect.each([
failEventStream: release === "crash",
capabilities: ExclusiveCapabilities,
continuationRequests: {
offer: (request) =>
Effect.sync(() => void offered.push(request)).pipe(
Effect.tap(() => Deferred.succeed(woken, undefined)),
),
offer: (request) => Effect.sync(() => void offered.push(request)),
take: Effect.never,
},
}),
Expand Down
6 changes: 3 additions & 3 deletions apps/server/src/orchestration-v2/ProviderSessionManager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1024,8 +1024,8 @@ export const layerWithOptions = (

// Dead processes cannot report task endings; stale rosters keep threads Waiting.
// Work the agent waited on is recorded on its run, so the agent hears about it on its
// next turn and status reads show the run interrupted. A crash or idle release, which
// nobody asked for, also wakes the agent to check what that work left behind.
// next turn and status reads show the run interrupted. An idle release, which nobody
// asked for, also wakes the agent; after a crash, writeReleasedWorkNote's continuation does.
const writeReleasedRosterEvents = Effect.fn(
"ProviderSessionManager.writeReleasedRosterEvents",
)(function* (
Expand Down Expand Up @@ -1099,7 +1099,7 @@ export const layerWithOptions = (
if (
noteRun !== undefined &&
!parked &&
(reason === "runtime_error" || reason === "idle_timeout") &&
reason === "idle_timeout" &&
latestRun?.id === noteRun.id &&
(noteRun.status === "completed" || noteRun.status === "waiting")
) {
Expand Down
Loading