From ae3d63756db5de38e88f77748f53a386a00ac954 Mon Sep 17 00:00:00 2001 From: nullStack65 <112618179+nullStack65@users.noreply.github.com> Date: Wed, 18 Nov 2026 21:12:02 +0000 Subject: [PATCH] fix(server): runs end when their provider session dies A run subscribes to its session's events after turn-start work such as the git baseline. If the provider died in that window, release had already failed and swapped out the subscriber map, so the late subscription waited forever and the run stayed running. A subscription made after release now fails itself (or is interrupted on shutdown), so the run settles. Co-Authored-By: Claude Opus 5.5 --- .../ProviderSessionManager.test.ts | 43 +++++++++++++++++++ .../ProviderSessionManager.ts | 28 ++++++++++-- 2 files changed, 68 insertions(+), 3 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index aa702dc52ed6..98745a12813c 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -2281,6 +2281,49 @@ it.effect("ProviderSessionManagerV2 drains subscribers when the provider stops", }), ); +it.effect("ProviderSessionManagerV2 fails a subscription made after its provider died", () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const effect = Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread-provider-session-manager-late-subscriber"); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + const runtime = yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + // A run holds the runtime it got at turn start and subscribes later. + const early = yield* runtime.subscribeEvents!; + const earlyExit = yield* early.events.pipe(Stream.runDrain, Effect.exit, Effect.forkScoped); + const adapterQueue = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId)); + assert.isDefined(adapterQueue); + yield* Queue.end(adapterQueue!); + assert.isTrue(Exit.isFailure(yield* Fiber.join(earlyExit))); + + const late = yield* runtime.subscribeEvents!; + const lateExit = yield* late.events.pipe(Stream.runDrain, Effect.exit); + assert.isTrue(Exit.isFailure(lateExit)); + if (Exit.isFailure(lateExit)) { + const error = Cause.squash(lateExit.cause); + assert.instanceOf(error, ProviderAdapter.ProviderAdapterEventStreamError); + } + }); + + yield* effect.pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 60_000 }))); + }), +); + it.effect("ProviderSessionManagerV2 fails a turn whose provider retry never resumes", () => Effect.gen(function* () { const state = yield* Ref.make(emptyState); diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index ba9ba7a2bd3f..abcb5fd88196 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -2270,6 +2270,7 @@ export const layerWithOptions = ( subscribers: Ref.Ref< ReadonlyMap> >, + runtime: ProviderAdapter.ProviderAdapterV2SessionRuntime, ): Effect.fn.Return { const queue = yield* Queue.unbounded(); const subscriberId = yield* Ref.getAndUpdate(nextSubscriberId, (value) => value + 1); @@ -2278,14 +2279,35 @@ export const layerWithOptions = ( updated.set(subscriberId, queue); return updated; }); - const close = Ref.modify(subscribers, (current) => { + const unregister = Ref.modify(subscribers, (current) => { if (!current.has(subscriberId)) { return [false, current] as const; } const updated = new Map(current); updated.delete(subscriberId); return [true, updated] as const; - }).pipe( + }); + // Release signals its subscribers once; a run that subscribes after it would wait forever. + // Still registered here means release took the map first, so this queue is ours to end. + const live = + (yield* Ref.get(sessions)).get(sessionKey(runtime.providerSessionId))?.runtime === + runtime; + if (!live && (yield* unregister)) { + yield* shutdownSignal.received + ? Queue.interrupt(queue) + : Queue.offer(queue, { + type: "failure", + cause: Cause.fail( + new ProviderAdapter.ProviderAdapterEventStreamError({ + driver: runtime.driver, + providerSessionId: runtime.providerSessionId, + raisedByT3: true, + cause: "The provider session was released before this subscription.", + }), + ), + }); + } + const close = unregister.pipe( Effect.flatMap((removed) => removed ? Queue.clear(queue).pipe(Effect.andThen(Queue.end(queue)), Effect.asVoid) @@ -2319,7 +2341,7 @@ export const layerWithOptions = ( >, ): ProviderAdapter.ProviderAdapterV2SessionRuntime => { const providerSessionId = runtime.providerSessionId; - const subscribeEvents = makeEventSubscription(eventSubscribers); + const subscribeEvents = makeEventSubscription(eventSubscribers, runtime); const turnMetrics = (operation: string, model?: string) => withMetrics({ counter: providerTurnsTotal,