From 1fd3fae5bc20cd65490eb1939e01b5e5b26f72d4 Mon Sep 17 00:00:00 2001 From: Kameron Smith Date: Thu, 19 Nov 2026 20:17:36 -0500 Subject: [PATCH 1/2] fix(server): stop Codex turns stuck in a compaction loop Track Codex compactions per thread. Three within five minutes appends a visible error activity and interrupts the active turn without resuming. Model: claude-opus-5-5 Task: coding Agent: T3 Claude Code (CLO-921 lane) Lane: magpie Co-Authored-By: Claude Opus 5.5 --- .../orchestration/CompactionLoopGuard.test.ts | 46 ++++++++++++ .../src/orchestration/CompactionLoopGuard.ts | 30 ++++++++ .../Layers/ProviderRuntimeIngestion.test.ts | 73 +++++++++++++++++++ .../Layers/ProviderRuntimeIngestion.ts | 45 ++++++++++++ 4 files changed, 194 insertions(+) create mode 100644 apps/server/src/orchestration/CompactionLoopGuard.test.ts create mode 100644 apps/server/src/orchestration/CompactionLoopGuard.ts diff --git a/apps/server/src/orchestration/CompactionLoopGuard.test.ts b/apps/server/src/orchestration/CompactionLoopGuard.test.ts new file mode 100644 index 0000000000000..89653b9e8c17a --- /dev/null +++ b/apps/server/src/orchestration/CompactionLoopGuard.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { CODEX_COMPACTION_LOOP_LIMIT, recordCompaction } from "./CompactionLoopGuard.ts"; + +const replay = (offsetsMs: ReadonlyArray) => { + let history: ReadonlyArray = []; + return offsetsMs.map((offset) => { + const result = recordCompaction(history, offset); + history = result.history; + return result.tripped; + }); +}; + +describe("recordCompaction", () => { + it("defaults to three compactions within five minutes", () => { + expect(CODEX_COMPACTION_LOOP_LIMIT.compactions).toBe(3); + expect(CODEX_COMPACTION_LOOP_LIMIT.windowMs).toBe(300_000); + }); + + it("trips on the third compaction of a thrash sequence", () => { + expect(replay([0, 40_000, 80_000])).toEqual([false, false, true]); + }); + + it("trips once per loop and again if the loop continues", () => { + expect(replay([0, 30_000, 60_000, 60_005, 90_000, 120_000])).toEqual([ + false, + false, + true, + false, + false, + true, + ]); + }); + + it("does not trip on normally spaced compactions", () => { + expect(replay([0, 6 * 60_000, 12 * 60_000, 18 * 60_000])).toEqual([false, false, false, false]); + }); + + it("drops compactions that fell out of the window", () => { + expect(replay([0, 200_000, 310_000, 400_000])).toEqual([false, false, false, true]); + }); + + it("counts the paired signals of one compaction once", () => { + expect(replay([0, 5, 60_000, 60_010])).toEqual([false, false, false, false]); + }); +}); diff --git a/apps/server/src/orchestration/CompactionLoopGuard.ts b/apps/server/src/orchestration/CompactionLoopGuard.ts new file mode 100644 index 0000000000000..70ddce533daf8 --- /dev/null +++ b/apps/server/src/orchestration/CompactionLoopGuard.ts @@ -0,0 +1,30 @@ +export const CODEX_COMPACTION_LOOP_LIMIT = { + compactions: 3, + windowMs: 5 * 60_000, + duplicateSignalMs: 2_000, +} as const; + +export type CompactionLoopLimit = { + readonly compactions: number; + readonly windowMs: number; + readonly duplicateSignalMs: number; +}; + +export function recordCompaction( + history: ReadonlyArray, + atMs: number, + limit: CompactionLoopLimit = CODEX_COMPACTION_LOOP_LIMIT, +): { readonly history: ReadonlyArray; readonly tripped: boolean } { + const last = history.at(-1); + if (last !== undefined && Math.abs(atMs - last) < limit.duplicateSignalMs) { + return { history, tripped: false }; + } + const recent = [...history.filter((at) => atMs - at < limit.windowMs), atMs]; + const tripped = recent.length >= limit.compactions; + return { history: tripped ? [atMs] : recent, tripped }; +} + +export function describeCompactionLoop(limit: CompactionLoopLimit = CODEX_COMPACTION_LOOP_LIMIT) { + const minutes = limit.windowMs / 60_000; + return `Context compacted ${limit.compactions} times within ${minutes} minute${minutes === 1 ? "" : "s"}, so the turn was stopped instead of re-reading the full context again. It will not resume on its own. Send a new message, ideally with a narrower request, to continue.`; +} diff --git a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts index bf1425848bca4..460e9139b3152 100644 --- a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts +++ b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts @@ -28,6 +28,7 @@ import { TurnId, } from "@t3tools/contracts"; import * as Clock from "effect/Clock"; +import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; @@ -5220,6 +5221,78 @@ describe("ProviderRuntimeIngestion", () => { expect(activity?.payload).toMatchObject({ requestId: "message-compact" }); }); + const replayCodexCompactions = async (offsetsSeconds: ReadonlyArray) => { + const harness = await createHarness(); + const threadId = asThreadId("thread-1"); + const turnId = asTurnId("turn-compaction-loop"); + const startedAt = DateTime.makeUnsafe("2026-01-01T00:00:00.000Z"); + const at = (seconds: number) => DateTime.formatIso(DateTime.add(startedAt, { seconds })); + harness.emit({ + type: "turn.started", + eventId: asEventId("evt-compaction-loop-turn-started"), + provider: ProviderDriverKind.make("codex"), + threadId, + createdAt: at(0), + turnId, + }); + await waitForThread(harness.readModel, (thread) => thread.session?.activeTurnId === turnId); + await harness.emitAndDrain( + offsetsSeconds.flatMap((seconds, index) => [ + { + type: "thread.state.changed" as const, + eventId: asEventId(`evt-compaction-loop-${index}`), + provider: ProviderDriverKind.make("codex"), + providerInstanceId: ProviderInstanceId.make("codex"), + createdAt: at(seconds), + threadId, + turnId, + payload: { state: "compacted" as const }, + }, + { + type: "thread.state.changed" as const, + eventId: asEventId(`evt-compaction-loop-${index}:thread-compacted`), + provider: ProviderDriverKind.make("codex"), + providerInstanceId: ProviderInstanceId.make("codex"), + createdAt: at(seconds), + threadId, + turnId, + payload: { state: "compacted" as const }, + }, + ]), + ); + const events = await Effect.runPromise( + Stream.runCollect(harness.engine.readEvents(0)).pipe( + Effect.map((chunk) => Array.from(chunk)), + ), + ); + const thread = (await harness.readModel()).threads.find((entry) => entry.id === threadId); + return { + turnId, + interrupts: events.filter((event) => event.type === "thread.turn-interrupt-requested"), + stops: (thread?.activities ?? []).filter( + (activity: ProviderRuntimeTestActivity) => + activity.kind === "provider.compaction-loop.stopped", + ), + }; + }; + + it("interrupts a Codex turn that compacts three times within five minutes", async () => { + const result = await replayCodexCompactions([10, 50, 90]); + expect(result.interrupts).toHaveLength(1); + expect(result.interrupts[0]?.payload).toMatchObject({ turnId: result.turnId }); + expect(result.stops).toHaveLength(1); + expect(result.stops[0]?.tone).toBe("error"); + expect(result.stops[0]?.payload).toMatchObject({ + detail: expect.stringContaining("will not resume on its own"), + }); + }); + + it("lets a Codex turn with normally spaced compactions keep running", async () => { + const result = await replayCodexCompactions([10, 400, 800, 1_200]); + expect(result.interrupts).toHaveLength(0); + expect(result.stops).toHaveLength(0); + }); + it("projects Codex task lifecycle chunks into thread activities", async () => { const harness = await createHarness(); const now = "2026-01-01T00:00:00.000Z"; diff --git a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts index 925e26236757a..c091ddd0bc285 100644 --- a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts +++ b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts @@ -53,6 +53,7 @@ import { type ProviderRuntimeIngestionShape, } from "../Services/ProviderRuntimeIngestion.ts"; import { projectActivityPayload } from "../ActivityPayloadProjection.ts"; +import { describeCompactionLoop, recordCompaction } from "../CompactionLoopGuard.ts"; import { forkParked } from "../../serverActivation.ts"; import { ServerSettingsService } from "../../serverSettings.ts"; import { resolveProjectSettings } from "@t3tools/shared/projectSettings"; @@ -1036,6 +1037,42 @@ const make = Effect.gen(function* () { Effect.map((uuid) => CommandId.make(`provider:${event.eventId}:${tag}:${uuid}`)), ); + const compactionHistoryByThread = new Map>(); + const guardCompactionLoop = Effect.fn("guardCompactionLoop")(function* ( + threadId: ThreadId, + event: ProviderRuntimeEvent, + activeTurnId: TurnId | null, + ) { + const result = recordCompaction( + compactionHistoryByThread.get(threadId) ?? [], + Date.parse(event.createdAt), + ); + compactionHistoryByThread.set(threadId, result.history); + if (!result.tripped || activeTurnId === null) return; + yield* orchestrationEngine.dispatch({ + type: "thread.activity.append", + commandId: yield* providerCommandId(event, "compaction-loop-stopped"), + threadId, + activity: { + id: EventId.make(`${event.eventId}:compaction-loop-stopped`), + tone: "error", + kind: "provider.compaction-loop.stopped", + summary: "Stopped a context compaction loop", + payload: { detail: describeCompactionLoop() }, + turnId: activeTurnId, + createdAt: event.createdAt, + }, + createdAt: event.createdAt, + }); + yield* orchestrationEngine.dispatch({ + type: "thread.turn.interrupt", + commandId: yield* providerCommandId(event, "compaction-loop-interrupt"), + threadId, + turnId: activeTurnId, + createdAt: event.createdAt, + }); + }); + // Post-start delivery: buffered `turn`-mode content and parent tool // heartbeats advance the in-memory observation without dispatching an // activity row, so an already-subscribed shell never refetches and can @@ -2723,11 +2760,19 @@ const make = Effect.gen(function* () { ), ), ).pipe(Effect.asVoid); + if ( + isCompactedThreadState && + event.provider === "codex" && + activityEvent.requestId === undefined + ) { + yield* guardCompactionLoop(thread.id, event, activeTurnId); + } // The turn is over: drop the live observation so the settled shell does // not keep advertising provider progress. Only the accepted lifecycle // owner may clear: a delayed completion rejected for a superseded turn // must not erase the current turn's evidence. if (event.type === "session.exited") { + compactionHistoryByThread.delete(thread.id); threadPostStartActivity.clearThread(thread.id); } else if (isTerminalTurn && shouldApplyThreadLifecycle) { threadPostStartActivity.clearThread(thread.id, observationTurnId); From 551fab33efd8a6aa41070c400ff9e159f17f516c Mon Sep 17 00:00:00 2001 From: nullStack65 <112618179+nullStack65@users.noreply.github.com> Date: Fri, 20 Nov 2026 03:40:05 +0000 Subject: [PATCH 2/2] fix(server): keep manual compacts out of the loop guard A manual /compact tags only the first of Codex's two compaction signals, so the untagged pair was still counting. Three manual compacts, or one manual plus two automatic ones, could stop the turn. That pair is now ignored. A loop that trips while no turn is active keeps its history, so the next compaction on a live turn still stops. Model: grok/grok-4.7 Task: coding Agent: T3 Codex (CLO-921 lane) Lane: magpie --- .../Layers/ProviderRuntimeIngestion.test.ts | 80 ++++++++++++++----- .../Layers/ProviderRuntimeIngestion.ts | 41 +++++++--- 2 files changed, 89 insertions(+), 32 deletions(-) diff --git a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts index 460e9139b3152..f3a6385d58dc3 100644 --- a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts +++ b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts @@ -5221,45 +5221,65 @@ describe("ProviderRuntimeIngestion", () => { expect(activity?.payload).toMatchObject({ requestId: "message-compact" }); }); - const replayCodexCompactions = async (offsetsSeconds: ReadonlyArray) => { + const replayCodexCompactions = async ( + offsetsSeconds: ReadonlyArray, + options?: { readonly beforeTurn?: boolean }, + ) => { const harness = await createHarness(); const threadId = asThreadId("thread-1"); const turnId = asTurnId("turn-compaction-loop"); const startedAt = DateTime.makeUnsafe("2026-01-01T00:00:00.000Z"); const at = (seconds: number) => DateTime.formatIso(DateTime.add(startedAt, { seconds })); - harness.emit({ - type: "turn.started", - eventId: asEventId("evt-compaction-loop-turn-started"), - provider: ProviderDriverKind.make("codex"), - threadId, - createdAt: at(0), - turnId, - }); - await waitForThread(harness.readModel, (thread) => thread.session?.activeTurnId === turnId); - await harness.emitAndDrain( - offsetsSeconds.flatMap((seconds, index) => [ - { + const compactionEvents = (phase: string) => + offsetsSeconds.flatMap((offset, index) => { + const seconds = typeof offset === "number" ? offset : offset.seconds; + const manual = typeof offset !== "number"; + return [0, 1].map((signal) => ({ type: "thread.state.changed" as const, - eventId: asEventId(`evt-compaction-loop-${index}`), + eventId: asEventId(`evt-compaction-loop-${phase}-${index}-${signal}`), provider: ProviderDriverKind.make("codex"), providerInstanceId: ProviderInstanceId.make("codex"), createdAt: at(seconds), threadId, turnId, + ...(manual && signal === 0 + ? { requestId: RuntimeRequestId.make(`manual-compact-${index}`) } + : {}), payload: { state: "compacted" as const }, - }, - { + })); + }); + const startTurn = () => { + harness.emit({ + type: "turn.started", + eventId: asEventId("evt-compaction-loop-turn-started"), + provider: ProviderDriverKind.make("codex"), + threadId, + createdAt: at(0), + turnId, + }); + return waitForThread(harness.readModel, (thread) => thread.session?.activeTurnId === turnId); + }; + if (options?.beforeTurn) { + await harness.emitAndDrain(compactionEvents("before")); + await startTurn(); + const last = offsetsSeconds.at(-1); + const lastSeconds = typeof last === "number" ? last : (last?.seconds ?? 0); + await harness.emitAndDrain( + [0, 1].map((signal) => ({ type: "thread.state.changed" as const, - eventId: asEventId(`evt-compaction-loop-${index}:thread-compacted`), + eventId: asEventId(`evt-compaction-loop-after-${signal}`), provider: ProviderDriverKind.make("codex"), providerInstanceId: ProviderInstanceId.make("codex"), - createdAt: at(seconds), + createdAt: at(lastSeconds + 10), threadId, turnId, payload: { state: "compacted" as const }, - }, - ]), - ); + })), + ); + } else { + await startTurn(); + await harness.emitAndDrain(compactionEvents("during")); + } const events = await Effect.runPromise( Stream.runCollect(harness.engine.readEvents(0)).pipe( Effect.map((chunk) => Array.from(chunk)), @@ -5293,6 +5313,24 @@ describe("ProviderRuntimeIngestion", () => { expect(result.stops).toHaveLength(0); }); + it("does not count a manual compact or its untagged pair", async () => { + const result = await replayCodexCompactions([ + { seconds: 10, manual: true }, + { seconds: 50, manual: true }, + { seconds: 90, manual: true }, + 100, + 130, + ]); + expect(result.interrupts).toHaveLength(0); + expect(result.stops).toHaveLength(0); + }); + + it("interrupts the next active turn after compactions that had no turn to stop", async () => { + const result = await replayCodexCompactions([10, 50, 90], { beforeTurn: true }); + expect(result.interrupts).toHaveLength(1); + expect(result.stops).toHaveLength(1); + }); + it("projects Codex task lifecycle chunks into thread activities", async () => { const harness = await createHarness(); const now = "2026-01-01T00:00:00.000Z"; diff --git a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts index c091ddd0bc285..ba5eb94bcc2f1 100644 --- a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts +++ b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts @@ -53,7 +53,11 @@ import { type ProviderRuntimeIngestionShape, } from "../Services/ProviderRuntimeIngestion.ts"; import { projectActivityPayload } from "../ActivityPayloadProjection.ts"; -import { describeCompactionLoop, recordCompaction } from "../CompactionLoopGuard.ts"; +import { + CODEX_COMPACTION_LOOP_LIMIT, + describeCompactionLoop, + recordCompaction, +} from "../CompactionLoopGuard.ts"; import { forkParked } from "../../serverActivation.ts"; import { ServerSettingsService } from "../../serverSettings.ts"; import { resolveProjectSettings } from "@t3tools/shared/projectSettings"; @@ -1038,15 +1042,22 @@ const make = Effect.gen(function* () { ); const compactionHistoryByThread = new Map>(); + const manualCompactionAtByThread = new Map(); const guardCompactionLoop = Effect.fn("guardCompactionLoop")(function* ( threadId: ThreadId, event: ProviderRuntimeEvent, activeTurnId: TurnId | null, ) { - const result = recordCompaction( - compactionHistoryByThread.get(threadId) ?? [], - Date.parse(event.createdAt), - ); + const previous = compactionHistoryByThread.get(threadId) ?? []; + const atMs = Date.parse(event.createdAt); + const result = recordCompaction(previous, atMs); + if (result.tripped && activeTurnId === null) { + compactionHistoryByThread.set(threadId, [ + ...previous.filter((at) => atMs - at < CODEX_COMPACTION_LOOP_LIMIT.windowMs), + atMs, + ]); + return; + } compactionHistoryByThread.set(threadId, result.history); if (!result.tripped || activeTurnId === null) return; yield* orchestrationEngine.dispatch({ @@ -2760,12 +2771,19 @@ const make = Effect.gen(function* () { ), ), ).pipe(Effect.asVoid); - if ( - isCompactedThreadState && - event.provider === "codex" && - activityEvent.requestId === undefined - ) { - yield* guardCompactionLoop(thread.id, event, activeTurnId); + if (isCompactedThreadState && event.provider === "codex") { + const atMs = Date.parse(event.createdAt); + if (activityEvent.requestId !== undefined) { + manualCompactionAtByThread.set(thread.id, atMs); + } else { + const manualAt = manualCompactionAtByThread.get(thread.id); + const pairedManualSignal = + manualAt !== undefined && + Math.abs(atMs - manualAt) < CODEX_COMPACTION_LOOP_LIMIT.duplicateSignalMs; + if (!pairedManualSignal) { + yield* guardCompactionLoop(thread.id, event, activeTurnId); + } + } } // The turn is over: drop the live observation so the settled shell does // not keep advertising provider progress. Only the accepted lifecycle @@ -2773,6 +2791,7 @@ const make = Effect.gen(function* () { // must not erase the current turn's evidence. if (event.type === "session.exited") { compactionHistoryByThread.delete(thread.id); + manualCompactionAtByThread.delete(thread.id); threadPostStartActivity.clearThread(thread.id); } else if (isTerminalTurn && shouldApplyThreadLifecycle) { threadPostStartActivity.clearThread(thread.id, observationTurnId);