diff --git a/apps/server/src/orchestration/CompactionLoopGuard.test.ts b/apps/server/src/orchestration/CompactionLoopGuard.test.ts new file mode 100644 index 0000000000000..89653b9e8c17a --- /dev/null +++ b/apps/server/src/orchestration/CompactionLoopGuard.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { CODEX_COMPACTION_LOOP_LIMIT, recordCompaction } from "./CompactionLoopGuard.ts"; + +const replay = (offsetsMs: ReadonlyArray) => { + let history: ReadonlyArray = []; + return offsetsMs.map((offset) => { + const result = recordCompaction(history, offset); + history = result.history; + return result.tripped; + }); +}; + +describe("recordCompaction", () => { + it("defaults to three compactions within five minutes", () => { + expect(CODEX_COMPACTION_LOOP_LIMIT.compactions).toBe(3); + expect(CODEX_COMPACTION_LOOP_LIMIT.windowMs).toBe(300_000); + }); + + it("trips on the third compaction of a thrash sequence", () => { + expect(replay([0, 40_000, 80_000])).toEqual([false, false, true]); + }); + + it("trips once per loop and again if the loop continues", () => { + expect(replay([0, 30_000, 60_000, 60_005, 90_000, 120_000])).toEqual([ + false, + false, + true, + false, + false, + true, + ]); + }); + + it("does not trip on normally spaced compactions", () => { + expect(replay([0, 6 * 60_000, 12 * 60_000, 18 * 60_000])).toEqual([false, false, false, false]); + }); + + it("drops compactions that fell out of the window", () => { + expect(replay([0, 200_000, 310_000, 400_000])).toEqual([false, false, false, true]); + }); + + it("counts the paired signals of one compaction once", () => { + expect(replay([0, 5, 60_000, 60_010])).toEqual([false, false, false, false]); + }); +}); diff --git a/apps/server/src/orchestration/CompactionLoopGuard.ts b/apps/server/src/orchestration/CompactionLoopGuard.ts new file mode 100644 index 0000000000000..70ddce533daf8 --- /dev/null +++ b/apps/server/src/orchestration/CompactionLoopGuard.ts @@ -0,0 +1,30 @@ +export const CODEX_COMPACTION_LOOP_LIMIT = { + compactions: 3, + windowMs: 5 * 60_000, + duplicateSignalMs: 2_000, +} as const; + +export type CompactionLoopLimit = { + readonly compactions: number; + readonly windowMs: number; + readonly duplicateSignalMs: number; +}; + +export function recordCompaction( + history: ReadonlyArray, + atMs: number, + limit: CompactionLoopLimit = CODEX_COMPACTION_LOOP_LIMIT, +): { readonly history: ReadonlyArray; readonly tripped: boolean } { + const last = history.at(-1); + if (last !== undefined && Math.abs(atMs - last) < limit.duplicateSignalMs) { + return { history, tripped: false }; + } + const recent = [...history.filter((at) => atMs - at < limit.windowMs), atMs]; + const tripped = recent.length >= limit.compactions; + return { history: tripped ? [atMs] : recent, tripped }; +} + +export function describeCompactionLoop(limit: CompactionLoopLimit = CODEX_COMPACTION_LOOP_LIMIT) { + const minutes = limit.windowMs / 60_000; + return `Context compacted ${limit.compactions} times within ${minutes} minute${minutes === 1 ? "" : "s"}, so the turn was stopped instead of re-reading the full context again. It will not resume on its own. Send a new message, ideally with a narrower request, to continue.`; +} diff --git a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts index bf1425848bca4..f3a6385d58dc3 100644 --- a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts +++ b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts @@ -28,6 +28,7 @@ import { TurnId, } from "@t3tools/contracts"; import * as Clock from "effect/Clock"; +import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; @@ -5220,6 +5221,116 @@ describe("ProviderRuntimeIngestion", () => { expect(activity?.payload).toMatchObject({ requestId: "message-compact" }); }); + const replayCodexCompactions = async ( + offsetsSeconds: ReadonlyArray, + options?: { readonly beforeTurn?: boolean }, + ) => { + const harness = await createHarness(); + const threadId = asThreadId("thread-1"); + const turnId = asTurnId("turn-compaction-loop"); + const startedAt = DateTime.makeUnsafe("2026-01-01T00:00:00.000Z"); + const at = (seconds: number) => DateTime.formatIso(DateTime.add(startedAt, { seconds })); + const compactionEvents = (phase: string) => + offsetsSeconds.flatMap((offset, index) => { + const seconds = typeof offset === "number" ? offset : offset.seconds; + const manual = typeof offset !== "number"; + return [0, 1].map((signal) => ({ + type: "thread.state.changed" as const, + eventId: asEventId(`evt-compaction-loop-${phase}-${index}-${signal}`), + provider: ProviderDriverKind.make("codex"), + providerInstanceId: ProviderInstanceId.make("codex"), + createdAt: at(seconds), + threadId, + turnId, + ...(manual && signal === 0 + ? { requestId: RuntimeRequestId.make(`manual-compact-${index}`) } + : {}), + payload: { state: "compacted" as const }, + })); + }); + const startTurn = () => { + harness.emit({ + type: "turn.started", + eventId: asEventId("evt-compaction-loop-turn-started"), + provider: ProviderDriverKind.make("codex"), + threadId, + createdAt: at(0), + turnId, + }); + return waitForThread(harness.readModel, (thread) => thread.session?.activeTurnId === turnId); + }; + if (options?.beforeTurn) { + await harness.emitAndDrain(compactionEvents("before")); + await startTurn(); + const last = offsetsSeconds.at(-1); + const lastSeconds = typeof last === "number" ? last : (last?.seconds ?? 0); + await harness.emitAndDrain( + [0, 1].map((signal) => ({ + type: "thread.state.changed" as const, + eventId: asEventId(`evt-compaction-loop-after-${signal}`), + provider: ProviderDriverKind.make("codex"), + providerInstanceId: ProviderInstanceId.make("codex"), + createdAt: at(lastSeconds + 10), + threadId, + turnId, + payload: { state: "compacted" as const }, + })), + ); + } else { + await startTurn(); + await harness.emitAndDrain(compactionEvents("during")); + } + const events = await Effect.runPromise( + Stream.runCollect(harness.engine.readEvents(0)).pipe( + Effect.map((chunk) => Array.from(chunk)), + ), + ); + const thread = (await harness.readModel()).threads.find((entry) => entry.id === threadId); + return { + turnId, + interrupts: events.filter((event) => event.type === "thread.turn-interrupt-requested"), + stops: (thread?.activities ?? []).filter( + (activity: ProviderRuntimeTestActivity) => + activity.kind === "provider.compaction-loop.stopped", + ), + }; + }; + + it("interrupts a Codex turn that compacts three times within five minutes", async () => { + const result = await replayCodexCompactions([10, 50, 90]); + expect(result.interrupts).toHaveLength(1); + expect(result.interrupts[0]?.payload).toMatchObject({ turnId: result.turnId }); + expect(result.stops).toHaveLength(1); + expect(result.stops[0]?.tone).toBe("error"); + expect(result.stops[0]?.payload).toMatchObject({ + detail: expect.stringContaining("will not resume on its own"), + }); + }); + + it("lets a Codex turn with normally spaced compactions keep running", async () => { + const result = await replayCodexCompactions([10, 400, 800, 1_200]); + expect(result.interrupts).toHaveLength(0); + expect(result.stops).toHaveLength(0); + }); + + it("does not count a manual compact or its untagged pair", async () => { + const result = await replayCodexCompactions([ + { seconds: 10, manual: true }, + { seconds: 50, manual: true }, + { seconds: 90, manual: true }, + 100, + 130, + ]); + expect(result.interrupts).toHaveLength(0); + expect(result.stops).toHaveLength(0); + }); + + it("interrupts the next active turn after compactions that had no turn to stop", async () => { + const result = await replayCodexCompactions([10, 50, 90], { beforeTurn: true }); + expect(result.interrupts).toHaveLength(1); + expect(result.stops).toHaveLength(1); + }); + it("projects Codex task lifecycle chunks into thread activities", async () => { const harness = await createHarness(); const now = "2026-01-01T00:00:00.000Z"; diff --git a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts index 925e26236757a..ba5eb94bcc2f1 100644 --- a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts +++ b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts @@ -53,6 +53,11 @@ import { type ProviderRuntimeIngestionShape, } from "../Services/ProviderRuntimeIngestion.ts"; import { projectActivityPayload } from "../ActivityPayloadProjection.ts"; +import { + CODEX_COMPACTION_LOOP_LIMIT, + describeCompactionLoop, + recordCompaction, +} from "../CompactionLoopGuard.ts"; import { forkParked } from "../../serverActivation.ts"; import { ServerSettingsService } from "../../serverSettings.ts"; import { resolveProjectSettings } from "@t3tools/shared/projectSettings"; @@ -1036,6 +1041,49 @@ const make = Effect.gen(function* () { Effect.map((uuid) => CommandId.make(`provider:${event.eventId}:${tag}:${uuid}`)), ); + const compactionHistoryByThread = new Map>(); + const manualCompactionAtByThread = new Map(); + const guardCompactionLoop = Effect.fn("guardCompactionLoop")(function* ( + threadId: ThreadId, + event: ProviderRuntimeEvent, + activeTurnId: TurnId | null, + ) { + const previous = compactionHistoryByThread.get(threadId) ?? []; + const atMs = Date.parse(event.createdAt); + const result = recordCompaction(previous, atMs); + if (result.tripped && activeTurnId === null) { + compactionHistoryByThread.set(threadId, [ + ...previous.filter((at) => atMs - at < CODEX_COMPACTION_LOOP_LIMIT.windowMs), + atMs, + ]); + return; + } + compactionHistoryByThread.set(threadId, result.history); + if (!result.tripped || activeTurnId === null) return; + yield* orchestrationEngine.dispatch({ + type: "thread.activity.append", + commandId: yield* providerCommandId(event, "compaction-loop-stopped"), + threadId, + activity: { + id: EventId.make(`${event.eventId}:compaction-loop-stopped`), + tone: "error", + kind: "provider.compaction-loop.stopped", + summary: "Stopped a context compaction loop", + payload: { detail: describeCompactionLoop() }, + turnId: activeTurnId, + createdAt: event.createdAt, + }, + createdAt: event.createdAt, + }); + yield* orchestrationEngine.dispatch({ + type: "thread.turn.interrupt", + commandId: yield* providerCommandId(event, "compaction-loop-interrupt"), + threadId, + turnId: activeTurnId, + createdAt: event.createdAt, + }); + }); + // Post-start delivery: buffered `turn`-mode content and parent tool // heartbeats advance the in-memory observation without dispatching an // activity row, so an already-subscribed shell never refetches and can @@ -2723,11 +2771,27 @@ const make = Effect.gen(function* () { ), ), ).pipe(Effect.asVoid); + if (isCompactedThreadState && event.provider === "codex") { + const atMs = Date.parse(event.createdAt); + if (activityEvent.requestId !== undefined) { + manualCompactionAtByThread.set(thread.id, atMs); + } else { + const manualAt = manualCompactionAtByThread.get(thread.id); + const pairedManualSignal = + manualAt !== undefined && + Math.abs(atMs - manualAt) < CODEX_COMPACTION_LOOP_LIMIT.duplicateSignalMs; + if (!pairedManualSignal) { + yield* guardCompactionLoop(thread.id, event, activeTurnId); + } + } + } // The turn is over: drop the live observation so the settled shell does // not keep advertising provider progress. Only the accepted lifecycle // owner may clear: a delayed completion rejected for a superseded turn // must not erase the current turn's evidence. if (event.type === "session.exited") { + compactionHistoryByThread.delete(thread.id); + manualCompactionAtByThread.delete(thread.id); threadPostStartActivity.clearThread(thread.id); } else if (isTerminalTurn && shouldApplyThreadLifecycle) { threadPostStartActivity.clearThread(thread.id, observationTurnId);