Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 46 additions & 0 deletions apps/server/src/orchestration/CompactionLoopGuard.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
import { describe, expect, it } from "vite-plus/test";

import { CODEX_COMPACTION_LOOP_LIMIT, recordCompaction } from "./CompactionLoopGuard.ts";

const replay = (offsetsMs: ReadonlyArray<number>) => {
let history: ReadonlyArray<number> = [];
return offsetsMs.map((offset) => {
const result = recordCompaction(history, offset);
history = result.history;
return result.tripped;
});
};

describe("recordCompaction", () => {
it("defaults to three compactions within five minutes", () => {
expect(CODEX_COMPACTION_LOOP_LIMIT.compactions).toBe(3);
expect(CODEX_COMPACTION_LOOP_LIMIT.windowMs).toBe(300_000);
});

it("trips on the third compaction of a thrash sequence", () => {
expect(replay([0, 40_000, 80_000])).toEqual([false, false, true]);
});

it("trips once per loop and again if the loop continues", () => {
expect(replay([0, 30_000, 60_000, 60_005, 90_000, 120_000])).toEqual([
false,
false,
true,
false,
false,
true,
]);
});

it("does not trip on normally spaced compactions", () => {
expect(replay([0, 6 * 60_000, 12 * 60_000, 18 * 60_000])).toEqual([false, false, false, false]);
});

it("drops compactions that fell out of the window", () => {
expect(replay([0, 200_000, 310_000, 400_000])).toEqual([false, false, false, true]);
});

it("counts the paired signals of one compaction once", () => {
expect(replay([0, 5, 60_000, 60_010])).toEqual([false, false, false, false]);
});
});
30 changes: 30 additions & 0 deletions apps/server/src/orchestration/CompactionLoopGuard.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
export const CODEX_COMPACTION_LOOP_LIMIT = {
compactions: 3,
windowMs: 5 * 60_000,
duplicateSignalMs: 2_000,
} as const;

export type CompactionLoopLimit = {
readonly compactions: number;
readonly windowMs: number;
readonly duplicateSignalMs: number;
};

export function recordCompaction(
history: ReadonlyArray<number>,
atMs: number,
limit: CompactionLoopLimit = CODEX_COMPACTION_LOOP_LIMIT,
): { readonly history: ReadonlyArray<number>; readonly tripped: boolean } {
const last = history.at(-1);
if (last !== undefined && Math.abs(atMs - last) < limit.duplicateSignalMs) {
return { history, tripped: false };
}
const recent = [...history.filter((at) => atMs - at < limit.windowMs), atMs];
const tripped = recent.length >= limit.compactions;
return { history: tripped ? [atMs] : recent, tripped };
}

export function describeCompactionLoop(limit: CompactionLoopLimit = CODEX_COMPACTION_LOOP_LIMIT) {
const minutes = limit.windowMs / 60_000;
return `Context compacted ${limit.compactions} times within ${minutes} minute${minutes === 1 ? "" : "s"}, so the turn was stopped instead of re-reading the full context again. It will not resume on its own. Send a new message, ideally with a narrower request, to continue.`;
}
111 changes: 111 additions & 0 deletions apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import {
TurnId,
} from "@t3tools/contracts";
import * as Clock from "effect/Clock";
import * as DateTime from "effect/DateTime";
import * as Deferred from "effect/Deferred";
import * as Effect from "effect/Effect";
import * as Exit from "effect/Exit";
Expand Down Expand Up @@ -5220,6 +5221,116 @@ describe("ProviderRuntimeIngestion", () => {
expect(activity?.payload).toMatchObject({ requestId: "message-compact" });
});

const replayCodexCompactions = async (
offsetsSeconds: ReadonlyArray<number | { readonly seconds: number; readonly manual: true }>,
options?: { readonly beforeTurn?: boolean },
) => {
const harness = await createHarness();
const threadId = asThreadId("thread-1");
const turnId = asTurnId("turn-compaction-loop");
const startedAt = DateTime.makeUnsafe("2026-01-01T00:00:00.000Z");
const at = (seconds: number) => DateTime.formatIso(DateTime.add(startedAt, { seconds }));
const compactionEvents = (phase: string) =>
offsetsSeconds.flatMap((offset, index) => {
const seconds = typeof offset === "number" ? offset : offset.seconds;
const manual = typeof offset !== "number";
return [0, 1].map((signal) => ({
type: "thread.state.changed" as const,
eventId: asEventId(`evt-compaction-loop-${phase}-${index}-${signal}`),
provider: ProviderDriverKind.make("codex"),
providerInstanceId: ProviderInstanceId.make("codex"),
createdAt: at(seconds),
threadId,
turnId,
...(manual && signal === 0
? { requestId: RuntimeRequestId.make(`manual-compact-${index}`) }
: {}),
payload: { state: "compacted" as const },
}));
});
const startTurn = () => {
harness.emit({
type: "turn.started",
eventId: asEventId("evt-compaction-loop-turn-started"),
provider: ProviderDriverKind.make("codex"),
threadId,
createdAt: at(0),
turnId,
});
return waitForThread(harness.readModel, (thread) => thread.session?.activeTurnId === turnId);
};
if (options?.beforeTurn) {
await harness.emitAndDrain(compactionEvents("before"));
await startTurn();
const last = offsetsSeconds.at(-1);
const lastSeconds = typeof last === "number" ? last : (last?.seconds ?? 0);
await harness.emitAndDrain(
[0, 1].map((signal) => ({
type: "thread.state.changed" as const,
eventId: asEventId(`evt-compaction-loop-after-${signal}`),
provider: ProviderDriverKind.make("codex"),
providerInstanceId: ProviderInstanceId.make("codex"),
createdAt: at(lastSeconds + 10),
threadId,
turnId,
payload: { state: "compacted" as const },
})),
);
} else {
await startTurn();
await harness.emitAndDrain(compactionEvents("during"));
}
const events = await Effect.runPromise(
Stream.runCollect(harness.engine.readEvents(0)).pipe(
Effect.map((chunk) => Array.from(chunk)),
),
);
const thread = (await harness.readModel()).threads.find((entry) => entry.id === threadId);
return {
turnId,
interrupts: events.filter((event) => event.type === "thread.turn-interrupt-requested"),
stops: (thread?.activities ?? []).filter(
(activity: ProviderRuntimeTestActivity) =>
activity.kind === "provider.compaction-loop.stopped",
),
};
};

it("interrupts a Codex turn that compacts three times within five minutes", async () => {
const result = await replayCodexCompactions([10, 50, 90]);
expect(result.interrupts).toHaveLength(1);
expect(result.interrupts[0]?.payload).toMatchObject({ turnId: result.turnId });
expect(result.stops).toHaveLength(1);
expect(result.stops[0]?.tone).toBe("error");
expect(result.stops[0]?.payload).toMatchObject({
detail: expect.stringContaining("will not resume on its own"),
});
});

it("lets a Codex turn with normally spaced compactions keep running", async () => {
const result = await replayCodexCompactions([10, 400, 800, 1_200]);
expect(result.interrupts).toHaveLength(0);
expect(result.stops).toHaveLength(0);
});

it("does not count a manual compact or its untagged pair", async () => {
const result = await replayCodexCompactions([
{ seconds: 10, manual: true },
{ seconds: 50, manual: true },
{ seconds: 90, manual: true },
100,
130,
]);
expect(result.interrupts).toHaveLength(0);
expect(result.stops).toHaveLength(0);
});

it("interrupts the next active turn after compactions that had no turn to stop", async () => {
const result = await replayCodexCompactions([10, 50, 90], { beforeTurn: true });
expect(result.interrupts).toHaveLength(1);
expect(result.stops).toHaveLength(1);
});

it("projects Codex task lifecycle chunks into thread activities", async () => {
const harness = await createHarness();
const now = "2026-01-01T00:00:00.000Z";
Expand Down
64 changes: 64 additions & 0 deletions apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,11 @@ import {
type ProviderRuntimeIngestionShape,
} from "../Services/ProviderRuntimeIngestion.ts";
import { projectActivityPayload } from "../ActivityPayloadProjection.ts";
import {
CODEX_COMPACTION_LOOP_LIMIT,
describeCompactionLoop,
recordCompaction,
} from "../CompactionLoopGuard.ts";
import { forkParked } from "../../serverActivation.ts";
import { ServerSettingsService } from "../../serverSettings.ts";
import { resolveProjectSettings } from "@t3tools/shared/projectSettings";
Expand Down Expand Up @@ -1036,6 +1041,49 @@ const make = Effect.gen(function* () {
Effect.map((uuid) => CommandId.make(`provider:${event.eventId}:${tag}:${uuid}`)),
);

const compactionHistoryByThread = new Map<ThreadId, ReadonlyArray<number>>();
const manualCompactionAtByThread = new Map<ThreadId, number>();
const guardCompactionLoop = Effect.fn("guardCompactionLoop")(function* (
threadId: ThreadId,
event: ProviderRuntimeEvent,
activeTurnId: TurnId | null,
) {
const previous = compactionHistoryByThread.get(threadId) ?? [];
const atMs = Date.parse(event.createdAt);
const result = recordCompaction(previous, atMs);
if (result.tripped && activeTurnId === null) {
compactionHistoryByThread.set(threadId, [
...previous.filter((at) => atMs - at < CODEX_COMPACTION_LOOP_LIMIT.windowMs),
atMs,
]);
return;
}
compactionHistoryByThread.set(threadId, result.history);
if (!result.tripped || activeTurnId === null) return;
yield* orchestrationEngine.dispatch({
type: "thread.activity.append",
commandId: yield* providerCommandId(event, "compaction-loop-stopped"),
threadId,
activity: {
id: EventId.make(`${event.eventId}:compaction-loop-stopped`),
tone: "error",
kind: "provider.compaction-loop.stopped",
summary: "Stopped a context compaction loop",
payload: { detail: describeCompactionLoop() },
turnId: activeTurnId,
createdAt: event.createdAt,
},
createdAt: event.createdAt,
});
yield* orchestrationEngine.dispatch({
type: "thread.turn.interrupt",
commandId: yield* providerCommandId(event, "compaction-loop-interrupt"),
threadId,
turnId: activeTurnId,
createdAt: event.createdAt,
});
});

// Post-start delivery: buffered `turn`-mode content and parent tool
// heartbeats advance the in-memory observation without dispatching an
// activity row, so an already-subscribed shell never refetches and can
Expand Down Expand Up @@ -2723,11 +2771,27 @@ const make = Effect.gen(function* () {
),
),
).pipe(Effect.asVoid);
if (isCompactedThreadState && event.provider === "codex") {
const atMs = Date.parse(event.createdAt);
if (activityEvent.requestId !== undefined) {
manualCompactionAtByThread.set(thread.id, atMs);
} else {
const manualAt = manualCompactionAtByThread.get(thread.id);
const pairedManualSignal =
manualAt !== undefined &&
Math.abs(atMs - manualAt) < CODEX_COMPACTION_LOOP_LIMIT.duplicateSignalMs;
if (!pairedManualSignal) {
yield* guardCompactionLoop(thread.id, event, activeTurnId);
}
}
}
// The turn is over: drop the live observation so the settled shell does
// not keep advertising provider progress. Only the accepted lifecycle
// owner may clear: a delayed completion rejected for a superseded turn
// must not erase the current turn's evidence.
if (event.type === "session.exited") {
compactionHistoryByThread.delete(thread.id);
manualCompactionAtByThread.delete(thread.id);
threadPostStartActivity.clearThread(thread.id);
} else if (isTerminalTurn && shouldApplyThreadLifecycle) {
threadPostStartActivity.clearThread(thread.id, observationTurnId);
Expand Down