Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions apps/server/src/orchestration-v2/ModelAlertReactor.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -481,6 +481,23 @@ it("a held item waits without failing, and a dropped one leaves without counting
assert.deepEqual(dropped.incidentSentAt, {});
});

it("a model alert survives a newer run that fell back too, unlike a thread event", () => {
const shell = {
latestRunId: "run:2",
activeRunId: null,
status: "completed",
archivedAt: null,
deletedAt: null,
settledOverride: null,
providerInstanceId: "codex",
} as never;
assert.equal(
alertIsStale(shell, "run:1", { providerInstanceId: "codex", newerRunSupersedes: false }),
false,
);
assert.equal(alertIsStale(shell, "run:1", { providerInstanceId: "codex" }), true);
});

it.each([
["the same run", { latestRunId: "run:1", activeRunId: null, status: "failed" }, false],
[
Expand Down
13 changes: 11 additions & 2 deletions apps/server/src/orchestration-v2/ModelAlertReactor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -734,7 +734,9 @@ export const make = Effect.gen(function* () {
const stillCurrent = Effect.fnUntraced(function* (
threadId: string,
runId: string | null,
about: { readonly providerInstanceId: string } | undefined,
about:
| { readonly providerInstanceId: string; readonly newerRunSupersedes?: boolean }
| undefined,
) {
const shell = yield* projections.getThreadShell(ThreadId.make(threadId));
if (alertIsStale(shell, runId, about)) return false;
Expand All @@ -749,9 +751,16 @@ export const make = Effect.gen(function* () {
if (fresh.length === 0) return null;
return fresh.length === alert.members.length ? alert : { ...alert, members: fresh };
}
// A healthy run clears the thread's degraded mark; until then a newer run is no fix.
if (
alert.type === "model-degraded" &&
(yield* Ref.get(state)).degraded[alert.threadId] === undefined
) {
return null;
}
const about =
alert.type === "model-degraded"
? { providerInstanceId: alert.providerInstanceId }
? { providerInstanceId: alert.providerInstanceId, newerRunSupersedes: false }
: undefined;
return (yield* stillCurrent(alert.threadId, alert.runId, about)) ? alert : null;
});
Expand Down
7 changes: 6 additions & 1 deletion apps/server/src/orchestration-v2/UsageLimitIncidents.ts
Original file line number Diff line number Diff line change
Expand Up @@ -79,14 +79,19 @@ const LIVE_OR_DONE = new Set<string>(["preparing", "starting", "running", "waiti
export function alertIsStale(
shell: StaleShell | null,
runId: string | null,
about?: { readonly providerInstanceId: string },
about?: {
readonly providerInstanceId: string;
/** False for a model alert: a newer run that fell back too is the same problem, not a fix. */
readonly newerRunSupersedes?: boolean;
},
): boolean {
if (shell === null || shell.deletedAt !== null || shell.archivedAt !== null) return true;
if (about !== undefined) {
if (shell.settledOverride === "settled") return true;
if (shell.providerInstanceId !== about.providerInstanceId) return true;
}
if (runId === null || shell.latestRunId === runId) return false;
if (about?.newerRunSupersedes === false) return false;
return shell.activeRunId !== null || LIVE_OR_DONE.has(shell.status);
}

Expand Down
Loading