diff --git a/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts b/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts index 3f43eb49ad75..3566598da64a 100644 --- a/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts +++ b/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts @@ -481,6 +481,23 @@ it("a held item waits without failing, and a dropped one leaves without counting assert.deepEqual(dropped.incidentSentAt, {}); }); +it("a model alert survives a newer run that fell back too, unlike a thread event", () => { + const shell = { + latestRunId: "run:2", + activeRunId: null, + status: "completed", + archivedAt: null, + deletedAt: null, + settledOverride: null, + providerInstanceId: "codex", + } as never; + assert.equal( + alertIsStale(shell, "run:1", { providerInstanceId: "codex", newerRunSupersedes: false }), + false, + ); + assert.equal(alertIsStale(shell, "run:1", { providerInstanceId: "codex" }), true); +}); + it.each([ ["the same run", { latestRunId: "run:1", activeRunId: null, status: "failed" }, false], [ diff --git a/apps/server/src/orchestration-v2/ModelAlertReactor.ts b/apps/server/src/orchestration-v2/ModelAlertReactor.ts index 0b48d5169e93..a4909326c652 100644 --- a/apps/server/src/orchestration-v2/ModelAlertReactor.ts +++ b/apps/server/src/orchestration-v2/ModelAlertReactor.ts @@ -734,7 +734,9 @@ export const make = Effect.gen(function* () { const stillCurrent = Effect.fnUntraced(function* ( threadId: string, runId: string | null, - about: { readonly providerInstanceId: string } | undefined, + about: + | { readonly providerInstanceId: string; readonly newerRunSupersedes?: boolean } + | undefined, ) { const shell = yield* projections.getThreadShell(ThreadId.make(threadId)); if (alertIsStale(shell, runId, about)) return false; @@ -749,9 +751,16 @@ export const make = Effect.gen(function* () { if (fresh.length === 0) return null; return fresh.length === alert.members.length ? alert : { ...alert, members: fresh }; } + // A healthy run clears the thread's degraded mark; until then a newer run is no fix. + if ( + alert.type === "model-degraded" && + (yield* Ref.get(state)).degraded[alert.threadId] === undefined + ) { + return null; + } const about = alert.type === "model-degraded" - ? { providerInstanceId: alert.providerInstanceId } + ? { providerInstanceId: alert.providerInstanceId, newerRunSupersedes: false } : undefined; return (yield* stillCurrent(alert.threadId, alert.runId, about)) ? alert : null; }); diff --git a/apps/server/src/orchestration-v2/UsageLimitIncidents.ts b/apps/server/src/orchestration-v2/UsageLimitIncidents.ts index 9280b577bb81..ec01a7013533 100644 --- a/apps/server/src/orchestration-v2/UsageLimitIncidents.ts +++ b/apps/server/src/orchestration-v2/UsageLimitIncidents.ts @@ -79,7 +79,11 @@ const LIVE_OR_DONE = new Set(["preparing", "starting", "running", "waiti export function alertIsStale( shell: StaleShell | null, runId: string | null, - about?: { readonly providerInstanceId: string }, + about?: { + readonly providerInstanceId: string; + /** False for a model alert: a newer run that fell back too is the same problem, not a fix. */ + readonly newerRunSupersedes?: boolean; + }, ): boolean { if (shell === null || shell.deletedAt !== null || shell.archivedAt !== null) return true; if (about !== undefined) { @@ -87,6 +91,7 @@ export function alertIsStale( if (shell.providerInstanceId !== about.providerInstanceId) return true; } if (runId === null || shell.latestRunId === runId) return false; + if (about?.newerRunSupersedes === false) return false; return shell.activeRunId !== null || LIVE_OR_DONE.has(shell.status); }