From a5b04074d7f31af4b81ff0f48ad5b51a4ef0b7f3 Mon Sep 17 00:00:00 2001 From: nullStack65 <112618179+nullStack65@users.noreply.github.com> Date: Thu, 19 Nov 2026 17:30:28 +0000 Subject: [PATCH] fix(server): a model alert is not dropped because a newer run fell back too Delivery-time staleness treated any newer completed run as a fix. A thread that falls back on every run then lost its alert whenever a second run finished before delivery, which broke the delegated-completion model alert test on the release branch. A model alert now stays until a healthy run clears the thread's degraded mark, or the thread is settled, archived, deleted or on another provider. Co-Authored-By: Claude Sonnet 5.5 --- .../orchestration-v2/ModelAlertReactor.test.ts | 17 +++++++++++++++++ .../src/orchestration-v2/ModelAlertReactor.ts | 13 +++++++++++-- .../src/orchestration-v2/UsageLimitIncidents.ts | 7 ++++++- 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts b/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts index 3f43eb49ad75..3566598da64a 100644 --- a/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts +++ b/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts @@ -481,6 +481,23 @@ it("a held item waits without failing, and a dropped one leaves without counting assert.deepEqual(dropped.incidentSentAt, {}); }); +it("a model alert survives a newer run that fell back too, unlike a thread event", () => { + const shell = { + latestRunId: "run:2", + activeRunId: null, + status: "completed", + archivedAt: null, + deletedAt: null, + settledOverride: null, + providerInstanceId: "codex", + } as never; + assert.equal( + alertIsStale(shell, "run:1", { providerInstanceId: "codex", newerRunSupersedes: false }), + false, + ); + assert.equal(alertIsStale(shell, "run:1", { providerInstanceId: "codex" }), true); +}); + it.each([ ["the same run", { latestRunId: "run:1", activeRunId: null, status: "failed" }, false], [ diff --git a/apps/server/src/orchestration-v2/ModelAlertReactor.ts b/apps/server/src/orchestration-v2/ModelAlertReactor.ts index 0b48d5169e93..a4909326c652 100644 --- a/apps/server/src/orchestration-v2/ModelAlertReactor.ts +++ b/apps/server/src/orchestration-v2/ModelAlertReactor.ts @@ -734,7 +734,9 @@ export const make = Effect.gen(function* () { const stillCurrent = Effect.fnUntraced(function* ( threadId: string, runId: string | null, - about: { readonly providerInstanceId: string } | undefined, + about: + | { readonly providerInstanceId: string; readonly newerRunSupersedes?: boolean } + | undefined, ) { const shell = yield* projections.getThreadShell(ThreadId.make(threadId)); if (alertIsStale(shell, runId, about)) return false; @@ -749,9 +751,16 @@ export const make = Effect.gen(function* () { if (fresh.length === 0) return null; return fresh.length === alert.members.length ? alert : { ...alert, members: fresh }; } + // A healthy run clears the thread's degraded mark; until then a newer run is no fix. + if ( + alert.type === "model-degraded" && + (yield* Ref.get(state)).degraded[alert.threadId] === undefined + ) { + return null; + } const about = alert.type === "model-degraded" - ? { providerInstanceId: alert.providerInstanceId } + ? { providerInstanceId: alert.providerInstanceId, newerRunSupersedes: false } : undefined; return (yield* stillCurrent(alert.threadId, alert.runId, about)) ? alert : null; }); diff --git a/apps/server/src/orchestration-v2/UsageLimitIncidents.ts b/apps/server/src/orchestration-v2/UsageLimitIncidents.ts index 9280b577bb81..ec01a7013533 100644 --- a/apps/server/src/orchestration-v2/UsageLimitIncidents.ts +++ b/apps/server/src/orchestration-v2/UsageLimitIncidents.ts @@ -79,7 +79,11 @@ const LIVE_OR_DONE = new Set(["preparing", "starting", "running", "waiti export function alertIsStale( shell: StaleShell | null, runId: string | null, - about?: { readonly providerInstanceId: string }, + about?: { + readonly providerInstanceId: string; + /** False for a model alert: a newer run that fell back too is the same problem, not a fix. */ + readonly newerRunSupersedes?: boolean; + }, ): boolean { if (shell === null || shell.deletedAt !== null || shell.archivedAt !== null) return true; if (about !== undefined) { @@ -87,6 +91,7 @@ export function alertIsStale( if (shell.providerInstanceId !== about.providerInstanceId) return true; } if (runId === null || shell.latestRunId === runId) return false; + if (about?.newerRunSupersedes === false) return false; return shell.activeRunId !== null || LIVE_OR_DONE.has(shell.status); }