From 7c6f479639eb9587529a68d6253221f983aa63de Mon Sep 17 00:00:00 2001 From: nullStack65 <112618179+nullStack65@users.noreply.github.com> Date: Thu, 19 Nov 2026 06:17:46 +0000 Subject: [PATCH 1/3] feat(server): one Lead alert per usage-limit incident, resume by itself, refuse launches on an exhausted provider Group usage-limit alerts by (environment, provider account) into one queued incident that lists threads and the reset time, merging late members instead of queuing more turns. Threads resume themselves at the reset (auto-resume now on by default); Lead hears only about a thread that does not come back. Drop alerts at delivery when the thread has a newer live run, is settled or archived, moved provider, or was replaced by its parent. Hold delivery while the watcher thread is out on the same account. Refuse delegate_task and thread launches on a provider known to be out of usage. Co-Authored-By: Claude Sonnet 5.5 --- apps/server/src/mcp/OrchestratorMcpService.ts | 1 + .../ModelAlertReactor.test.ts | 268 ++++++++++ .../src/orchestration-v2/ModelAlertReactor.ts | 506 +++++++++++++++++- .../orchestration-v2/ProviderExhaustion.ts | 33 ++ .../ThreadLaunchService.test.ts | 42 ++ .../orchestration-v2/ThreadLaunchService.ts | 1 + .../ThreadManagementService.ts | 32 +- .../orchestration-v2/UsageLimitIncidents.ts | 144 +++++ packages/contracts/src/settings.ts | 4 +- 9 files changed, 1006 insertions(+), 25 deletions(-) create mode 100644 apps/server/src/orchestration-v2/ProviderExhaustion.ts create mode 100644 apps/server/src/orchestration-v2/UsageLimitIncidents.ts diff --git a/apps/server/src/mcp/OrchestratorMcpService.ts b/apps/server/src/mcp/OrchestratorMcpService.ts index 3a774671e841..8698aa87b82b 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.ts @@ -1521,6 +1521,7 @@ const make = Effect.gen(function* () { projectId: parent.thread.projectId, commandId, instanceId: target.modelSelection.instanceId, + launch: true, }) .pipe(Effect.mapError(threadManagementFailure)); diff --git a/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts b/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts index 3744fb10861b..b37fc017c9e3 100644 --- a/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts +++ b/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts @@ -4,15 +4,24 @@ import * as Effect from "effect/Effect"; import { afterDeliveryAttempt, + afterDrop, + afterHold, deliverAlertToWatcher, + INCIDENT_GROUP_INTERVAL_MS, + INCIDENT_GROUP_WAIT_MS, EMPTY_MODEL_ALERT_STATE, type ModelAlert, modelAlertDestinations, modelAlertFor, + overdueResumes, recordModelAlert, + recordUsageLimit, + RESUME_GRACE_MS, recordThreadEvents, watcherAlertMessage, } from "./ModelAlertReactor.ts"; +import { ProviderExhaustion } from "./ProviderExhaustion.ts"; +import { alertIsStale, expectsAutoResume, replacedBySibling } from "./UsageLimitIncidents.ts"; import type { ThreadEventAlert, ThreadEventKind, ThreadFinding } from "./ThreadEventAlerts.ts"; const alertFor = (runId: string, overrides: Partial = {}): ModelAlert => ({ @@ -288,3 +297,262 @@ it.effect("forwards a thread event to a watcher on another environment with one }); }), ); + +const limitInput = ( + index: number, + overrides: Partial[1]> = {}, +) => ({ + member: { + threadId: `thread:${index}`, + threadTitle: `Lane ${index}`, + threadLink: `[Lane ${index}](t3-thread://v2/env/thread:${index})`, + runId: `run:${index}`, + }, + environmentId: "env", + environmentLabel: "primary-r720", + providerInstanceId: "claudeAgent", + resetAt: "2026-11-19T09:00:00.000Z", + resumeFailed: false, + willResume: false, + signature: `usage_limit:limit:run:${index}`, + occurredAt: "2026-11-19T04:00:00.000Z", + ...overrides, +}); + +const stormOf = ( + count: number, + overrides: Partial[1]> = {}, + destinations: ReadonlyArray<"watcher" | "webhook"> = ["watcher"], +) => { + let state = EMPTY_MODEL_ALERT_STATE; + for (let index = 0; index < count; index += 1) { + state = recordUsageLimit( + state, + limitInput(index, overrides), + destinations, + 1_000 + index, + ).state; + } + return state; +}; + +it("20 threads hitting one usage limit queue exactly one Lead-facing item", () => { + const state = stormOf(20); + assert.equal(state.outbox.length, 1); + const [entry] = state.outbox; + assert.equal(entry!.destination, "watcher"); + assert.equal(entry!.alert.type, "usage-limit-incident"); + if (entry!.alert.type !== "usage-limit-incident") return; + assert.equal(entry!.alert.members.length, 20); + assert.equal(entry!.alert.resetAt, "2026-11-19T09:00:00.000Z"); + // The first member waits for the rest of the storm. + assert.equal(entry!.nextAttemptAt, 1_000 + INCIDENT_GROUP_WAIT_MS); + const message = watcherAlertMessage(entry!.alert); + assert.include(message, "20 threads stopped"); + assert.include(message, "resets at 2026-11-19T09:00:00.000Z"); + assert.include(message, "[Lane 19]"); + assert.notInclude(message, "Model alert from"); +}); + +it("a repeat on the same run, or another run of an already listed thread, adds no item", () => { + let state = stormOf(3); + state = recordUsageLimit(state, limitInput(1), ["watcher"], 5_000).state; + state = recordUsageLimit( + state, + limitInput(1, { + signature: "usage_limit:limit:run:99", + member: { ...limitInput(1).member, runId: "run:99" }, + }), + ["watcher"], + 6_000, + ).state; + assert.equal(state.outbox.length, 1); + const alert = state.outbox[0]!.alert; + assert.equal(alert.type === "usage-limit-incident" && alert.members.length, 3); +}); + +it("a limit with auto-resume ahead sends Lead nothing, and a thread that comes back clears", () => { + let state = stormOf(20, { willResume: true }); + assert.equal(state.outbox.length, 0); + assert.equal(Object.keys(state.resuming).length, 20); + state = recordModelAlert( + state, + { alert: undefined, healthy: true, threadId: "thread:3" }, + ["watcher"], + 2_000, + ).state; + assert.equal(Object.keys(state.resuming).length, 19); + assert.equal(state.resuming["thread:3"], undefined); +}); + +it("threads that do not come back after the reset make one separate incident", () => { + let state = stormOf(20, { willResume: true }); + const nowMs = Date.parse("2026-11-19T09:00:00.000Z") + RESUME_GRACE_MS + 1; + assert.deepEqual(overdueResumes(state, nowMs - 2).length, 0); + const overdue = overdueResumes(state, nowMs); + assert.equal(overdue.length, 20); + for (const [threadId, entry] of overdue) { + const index = Number(threadId.split(":")[1]); + const { [threadId]: _done, ...resuming } = state.resuming; + state = recordUsageLimit( + { ...state, resuming }, + limitInput(index, { + resumeFailed: true, + signature: `usage_limit:resume-missed:${entry.runId}`, + }), + ["watcher"], + nowMs, + ).state; + } + assert.equal(state.outbox.length, 1); + const alert = state.outbox[0]!.alert; + assert.equal(alert.type === "usage-limit-incident" && alert.resumeFailed, true); + assert.include(watcherAlertMessage(alert), "did not come back after the limit reset"); +}); + +it("members joining while a digest is sent wait for the next one, and stragglers wait a group interval", () => { + const state = stormOf(2); + const sent = state.outbox[0]!; + const joined = recordUsageLimit(state, limitInput(2), ["watcher"], 40_000).state; + assert.equal(joined.outbox.length, 1); + const afterSend = afterDeliveryAttempt(joined, sent, true, 41_000); + assert.equal(afterSend.outbox.length, 1); + const rest = afterSend.outbox[0]!; + assert.equal(rest.alert.type === "usage-limit-incident" && rest.alert.members.length, 1); + assert.equal(rest.nextAttemptAt, 41_000 + INCIDENT_GROUP_INTERVAL_MS); + // Once nothing joined, the sent digest is gone and a later thread starts a delayed new item. + const quiet = afterDeliveryAttempt(state, sent, true, 41_000); + assert.equal(quiet.outbox.length, 0); + const later = recordUsageLimit(quiet, limitInput(7), ["watcher"], 50_000).state; + assert.equal(later.outbox.length, 1); + assert.equal(later.outbox[0]!.nextAttemptAt, 41_000 + INCIDENT_GROUP_INTERVAL_MS); + assert.notEqual(later.outbox[0]!.alert.id, sent.alert.id); +}); + +it("a held item waits without failing, and a dropped one leaves without counting as sent", () => { + const state = stormOf(2); + const entry = state.outbox[0]!; + const held = afterHold(state, entry, 999_000); + assert.deepEqual([held.outbox[0]!.attempts, held.outbox[0]!.nextAttemptAt], [0, 999_000]); + // Members still merge into a held item, so Lead gets one digest when it is back. + const merged = recordUsageLimit(held, limitInput(5), ["watcher"], 70_000).state; + assert.equal(merged.outbox.length, 1); + assert.equal(merged.outbox[0]!.nextAttemptAt, 999_000); + const dropped = afterDrop(merged, entry); + assert.equal(dropped.outbox.length, 0); + assert.deepEqual(dropped.incidentSentAt, {}); +}); + +it.each([ + ["the same run", { latestRunId: "run:1", activeRunId: null, status: "failed" }, false], + [ + "a newer run that is running", + { latestRunId: "run:2", activeRunId: "run:2", status: "running" }, + true, + ], + [ + "a newer run that completed", + { latestRunId: "run:2", activeRunId: null, status: "completed" }, + true, + ], + [ + "a newer run that failed again", + { latestRunId: "run:2", activeRunId: null, status: "failed" }, + false, + ], + ["an archived thread", { latestRunId: "run:1", status: "failed", archivedAt: "x" }, true], +] as const)("alert staleness at delivery: %s -> %s", (_name, shell, stale) => { + assert.equal( + alertIsStale( + { activeRunId: null, archivedAt: null, deletedAt: null, ...shell } as never, + "run:1", + ), + stale, + ); + assert.equal(alertIsStale(null, "run:1"), true); +}); + +it("auto-resume is expected only for a first limit on a resumable thread with a known reset", () => { + const shell = { + usageLimitResetAt: "2026-11-19T09:00:00.000Z", + archivedAt: null, + settledOverride: null, + pendingRuntimeRequest: null, + } as never; + const nowMs = Date.parse("2026-11-19T04:00:00.000Z"); + const expects = (overrides: object) => + expectsAutoResume({ autoResumeEnabled: true, runWasResume: false, shell, nowMs, ...overrides }); + assert.equal(expects({}), true); + assert.equal(expects({ autoResumeEnabled: false }), false); + assert.equal(expects({ runWasResume: true }), false); + assert.equal(expects({ shell: { ...(shell as object), usageLimitResetAt: null } }), false); + assert.equal(expects({ shell: { ...(shell as object), settledOverride: "settled" } }), false); +}); + +it.effect("delivers a 20-thread storm to Lead as one message", () => + Effect.gen(function* () { + const state = stormOf(20, {}, ["watcher", "webhook"]); + const dispatched: Array = []; + const deps = { + dispatch: (command: unknown) => Effect.sync(() => void dispatched.push(command)), + call: () => Effect.void, + sha256Hex: (identity: string) => Effect.succeed(`digest(${identity.length})`), + } as never; + const lead = { environmentId: EnvironmentId.make("env"), threadId: ThreadId.make("lead") }; + const due = state.outbox.filter((entry) => entry.destination === "watcher"); + for (const entry of due) { + yield* deliverAlertToWatcher(deps, entry.alert, lead, EnvironmentId.make("env")); + } + assert.equal(dispatched.length, 1); + assert.include((dispatched[0] as { text: string }).text, "20 threads stopped"); + }), +); + +it("a model alert is stale once its thread is settled or runs on another provider", () => { + const base = { + latestRunId: "run:1", + activeRunId: null, + status: "failed", + archivedAt: null, + deletedAt: null, + settledOverride: null, + providerInstanceId: "magpie-primary-codex", + } as never; + const about = { providerInstanceId: "magpie-primary-codex" }; + assert.equal(alertIsStale(base, "run:1", about), false); + assert.equal( + alertIsStale({ ...(base as object), settledOverride: "settled" } as never, "run:1", about), + true, + ); + assert.equal( + alertIsStale( + { ...(base as object), providerInstanceId: "claudeAgent" } as never, + "run:1", + about, + ), + true, + ); + // Thread events are not about a provider, and a settle is what some of them report. + assert.equal( + alertIsStale({ ...(base as object), settledOverride: "settled" } as never, "run:1"), + false, + ); +}); + +it("a delegated child is replaced when its parent started a later one with the same task title", () => { + assert.equal(replacedBySibling({ title: "Review PR 98" }, [{ title: " review pr 98 " }]), true); + assert.equal(replacedBySibling({ title: "Review PR 98" }, [{ title: "Build ASUS" }]), false); + assert.equal(replacedBySibling({ title: "Review PR 98" }, []), false); +}); + +it("a provider out of usage is tracked until its reset, and a healthy run clears it", () => { + ProviderExhaustion.clear(); + assert.equal(ProviderExhaustion.until("magpie-primary-codex", 1_000), null); + ProviderExhaustion.mark("magpie-primary-codex", 5_000); + assert.equal(ProviderExhaustion.until("magpie-primary-codex", 1_000), 5_000); + assert.equal(ProviderExhaustion.until("claudeAgent", 1_000), null); + assert.equal(ProviderExhaustion.until("magpie-primary-codex", 5_000), null); + ProviderExhaustion.mark("magpie-primary-codex", 9_000); + ProviderExhaustion.clear("magpie-primary-codex"); + assert.equal(ProviderExhaustion.until("magpie-primary-codex", 1_000), null); +}); diff --git a/apps/server/src/orchestration-v2/ModelAlertReactor.ts b/apps/server/src/orchestration-v2/ModelAlertReactor.ts index 71e7822885ef..b1b9b2a4df9f 100644 --- a/apps/server/src/orchestration-v2/ModelAlertReactor.ts +++ b/apps/server/src/orchestration-v2/ModelAlertReactor.ts @@ -6,10 +6,11 @@ import { type ModelAlertWatcher, type OrchestrationV2DomainEvent, type OrchestrationV2Run, + type OrchestrationV2ThreadShell, type OrchestrationV2ThreadProjection, OrchestrationV2WorkerAlertReason, OrchestratorMcpThreadSendResult, - type ThreadId, + ThreadId, } from "@t3tools/contracts"; import { runModelSignals, workerAlert } from "@t3tools/provider-core/server/subagentProjection"; import { writeFileStringAtomically } from "@t3tools/shared/atomicWrite"; @@ -39,6 +40,7 @@ import * as PeerMcpClient from "../peer/PeerMcpClient.ts"; import { forkParked } from "../serverActivation.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as Orchestrator from "./Orchestrator.ts"; +import { ProviderExhaustion } from "./ProviderExhaustion.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; import { judgeThread, @@ -49,6 +51,16 @@ import { type ThreadEventTrigger, type ThreadFinding, } from "./ThreadEventAlerts.ts"; +import { + alertIsStale, + expectsAutoResume, + incidentGroupKey, + incidentMessage, + type IncidentMember, + replacedBySibling, + UsageLimitIncident, + withIncidentMember, +} from "./UsageLimitIncidents.ts"; export const ModelAlert = Schema.Struct({ type: Schema.Literal("model-degraded"), @@ -70,8 +82,8 @@ export const ModelAlert = Schema.Struct({ }); export type ModelAlert = typeof ModelAlert.Type; -/** Everything the watcher thread hears: model alerts and deterministic thread events. */ -export const WatcherAlert = Schema.Union([ModelAlert, ThreadEventAlert]); +/** Everything the watcher thread hears: model alerts, thread events and usage limit incidents. */ +export const WatcherAlert = Schema.Union([ModelAlert, ThreadEventAlert, UsageLimitIncident]); export type WatcherAlert = typeof WatcherAlert.Type; const Destination = Schema.Literals(["watcher", "webhook"]); @@ -94,6 +106,18 @@ export const ModelAlertState = Schema.Struct({ reported: Schema.Record(Schema.String, Schema.String).pipe( Schema.withDecodingDefault(Effect.succeed({})), ), + /** + * Threads that hit a usage limit the server will resume at the reset. Lead hears of one only if + * it fails to come back. + */ + resuming: Schema.Record( + Schema.String, + Schema.Struct({ runId: Schema.String, resetAt: Schema.String, since: Schema.Number }), + ).pipe(Schema.withDecodingDefault(Effect.succeed({}))), + /** When each incident group last reached its destination, so later members wait a group interval. */ + incidentSentAt: Schema.Record(Schema.String, Schema.Number).pipe( + Schema.withDecodingDefault(Effect.succeed({})), + ), outbox: Schema.Array(Delivery), }); export type ModelAlertState = typeof ModelAlertState.Type; @@ -102,12 +126,23 @@ export const EMPTY_MODEL_ALERT_STATE: ModelAlertState = { version: 1, degraded: {}, reported: {}, + resuming: {}, + incidentSentAt: {}, outbox: [], }; const MAX_DEGRADED_THREADS = 1_000; const MAX_REPORTED = 2_000; const MAX_OUTBOX = 200; +const MAX_RESUMING = 2_000; +const MAX_INCIDENT_GROUPS = 200; +/** New incident members wait this long for the rest of the storm before the digest goes out. */ +export const INCIDENT_GROUP_WAIT_MS = 30_000; +/** A group that just reported waits this long before it reports its stragglers. */ +export const INCIDENT_GROUP_INTERVAL_MS = 5 * 60 * 1000; +/** How long past its reset a resumable thread may stay stopped before Lead hears it did not come back. */ +export const RESUME_GRACE_MS = 30 * 60 * 1000; +const HOLD_RECHECK_MS = 5 * 60 * 1000; const OUTBOX_TTL_MS = 7 * 24 * 60 * 60 * 1000; const RETRY_BASE_MS = 15_000; const RETRY_MAX_MS = 15 * 60 * 1000; @@ -190,11 +225,16 @@ export function recordModelAlert( ): { readonly state: ModelAlertState; readonly enqueued: boolean } { const { alert } = outcome; if (alert === undefined) { - if (!outcome.healthy || state.degraded[outcome.threadId] === undefined) { + if ( + !outcome.healthy || + (state.degraded[outcome.threadId] === undefined && + state.resuming[outcome.threadId] === undefined) + ) { return { state, enqueued: false }; } const { [outcome.threadId]: _cleared, ...degraded } = state.degraded; - return { state: { ...state, degraded }, enqueued: false }; + const { [outcome.threadId]: _resumed, ...resuming } = state.resuming; + return { state: { ...state, degraded, resuming }, enqueued: false }; } const next = signature(alert); if (state.degraded[alert.threadId] === next) return { state, enqueued: false }; @@ -288,18 +328,59 @@ const pruneOutbox = (outbox: ReadonlyArray, nowMs: number) => const sameDelivery = (left: Delivery, right: Delivery) => left.alert.id === right.alert.id && left.destination === right.destination; -/** The state after one delivery attempt: a delivered entry leaves, a failed one waits longer. */ +/** The members of a stored incident that a delivered snapshot did not carry. */ +const lateMembers = (stored: UsageLimitIncident, sent: UsageLimitIncident) => + stored.members.filter( + (member) => + !sent.members.some( + (known) => known.threadId === member.threadId && known.runId === member.runId, + ), + ); + +/** + * The state after one delivery attempt: a delivered entry leaves, a failed one waits longer. An + * incident that gained members while it was being sent keeps those for a later digest. + */ export function afterDeliveryAttempt( state: ModelAlertState, entry: Delivery, delivered: boolean, nowMs: number, ): ModelAlertState { + const sentIncident = delivered && entry.alert.type === "usage-limit-incident"; + const incidentSentAt = + sentIncident && entry.destination === "watcher" + ? Object.fromEntries( + Object.entries({ + ...state.incidentSentAt, + [entry.alert.groupKey]: nowMs, + }).slice(-MAX_INCIDENT_GROUPS), + ) + : state.incidentSentAt; return { ...state, + incidentSentAt, outbox: pruneOutbox(state.outbox, nowMs).flatMap((candidate) => { if (!sameDelivery(candidate, entry)) return [candidate]; - if (delivered) return []; + if (delivered) { + if ( + candidate.alert.type === "usage-limit-incident" && + entry.alert.type === "usage-limit-incident" + ) { + const late = lateMembers(candidate.alert, entry.alert); + if (late.length > 0) { + return [ + { + ...candidate, + alert: { ...candidate.alert, members: late }, + attempts: 0, + nextAttemptAt: nowMs + INCIDENT_GROUP_INTERVAL_MS, + }, + ]; + } + } + return []; + } const attempts = candidate.attempts + 1; return [ { @@ -312,6 +393,128 @@ export function afterDeliveryAttempt( }; } +/** An entry nobody needs any more leaves without counting as sent. */ +export const afterDrop = (state: ModelAlertState, entry: Delivery): ModelAlertState => ({ + ...state, + outbox: state.outbox.filter((candidate) => !sameDelivery(candidate, entry)), +}); + +/** An entry held back, not failed: it waits for `untilMs` without counting an attempt. */ +export function afterHold( + state: ModelAlertState, + entry: Delivery, + untilMs: number, +): ModelAlertState { + return { + ...state, + outbox: state.outbox.map((candidate) => + sameDelivery(candidate, entry) ? { ...candidate, nextAttemptAt: untilMs } : candidate, + ), + }; +} + +/** + * Records a thread that stopped at a usage limit. The server resumes it at the reset, so Lead + * hears nothing unless it does not come back; otherwise every thread of one provider account + * joins one queued incident, which waits briefly for the rest of the storm. + */ +export function recordUsageLimit( + state: ModelAlertState, + input: { + readonly member: IncidentMember; + readonly environmentId: string; + readonly environmentLabel: string; + readonly providerInstanceId: string; + readonly resetAt: string | null; + readonly resumeFailed: boolean; + readonly willResume: boolean; + /** Repeats of the same problem on the same run stay quiet. */ + readonly signature: string; + readonly occurredAt: string; + }, + destinations: ReadonlyArray, + nowMs: number, +): { readonly state: ModelAlertState; readonly enqueued: boolean } { + const { member } = input; + if (state.degraded[member.threadId] === input.signature) return { state, enqueued: false }; + const { [member.threadId]: _previous, ...restDegraded } = state.degraded; + const { [member.threadId]: _resumed, ...restResuming } = state.resuming; + const degraded = { + ...Object.fromEntries(Object.entries(restDegraded).slice(-(MAX_DEGRADED_THREADS - 1))), + [member.threadId]: input.signature, + }; + if (input.willResume && input.resetAt !== null) { + return { + state: { + ...state, + degraded, + resuming: { + ...Object.fromEntries(Object.entries(restResuming).slice(-(MAX_RESUMING - 1))), + [member.threadId]: { runId: member.runId, resetAt: input.resetAt, since: nowMs }, + }, + }, + enqueued: false, + }; + } + const groupKey = incidentGroupKey(input); + let outbox = pruneOutbox(state.outbox, nowMs); + let created = false; + for (const destination of destinations) { + const index = outbox.findIndex( + (entry) => + entry.destination === destination && + entry.alert.type === "usage-limit-incident" && + entry.alert.groupKey === groupKey, + ); + const queued = index === -1 ? undefined : outbox[index]; + if (queued !== undefined && queued.alert.type === "usage-limit-incident") { + outbox = outbox.map((entry, at) => + at === index + ? { ...queued, alert: withIncidentMember(queued.alert, member, input.resetAt) } + : entry, + ); + continue; + } + created = true; + const sentAt = state.incidentSentAt[groupKey]; + const notBefore = Math.max( + nowMs + INCIDENT_GROUP_WAIT_MS, + sentAt === undefined ? 0 : sentAt + INCIDENT_GROUP_INTERVAL_MS, + ); + outbox = [ + ...outbox, + { + destination, + alert: { + type: "usage-limit-incident", + id: `${groupKey}:${nowMs}`, + groupKey, + environmentId: input.environmentId, + environmentLabel: input.environmentLabel, + providerInstanceId: input.providerInstanceId, + resumeFailed: input.resumeFailed, + resetAt: input.resetAt, + members: [member], + occurredAt: input.occurredAt, + }, + attempts: 0, + nextAttemptAt: notBefore, + createdAt: nowMs, + }, + ].slice(-MAX_OUTBOX); + } + return { + state: { ...state, degraded, resuming: restResuming, outbox }, + enqueued: created, + }; +} + +/** Threads whose usage limit reset long enough ago that they should have been resumed. */ +export const overdueResumes = (state: ModelAlertState, nowMs: number) => + Object.entries(state.resuming).filter( + ([, entry]) => Date.parse(entry.resetAt) + RESUME_GRACE_MS < nowMs, + ); + const REASON_TEXT: Record string> = { model_substituted: (alert) => `it ran ${alert.executedModel ?? "another model"} instead of ${alert.selectedModel}`, @@ -330,8 +533,16 @@ function modelAlertMessage(alert: ModelAlert): string { ].join("\n\n"); } -export const watcherAlertMessage = (alert: WatcherAlert) => - alert.type === "thread-event" ? threadEventMessage(alert) : modelAlertMessage(alert); +export const watcherAlertMessage = (alert: WatcherAlert) => { + switch (alert.type) { + case "thread-event": + return threadEventMessage(alert); + case "usage-limit-incident": + return incidentMessage(alert); + case "model-degraded": + return modelAlertMessage(alert); + } +}; /** * Puts an alert in the watcher thread, here or on the linked machine that owns it. The retry key is @@ -350,7 +561,7 @@ export const deliverAlertToWatcher = Effect.fnUntraced(function* ( const message = watcherAlertMessage(alert); // Run ids can be long; a retry key must stay under the 256 characters the peer accepts. const identity = `${alert.id}\n${watcher.environmentId}\n${watcher.threadId}`; - const key = `${alert.type === "thread-event" ? "thread-event" : "model-alert"}:${yield* deps.sha256Hex(identity)}`; + const key = `${alert.type === "model-degraded" ? "model-alert" : alert.type}:${yield* deps.sha256Hex(identity)}`; if (watcher.environmentId === here) { yield* deps.dispatch({ type: "message.dispatch", @@ -450,21 +661,136 @@ export const make = Effect.gen(function* () { here, ); + const WatcherThreadStatus = Schema.Struct({ + thread: Schema.Struct({ status: Schema.String, providerInstanceId: Schema.String }), + }); + + /** + * Whether the parent already started a replacement for this delegated child: a later child of + * the same parent with the same task title. + */ + const replacedByParent = Effect.fnUntraced(function* (shell: OrchestrationV2ThreadShell) { + const parentId = shell.lineage.parentThreadId; + if (parentId === null || shell.lineage.relationshipToParent !== "subagent") return false; + const children = yield* projections.listAppOwnedChildThreadIds(parentId); + const at = children.indexOf(shell.id); + if (at === -1) return false; + const later = yield* Effect.forEach(children.slice(at + 1, at + 21), (id) => + projections.getThreadShell(id), + ); + return replacedBySibling( + shell, + later.flatMap((sibling) => (sibling === null ? [] : [sibling])), + ); + }); + + /** + * The alert as it stands now: threads that moved on leave it, and null means nothing is left to + * say. A thread moved on when it has a newer run that runs or finished, is settled, archived or + * on another provider, or its parent replaced it. + */ + const currentAlert = Effect.fnUntraced(function* (alert: WatcherAlert) { + const stillCurrent = Effect.fnUntraced(function* ( + threadId: string, + runId: string | null, + about: { readonly providerInstanceId: string } | undefined, + ) { + const shell = yield* projections.getThreadShell(ThreadId.make(threadId)); + if (alertIsStale(shell, runId, about)) return false; + return about === undefined || shell === null || !(yield* replacedByParent(shell)); + }); + if (alert.type === "usage-limit-incident") { + const fresh = yield* Effect.filter(alert.members, (member) => + stillCurrent(member.threadId, member.runId, { + providerInstanceId: alert.providerInstanceId, + }), + ); + if (fresh.length === 0) return null; + return fresh.length === alert.members.length ? alert : { ...alert, members: fresh }; + } + const about = + alert.type === "model-degraded" + ? { providerInstanceId: alert.providerInstanceId } + : undefined; + return (yield* stillCurrent(alert.threadId, alert.runId, about)) ? alert : null; + }); + + /** + * When the watcher's own thread is stopped at the same account limit, the time to look again. + * Anything sent now would only queue a turn that fails the same way. + */ + const holdUntil = Effect.fnUntraced(function* ( + watcher: ModelAlertWatcher, + here: EnvironmentId, + alert: WatcherAlert, + nowMs: number, + ) { + const providerInstanceId = + alert.type === "usage-limit-incident" || + (alert.type === "model-degraded" && alert.reasons.includes("usage_limit")) + ? alert.providerInstanceId + : null; + if (providerInstanceId === null) return null; + let down = false; + let resetAt: string | null = alert.type === "usage-limit-incident" ? alert.resetAt : null; + if (watcher.environmentId === here) { + const shell = yield* projections.getThreadShell(watcher.threadId); + down = + shell !== null && + shell.status === "failed" && + shell.providerInstanceId === providerInstanceId && + shell.lastErrorClass === "usage_limit"; + resetAt = shell?.usageLimitResetAt ?? resetAt; + } else { + down = yield* peers + .call({ + environmentId: watcher.environmentId, + tool: "t3_thread_read", + arguments: { threadId: watcher.threadId, limit: 1, runLimit: 1 }, + limits: { runtimeMode: "full-access", interactionMode: "default" }, + success: WatcherThreadStatus, + }) + .pipe( + Effect.map( + ({ thread }) => + thread.status === "failed" && thread.providerInstanceId === providerInstanceId, + ), + // An unreadable watcher is the delivery's problem, not a reason to hold. + Effect.orElseSucceed(() => false), + ); + } + if (!down) return null; + const resetMs = resetAt === null ? Number.NaN : Date.parse(resetAt); + const recheck = nowMs + HOLD_RECHECK_MS; + return Number.isFinite(resetMs) && resetMs > nowMs ? Math.min(resetMs, recheck) : recheck; + }); + + type DeliveryResult = + | { readonly _tag: "sent" } + | { readonly _tag: "dropped" } + | { readonly _tag: "held"; readonly until: number }; + const deliver = Effect.fnUntraced(function* (entry: Delivery) { const current = yield* settings.getSettings; + const alert = yield* currentAlert(entry.alert); + if (alert === null) return { _tag: "dropped" } satisfies DeliveryResult; if (entry.destination === "webhook") { const url = current.modelAlertWebhookUrl.trim(); - if (url === "") return; + if (url === "") return { _tag: "sent" } satisfies DeliveryResult; // A webhook URL can embed a credential, so its failure names only the host. const host = URL.canParse(url) ? new URL(url).host : "an unparsable URL"; yield* http - .execute(HttpClientRequest.post(url).pipe(HttpClientRequest.bodyJsonUnsafe(entry.alert))) + .execute(HttpClientRequest.post(url).pipe(HttpClientRequest.bodyJsonUnsafe(alert))) .pipe(Effect.catchCause(() => Effect.fail(`model alert webhook POST to ${host} failed`))); - return; + return { _tag: "sent" } satisfies DeliveryResult; } const watcher = current.modelAlertWatcher; - if (watcher === null) return; - yield* deliverToWatcher(entry.alert, watcher, yield* environment.getEnvironmentId); + if (watcher === null) return { _tag: "sent" } satisfies DeliveryResult; + const here = yield* environment.getEnvironmentId; + const until = yield* holdUntil(watcher, here, alert, yield* Clock.currentTimeMillis); + if (until !== null) return { _tag: "held", until } satisfies DeliveryResult; + yield* deliverToWatcher(alert, watcher, here); + return { _tag: "sent" } satisfies DeliveryResult; }); const flush: ModelAlertReactor["Service"]["flush"] = Effect.gen(function* () { @@ -474,28 +800,95 @@ export const make = Effect.gen(function* () { due, (entry) => Effect.gen(function* () { - const delivered = yield* deliver(entry).pipe( + const result = yield* deliver(entry).pipe( Effect.timeout("30 seconds"), - Effect.as(true), Effect.catchCause((cause) => Effect.logWarning("delivering a model alert failed; it will be retried", { alertId: entry.alert.id, destination: entry.destination, attempts: entry.attempts + 1, cause, - }).pipe(Effect.as(false)), + }).pipe(Effect.as(undefined)), ), ); const attemptedAt = yield* Clock.currentTimeMillis; yield* update((current) => [ undefined, - afterDeliveryAttempt(current, entry, delivered, attemptedAt), + result === undefined + ? afterDeliveryAttempt(current, entry, false, attemptedAt) + : result._tag === "held" + ? afterHold(current, entry, result.until) + : result._tag === "dropped" + ? afterDrop(current, entry) + : afterDeliveryAttempt(current, entry, true, attemptedAt), ]); }), { concurrency: 4, discard: true }, ); }); + /** Folds a thread that stopped at a usage limit into its provider account's one incident. */ + const handleUsageLimit = Effect.fnUntraced(function* ( + event: Extract, + alert: ModelAlert, + ) { + const current = yield* settings.getSettings; + const shell = yield* projections.getThreadShell(event.threadId); + const nowMs = yield* Clock.currentTimeMillis; + const resumeFailed = event.payload.usageLimitContinuationOfRunId !== undefined; + const resetMs = + shell?.usageLimitResetAt == null ? Number.NaN : Date.parse(shell.usageLimitResetAt); + // New launches on this provider are refused until the reset, so they alert nobody. + if (Number.isFinite(resetMs) && resetMs > nowMs) { + ProviderExhaustion.mark(alert.providerInstanceId, resetMs); + } + const willResume = expectsAutoResume({ + autoResumeEnabled: current.autoResumeLimitedThreads, + runWasResume: resumeFailed, + shell, + nowMs, + }); + const here = yield* environment.getEnvironmentId; + const destinations = modelAlertDestinations({ + watcher: current.modelAlertWatcher, + webhookUrl: current.modelAlertWebhookUrl, + here, + lineage: shell?.lineage ?? { parentThreadId: null, relationshipToParent: null }, + }); + const label = (yield* environment.getDescriptor).label; + const enqueued = yield* update((state) => { + const result = recordUsageLimit( + state, + { + member: { + threadId: alert.threadId, + threadTitle: alert.threadTitle, + threadLink: alert.threadLink, + runId: alert.runId, + }, + environmentId: here, + environmentLabel: label, + providerInstanceId: alert.providerInstanceId, + resetAt: shell?.usageLimitResetAt ?? null, + resumeFailed, + willResume, + signature: `usage_limit:${resumeFailed ? "resume-failed" : "limit"}:${alert.runId}`, + occurredAt: alert.occurredAt, + }, + destinations, + nowMs, + ); + return [result.enqueued, result.state]; + }); + if (willResume) { + yield* Effect.logInfo("a usage limit stopped a thread; it resumes by itself at the reset", { + threadId: alert.threadId, + resetAt: shell?.usageLimitResetAt, + }); + } + if (enqueued) yield* Queue.offer(wake, undefined); + }); + /** Queues a run's model alert, and says whether it reported a usage limit. */ const handleModelAlert = Effect.fnUntraced(function* ( event: Extract, @@ -519,7 +912,14 @@ export const make = Effect.gen(function* () { projection: records, occurredAt: DateTime.formatIso(yield* DateTime.now), }); + if (judged.alert === undefined && judged.healthy) { + ProviderExhaustion.clear(run.providerInstanceId); + } if (judged.alert === undefined && !judged.healthy) return { usageLimited: false }; + if (judged.alert?.reasons.every((reason) => reason === "usage_limit") === true) { + yield* handleUsageLimit(event, judged.alert); + return { usageLimited: true }; + } const outcome = judged.alert === undefined ? judged @@ -672,7 +1072,75 @@ export const make = Effect.gen(function* () { }); /** Looks at every thread that may be live for the time-based kinds no event announces. */ + /** + * A thread the server was to resume is still stopped long after its reset: that is the one + * usage limit outcome Lead hears about. + */ + const sweepResumes = Effect.gen(function* () { + const nowMs = yield* Clock.currentTimeMillis; + const overdue = overdueResumes(yield* Ref.get(state), nowMs); + if (overdue.length === 0) return; + const current = yield* settings.getSettings; + const here = yield* environment.getEnvironmentId; + const label = (yield* environment.getDescriptor).label; + const occurredAt = DateTime.formatIso(yield* DateTime.now); + yield* Effect.forEach( + overdue, + ([threadId, entry]) => + Effect.gen(function* () { + const shell = yield* projections.getThreadShell(ThreadId.make(threadId)); + const notResumed = + shell !== null && + shell.latestRunId === entry.runId && + shell.status === "failed" && + shell.archivedAt === null && + shell.settledOverride !== "settled"; + const destinations = modelAlertDestinations({ + watcher: current.modelAlertWatcher, + webhookUrl: current.modelAlertWebhookUrl, + here, + lineage: shell?.lineage ?? { parentThreadId: null, relationshipToParent: null }, + }); + const enqueued = yield* update((state) => { + const { [threadId]: _done, ...resuming } = state.resuming; + const cleared = { ...state, resuming }; + if (!notResumed) return [false, cleared]; + const result = recordUsageLimit( + cleared, + { + member: { + threadId, + threadTitle: shell.title, + threadLink: formatThreadLink( + { environmentId: here, threadId: shell.id }, + shell.title, + ), + runId: entry.runId, + }, + environmentId: here, + environmentLabel: label, + providerInstanceId: shell.providerInstanceId, + resetAt: entry.resetAt, + resumeFailed: true, + willResume: false, + signature: `usage_limit:resume-missed:${entry.runId}`, + occurredAt, + }, + destinations, + nowMs, + ); + return [result.enqueued, result.state]; + }); + if (enqueued) yield* Queue.offer(wake, undefined); + }), + { discard: true }, + ); + }); + const sweep: ModelAlertReactor["Service"]["sweep"] = Effect.gen(function* () { + yield* sweepResumes.pipe( + Effect.catchCause((cause) => Effect.logWarning("resume sweep failed", { cause })), + ); const index = yield* projections.listActiveShellIndex(); yield* Effect.forEach( index.filter((entry) => entry.mayBeLive && isListedThreadLineage(entry.lineage)), diff --git a/apps/server/src/orchestration-v2/ProviderExhaustion.ts b/apps/server/src/orchestration-v2/ProviderExhaustion.ts new file mode 100644 index 000000000000..abb46b008791 --- /dev/null +++ b/apps/server/src/orchestration-v2/ProviderExhaustion.ts @@ -0,0 +1,33 @@ +/** + * Provider instances this server has seen stop at a usage limit, and until when. A new launch on + * one is refused up front instead of starting a run that fails at once and alerts. In memory and + * process-wide on purpose: the first failure after a restart teaches it again, and every thread + * on the account shares one limit. + */ +const exhaustedUntil = new Map(); +const MAX_TRACKED = 100; + +export const ProviderExhaustion = { + mark(instanceId: string, untilMs: number): void { + exhaustedUntil.delete(instanceId); + exhaustedUntil.set(instanceId, untilMs); + for (const [key] of exhaustedUntil) { + if (exhaustedUntil.size <= MAX_TRACKED) break; + exhaustedUntil.delete(key); + } + }, + /** The epoch millisecond the instance is out until, or null when it is usable. */ + until(instanceId: string, nowMs: number): number | null { + const until = exhaustedUntil.get(instanceId); + if (until === undefined) return null; + if (until <= nowMs) { + exhaustedUntil.delete(instanceId); + return null; + } + return until; + }, + clear(instanceId?: string): void { + if (instanceId === undefined) exhaustedUntil.clear(); + else exhaustedUntil.delete(instanceId); + }, +}; diff --git a/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts b/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts index 3f9eb8c43997..56cd25c36310 100644 --- a/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts +++ b/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts @@ -65,6 +65,7 @@ import * as EffectOutbox from "./EffectOutbox.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts"; import * as ThreadLaunch from "./ThreadLaunchService.ts"; +import { ProviderExhaustion } from "./ProviderExhaustion.ts"; import * as GitVcsDriver from "../vcs/GitVcsDriver.ts"; import * as ThreadManagement from "./ThreadManagementService.ts"; import { OrchestratorProjectionError } from "./Orchestrator.ts"; @@ -1649,6 +1650,47 @@ it.effect("names the missing base branch when a worktree cannot be created from ), ); +it.effect("refuses 6 launches on a provider that is out of usage, and starts no thread", () => { + const harness = makeHarness({}); + return Effect.gen(function* () { + const launches = yield* ThreadLaunch.ThreadLaunchService; + const threads = yield* ThreadManagement.ThreadManagementService; + const before = (yield* threads.listProjectThreads({ projectId, includeSubagents: true })) + .length; + ProviderExhaustion.mark(modelSelection.instanceId, Date.now() + 60 * 60 * 1000); + const refusals = yield* Effect.forEach([1, 2, 3, 4, 5, 6], (index) => + launches + .launch( + launchInput({ + command: `command:launch:exhausted:${index}`, + thread: `thread:launch:exhausted:${index}`, + message: `Helper ${index}`, + }), + ) + .pipe(Effect.flip), + ); + for (const error of refusals) { + assert.equal(error._tag, "ThreadManagementProviderInstanceDisabledError"); + assert.match(error.message, /is out of usage until .*; pick another provider/); + } + const after = (yield* threads.listProjectThreads({ projectId, includeSubagents: true })).length; + assert.equal(after, before); + // Once the provider is usable again the same launch goes through. + ProviderExhaustion.clear(modelSelection.instanceId); + const launched = yield* launches.launch( + launchInput({ + command: "command:launch:exhausted:after", + thread: "thread:launch:exhausted:after", + message: "Helper again", + }), + ); + assert.equal(launched.threadId, ThreadId.make("thread:launch:exhausted:after")); + }).pipe( + Effect.ensuring(Effect.sync(() => ProviderExhaustion.clear())), + Effect.provide(harness.layer), + ); +}); + it.effect("rejects a preparation retry after the project disables its provider", () => { let fetchAttempts = 0; const harness = makeHarness({ diff --git a/apps/server/src/orchestration-v2/ThreadLaunchService.ts b/apps/server/src/orchestration-v2/ThreadLaunchService.ts index e02256286955..5810575fe67a 100644 --- a/apps/server/src/orchestration-v2/ThreadLaunchService.ts +++ b/apps/server/src/orchestration-v2/ThreadLaunchService.ts @@ -1218,6 +1218,7 @@ const make = Effect.gen(function* () { projectId: input.projectId, commandId: input.commandId, instanceId: input.modelSelection.instanceId, + launch: true, }); return yield* Effect.gen(function* () { // Recover ids from accepted launch receipts before allocating; retries have no client id and new ids collide with receipts. diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index 4d9b95b20d94..9891d07e3677 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -61,6 +61,7 @@ import * as Stream from "effect/Stream"; import * as CommandReceiptStore from "./CommandReceiptStore.ts"; import * as Orchestrator from "./Orchestrator.ts"; +import { ProviderExhaustion } from "./ProviderExhaustion.ts"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; import { projectedRowEncodedBytes } from "./threadHistoryPaging.ts"; import { threadPullRequestsOf } from "../sourceControl/threadPullRequests.ts"; @@ -341,10 +342,14 @@ export class ThreadManagementProviderInstanceDisabledError extends Schema.Tagged { projectId: ProjectId, instanceId: ProviderInstanceId, + /** Set when the instance is enabled but out of usage until then. */ + exhaustedUntil: Schema.optional(Schema.String), }, ) { override get message(): string { - return `Provider instance "${this.instanceId}" is disabled for project ${this.projectId}.`; + return this.exhaustedUntil === undefined + ? `Provider instance "${this.instanceId}" is disabled for project ${this.projectId}.` + : `Provider "${this.instanceId}" is out of usage until ${this.exhaustedUntil}; pick another provider.`; } } @@ -465,6 +470,8 @@ export interface ThreadManagementServiceShape { readonly projectId: ProjectId; readonly commandId: CommandId; readonly instanceId: ProviderInstanceId; + /** New work (a launch or a delegated task) is also refused while the provider is out of usage. */ + readonly launch?: boolean; }) => Effect.Effect; readonly isProviderInstanceEnabledForProject: (input: { readonly projectId: ProjectId; @@ -644,12 +651,26 @@ const make = Effect.gen(function* () { : serverSettings.getSettings.pipe( Effect.orDie, Effect.flatMap((current) => - resolveProjectProviderInstanceEnabled(current, input.projectId, input.instanceId) - ? Effect.void - : new ThreadManagementProviderInstanceDisabledError({ + !resolveProjectProviderInstanceEnabled(current, input.projectId, input.instanceId) + ? new ThreadManagementProviderInstanceDisabledError({ projectId: input.projectId, instanceId: input.instanceId, - }), + }) + : Effect.sync(() => + input.launch === true + ? ProviderExhaustion.until(input.instanceId, Date.now()) + : null, + ).pipe( + Effect.flatMap((until) => + until === null + ? Effect.void + : new ThreadManagementProviderInstanceDisabledError({ + projectId: input.projectId, + instanceId: input.instanceId, + exhaustedUntil: new Date(until).toISOString(), + }), + ), + ), ), ), ), @@ -852,6 +873,7 @@ const make = Effect.gen(function* () { projectId: parent.thread.projectId, commandId: command.commandId, instanceId: command.modelSelection.instanceId, + launch: true, }), ), ); diff --git a/apps/server/src/orchestration-v2/UsageLimitIncidents.ts b/apps/server/src/orchestration-v2/UsageLimitIncidents.ts new file mode 100644 index 000000000000..89ca4235f994 --- /dev/null +++ b/apps/server/src/orchestration-v2/UsageLimitIncidents.ts @@ -0,0 +1,144 @@ +import type { OrchestrationV2ThreadShell } from "@t3tools/contracts"; +import * as Schema from "effect/Schema"; + +/** + * One usage limit hits every thread on an account at once. The watcher hears one incident per + * (provider account, outcome), the way Alertmanager groups by labels, not one turn per thread. + * Delivery and state live in ModelAlertReactor. + */ +export const IncidentMember = Schema.Struct({ + threadId: Schema.String, + threadTitle: Schema.String, + threadLink: Schema.String, + runId: Schema.String, +}); +export type IncidentMember = typeof IncidentMember.Type; + +export const UsageLimitIncident = Schema.Struct({ + type: Schema.Literal("usage-limit-incident"), + /** Unique per queued item, so a later incident for the same group is a new message. */ + id: Schema.String, + /** What merges: environment, provider account and outcome. */ + groupKey: Schema.String, + environmentId: Schema.String, + environmentLabel: Schema.String, + providerInstanceId: Schema.String, + /** False: the threads stopped at the limit. True: they did not come back after the reset. */ + resumeFailed: Schema.Boolean, + resetAt: Schema.NullOr(Schema.String), + members: Schema.Array(IncidentMember), + occurredAt: Schema.String, +}); +export type UsageLimitIncident = typeof UsageLimitIncident.Type; + +export const incidentGroupKey = (input: { + readonly environmentId: string; + readonly providerInstanceId: string; + readonly resumeFailed: boolean; +}) => + `${input.environmentId}:usage-limit${input.resumeFailed ? "-resume-failed" : ""}:${input.providerInstanceId}`; + +const laterIso = (left: string | null, right: string | null) => { + if (left === null) return right; + if (right === null) return left; + return Date.parse(right) > Date.parse(left) ? right : left; +}; + +/** The incident with one more thread; a thread already in it keeps one row, at its newest run. */ +export function withIncidentMember( + incident: UsageLimitIncident, + member: IncidentMember, + resetAt: string | null, +): UsageLimitIncident { + return { + ...incident, + resetAt: laterIso(incident.resetAt, resetAt), + members: [...incident.members.filter((known) => known.threadId !== member.threadId), member], + }; +} + +type StaleShell = Pick< + OrchestrationV2ThreadShell, + | "latestRunId" + | "activeRunId" + | "status" + | "archivedAt" + | "deletedAt" + | "settledOverride" + | "providerInstanceId" +>; + +const LIVE_OR_DONE = new Set(["preparing", "starting", "running", "waiting", "completed"]); + +/** + * Whether an alert about a run is out of date: the thread has a newer run that is running or + * finished, or it is gone. A model alert is also out of date once the thread is settled or runs on + * another provider. Checked when the alert is delivered, not when it was judged. + */ +export function alertIsStale( + shell: StaleShell | null, + runId: string | null, + about?: { readonly providerInstanceId: string }, +): boolean { + if (shell === null || shell.deletedAt !== null || shell.archivedAt !== null) return true; + if (about !== undefined) { + if (shell.settledOverride === "settled") return true; + if (shell.providerInstanceId !== about.providerInstanceId) return true; + } + if (runId === null || shell.latestRunId === runId) return false; + return shell.activeRunId !== null || LIVE_OR_DONE.has(shell.status); +} + +/** + * A delegated child its parent replaced: a later sibling carries the same title, which is how + * delegate_task names the task. The replacement's own failure is its own alert. + */ +export function replacedBySibling( + child: { readonly title: string }, + later: ReadonlyArray<{ readonly title: string }>, +): boolean { + const title = child.title.trim().toLowerCase(); + return later.some((sibling) => sibling.title.trim().toLowerCase() === title); +} + +/** Whether the server will resume a thread by itself when its limit resets. */ +export function expectsAutoResume(input: { + readonly autoResumeEnabled: boolean; + /** The run was already a resume after an earlier reset: a second failure is news. */ + readonly runWasResume: boolean; + readonly shell: Pick< + OrchestrationV2ThreadShell, + "usageLimitResetAt" | "archivedAt" | "settledOverride" | "pendingRuntimeRequest" + > | null; + readonly nowMs: number; +}): boolean { + const { shell } = input; + if (!input.autoResumeEnabled || input.runWasResume || shell === null) return false; + if (shell.archivedAt !== null || shell.settledOverride === "settled") return false; + if (shell.pendingRuntimeRequest !== null) return false; + const resetMs = + shell.usageLimitResetAt == null ? Number.NaN : Date.parse(shell.usageLimitResetAt); + return Number.isFinite(resetMs) && resetMs > input.nowMs - 60_000; +} + +const MAX_LISTED = 25; + +export function incidentMessage(incident: UsageLimitIncident): string { + const count = incident.members.length; + const threads = `${count} thread${count === 1 ? "" : "s"}`; + const reset = + incident.resetAt === null + ? "The reset time is not reported." + : `It resets at ${incident.resetAt}.`; + const lead = incident.resumeFailed + ? `Usage limit incident on ${incident.providerInstanceId} (${incident.environmentLabel}, environment ${incident.environmentId}): ${threads} did not come back after the limit reset. ${reset}` + : `Usage limit incident on ${incident.providerInstanceId} (${incident.environmentLabel}, environment ${incident.environmentId}): ${threads} stopped at the provider's usage limit. ${reset}`; + const listed = incident.members + .slice(0, MAX_LISTED) + .map((member, index) => `${index + 1}. ${member.threadLink} (run ${member.runId})`); + if (count > MAX_LISTED) listed.push(`...and ${count - MAX_LISTED} more.`); + const next = incident.resumeFailed + ? "Automatic resume did not work for these. Resume them yourself, switch their provider, or replace them. This is the only alert for this incident." + : "Nothing was resumed automatically for these. Resume them after the reset, or switch their provider. This is the only alert for this incident; later threads that hit the same limit join a later digest."; + return [lead, listed.join("\n"), next].join("\n\n"); +} diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index c31e13fffc98..57d324d5879a 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -1512,7 +1512,9 @@ export const ServerSettings = Schema.Struct({ ), idleCompaction: Schema.optionalKey(IdleCompactionMode), snoozeLimitedThreads: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), - autoResumeLimitedThreads: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), + // Closura: a usage limit resets for every thread on the account at once. Resuming them is + // the server's job, so Lead is told only about a thread that does not come back. + autoResumeLimitedThreads: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))), snoozePresets: SnoozePresetSettings.pipe( Schema.withDecodingDefault(Effect.succeed(DEFAULT_SNOOZE_PRESETS)), ), From be52528cd3b573c71d5c9829b1286f050aaa9877 Mon Sep 17 00:00:00 2001 From: nullStack65 <112618179+nullStack65@users.noreply.github.com> Date: Thu, 19 Nov 2026 06:27:42 +0000 Subject: [PATCH 2/3] fix(server): type errors and Effect clock/date lint in grouped limit alerts Narrow the queued incident outside the map callback, keep the incident literal type, read the exhaustion time through Clock and format it with DateTime, and fix three test typings (alert runId union, readonly expected kinds, Clock in the launch refusal test). Model: Claude Sonnet 5.5 Task: fix Agent: claude Co-Authored-By: Claude Sonnet 5.5 --- .../src/orchestration-v2/ModelAlertReactor.test.ts | 5 ++++- .../src/orchestration-v2/ModelAlertReactor.ts | 7 ++++--- .../src/orchestration-v2/ThreadEventAlerts.test.ts | 2 +- .../orchestration-v2/ThreadLaunchService.test.ts | 6 +++++- .../src/orchestration-v2/ThreadManagementService.ts | 13 +++++++------ 5 files changed, 21 insertions(+), 12 deletions(-) diff --git a/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts b/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts index 469db3915e7d..2512f43dc9c2 100644 --- a/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts +++ b/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts @@ -71,7 +71,10 @@ it("alerts a thread that falls back every run once, until it recovers or changes step(alertFor("run:6")); assert.deepEqual(enqueued, [true, false, false, false, false, true, false, true]); assert.deepEqual( - state.outbox.map((entry) => [entry.alert.runId, entry.destination]), + state.outbox.map((entry) => [ + entry.alert.type === "model-degraded" ? entry.alert.runId : entry.alert.id, + entry.destination, + ]), [ ["run:1", "watcher"], ["run:1", "webhook"], diff --git a/apps/server/src/orchestration-v2/ModelAlertReactor.ts b/apps/server/src/orchestration-v2/ModelAlertReactor.ts index a10a708c70ca..d730f6038e22 100644 --- a/apps/server/src/orchestration-v2/ModelAlertReactor.ts +++ b/apps/server/src/orchestration-v2/ModelAlertReactor.ts @@ -474,10 +474,11 @@ export function recordUsageLimit( entry.alert.groupKey === groupKey, ); const queued = index === -1 ? undefined : outbox[index]; - if (queued !== undefined && queued.alert.type === "usage-limit-incident") { + const queuedAlert = queued?.alert; + if (queued !== undefined && queuedAlert?.type === "usage-limit-incident") { outbox = outbox.map((entry, at) => at === index - ? { ...queued, alert: withIncidentMember(queued.alert, member, input.resetAt) } + ? { ...queued, alert: withIncidentMember(queuedAlert, member, input.resetAt) } : entry, ); continue; @@ -493,7 +494,7 @@ export function recordUsageLimit( { destination, alert: { - type: "usage-limit-incident", + type: "usage-limit-incident" as const, id: `${groupKey}:${nowMs}`, groupKey, environmentId: input.environmentId, diff --git a/apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts b/apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts index 5dfb585dfd96..80abe7f06abf 100644 --- a/apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts +++ b/apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts @@ -96,7 +96,7 @@ it.each([ const { findings } = judge(shellWith(overrides as never), "run-ended"); assert.deepEqual( findings.filter((finding) => finding.kind === "lane-ended").map((finding) => finding.kind), - expected, + [...expected], ); }); diff --git a/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts b/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts index 56cd25c36310..42e214b8a0f5 100644 --- a/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts +++ b/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts @@ -33,6 +33,7 @@ import { } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; import * as Deferred from "effect/Deferred"; +import * as Clock from "effect/Clock"; import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; @@ -1657,7 +1658,10 @@ it.effect("refuses 6 launches on a provider that is out of usage, and starts no const threads = yield* ThreadManagement.ThreadManagementService; const before = (yield* threads.listProjectThreads({ projectId, includeSubagents: true })) .length; - ProviderExhaustion.mark(modelSelection.instanceId, Date.now() + 60 * 60 * 1000); + ProviderExhaustion.mark( + modelSelection.instanceId, + (yield* Clock.currentTimeMillis) + 60 * 60 * 1000, + ); const refusals = yield* Effect.forEach([1, 2, 3, 4, 5, 6], (index) => launches .launch( diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index 9891d07e3677..5d9eb0005297 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -656,18 +656,19 @@ const make = Effect.gen(function* () { projectId: input.projectId, instanceId: input.instanceId, }) - : Effect.sync(() => - input.launch === true - ? ProviderExhaustion.until(input.instanceId, Date.now()) - : null, - ).pipe( + : Clock.currentTimeMillis.pipe( + Effect.map((now) => + input.launch === true + ? ProviderExhaustion.until(input.instanceId, now) + : null, + ), Effect.flatMap((until) => until === null ? Effect.void : new ThreadManagementProviderInstanceDisabledError({ projectId: input.projectId, instanceId: input.instanceId, - exhaustedUntil: new Date(until).toISOString(), + exhaustedUntil: DateTime.formatIso(DateTime.makeUnsafe(until)), }), ), ), From c706093072b75bb66ee468d27608b7a8bf19345d Mon Sep 17 00:00:00 2001 From: nullStack65 <112618179+nullStack65@users.noreply.github.com> Date: Thu, 19 Nov 2026 06:35:50 +0000 Subject: [PATCH 3/3] fix(server): grouped limit alerts review fixes - A leftover digest gets its own id, so its delivery key is not a replay of the first. - A dropped incident keeps members that joined after its snapshot. - A watcher hold lasts only while the reset it reports is ahead, and a remote watcher no longer holds on any failure. - A sibling replaces a delegated child only when it is running or finished and not archived or deleted. - Auto-resume is not expected for a child of a closed parent or a thread snoozed past the grace period. Co-Authored-By: Claude Sonnet 5.5 --- .../ModelAlertReactor.test.ts | 43 ++++++++++++-- .../src/orchestration-v2/ModelAlertReactor.ts | 59 ++++++++++++++++--- .../orchestration-v2/UsageLimitIncidents.ts | 36 +++++++++-- 3 files changed, 120 insertions(+), 18 deletions(-) diff --git a/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts b/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts index 2512f43dc9c2..3f43eb49ad75 100644 --- a/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts +++ b/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts @@ -1,5 +1,6 @@ import { assert, it } from "@effect/vitest"; import { EnvironmentId, type OrchestrationV2Run, ThreadId } from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import { @@ -454,6 +455,8 @@ it("members joining while a digest is sent wait for the next one, and stragglers assert.equal(later.outbox.length, 1); assert.equal(later.outbox[0]!.nextAttemptAt, 41_000 + INCIDENT_GROUP_INTERVAL_MS); assert.notEqual(later.outbox[0]!.alert.id, sent.alert.id); + // The leftover digest has its own id, so its delivery key is not a replay of the first. + assert.notEqual(rest.alert.id, sent.alert.id); }); it("a held item waits without failing, and a dropped one leaves without counting as sent", () => { @@ -465,8 +468,16 @@ it("a held item waits without failing, and a dropped one leaves without counting const merged = recordUsageLimit(held, limitInput(5), ["watcher"], 70_000).state; assert.equal(merged.outbox.length, 1); assert.equal(merged.outbox[0]!.nextAttemptAt, 999_000); - const dropped = afterDrop(merged, entry); - assert.equal(dropped.outbox.length, 0); + // The snapshot was judged stale, but a thread joined after it: that one still reaches Lead. + const dropped = afterDrop(merged, entry, 80_000); + assert.equal(dropped.outbox.length, 1); + const rest = dropped.outbox[0]!.alert; + assert.deepEqual( + rest.type === "usage-limit-incident" && rest.members.map((member) => member.threadId), + ["thread:5"], + ); + assert.notEqual(rest.id, entry.alert.id); + assert.equal(afterDrop(held, entry, 80_000).outbox.length, 0); assert.deepEqual(dropped.incidentSentAt, {}); }); @@ -505,6 +516,7 @@ it("auto-resume is expected only for a first limit on a resumable thread with a archivedAt: null, settledOverride: null, pendingRuntimeRequest: null, + snoozedUntil: null, } as never; const nowMs = Date.parse("2026-11-19T04:00:00.000Z"); const expects = (overrides: object) => @@ -514,6 +526,14 @@ it("auto-resume is expected only for a first limit on a resumable thread with a assert.equal(expects({ runWasResume: true }), false); assert.equal(expects({ shell: { ...(shell as object), usageLimitResetAt: null } }), false); assert.equal(expects({ shell: { ...(shell as object), settledOverride: "settled" } }), false); + // The worker skips a child of a closed parent and waits out a snooze, so neither resumes in time. + assert.equal(expects({ parentClosed: true }), false); + const snoozedTo = (iso: string) => ({ + shell: { ...(shell as object), snoozedUntil: DateTime.makeUnsafe(iso) }, + graceMs: RESUME_GRACE_MS, + }); + assert.equal(expects(snoozedTo("2026-11-19T09:10:00.000Z")), true); + assert.equal(expects(snoozedTo("2026-11-19T12:00:00.000Z")), false); }); it.effect("delivers a 20-thread storm to Lead as one message", () => @@ -567,9 +587,22 @@ it("a model alert is stale once its thread is settled or runs on another provide }); it("a delegated child is replaced when its parent started a later one with the same task title", () => { - assert.equal(replacedBySibling({ title: "Review PR 98" }, [{ title: " review pr 98 " }]), true); - assert.equal(replacedBySibling({ title: "Review PR 98" }, [{ title: "Build ASUS" }]), false); - assert.equal(replacedBySibling({ title: "Review PR 98" }, []), false); + const sibling = (overrides: object) => + ({ + title: " review pr 98 ", + status: "running", + activeRunId: "run:2", + archivedAt: null, + deletedAt: null, + ...overrides, + }) as never; + const child = { title: "Review PR 98" }; + assert.equal(replacedBySibling(child, [sibling({})]), true); + assert.equal(replacedBySibling(child, [sibling({ title: "Build ASUS" })]), false); + assert.equal(replacedBySibling(child, []), false); + // Templated siblings that also failed, or are gone, replace nothing. + assert.equal(replacedBySibling(child, [sibling({ status: "failed", activeRunId: null })]), false); + assert.equal(replacedBySibling(child, [sibling({ archivedAt: "x" })]), false); }); it("a provider out of usage is tracked until its reset, and a healthy run clears it", () => { diff --git a/apps/server/src/orchestration-v2/ModelAlertReactor.ts b/apps/server/src/orchestration-v2/ModelAlertReactor.ts index d730f6038e22..0b48d5169e93 100644 --- a/apps/server/src/orchestration-v2/ModelAlertReactor.ts +++ b/apps/server/src/orchestration-v2/ModelAlertReactor.ts @@ -379,7 +379,12 @@ export function afterDeliveryAttempt( return [ { ...candidate, - alert: { ...candidate.alert, members: late }, + // A new id: the delivery key hashes it, and a second digest must not replay the first. + alert: { + ...candidate.alert, + id: `${candidate.alert.groupKey}:${nowMs}`, + members: late, + }, attempts: 0, nextAttemptAt: nowMs + INCIDENT_GROUP_INTERVAL_MS, }, @@ -400,10 +405,38 @@ export function afterDeliveryAttempt( }; } -/** An entry nobody needs any more leaves without counting as sent. */ -export const afterDrop = (state: ModelAlertState, entry: Delivery): ModelAlertState => ({ +/** + * An entry nobody needs any more leaves without counting as sent. An incident keeps the members + * that joined after the snapshot was judged, under a new id. + */ +export const afterDrop = ( + state: ModelAlertState, + entry: Delivery, + nowMs: number, +): ModelAlertState => ({ ...state, - outbox: state.outbox.filter((candidate) => !sameDelivery(candidate, entry)), + outbox: state.outbox.flatMap((candidate) => { + if (!sameDelivery(candidate, entry)) return [candidate]; + if ( + candidate.alert.type !== "usage-limit-incident" || + entry.alert.type !== "usage-limit-incident" + ) { + return []; + } + const late = lateMembers(candidate.alert, entry.alert); + return late.length === 0 + ? [] + : [ + { + ...candidate, + alert: { + ...candidate.alert, + id: `${candidate.alert.groupKey}:${nowMs}`, + members: late, + }, + }, + ]; + }), }); /** An entry held back, not failed: it waits for `untilMs` without counting an attempt. */ @@ -750,6 +783,8 @@ export const make = Effect.gen(function* () { shell.lastErrorClass === "usage_limit"; resetAt = shell?.usageLimitResetAt ?? resetAt; } else { + // The read result carries no error class, so a remote watcher counts as down only while + // the reset this alert reports is still ahead. down = yield* peers .call({ environmentId: watcher.environmentId, @@ -768,9 +803,10 @@ export const make = Effect.gen(function* () { ); } if (!down) return null; + // Past the reset the limit is over: whatever keeps the watcher down, a message may restart it. const resetMs = resetAt === null ? Number.NaN : Date.parse(resetAt); - const recheck = nowMs + HOLD_RECHECK_MS; - return Number.isFinite(resetMs) && resetMs > nowMs ? Math.min(resetMs, recheck) : recheck; + if (!Number.isFinite(resetMs) || resetMs <= nowMs) return null; + return Math.min(resetMs, nowMs + HOLD_RECHECK_MS); }); type DeliveryResult = @@ -827,7 +863,7 @@ export const make = Effect.gen(function* () { : result._tag === "held" ? afterHold(current, entry, result.until) : result._tag === "dropped" - ? afterDrop(current, entry) + ? afterDrop(current, entry, attemptedAt) : afterDeliveryAttempt(current, entry, true, attemptedAt), ]); }), @@ -850,11 +886,20 @@ export const make = Effect.gen(function* () { if (Number.isFinite(resetMs) && resetMs > nowMs) { ProviderExhaustion.mark(alert.providerInstanceId, resetMs); } + const parentId = shell?.lineage.parentThreadId ?? null; + const parent = + parentId !== null && shell?.lineage.relationshipToParent === "subagent" + ? yield* projections.getThreadShell(parentId) + : undefined; const willResume = expectsAutoResume({ autoResumeEnabled: current.autoResumeLimitedThreads, runWasResume: resumeFailed, shell, + parentClosed: + parent !== undefined && + (parent === null || parent.archivedAt !== null || parent.settledOverride === "settled"), nowMs, + graceMs: RESUME_GRACE_MS, }); const here = yield* environment.getEnvironmentId; const destinations = modelAlertDestinations({ diff --git a/apps/server/src/orchestration-v2/UsageLimitIncidents.ts b/apps/server/src/orchestration-v2/UsageLimitIncidents.ts index 89ca4235f994..9280b577bb81 100644 --- a/apps/server/src/orchestration-v2/UsageLimitIncidents.ts +++ b/apps/server/src/orchestration-v2/UsageLimitIncidents.ts @@ -1,4 +1,5 @@ import type { OrchestrationV2ThreadShell } from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; import * as Schema from "effect/Schema"; /** @@ -89,16 +90,28 @@ export function alertIsStale( return shell.activeRunId !== null || LIVE_OR_DONE.has(shell.status); } +type SiblingShell = Pick< + OrchestrationV2ThreadShell, + "title" | "status" | "activeRunId" | "archivedAt" | "deletedAt" +>; + /** * A delegated child its parent replaced: a later sibling carries the same title, which is how - * delegate_task names the task. The replacement's own failure is its own alert. + * delegate_task names the task, and is itself running or finished. A sibling that also failed, or + * is gone, replaces nothing: templated prompts share a title and each still needs its alert. */ export function replacedBySibling( child: { readonly title: string }, - later: ReadonlyArray<{ readonly title: string }>, + later: ReadonlyArray, ): boolean { const title = child.title.trim().toLowerCase(); - return later.some((sibling) => sibling.title.trim().toLowerCase() === title); + return later.some( + (sibling) => + sibling.title.trim().toLowerCase() === title && + sibling.archivedAt === null && + sibling.deletedAt === null && + (sibling.activeRunId !== null || LIVE_OR_DONE.has(sibling.status)), + ); } /** Whether the server will resume a thread by itself when its limit resets. */ @@ -108,17 +121,28 @@ export function expectsAutoResume(input: { readonly runWasResume: boolean; readonly shell: Pick< OrchestrationV2ThreadShell, - "usageLimitResetAt" | "archivedAt" | "settledOverride" | "pendingRuntimeRequest" + | "usageLimitResetAt" + | "archivedAt" + | "settledOverride" + | "pendingRuntimeRequest" + | "snoozedUntil" > | null; + /** The recovery worker never arms a delegated child whose parent is settled, archived or gone. */ + readonly parentClosed?: boolean; readonly nowMs: number; + /** How long past the reset the thread may stay stopped before it counts as not resumed. */ + readonly graceMs?: number; }): boolean { const { shell } = input; if (!input.autoResumeEnabled || input.runWasResume || shell === null) return false; if (shell.archivedAt !== null || shell.settledOverride === "settled") return false; - if (shell.pendingRuntimeRequest !== null) return false; + if (shell.pendingRuntimeRequest !== null || input.parentClosed === true) return false; const resetMs = shell.usageLimitResetAt == null ? Number.NaN : Date.parse(shell.usageLimitResetAt); - return Number.isFinite(resetMs) && resetMs > input.nowMs - 60_000; + if (!Number.isFinite(resetMs) || resetMs <= input.nowMs - 60_000) return false; + // The worker waits out a snooze, so one that outlasts the grace period means no resume in time. + const snoozedMs = shell.snoozedUntil == null ? null : DateTime.toEpochMillis(shell.snoozedUntil); + return snoozedMs === null || snoozedMs <= resetMs + (input.graceMs ?? 0); } const MAX_LISTED = 25;