diff --git a/apps/server/src/mcp/OrchestratorMcpService.ts b/apps/server/src/mcp/OrchestratorMcpService.ts index 3a774671e841..8698aa87b82b 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.ts @@ -1521,6 +1521,7 @@ const make = Effect.gen(function* () { projectId: parent.thread.projectId, commandId, instanceId: target.modelSelection.instanceId, + launch: true, }) .pipe(Effect.mapError(threadManagementFailure)); diff --git a/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts b/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts index 5a84289a1842..3f43eb49ad75 100644 --- a/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts +++ b/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts @@ -1,18 +1,28 @@ import { assert, it } from "@effect/vitest"; import { EnvironmentId, type OrchestrationV2Run, ThreadId } from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import { afterDeliveryAttempt, + afterDrop, + afterHold, deliverAlertToWatcher, + INCIDENT_GROUP_INTERVAL_MS, + INCIDENT_GROUP_WAIT_MS, EMPTY_MODEL_ALERT_STATE, type ModelAlert, modelAlertDestinations, modelAlertFor, + overdueResumes, recordModelAlert, + recordUsageLimit, + RESUME_GRACE_MS, recordThreadEvents, watcherAlertMessage, } from "./ModelAlertReactor.ts"; +import { ProviderExhaustion } from "./ProviderExhaustion.ts"; +import { alertIsStale, expectsAutoResume, replacedBySibling } from "./UsageLimitIncidents.ts"; import type { ThreadEventAlert, ThreadEventKind, ThreadFinding } from "./ThreadEventAlerts.ts"; const alertFor = (runId: string, overrides: Partial = {}): ModelAlert => ({ @@ -62,7 +72,10 @@ it("alerts a thread that falls back every run once, until it recovers or changes step(alertFor("run:6")); assert.deepEqual(enqueued, [true, false, false, false, false, true, false, true]); assert.deepEqual( - state.outbox.map((entry) => [entry.alert.runId, entry.destination]), + state.outbox.map((entry) => [ + entry.alert.type === "model-degraded" ? entry.alert.runId : entry.alert.id, + entry.destination, + ]), [ ["run:1", "watcher"], ["run:1", "webhook"], @@ -312,3 +325,294 @@ it.effect("forwards a thread event to a watcher on another environment with one }); }), ); + +const limitInput = ( + index: number, + overrides: Partial[1]> = {}, +) => ({ + member: { + threadId: `thread:${index}`, + threadTitle: `Lane ${index}`, + threadLink: `[Lane ${index}](t3-thread://v2/env/thread:${index})`, + runId: `run:${index}`, + }, + environmentId: "env", + environmentLabel: "primary-r720", + providerInstanceId: "claudeAgent", + resetAt: "2026-11-19T09:00:00.000Z", + resumeFailed: false, + willResume: false, + signature: `usage_limit:limit:run:${index}`, + occurredAt: "2026-11-19T04:00:00.000Z", + ...overrides, +}); + +const stormOf = ( + count: number, + overrides: Partial[1]> = {}, + destinations: ReadonlyArray<"watcher" | "webhook"> = ["watcher"], +) => { + let state = EMPTY_MODEL_ALERT_STATE; + for (let index = 0; index < count; index += 1) { + state = recordUsageLimit( + state, + limitInput(index, overrides), + destinations, + 1_000 + index, + ).state; + } + return state; +}; + +it("20 threads hitting one usage limit queue exactly one Lead-facing item", () => { + const state = stormOf(20); + assert.equal(state.outbox.length, 1); + const [entry] = state.outbox; + assert.equal(entry!.destination, "watcher"); + assert.equal(entry!.alert.type, "usage-limit-incident"); + if (entry!.alert.type !== "usage-limit-incident") return; + assert.equal(entry!.alert.members.length, 20); + assert.equal(entry!.alert.resetAt, "2026-11-19T09:00:00.000Z"); + // The first member waits for the rest of the storm. + assert.equal(entry!.nextAttemptAt, 1_000 + INCIDENT_GROUP_WAIT_MS); + const message = watcherAlertMessage(entry!.alert); + assert.include(message, "20 threads stopped"); + assert.include(message, "resets at 2026-11-19T09:00:00.000Z"); + assert.include(message, "[Lane 19]"); + assert.notInclude(message, "Model alert from"); +}); + +it("a repeat on the same run, or another run of an already listed thread, adds no item", () => { + let state = stormOf(3); + state = recordUsageLimit(state, limitInput(1), ["watcher"], 5_000).state; + state = recordUsageLimit( + state, + limitInput(1, { + signature: "usage_limit:limit:run:99", + member: { ...limitInput(1).member, runId: "run:99" }, + }), + ["watcher"], + 6_000, + ).state; + assert.equal(state.outbox.length, 1); + const alert = state.outbox[0]!.alert; + assert.equal(alert.type === "usage-limit-incident" && alert.members.length, 3); +}); + +it("a limit with auto-resume ahead sends Lead nothing, and a thread that comes back clears", () => { + let state = stormOf(20, { willResume: true }); + assert.equal(state.outbox.length, 0); + assert.equal(Object.keys(state.resuming).length, 20); + state = recordModelAlert( + state, + { alert: undefined, healthy: true, threadId: "thread:3" }, + ["watcher"], + 2_000, + ).state; + assert.equal(Object.keys(state.resuming).length, 19); + assert.equal(state.resuming["thread:3"], undefined); +}); + +it("threads that do not come back after the reset make one separate incident", () => { + let state = stormOf(20, { willResume: true }); + const nowMs = Date.parse("2026-11-19T09:00:00.000Z") + RESUME_GRACE_MS + 1; + assert.deepEqual(overdueResumes(state, nowMs - 2).length, 0); + const overdue = overdueResumes(state, nowMs); + assert.equal(overdue.length, 20); + for (const [threadId, entry] of overdue) { + const index = Number(threadId.split(":")[1]); + const { [threadId]: _done, ...resuming } = state.resuming; + state = recordUsageLimit( + { ...state, resuming }, + limitInput(index, { + resumeFailed: true, + signature: `usage_limit:resume-missed:${entry.runId}`, + }), + ["watcher"], + nowMs, + ).state; + } + assert.equal(state.outbox.length, 1); + const alert = state.outbox[0]!.alert; + assert.equal(alert.type === "usage-limit-incident" && alert.resumeFailed, true); + assert.include(watcherAlertMessage(alert), "did not come back after the limit reset"); +}); + +it("members joining while a digest is sent wait for the next one, and stragglers wait a group interval", () => { + const state = stormOf(2); + const sent = state.outbox[0]!; + const joined = recordUsageLimit(state, limitInput(2), ["watcher"], 40_000).state; + assert.equal(joined.outbox.length, 1); + const afterSend = afterDeliveryAttempt(joined, sent, true, 41_000); + assert.equal(afterSend.outbox.length, 1); + const rest = afterSend.outbox[0]!; + assert.equal(rest.alert.type === "usage-limit-incident" && rest.alert.members.length, 1); + assert.equal(rest.nextAttemptAt, 41_000 + INCIDENT_GROUP_INTERVAL_MS); + // Once nothing joined, the sent digest is gone and a later thread starts a delayed new item. + const quiet = afterDeliveryAttempt(state, sent, true, 41_000); + assert.equal(quiet.outbox.length, 0); + const later = recordUsageLimit(quiet, limitInput(7), ["watcher"], 50_000).state; + assert.equal(later.outbox.length, 1); + assert.equal(later.outbox[0]!.nextAttemptAt, 41_000 + INCIDENT_GROUP_INTERVAL_MS); + assert.notEqual(later.outbox[0]!.alert.id, sent.alert.id); + // The leftover digest has its own id, so its delivery key is not a replay of the first. + assert.notEqual(rest.alert.id, sent.alert.id); +}); + +it("a held item waits without failing, and a dropped one leaves without counting as sent", () => { + const state = stormOf(2); + const entry = state.outbox[0]!; + const held = afterHold(state, entry, 999_000); + assert.deepEqual([held.outbox[0]!.attempts, held.outbox[0]!.nextAttemptAt], [0, 999_000]); + // Members still merge into a held item, so Lead gets one digest when it is back. + const merged = recordUsageLimit(held, limitInput(5), ["watcher"], 70_000).state; + assert.equal(merged.outbox.length, 1); + assert.equal(merged.outbox[0]!.nextAttemptAt, 999_000); + // The snapshot was judged stale, but a thread joined after it: that one still reaches Lead. + const dropped = afterDrop(merged, entry, 80_000); + assert.equal(dropped.outbox.length, 1); + const rest = dropped.outbox[0]!.alert; + assert.deepEqual( + rest.type === "usage-limit-incident" && rest.members.map((member) => member.threadId), + ["thread:5"], + ); + assert.notEqual(rest.id, entry.alert.id); + assert.equal(afterDrop(held, entry, 80_000).outbox.length, 0); + assert.deepEqual(dropped.incidentSentAt, {}); +}); + +it.each([ + ["the same run", { latestRunId: "run:1", activeRunId: null, status: "failed" }, false], + [ + "a newer run that is running", + { latestRunId: "run:2", activeRunId: "run:2", status: "running" }, + true, + ], + [ + "a newer run that completed", + { latestRunId: "run:2", activeRunId: null, status: "completed" }, + true, + ], + [ + "a newer run that failed again", + { latestRunId: "run:2", activeRunId: null, status: "failed" }, + false, + ], + ["an archived thread", { latestRunId: "run:1", status: "failed", archivedAt: "x" }, true], +] as const)("alert staleness at delivery: %s -> %s", (_name, shell, stale) => { + assert.equal( + alertIsStale( + { activeRunId: null, archivedAt: null, deletedAt: null, ...shell } as never, + "run:1", + ), + stale, + ); + assert.equal(alertIsStale(null, "run:1"), true); +}); + +it("auto-resume is expected only for a first limit on a resumable thread with a known reset", () => { + const shell = { + usageLimitResetAt: "2026-11-19T09:00:00.000Z", + archivedAt: null, + settledOverride: null, + pendingRuntimeRequest: null, + snoozedUntil: null, + } as never; + const nowMs = Date.parse("2026-11-19T04:00:00.000Z"); + const expects = (overrides: object) => + expectsAutoResume({ autoResumeEnabled: true, runWasResume: false, shell, nowMs, ...overrides }); + assert.equal(expects({}), true); + assert.equal(expects({ autoResumeEnabled: false }), false); + assert.equal(expects({ runWasResume: true }), false); + assert.equal(expects({ shell: { ...(shell as object), usageLimitResetAt: null } }), false); + assert.equal(expects({ shell: { ...(shell as object), settledOverride: "settled" } }), false); + // The worker skips a child of a closed parent and waits out a snooze, so neither resumes in time. + assert.equal(expects({ parentClosed: true }), false); + const snoozedTo = (iso: string) => ({ + shell: { ...(shell as object), snoozedUntil: DateTime.makeUnsafe(iso) }, + graceMs: RESUME_GRACE_MS, + }); + assert.equal(expects(snoozedTo("2026-11-19T09:10:00.000Z")), true); + assert.equal(expects(snoozedTo("2026-11-19T12:00:00.000Z")), false); +}); + +it.effect("delivers a 20-thread storm to Lead as one message", () => + Effect.gen(function* () { + const state = stormOf(20, {}, ["watcher", "webhook"]); + const dispatched: Array = []; + const deps = { + dispatch: (command: unknown) => Effect.sync(() => void dispatched.push(command)), + call: () => Effect.void, + sha256Hex: (identity: string) => Effect.succeed(`digest(${identity.length})`), + } as never; + const lead = { environmentId: EnvironmentId.make("env"), threadId: ThreadId.make("lead") }; + const due = state.outbox.filter((entry) => entry.destination === "watcher"); + for (const entry of due) { + yield* deliverAlertToWatcher(deps, entry.alert, lead, EnvironmentId.make("env")); + } + assert.equal(dispatched.length, 1); + assert.include((dispatched[0] as { text: string }).text, "20 threads stopped"); + }), +); + +it("a model alert is stale once its thread is settled or runs on another provider", () => { + const base = { + latestRunId: "run:1", + activeRunId: null, + status: "failed", + archivedAt: null, + deletedAt: null, + settledOverride: null, + providerInstanceId: "magpie-primary-codex", + } as never; + const about = { providerInstanceId: "magpie-primary-codex" }; + assert.equal(alertIsStale(base, "run:1", about), false); + assert.equal( + alertIsStale({ ...(base as object), settledOverride: "settled" } as never, "run:1", about), + true, + ); + assert.equal( + alertIsStale( + { ...(base as object), providerInstanceId: "claudeAgent" } as never, + "run:1", + about, + ), + true, + ); + // Thread events are not about a provider, and a settle is what some of them report. + assert.equal( + alertIsStale({ ...(base as object), settledOverride: "settled" } as never, "run:1"), + false, + ); +}); + +it("a delegated child is replaced when its parent started a later one with the same task title", () => { + const sibling = (overrides: object) => + ({ + title: " review pr 98 ", + status: "running", + activeRunId: "run:2", + archivedAt: null, + deletedAt: null, + ...overrides, + }) as never; + const child = { title: "Review PR 98" }; + assert.equal(replacedBySibling(child, [sibling({})]), true); + assert.equal(replacedBySibling(child, [sibling({ title: "Build ASUS" })]), false); + assert.equal(replacedBySibling(child, []), false); + // Templated siblings that also failed, or are gone, replace nothing. + assert.equal(replacedBySibling(child, [sibling({ status: "failed", activeRunId: null })]), false); + assert.equal(replacedBySibling(child, [sibling({ archivedAt: "x" })]), false); +}); + +it("a provider out of usage is tracked until its reset, and a healthy run clears it", () => { + ProviderExhaustion.clear(); + assert.equal(ProviderExhaustion.until("magpie-primary-codex", 1_000), null); + ProviderExhaustion.mark("magpie-primary-codex", 5_000); + assert.equal(ProviderExhaustion.until("magpie-primary-codex", 1_000), 5_000); + assert.equal(ProviderExhaustion.until("claudeAgent", 1_000), null); + assert.equal(ProviderExhaustion.until("magpie-primary-codex", 5_000), null); + ProviderExhaustion.mark("magpie-primary-codex", 9_000); + ProviderExhaustion.clear("magpie-primary-codex"); + assert.equal(ProviderExhaustion.until("magpie-primary-codex", 1_000), null); +}); diff --git a/apps/server/src/orchestration-v2/ModelAlertReactor.ts b/apps/server/src/orchestration-v2/ModelAlertReactor.ts index 1e461f9def0d..0b48d5169e93 100644 --- a/apps/server/src/orchestration-v2/ModelAlertReactor.ts +++ b/apps/server/src/orchestration-v2/ModelAlertReactor.ts @@ -6,10 +6,11 @@ import { type ModelAlertWatcher, type OrchestrationV2DomainEvent, type OrchestrationV2Run, + type OrchestrationV2ThreadShell, type OrchestrationV2ThreadProjection, OrchestrationV2WorkerAlertReason, OrchestratorMcpThreadSendResult, - type ThreadId, + ThreadId, } from "@t3tools/contracts"; import { runModelSignals, workerAlert } from "@t3tools/provider-core/server/subagentProjection"; import { writeFileStringAtomically } from "@t3tools/shared/atomicWrite"; @@ -39,6 +40,7 @@ import * as PeerMcpClient from "../peer/PeerMcpClient.ts"; import { forkParked } from "../serverActivation.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as Orchestrator from "./Orchestrator.ts"; +import { ProviderExhaustion } from "./ProviderExhaustion.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; import { judgeThread, @@ -49,6 +51,16 @@ import { type ThreadEventTrigger, type ThreadFinding, } from "./ThreadEventAlerts.ts"; +import { + alertIsStale, + expectsAutoResume, + incidentGroupKey, + incidentMessage, + type IncidentMember, + replacedBySibling, + UsageLimitIncident, + withIncidentMember, +} from "./UsageLimitIncidents.ts"; export const ModelAlert = Schema.Struct({ type: Schema.Literal("model-degraded"), @@ -70,8 +82,8 @@ export const ModelAlert = Schema.Struct({ }); export type ModelAlert = typeof ModelAlert.Type; -/** Everything the watcher thread hears: model alerts and deterministic thread events. */ -export const WatcherAlert = Schema.Union([ModelAlert, ThreadEventAlert]); +/** Everything the watcher thread hears: model alerts, thread events and usage limit incidents. */ +export const WatcherAlert = Schema.Union([ModelAlert, ThreadEventAlert, UsageLimitIncident]); export type WatcherAlert = typeof WatcherAlert.Type; const Destination = Schema.Literals(["watcher", "webhook"]); @@ -94,6 +106,18 @@ export const ModelAlertState = Schema.Struct({ reported: Schema.Record(Schema.String, Schema.String).pipe( Schema.withDecodingDefault(Effect.succeed({})), ), + /** + * Threads that hit a usage limit the server will resume at the reset. Lead hears of one only if + * it fails to come back. + */ + resuming: Schema.Record( + Schema.String, + Schema.Struct({ runId: Schema.String, resetAt: Schema.String, since: Schema.Number }), + ).pipe(Schema.withDecodingDefault(Effect.succeed({}))), + /** When each incident group last reached its destination, so later members wait a group interval. */ + incidentSentAt: Schema.Record(Schema.String, Schema.Number).pipe( + Schema.withDecodingDefault(Effect.succeed({})), + ), outbox: Schema.Array(Delivery), }); export type ModelAlertState = typeof ModelAlertState.Type; @@ -102,12 +126,23 @@ export const EMPTY_MODEL_ALERT_STATE: ModelAlertState = { version: 1, degraded: {}, reported: {}, + resuming: {}, + incidentSentAt: {}, outbox: [], }; const MAX_DEGRADED_THREADS = 1_000; const MAX_REPORTED = 2_000; const MAX_OUTBOX = 200; +const MAX_RESUMING = 2_000; +const MAX_INCIDENT_GROUPS = 200; +/** New incident members wait this long for the rest of the storm before the digest goes out. */ +export const INCIDENT_GROUP_WAIT_MS = 30_000; +/** A group that just reported waits this long before it reports its stragglers. */ +export const INCIDENT_GROUP_INTERVAL_MS = 5 * 60 * 1000; +/** How long past its reset a resumable thread may stay stopped before Lead hears it did not come back. */ +export const RESUME_GRACE_MS = 30 * 60 * 1000; +const HOLD_RECHECK_MS = 5 * 60 * 1000; const OUTBOX_TTL_MS = 7 * 24 * 60 * 60 * 1000; const RETRY_BASE_MS = 15_000; const RETRY_MAX_MS = 15 * 60 * 1000; @@ -190,11 +225,16 @@ export function recordModelAlert( ): { readonly state: ModelAlertState; readonly enqueued: boolean } { const { alert } = outcome; if (alert === undefined) { - if (!outcome.healthy || state.degraded[outcome.threadId] === undefined) { + if ( + !outcome.healthy || + (state.degraded[outcome.threadId] === undefined && + state.resuming[outcome.threadId] === undefined) + ) { return { state, enqueued: false }; } const { [outcome.threadId]: _cleared, ...degraded } = state.degraded; - return { state: { ...state, degraded }, enqueued: false }; + const { [outcome.threadId]: _resumed, ...resuming } = state.resuming; + return { state: { ...state, degraded, resuming }, enqueued: false }; } const next = signature(alert); if (state.degraded[alert.threadId] === next) return { state, enqueued: false }; @@ -295,18 +335,64 @@ const pruneOutbox = (outbox: ReadonlyArray, nowMs: number) => const sameDelivery = (left: Delivery, right: Delivery) => left.alert.id === right.alert.id && left.destination === right.destination; -/** The state after one delivery attempt: a delivered entry leaves, a failed one waits longer. */ +/** The members of a stored incident that a delivered snapshot did not carry. */ +const lateMembers = (stored: UsageLimitIncident, sent: UsageLimitIncident) => + stored.members.filter( + (member) => + !sent.members.some( + (known) => known.threadId === member.threadId && known.runId === member.runId, + ), + ); + +/** + * The state after one delivery attempt: a delivered entry leaves, a failed one waits longer. An + * incident that gained members while it was being sent keeps those for a later digest. + */ export function afterDeliveryAttempt( state: ModelAlertState, entry: Delivery, delivered: boolean, nowMs: number, ): ModelAlertState { + const sentIncident = delivered && entry.alert.type === "usage-limit-incident"; + const incidentSentAt = + sentIncident && entry.destination === "watcher" + ? Object.fromEntries( + Object.entries({ + ...state.incidentSentAt, + [entry.alert.groupKey]: nowMs, + }).slice(-MAX_INCIDENT_GROUPS), + ) + : state.incidentSentAt; return { ...state, + incidentSentAt, outbox: pruneOutbox(state.outbox, nowMs).flatMap((candidate) => { if (!sameDelivery(candidate, entry)) return [candidate]; - if (delivered) return []; + if (delivered) { + if ( + candidate.alert.type === "usage-limit-incident" && + entry.alert.type === "usage-limit-incident" + ) { + const late = lateMembers(candidate.alert, entry.alert); + if (late.length > 0) { + return [ + { + ...candidate, + // A new id: the delivery key hashes it, and a second digest must not replay the first. + alert: { + ...candidate.alert, + id: `${candidate.alert.groupKey}:${nowMs}`, + members: late, + }, + attempts: 0, + nextAttemptAt: nowMs + INCIDENT_GROUP_INTERVAL_MS, + }, + ]; + } + } + return []; + } const attempts = candidate.attempts + 1; return [ { @@ -319,6 +405,157 @@ export function afterDeliveryAttempt( }; } +/** + * An entry nobody needs any more leaves without counting as sent. An incident keeps the members + * that joined after the snapshot was judged, under a new id. + */ +export const afterDrop = ( + state: ModelAlertState, + entry: Delivery, + nowMs: number, +): ModelAlertState => ({ + ...state, + outbox: state.outbox.flatMap((candidate) => { + if (!sameDelivery(candidate, entry)) return [candidate]; + if ( + candidate.alert.type !== "usage-limit-incident" || + entry.alert.type !== "usage-limit-incident" + ) { + return []; + } + const late = lateMembers(candidate.alert, entry.alert); + return late.length === 0 + ? [] + : [ + { + ...candidate, + alert: { + ...candidate.alert, + id: `${candidate.alert.groupKey}:${nowMs}`, + members: late, + }, + }, + ]; + }), +}); + +/** An entry held back, not failed: it waits for `untilMs` without counting an attempt. */ +export function afterHold( + state: ModelAlertState, + entry: Delivery, + untilMs: number, +): ModelAlertState { + return { + ...state, + outbox: state.outbox.map((candidate) => + sameDelivery(candidate, entry) ? { ...candidate, nextAttemptAt: untilMs } : candidate, + ), + }; +} + +/** + * Records a thread that stopped at a usage limit. The server resumes it at the reset, so Lead + * hears nothing unless it does not come back; otherwise every thread of one provider account + * joins one queued incident, which waits briefly for the rest of the storm. + */ +export function recordUsageLimit( + state: ModelAlertState, + input: { + readonly member: IncidentMember; + readonly environmentId: string; + readonly environmentLabel: string; + readonly providerInstanceId: string; + readonly resetAt: string | null; + readonly resumeFailed: boolean; + readonly willResume: boolean; + /** Repeats of the same problem on the same run stay quiet. */ + readonly signature: string; + readonly occurredAt: string; + }, + destinations: ReadonlyArray, + nowMs: number, +): { readonly state: ModelAlertState; readonly enqueued: boolean } { + const { member } = input; + if (state.degraded[member.threadId] === input.signature) return { state, enqueued: false }; + const { [member.threadId]: _previous, ...restDegraded } = state.degraded; + const { [member.threadId]: _resumed, ...restResuming } = state.resuming; + const degraded = { + ...Object.fromEntries(Object.entries(restDegraded).slice(-(MAX_DEGRADED_THREADS - 1))), + [member.threadId]: input.signature, + }; + if (input.willResume && input.resetAt !== null) { + return { + state: { + ...state, + degraded, + resuming: { + ...Object.fromEntries(Object.entries(restResuming).slice(-(MAX_RESUMING - 1))), + [member.threadId]: { runId: member.runId, resetAt: input.resetAt, since: nowMs }, + }, + }, + enqueued: false, + }; + } + const groupKey = incidentGroupKey(input); + let outbox = pruneOutbox(state.outbox, nowMs); + let created = false; + for (const destination of destinations) { + const index = outbox.findIndex( + (entry) => + entry.destination === destination && + entry.alert.type === "usage-limit-incident" && + entry.alert.groupKey === groupKey, + ); + const queued = index === -1 ? undefined : outbox[index]; + const queuedAlert = queued?.alert; + if (queued !== undefined && queuedAlert?.type === "usage-limit-incident") { + outbox = outbox.map((entry, at) => + at === index + ? { ...queued, alert: withIncidentMember(queuedAlert, member, input.resetAt) } + : entry, + ); + continue; + } + created = true; + const sentAt = state.incidentSentAt[groupKey]; + const notBefore = Math.max( + nowMs + INCIDENT_GROUP_WAIT_MS, + sentAt === undefined ? 0 : sentAt + INCIDENT_GROUP_INTERVAL_MS, + ); + outbox = [ + ...outbox, + { + destination, + alert: { + type: "usage-limit-incident" as const, + id: `${groupKey}:${nowMs}`, + groupKey, + environmentId: input.environmentId, + environmentLabel: input.environmentLabel, + providerInstanceId: input.providerInstanceId, + resumeFailed: input.resumeFailed, + resetAt: input.resetAt, + members: [member], + occurredAt: input.occurredAt, + }, + attempts: 0, + nextAttemptAt: notBefore, + createdAt: nowMs, + }, + ].slice(-MAX_OUTBOX); + } + return { + state: { ...state, degraded, resuming: restResuming, outbox }, + enqueued: created, + }; +} + +/** Threads whose usage limit reset long enough ago that they should have been resumed. */ +export const overdueResumes = (state: ModelAlertState, nowMs: number) => + Object.entries(state.resuming).filter( + ([, entry]) => Date.parse(entry.resetAt) + RESUME_GRACE_MS < nowMs, + ); + const REASON_TEXT: Record string> = { model_substituted: (alert) => `it ran ${alert.executedModel ?? "another model"} instead of ${alert.selectedModel}`, @@ -337,8 +574,16 @@ function modelAlertMessage(alert: ModelAlert): string { ].join("\n\n"); } -export const watcherAlertMessage = (alert: WatcherAlert) => - alert.type === "thread-event" ? threadEventMessage(alert) : modelAlertMessage(alert); +export const watcherAlertMessage = (alert: WatcherAlert) => { + switch (alert.type) { + case "thread-event": + return threadEventMessage(alert); + case "usage-limit-incident": + return incidentMessage(alert); + case "model-degraded": + return modelAlertMessage(alert); + } +}; /** * Puts an alert in the watcher thread, here or on the linked machine that owns it. The retry key is @@ -357,7 +602,7 @@ export const deliverAlertToWatcher = Effect.fnUntraced(function* ( const message = watcherAlertMessage(alert); // Run ids can be long; a retry key must stay under the 256 characters the peer accepts. const identity = `${alert.id}\n${watcher.environmentId}\n${watcher.threadId}`; - const key = `${alert.type === "thread-event" ? "thread-event" : "model-alert"}:${yield* deps.sha256Hex(identity)}`; + const key = `${alert.type === "model-degraded" ? "model-alert" : alert.type}:${yield* deps.sha256Hex(identity)}`; if (watcher.environmentId === here) { yield* deps.dispatch({ type: "message.dispatch", @@ -457,21 +702,139 @@ export const make = Effect.gen(function* () { here, ); + const WatcherThreadStatus = Schema.Struct({ + thread: Schema.Struct({ status: Schema.String, providerInstanceId: Schema.String }), + }); + + /** + * Whether the parent already started a replacement for this delegated child: a later child of + * the same parent with the same task title. + */ + const replacedByParent = Effect.fnUntraced(function* (shell: OrchestrationV2ThreadShell) { + const parentId = shell.lineage.parentThreadId; + if (parentId === null || shell.lineage.relationshipToParent !== "subagent") return false; + const children = yield* projections.listAppOwnedChildThreadIds(parentId); + const at = children.indexOf(shell.id); + if (at === -1) return false; + const later = yield* Effect.forEach(children.slice(at + 1, at + 21), (id) => + projections.getThreadShell(id), + ); + return replacedBySibling( + shell, + later.flatMap((sibling) => (sibling === null ? [] : [sibling])), + ); + }); + + /** + * The alert as it stands now: threads that moved on leave it, and null means nothing is left to + * say. A thread moved on when it has a newer run that runs or finished, is settled, archived or + * on another provider, or its parent replaced it. + */ + const currentAlert = Effect.fnUntraced(function* (alert: WatcherAlert) { + const stillCurrent = Effect.fnUntraced(function* ( + threadId: string, + runId: string | null, + about: { readonly providerInstanceId: string } | undefined, + ) { + const shell = yield* projections.getThreadShell(ThreadId.make(threadId)); + if (alertIsStale(shell, runId, about)) return false; + return about === undefined || shell === null || !(yield* replacedByParent(shell)); + }); + if (alert.type === "usage-limit-incident") { + const fresh = yield* Effect.filter(alert.members, (member) => + stillCurrent(member.threadId, member.runId, { + providerInstanceId: alert.providerInstanceId, + }), + ); + if (fresh.length === 0) return null; + return fresh.length === alert.members.length ? alert : { ...alert, members: fresh }; + } + const about = + alert.type === "model-degraded" + ? { providerInstanceId: alert.providerInstanceId } + : undefined; + return (yield* stillCurrent(alert.threadId, alert.runId, about)) ? alert : null; + }); + + /** + * When the watcher's own thread is stopped at the same account limit, the time to look again. + * Anything sent now would only queue a turn that fails the same way. + */ + const holdUntil = Effect.fnUntraced(function* ( + watcher: ModelAlertWatcher, + here: EnvironmentId, + alert: WatcherAlert, + nowMs: number, + ) { + const providerInstanceId = + alert.type === "usage-limit-incident" || + (alert.type === "model-degraded" && alert.reasons.includes("usage_limit")) + ? alert.providerInstanceId + : null; + if (providerInstanceId === null) return null; + let down = false; + let resetAt: string | null = alert.type === "usage-limit-incident" ? alert.resetAt : null; + if (watcher.environmentId === here) { + const shell = yield* projections.getThreadShell(watcher.threadId); + down = + shell !== null && + shell.status === "failed" && + shell.providerInstanceId === providerInstanceId && + shell.lastErrorClass === "usage_limit"; + resetAt = shell?.usageLimitResetAt ?? resetAt; + } else { + // The read result carries no error class, so a remote watcher counts as down only while + // the reset this alert reports is still ahead. + down = yield* peers + .call({ + environmentId: watcher.environmentId, + tool: "t3_thread_read", + arguments: { threadId: watcher.threadId, limit: 1, runLimit: 1 }, + limits: { runtimeMode: "full-access", interactionMode: "default" }, + success: WatcherThreadStatus, + }) + .pipe( + Effect.map( + ({ thread }) => + thread.status === "failed" && thread.providerInstanceId === providerInstanceId, + ), + // An unreadable watcher is the delivery's problem, not a reason to hold. + Effect.orElseSucceed(() => false), + ); + } + if (!down) return null; + // Past the reset the limit is over: whatever keeps the watcher down, a message may restart it. + const resetMs = resetAt === null ? Number.NaN : Date.parse(resetAt); + if (!Number.isFinite(resetMs) || resetMs <= nowMs) return null; + return Math.min(resetMs, nowMs + HOLD_RECHECK_MS); + }); + + type DeliveryResult = + | { readonly _tag: "sent" } + | { readonly _tag: "dropped" } + | { readonly _tag: "held"; readonly until: number }; + const deliver = Effect.fnUntraced(function* (entry: Delivery) { const current = yield* settings.getSettings; + const alert = yield* currentAlert(entry.alert); + if (alert === null) return { _tag: "dropped" } satisfies DeliveryResult; if (entry.destination === "webhook") { const url = current.modelAlertWebhookUrl.trim(); - if (url === "") return; + if (url === "") return { _tag: "sent" } satisfies DeliveryResult; // A webhook URL can embed a credential, so its failure names only the host. const host = URL.canParse(url) ? new URL(url).host : "an unparsable URL"; yield* http - .execute(HttpClientRequest.post(url).pipe(HttpClientRequest.bodyJsonUnsafe(entry.alert))) + .execute(HttpClientRequest.post(url).pipe(HttpClientRequest.bodyJsonUnsafe(alert))) .pipe(Effect.catchCause(() => Effect.fail(`model alert webhook POST to ${host} failed`))); - return; + return { _tag: "sent" } satisfies DeliveryResult; } const watcher = current.modelAlertWatcher; - if (watcher === null) return; - yield* deliverToWatcher(entry.alert, watcher, yield* environment.getEnvironmentId); + if (watcher === null) return { _tag: "sent" } satisfies DeliveryResult; + const here = yield* environment.getEnvironmentId; + const until = yield* holdUntil(watcher, here, alert, yield* Clock.currentTimeMillis); + if (until !== null) return { _tag: "held", until } satisfies DeliveryResult; + yield* deliverToWatcher(alert, watcher, here); + return { _tag: "sent" } satisfies DeliveryResult; }); const flush: ModelAlertReactor["Service"]["flush"] = Effect.gen(function* () { @@ -481,28 +844,104 @@ export const make = Effect.gen(function* () { due, (entry) => Effect.gen(function* () { - const delivered = yield* deliver(entry).pipe( + const result = yield* deliver(entry).pipe( Effect.timeout("30 seconds"), - Effect.as(true), Effect.catchCause((cause) => Effect.logWarning("delivering a model alert failed; it will be retried", { alertId: entry.alert.id, destination: entry.destination, attempts: entry.attempts + 1, cause, - }).pipe(Effect.as(false)), + }).pipe(Effect.as(undefined)), ), ); const attemptedAt = yield* Clock.currentTimeMillis; yield* update((current) => [ undefined, - afterDeliveryAttempt(current, entry, delivered, attemptedAt), + result === undefined + ? afterDeliveryAttempt(current, entry, false, attemptedAt) + : result._tag === "held" + ? afterHold(current, entry, result.until) + : result._tag === "dropped" + ? afterDrop(current, entry, attemptedAt) + : afterDeliveryAttempt(current, entry, true, attemptedAt), ]); }), { concurrency: 4, discard: true }, ); }); + /** Folds a thread that stopped at a usage limit into its provider account's one incident. */ + const handleUsageLimit = Effect.fnUntraced(function* ( + event: Extract, + alert: ModelAlert, + ) { + const current = yield* settings.getSettings; + const shell = yield* projections.getThreadShell(event.threadId); + const nowMs = yield* Clock.currentTimeMillis; + const resumeFailed = event.payload.usageLimitContinuationOfRunId !== undefined; + const resetMs = + shell?.usageLimitResetAt == null ? Number.NaN : Date.parse(shell.usageLimitResetAt); + // New launches on this provider are refused until the reset, so they alert nobody. + if (Number.isFinite(resetMs) && resetMs > nowMs) { + ProviderExhaustion.mark(alert.providerInstanceId, resetMs); + } + const parentId = shell?.lineage.parentThreadId ?? null; + const parent = + parentId !== null && shell?.lineage.relationshipToParent === "subagent" + ? yield* projections.getThreadShell(parentId) + : undefined; + const willResume = expectsAutoResume({ + autoResumeEnabled: current.autoResumeLimitedThreads, + runWasResume: resumeFailed, + shell, + parentClosed: + parent !== undefined && + (parent === null || parent.archivedAt !== null || parent.settledOverride === "settled"), + nowMs, + graceMs: RESUME_GRACE_MS, + }); + const here = yield* environment.getEnvironmentId; + const destinations = modelAlertDestinations({ + watcher: current.modelAlertWatcher, + webhookUrl: current.modelAlertWebhookUrl, + here, + lineage: shell?.lineage ?? { parentThreadId: null, relationshipToParent: null }, + }); + const label = (yield* environment.getDescriptor).label; + const enqueued = yield* update((state) => { + const result = recordUsageLimit( + state, + { + member: { + threadId: alert.threadId, + threadTitle: alert.threadTitle, + threadLink: alert.threadLink, + runId: alert.runId, + }, + environmentId: here, + environmentLabel: label, + providerInstanceId: alert.providerInstanceId, + resetAt: shell?.usageLimitResetAt ?? null, + resumeFailed, + willResume, + signature: `usage_limit:${resumeFailed ? "resume-failed" : "limit"}:${alert.runId}`, + occurredAt: alert.occurredAt, + }, + destinations, + nowMs, + ); + return [result.enqueued, result.state]; + }); + if (willResume) { + yield* Effect.logInfo("a usage limit stopped a thread; it resumes by itself at the reset", { + threadId: alert.threadId, + resetAt: shell?.usageLimitResetAt, + }); + } + if (enqueued) yield* Queue.offer(wake, undefined); + }); + /** Queues a run's model alert, and says whether it reported a usage limit. */ const handleModelAlert = Effect.fnUntraced(function* ( event: Extract, @@ -526,7 +965,14 @@ export const make = Effect.gen(function* () { projection: records, occurredAt: DateTime.formatIso(yield* DateTime.now), }); + if (judged.alert === undefined && judged.healthy) { + ProviderExhaustion.clear(run.providerInstanceId); + } if (judged.alert === undefined && !judged.healthy) return { usageLimited: false }; + if (judged.alert?.reasons.every((reason) => reason === "usage_limit") === true) { + yield* handleUsageLimit(event, judged.alert); + return { usageLimited: true }; + } const outcome = judged.alert === undefined ? judged @@ -683,7 +1129,75 @@ export const make = Effect.gen(function* () { }); /** Looks at every thread that may be live for the time-based kinds no event announces. */ + /** + * A thread the server was to resume is still stopped long after its reset: that is the one + * usage limit outcome Lead hears about. + */ + const sweepResumes = Effect.gen(function* () { + const nowMs = yield* Clock.currentTimeMillis; + const overdue = overdueResumes(yield* Ref.get(state), nowMs); + if (overdue.length === 0) return; + const current = yield* settings.getSettings; + const here = yield* environment.getEnvironmentId; + const label = (yield* environment.getDescriptor).label; + const occurredAt = DateTime.formatIso(yield* DateTime.now); + yield* Effect.forEach( + overdue, + ([threadId, entry]) => + Effect.gen(function* () { + const shell = yield* projections.getThreadShell(ThreadId.make(threadId)); + const notResumed = + shell !== null && + shell.latestRunId === entry.runId && + shell.status === "failed" && + shell.archivedAt === null && + shell.settledOverride !== "settled"; + const destinations = modelAlertDestinations({ + watcher: current.modelAlertWatcher, + webhookUrl: current.modelAlertWebhookUrl, + here, + lineage: shell?.lineage ?? { parentThreadId: null, relationshipToParent: null }, + }); + const enqueued = yield* update((state) => { + const { [threadId]: _done, ...resuming } = state.resuming; + const cleared = { ...state, resuming }; + if (!notResumed) return [false, cleared]; + const result = recordUsageLimit( + cleared, + { + member: { + threadId, + threadTitle: shell.title, + threadLink: formatThreadLink( + { environmentId: here, threadId: shell.id }, + shell.title, + ), + runId: entry.runId, + }, + environmentId: here, + environmentLabel: label, + providerInstanceId: shell.providerInstanceId, + resetAt: entry.resetAt, + resumeFailed: true, + willResume: false, + signature: `usage_limit:resume-missed:${entry.runId}`, + occurredAt, + }, + destinations, + nowMs, + ); + return [result.enqueued, result.state]; + }); + if (enqueued) yield* Queue.offer(wake, undefined); + }), + { discard: true }, + ); + }); + const sweep: ModelAlertReactor["Service"]["sweep"] = Effect.gen(function* () { + yield* sweepResumes.pipe( + Effect.catchCause((cause) => Effect.logWarning("resume sweep failed", { cause })), + ); const index = yield* projections.listActiveShellIndex(); // One scan of the delegated tasks for the whole sweep, not one per thread. const remoteTasks = new Map(); diff --git a/apps/server/src/orchestration-v2/ProviderExhaustion.ts b/apps/server/src/orchestration-v2/ProviderExhaustion.ts new file mode 100644 index 000000000000..abb46b008791 --- /dev/null +++ b/apps/server/src/orchestration-v2/ProviderExhaustion.ts @@ -0,0 +1,33 @@ +/** + * Provider instances this server has seen stop at a usage limit, and until when. A new launch on + * one is refused up front instead of starting a run that fails at once and alerts. In memory and + * process-wide on purpose: the first failure after a restart teaches it again, and every thread + * on the account shares one limit. + */ +const exhaustedUntil = new Map(); +const MAX_TRACKED = 100; + +export const ProviderExhaustion = { + mark(instanceId: string, untilMs: number): void { + exhaustedUntil.delete(instanceId); + exhaustedUntil.set(instanceId, untilMs); + for (const [key] of exhaustedUntil) { + if (exhaustedUntil.size <= MAX_TRACKED) break; + exhaustedUntil.delete(key); + } + }, + /** The epoch millisecond the instance is out until, or null when it is usable. */ + until(instanceId: string, nowMs: number): number | null { + const until = exhaustedUntil.get(instanceId); + if (until === undefined) return null; + if (until <= nowMs) { + exhaustedUntil.delete(instanceId); + return null; + } + return until; + }, + clear(instanceId?: string): void { + if (instanceId === undefined) exhaustedUntil.clear(); + else exhaustedUntil.delete(instanceId); + }, +}; diff --git a/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts b/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts index 3f9eb8c43997..42e214b8a0f5 100644 --- a/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts +++ b/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts @@ -33,6 +33,7 @@ import { } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; import * as Deferred from "effect/Deferred"; +import * as Clock from "effect/Clock"; import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; @@ -65,6 +66,7 @@ import * as EffectOutbox from "./EffectOutbox.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts"; import * as ThreadLaunch from "./ThreadLaunchService.ts"; +import { ProviderExhaustion } from "./ProviderExhaustion.ts"; import * as GitVcsDriver from "../vcs/GitVcsDriver.ts"; import * as ThreadManagement from "./ThreadManagementService.ts"; import { OrchestratorProjectionError } from "./Orchestrator.ts"; @@ -1649,6 +1651,50 @@ it.effect("names the missing base branch when a worktree cannot be created from ), ); +it.effect("refuses 6 launches on a provider that is out of usage, and starts no thread", () => { + const harness = makeHarness({}); + return Effect.gen(function* () { + const launches = yield* ThreadLaunch.ThreadLaunchService; + const threads = yield* ThreadManagement.ThreadManagementService; + const before = (yield* threads.listProjectThreads({ projectId, includeSubagents: true })) + .length; + ProviderExhaustion.mark( + modelSelection.instanceId, + (yield* Clock.currentTimeMillis) + 60 * 60 * 1000, + ); + const refusals = yield* Effect.forEach([1, 2, 3, 4, 5, 6], (index) => + launches + .launch( + launchInput({ + command: `command:launch:exhausted:${index}`, + thread: `thread:launch:exhausted:${index}`, + message: `Helper ${index}`, + }), + ) + .pipe(Effect.flip), + ); + for (const error of refusals) { + assert.equal(error._tag, "ThreadManagementProviderInstanceDisabledError"); + assert.match(error.message, /is out of usage until .*; pick another provider/); + } + const after = (yield* threads.listProjectThreads({ projectId, includeSubagents: true })).length; + assert.equal(after, before); + // Once the provider is usable again the same launch goes through. + ProviderExhaustion.clear(modelSelection.instanceId); + const launched = yield* launches.launch( + launchInput({ + command: "command:launch:exhausted:after", + thread: "thread:launch:exhausted:after", + message: "Helper again", + }), + ); + assert.equal(launched.threadId, ThreadId.make("thread:launch:exhausted:after")); + }).pipe( + Effect.ensuring(Effect.sync(() => ProviderExhaustion.clear())), + Effect.provide(harness.layer), + ); +}); + it.effect("rejects a preparation retry after the project disables its provider", () => { let fetchAttempts = 0; const harness = makeHarness({ diff --git a/apps/server/src/orchestration-v2/ThreadLaunchService.ts b/apps/server/src/orchestration-v2/ThreadLaunchService.ts index e02256286955..5810575fe67a 100644 --- a/apps/server/src/orchestration-v2/ThreadLaunchService.ts +++ b/apps/server/src/orchestration-v2/ThreadLaunchService.ts @@ -1218,6 +1218,7 @@ const make = Effect.gen(function* () { projectId: input.projectId, commandId: input.commandId, instanceId: input.modelSelection.instanceId, + launch: true, }); return yield* Effect.gen(function* () { // Recover ids from accepted launch receipts before allocating; retries have no client id and new ids collide with receipts. diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index 4d9b95b20d94..5d9eb0005297 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -61,6 +61,7 @@ import * as Stream from "effect/Stream"; import * as CommandReceiptStore from "./CommandReceiptStore.ts"; import * as Orchestrator from "./Orchestrator.ts"; +import { ProviderExhaustion } from "./ProviderExhaustion.ts"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; import { projectedRowEncodedBytes } from "./threadHistoryPaging.ts"; import { threadPullRequestsOf } from "../sourceControl/threadPullRequests.ts"; @@ -341,10 +342,14 @@ export class ThreadManagementProviderInstanceDisabledError extends Schema.Tagged { projectId: ProjectId, instanceId: ProviderInstanceId, + /** Set when the instance is enabled but out of usage until then. */ + exhaustedUntil: Schema.optional(Schema.String), }, ) { override get message(): string { - return `Provider instance "${this.instanceId}" is disabled for project ${this.projectId}.`; + return this.exhaustedUntil === undefined + ? `Provider instance "${this.instanceId}" is disabled for project ${this.projectId}.` + : `Provider "${this.instanceId}" is out of usage until ${this.exhaustedUntil}; pick another provider.`; } } @@ -465,6 +470,8 @@ export interface ThreadManagementServiceShape { readonly projectId: ProjectId; readonly commandId: CommandId; readonly instanceId: ProviderInstanceId; + /** New work (a launch or a delegated task) is also refused while the provider is out of usage. */ + readonly launch?: boolean; }) => Effect.Effect; readonly isProviderInstanceEnabledForProject: (input: { readonly projectId: ProjectId; @@ -644,12 +651,27 @@ const make = Effect.gen(function* () { : serverSettings.getSettings.pipe( Effect.orDie, Effect.flatMap((current) => - resolveProjectProviderInstanceEnabled(current, input.projectId, input.instanceId) - ? Effect.void - : new ThreadManagementProviderInstanceDisabledError({ + !resolveProjectProviderInstanceEnabled(current, input.projectId, input.instanceId) + ? new ThreadManagementProviderInstanceDisabledError({ projectId: input.projectId, instanceId: input.instanceId, - }), + }) + : Clock.currentTimeMillis.pipe( + Effect.map((now) => + input.launch === true + ? ProviderExhaustion.until(input.instanceId, now) + : null, + ), + Effect.flatMap((until) => + until === null + ? Effect.void + : new ThreadManagementProviderInstanceDisabledError({ + projectId: input.projectId, + instanceId: input.instanceId, + exhaustedUntil: DateTime.formatIso(DateTime.makeUnsafe(until)), + }), + ), + ), ), ), ), @@ -852,6 +874,7 @@ const make = Effect.gen(function* () { projectId: parent.thread.projectId, commandId: command.commandId, instanceId: command.modelSelection.instanceId, + launch: true, }), ), ); diff --git a/apps/server/src/orchestration-v2/UsageLimitIncidents.ts b/apps/server/src/orchestration-v2/UsageLimitIncidents.ts new file mode 100644 index 000000000000..9280b577bb81 --- /dev/null +++ b/apps/server/src/orchestration-v2/UsageLimitIncidents.ts @@ -0,0 +1,168 @@ +import type { OrchestrationV2ThreadShell } from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; +import * as Schema from "effect/Schema"; + +/** + * One usage limit hits every thread on an account at once. The watcher hears one incident per + * (provider account, outcome), the way Alertmanager groups by labels, not one turn per thread. + * Delivery and state live in ModelAlertReactor. + */ +export const IncidentMember = Schema.Struct({ + threadId: Schema.String, + threadTitle: Schema.String, + threadLink: Schema.String, + runId: Schema.String, +}); +export type IncidentMember = typeof IncidentMember.Type; + +export const UsageLimitIncident = Schema.Struct({ + type: Schema.Literal("usage-limit-incident"), + /** Unique per queued item, so a later incident for the same group is a new message. */ + id: Schema.String, + /** What merges: environment, provider account and outcome. */ + groupKey: Schema.String, + environmentId: Schema.String, + environmentLabel: Schema.String, + providerInstanceId: Schema.String, + /** False: the threads stopped at the limit. True: they did not come back after the reset. */ + resumeFailed: Schema.Boolean, + resetAt: Schema.NullOr(Schema.String), + members: Schema.Array(IncidentMember), + occurredAt: Schema.String, +}); +export type UsageLimitIncident = typeof UsageLimitIncident.Type; + +export const incidentGroupKey = (input: { + readonly environmentId: string; + readonly providerInstanceId: string; + readonly resumeFailed: boolean; +}) => + `${input.environmentId}:usage-limit${input.resumeFailed ? "-resume-failed" : ""}:${input.providerInstanceId}`; + +const laterIso = (left: string | null, right: string | null) => { + if (left === null) return right; + if (right === null) return left; + return Date.parse(right) > Date.parse(left) ? right : left; +}; + +/** The incident with one more thread; a thread already in it keeps one row, at its newest run. */ +export function withIncidentMember( + incident: UsageLimitIncident, + member: IncidentMember, + resetAt: string | null, +): UsageLimitIncident { + return { + ...incident, + resetAt: laterIso(incident.resetAt, resetAt), + members: [...incident.members.filter((known) => known.threadId !== member.threadId), member], + }; +} + +type StaleShell = Pick< + OrchestrationV2ThreadShell, + | "latestRunId" + | "activeRunId" + | "status" + | "archivedAt" + | "deletedAt" + | "settledOverride" + | "providerInstanceId" +>; + +const LIVE_OR_DONE = new Set(["preparing", "starting", "running", "waiting", "completed"]); + +/** + * Whether an alert about a run is out of date: the thread has a newer run that is running or + * finished, or it is gone. A model alert is also out of date once the thread is settled or runs on + * another provider. Checked when the alert is delivered, not when it was judged. + */ +export function alertIsStale( + shell: StaleShell | null, + runId: string | null, + about?: { readonly providerInstanceId: string }, +): boolean { + if (shell === null || shell.deletedAt !== null || shell.archivedAt !== null) return true; + if (about !== undefined) { + if (shell.settledOverride === "settled") return true; + if (shell.providerInstanceId !== about.providerInstanceId) return true; + } + if (runId === null || shell.latestRunId === runId) return false; + return shell.activeRunId !== null || LIVE_OR_DONE.has(shell.status); +} + +type SiblingShell = Pick< + OrchestrationV2ThreadShell, + "title" | "status" | "activeRunId" | "archivedAt" | "deletedAt" +>; + +/** + * A delegated child its parent replaced: a later sibling carries the same title, which is how + * delegate_task names the task, and is itself running or finished. A sibling that also failed, or + * is gone, replaces nothing: templated prompts share a title and each still needs its alert. + */ +export function replacedBySibling( + child: { readonly title: string }, + later: ReadonlyArray, +): boolean { + const title = child.title.trim().toLowerCase(); + return later.some( + (sibling) => + sibling.title.trim().toLowerCase() === title && + sibling.archivedAt === null && + sibling.deletedAt === null && + (sibling.activeRunId !== null || LIVE_OR_DONE.has(sibling.status)), + ); +} + +/** Whether the server will resume a thread by itself when its limit resets. */ +export function expectsAutoResume(input: { + readonly autoResumeEnabled: boolean; + /** The run was already a resume after an earlier reset: a second failure is news. */ + readonly runWasResume: boolean; + readonly shell: Pick< + OrchestrationV2ThreadShell, + | "usageLimitResetAt" + | "archivedAt" + | "settledOverride" + | "pendingRuntimeRequest" + | "snoozedUntil" + > | null; + /** The recovery worker never arms a delegated child whose parent is settled, archived or gone. */ + readonly parentClosed?: boolean; + readonly nowMs: number; + /** How long past the reset the thread may stay stopped before it counts as not resumed. */ + readonly graceMs?: number; +}): boolean { + const { shell } = input; + if (!input.autoResumeEnabled || input.runWasResume || shell === null) return false; + if (shell.archivedAt !== null || shell.settledOverride === "settled") return false; + if (shell.pendingRuntimeRequest !== null || input.parentClosed === true) return false; + const resetMs = + shell.usageLimitResetAt == null ? Number.NaN : Date.parse(shell.usageLimitResetAt); + if (!Number.isFinite(resetMs) || resetMs <= input.nowMs - 60_000) return false; + // The worker waits out a snooze, so one that outlasts the grace period means no resume in time. + const snoozedMs = shell.snoozedUntil == null ? null : DateTime.toEpochMillis(shell.snoozedUntil); + return snoozedMs === null || snoozedMs <= resetMs + (input.graceMs ?? 0); +} + +const MAX_LISTED = 25; + +export function incidentMessage(incident: UsageLimitIncident): string { + const count = incident.members.length; + const threads = `${count} thread${count === 1 ? "" : "s"}`; + const reset = + incident.resetAt === null + ? "The reset time is not reported." + : `It resets at ${incident.resetAt}.`; + const lead = incident.resumeFailed + ? `Usage limit incident on ${incident.providerInstanceId} (${incident.environmentLabel}, environment ${incident.environmentId}): ${threads} did not come back after the limit reset. ${reset}` + : `Usage limit incident on ${incident.providerInstanceId} (${incident.environmentLabel}, environment ${incident.environmentId}): ${threads} stopped at the provider's usage limit. ${reset}`; + const listed = incident.members + .slice(0, MAX_LISTED) + .map((member, index) => `${index + 1}. ${member.threadLink} (run ${member.runId})`); + if (count > MAX_LISTED) listed.push(`...and ${count - MAX_LISTED} more.`); + const next = incident.resumeFailed + ? "Automatic resume did not work for these. Resume them yourself, switch their provider, or replace them. This is the only alert for this incident." + : "Nothing was resumed automatically for these. Resume them after the reset, or switch their provider. This is the only alert for this incident; later threads that hit the same limit join a later digest."; + return [lead, listed.join("\n"), next].join("\n\n"); +} diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index c31e13fffc98..57d324d5879a 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -1512,7 +1512,9 @@ export const ServerSettings = Schema.Struct({ ), idleCompaction: Schema.optionalKey(IdleCompactionMode), snoozeLimitedThreads: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), - autoResumeLimitedThreads: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), + // Closura: a usage limit resets for every thread on the account at once. Resuming them is + // the server's job, so Lead is told only about a thread that does not come back. + autoResumeLimitedThreads: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))), snoozePresets: SnoozePresetSettings.pipe( Schema.withDecodingDefault(Effect.succeed(DEFAULT_SNOOZE_PRESETS)), ),