diff --git a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts index fce4fb9354ec6..0433de375232c 100644 --- a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts +++ b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts @@ -1432,6 +1432,9 @@ const makeModelAlertReactor = (input: { Layer.mock(PeerMcpClient.PeerMcpClient)({ call: (request) => Effect.gen(function* () { + // The busy-watcher check reads the remote watcher first; it finds it idle. + if (request.tool === "t3_thread_read") + return { thread: { status: "idle" } } as never; yield* Ref.update(sends, (all) => [ ...all, { @@ -5088,6 +5091,13 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => { now, }), ); + // The first alert queued a turn on the watcher, so the next one waits for that turn, + // up to the hold cap, instead of queueing behind it. + assert.deepEqual( + (yield* alerts).map((message) => message.text.includes(`/${topLevelThreadId})`)), + [false], + ); + yield* TestClock.adjust("11 minutes"); assert.deepEqual( (yield* alerts).map((message) => message.text.includes(`/${topLevelThreadId})`)), [false, true], diff --git a/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts b/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts index 3566598da64ac..f1d781547c70c 100644 --- a/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts +++ b/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts @@ -7,6 +7,9 @@ import { afterDeliveryAttempt, afterDrop, afterHold, + BUSY_WATCHER_MAX_HOLD_MS, + BUSY_WATCHER_RECHECK_MS, + busyWatcherHold, deliverAlertToWatcher, INCIDENT_GROUP_INTERVAL_MS, INCIDENT_GROUP_WAIT_MS, @@ -481,6 +484,20 @@ it("a held item waits without failing, and a dropped one leaves without counting assert.deepEqual(dropped.incidentSentAt, {}); }); +it.each([ + ["an idle watcher", "idle", 0, null], + ["a watcher that cannot be read", null, 0, null], + ["a watcher mid-turn", "running", 0, BUSY_WATCHER_RECHECK_MS], + ["a watcher with a queued turn", "queued", 0, BUSY_WATCHER_RECHECK_MS], + ["a watcher waiting on its own background work", "waiting", 0, BUSY_WATCHER_RECHECK_MS], + ["a watcher busy near the cap", "running", BUSY_WATCHER_MAX_HOLD_MS - 1_000, 1_000], + ["a watcher busy past the cap", "running", BUSY_WATCHER_MAX_HOLD_MS, null], +] as const)("busy watcher hold: %s", (_name, status, ageMs, waitMs) => { + const createdAt = 1_000_000; + const nowMs = createdAt + ageMs; + assert.equal(busyWatcherHold(status, createdAt, nowMs), waitMs === null ? null : nowMs + waitMs); +}); + it("a model alert survives a newer run that fell back too, unlike a thread event", () => { const shell = { latestRunId: "run:2", diff --git a/apps/server/src/orchestration-v2/ModelAlertReactor.ts b/apps/server/src/orchestration-v2/ModelAlertReactor.ts index a4909326c6526..fd23f6a7628f8 100644 --- a/apps/server/src/orchestration-v2/ModelAlertReactor.ts +++ b/apps/server/src/orchestration-v2/ModelAlertReactor.ts @@ -48,6 +48,8 @@ import { ThreadEventAlert, type ThreadEventKind, threadEventMessage, + REJUDGE_TRIGGER, + threadEventStillHolds, type ThreadEventTrigger, type ThreadFinding, } from "./ThreadEventAlerts.ts"; @@ -453,6 +455,33 @@ export function afterHold( }; } +const BUSY_WATCHER_STATUSES = new Set([ + "preparing", + "queued", + "starting", + "running", + "waiting", +]); +/** Matches the flush interval, so a held alert is looked at again on the next flush. */ +export const BUSY_WATCHER_RECHECK_MS = 30_000; +export const BUSY_WATCHER_MAX_HOLD_MS = 10 * 60 * 1000; + +/** + * While the watcher is mid-turn, a sent alert only queues behind that turn and is judged stale or + * not when it was sent. Holding it until the turn ends lets the delivery check drop what went stale + * meanwhile. Bounded, so a watcher that never goes idle still hears everything. + */ +export function busyWatcherHold( + watcherStatus: string | null, + createdAt: number, + nowMs: number, +): number | null { + if (watcherStatus === null || !BUSY_WATCHER_STATUSES.has(watcherStatus)) return null; + const deadline = createdAt + BUSY_WATCHER_MAX_HOLD_MS; + if (nowMs >= deadline) return null; + return Math.min(nowMs + BUSY_WATCHER_RECHECK_MS, deadline); +} + /** * Records a thread that stopped at a usage limit. The server resumes it at the reset, so Lead * hears nothing unless it does not come back; otherwise every thread of one provider account @@ -702,6 +731,13 @@ export const make = Effect.gen(function* () { here, ); + type WatcherView = { + readonly status: string; + readonly providerInstanceId: string | null; + /** Null when unknown, as for a remote watcher. */ + readonly usageLimited: boolean | null; + readonly resetAt: string | null; + }; const WatcherThreadStatus = Schema.Struct({ thread: Schema.Struct({ status: Schema.String, providerInstanceId: Schema.String }), }); @@ -762,71 +798,140 @@ export const make = Effect.gen(function* () { alert.type === "model-degraded" ? { providerInstanceId: alert.providerInstanceId, newerRunSupersedes: false } : undefined; - return (yield* stillCurrent(alert.threadId, alert.runId, about)) ? alert : null; + if (!(yield* stillCurrent(alert.threadId, alert.runId, about))) return null; + if (alert.type === "thread-event") { + const shell = yield* projections.getThreadShell(ThreadId.make(alert.threadId)); + if (shell === null) return null; + const holds = threadEventStillHolds({ + alert, + shell, + stuckMinutes: (yield* settings.getSettings).watcherStuckMinutes, + ...(yield* judgeContext(shell, REJUDGE_TRIGGER[alert.kind])), + }); + if (!holds) return null; + } + return alert; + }); + + /** What the shell does not carry: open delegated tasks elsewhere, and children's last activity. */ + const judgeContext = Effect.fnUntraced(function* ( + shell: OrchestrationV2ThreadShell, + trigger: ThreadEventTrigger, + remoteTasks?: ReadonlyMap, + ) { + const remoteTaskCount = + trigger === "request" || trigger === "inbox" + ? 0 + : remoteTasks !== undefined + ? (remoteTasks.get(shell.id) ?? 0) + : (yield* projections.getOpenRemoteDelegatedTasks).filter( + (task) => task.parentThreadId === shell.id, + ).length; + // A parent that waits on children is not stuck while they work. + const childActivityMs = + trigger === "sweep" + ? Math.max( + 0, + ...(yield* Effect.forEach( + (shell.pendingBackgroundTasks ?? []).flatMap((task) => + task.kind === "subagent" && task.childThreadId !== undefined + ? [task.childThreadId] + : [], + ), + (childId) => + projections + .getThreadShell(childId) + .pipe( + Effect.map((child) => + child === null ? 0 : DateTime.toEpochMillis(child.updatedAt), + ), + ), + )), + ) + : null; + return { remoteTaskCount, childActivityMs }; + }); + + /** The watcher thread as one delivery needs it, or null when it cannot be read. */ + const readWatcher = Effect.fnUntraced(function* ( + watcher: ModelAlertWatcher, + here: EnvironmentId, + ) { + if (watcher.environmentId === here) { + const shell = yield* projections.getThreadShell(watcher.threadId); + if (shell === null) return null; + return { + status: shell.status as string, + providerInstanceId: shell.providerInstanceId as string | null, + usageLimited: shell.lastErrorClass === "usage_limit", + resetAt: shell.usageLimitResetAt ?? null, + } satisfies WatcherView; + } + return yield* peers + .call({ + environmentId: watcher.environmentId, + tool: "t3_thread_read", + arguments: { threadId: watcher.threadId, limit: 1, runLimit: 1 }, + limits: { runtimeMode: "full-access", interactionMode: "default" }, + success: WatcherThreadStatus, + }) + .pipe( + // The read result carries no error class or reset time. + Effect.map(({ thread }): WatcherView => ({ + status: thread.status, + providerInstanceId: thread.providerInstanceId, + usageLimited: null, + resetAt: null, + })), + // A read that fails or dies must never block the alert; deliver as if idle. + Effect.catchCause(() => Effect.succeed(null)), + ); }); /** * When the watcher's own thread is stopped at the same account limit, the time to look again. * Anything sent now would only queue a turn that fails the same way. */ - const holdUntil = Effect.fnUntraced(function* ( - watcher: ModelAlertWatcher, - here: EnvironmentId, - alert: WatcherAlert, - nowMs: number, - ) { + const holdUntil = (view: WatcherView | null, alert: WatcherAlert, nowMs: number) => { const providerInstanceId = alert.type === "usage-limit-incident" || (alert.type === "model-degraded" && alert.reasons.includes("usage_limit")) ? alert.providerInstanceId : null; - if (providerInstanceId === null) return null; - let down = false; - let resetAt: string | null = alert.type === "usage-limit-incident" ? alert.resetAt : null; - if (watcher.environmentId === here) { - const shell = yield* projections.getThreadShell(watcher.threadId); - down = - shell !== null && - shell.status === "failed" && - shell.providerInstanceId === providerInstanceId && - shell.lastErrorClass === "usage_limit"; - resetAt = shell?.usageLimitResetAt ?? resetAt; - } else { - // The read result carries no error class, so a remote watcher counts as down only while - // the reset this alert reports is still ahead. - down = yield* peers - .call({ - environmentId: watcher.environmentId, - tool: "t3_thread_read", - arguments: { threadId: watcher.threadId, limit: 1, runLimit: 1 }, - limits: { runtimeMode: "full-access", interactionMode: "default" }, - success: WatcherThreadStatus, - }) - .pipe( - Effect.map( - ({ thread }) => - thread.status === "failed" && thread.providerInstanceId === providerInstanceId, - ), - // An unreadable watcher is the delivery's problem, not a reason to hold. - Effect.orElseSucceed(() => false), - ); - } + if (providerInstanceId === null || view === null) return null; + // A remote watcher counts as down only while the reset this alert reports is still ahead. + const down = + view.status === "failed" && + view.providerInstanceId === providerInstanceId && + view.usageLimited !== false; + const resetAt = view.resetAt ?? (alert.type === "usage-limit-incident" ? alert.resetAt : null); if (!down) return null; // Past the reset the limit is over: whatever keeps the watcher down, a message may restart it. const resetMs = resetAt === null ? Number.NaN : Date.parse(resetAt); if (!Number.isFinite(resetMs) || resetMs <= nowMs) return null; return Math.min(resetMs, nowMs + HOLD_RECHECK_MS); - }); + }; type DeliveryResult = | { readonly _tag: "sent" } | { readonly _tag: "dropped" } | { readonly _tag: "held"; readonly until: number }; - const deliver = Effect.fnUntraced(function* (entry: Delivery) { + const deliver = Effect.fnUntraced(function* ( + entry: Delivery, + watcherView: Effect.Effect, + ) { const current = yield* settings.getSettings; const alert = yield* currentAlert(entry.alert); - if (alert === null) return { _tag: "dropped" } satisfies DeliveryResult; + if (alert === null) { + yield* Effect.logInfo("watcher alert no longer true; dropped without waking anyone", { + alertId: entry.alert.id, + type: entry.alert.type, + kind: entry.alert.type === "thread-event" ? entry.alert.kind : null, + destination: entry.destination, + }); + return { _tag: "dropped" } satisfies DeliveryResult; + } if (entry.destination === "webhook") { const url = current.modelAlertWebhookUrl.trim(); if (url === "") return { _tag: "sent" } satisfies DeliveryResult; @@ -840,7 +945,11 @@ export const make = Effect.gen(function* () { const watcher = current.modelAlertWatcher; if (watcher === null) return { _tag: "sent" } satisfies DeliveryResult; const here = yield* environment.getEnvironmentId; - const until = yield* holdUntil(watcher, here, alert, yield* Clock.currentTimeMillis); + const nowMs = yield* Clock.currentTimeMillis; + const view = yield* watcherView; + const until = + holdUntil(view, alert, nowMs) ?? + busyWatcherHold(view?.status ?? null, entry.createdAt, nowMs); if (until !== null) return { _tag: "held", until } satisfies DeliveryResult; yield* deliverToWatcher(alert, watcher, here); return { _tag: "sent" } satisfies DeliveryResult; @@ -849,11 +958,20 @@ export const make = Effect.gen(function* () { const flush: ModelAlertReactor["Service"]["flush"] = Effect.gen(function* () { const nowMs = yield* Clock.currentTimeMillis; const due = (yield* Ref.get(state)).outbox.filter((entry) => entry.nextAttemptAt <= nowMs); + if (due.length === 0) return; + // One read of the watcher per flush, shared by every alert due in it. + const watcherView = yield* Effect.cached( + Effect.gen(function* () { + const watcher = (yield* settings.getSettings).modelAlertWatcher; + if (watcher === null) return null; + return yield* readWatcher(watcher, yield* environment.getEnvironmentId); + }).pipe(Effect.catchCause(() => Effect.succeed(null))), + ); yield* Effect.forEach( due, (entry) => Effect.gen(function* () { - const result = yield* deliver(entry).pipe( + const result = yield* deliver(entry, watcherView).pipe( Effect.timeout("30 seconds"), Effect.catchCause((cause) => Effect.logWarning("delivering a model alert failed; it will be retried", { @@ -1029,44 +1147,13 @@ export const make = Effect.gen(function* () { if (watcher === null && current.modelAlertWebhookUrl.trim() === "") return; const shell = yield* projections.getThreadShell(threadId); if (shell === null || shell.deletedAt !== null) return; - const remoteTaskCount = - trigger === "request" || trigger === "inbox" - ? 0 - : remoteTasks !== undefined - ? (remoteTasks.get(threadId) ?? 0) - : (yield* projections.getOpenRemoteDelegatedTasks).filter( - (task) => task.parentThreadId === threadId, - ).length; - // A parent that waits on children is not stuck while they work. - const childActivityMs = - trigger === "sweep" - ? Math.max( - 0, - ...(yield* Effect.forEach( - (shell.pendingBackgroundTasks ?? []).flatMap((task) => - task.kind === "subagent" && task.childThreadId !== undefined - ? [task.childThreadId] - : [], - ), - (childId) => - projections - .getThreadShell(childId) - .pipe( - Effect.map((child) => - child === null ? 0 : DateTime.toEpochMillis(child.updatedAt), - ), - ), - )), - ) - : null; const nowMs = yield* Clock.currentTimeMillis; const judged = judgeThread({ shell, trigger, nowMs, stuckMinutes: current.watcherStuckMinutes, - remoteTaskCount, - childActivityMs, + ...(yield* judgeContext(shell, trigger, remoteTasks)), usageLimitReported, }); const label = (yield* environment.getDescriptor).label; diff --git a/apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts b/apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts index 80abe7f06abf2..24fb597c21ce8 100644 --- a/apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts +++ b/apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts @@ -7,6 +7,7 @@ import { promisesToContinue, threadEventAlert, threadEventMessage, + threadEventStillHolds, type ThreadEventTrigger, type WatchedShell, } from "./ThreadEventAlerts.ts"; @@ -303,6 +304,94 @@ it("a recurrence of the same state is a new message", () => { assert.notEqual(make("2026-11-19T05:00:00.000Z").id, make("2026-11-19T05:30:00.000Z").id); }); +const fired = (overrides: Partial, trigger: ThreadEventTrigger) => { + const shell = shellWith(overrides); + const finding = judge(shell, trigger).findings[0]!; + return threadEventAlert({ + environmentId: "env-r720" as never, + environmentLabel: "primary-r720", + shell, + finding, + occurredAt: DateTime.formatIso(DateTime.makeUnsafe(NOW)), + }); +}; +const stillHolds = ( + alert: ReturnType, + overrides: Partial, + base: Partial, +) => + threadEventStillHolds({ alert, shell: shellWith({ ...base, ...overrides }), stuckMinutes: 20 }); + +const failed = { status: "failed" } as never; +const openWithPr = { status: "completed", pullRequests: [openPullRequest] } as never; +const quietRun = { + activeRunId: "run:8", + status: "running", + activityRunStatus: "running", + updatedAt: at(25), +} as never; +const asking = { + pendingRuntimeRequest: { id: "request:9", kind: "approval", userOnly: false }, +} as never; +const promised = { status: "idle", latestVisibleMessage: assistantSays(FLEET_PROMISE) } as never; + +it.each([ + ["lane-ended, nothing changed", failed, "run-ended", {}, true], + ["lane-ended, the lane restarted itself", failed, "run-ended", quietRun, false], + [ + "lane-ended, a newer run already finished", + failed, + "run-ended", + { latestRunId: "run:8", status: "completed" }, + false, + ], + ["lane-ended, the lane was settled", failed, "run-ended", { settledOverride: "settled" }, false], + ["lane-ended, the lane was archived", failed, "run-ended", { archivedAt: at(1) }, false], + [ + "lane-ended with an open PR, a PR watch registered since", + openWithPr, + "run-ended", + { pendingBackgroundTasks: [{ taskId: "pull-request-watch:97", kind: "monitor" }] }, + false, + ], + [ + "lane-ended with an open PR, the PR merged since", + openWithPr, + "run-ended", + { pullRequests: [{ ...(openPullRequest as object), snapshot: { state: "merged" } }] }, + false, + ], + ["lane-ended with an open PR, nothing changed", openWithPr, "run-ended", {}, true], + ["stuck, nothing changed", quietRun, "sweep", {}, true], + ["stuck, new activity since", quietRun, "sweep", { updatedAt: at(-1) }, false], + [ + "needs-person, the question was answered", + asking, + "request", + { pendingRuntimeRequest: null }, + false, + ], + ["needs-person, still unanswered", asking, "request", {}, true], + ["promise-without-wake, nothing changed", promised, "settled", {}, true], + [ + "promise-without-wake, a real wait registered since", + promised, + "settled", + { pendingBackgroundTasks: [{ taskId: "b1", kind: "command" }] }, + false, + ], +] as const)("at delivery, %s -> still holds %s", (_name, base, trigger, now, expected) => { + const alert = fired(base, trigger); + assert.equal(stillHolds(alert, now as never, base), expected); +}); + +it("a thread event still holds when delivery waited past the recent-run window", () => { + // Judged at the time it fired, not now: a watcher offline for two hours still hears it. + const alert = fired(failed, "run-ended"); + assert.equal(stillHolds(alert, {}, failed), true); + assert.equal(stillHolds({ ...alert, occurredAt: "not a time" }, quietRun, failed), true); +}); + it("each trigger evaluates only the kinds it can change", () => { assert.deepEqual(judge(shellWith(), "run-ended").evaluated, [ "lane-ended", diff --git a/apps/server/src/orchestration-v2/ThreadEventAlerts.ts b/apps/server/src/orchestration-v2/ThreadEventAlerts.ts index 502c4b2bc3141..aa61f31970dc4 100644 --- a/apps/server/src/orchestration-v2/ThreadEventAlerts.ts +++ b/apps/server/src/orchestration-v2/ThreadEventAlerts.ts @@ -326,6 +326,45 @@ export function threadEventAlert(input: { }; } +/** The trigger that judges each kind again at delivery: any trigger whose kinds include it. */ +export const REJUDGE_TRIGGER: Record = { + "lane-ended": "run-ended", + "promise-without-wake": "settled", + stuck: "sweep", + "needs-person": "request", +}; + +/** + * Whether a queued thread event is still true of the thread as it stands now, judged at the time it + * fired so the recent-run window does not expire while delivery waits. A lane that restarted, + * settled, registered a wait, merged its pull request or got its answer since then is no news. + */ +export function threadEventStillHolds(input: { + readonly alert: ThreadEventAlert; + readonly shell: WatchedShell; + readonly stuckMinutes: number; + readonly remoteTaskCount?: number; + readonly childActivityMs?: number | null; +}): boolean { + const { alert } = input; + const firedMs = Date.parse(alert.occurredAt); + if (!Number.isFinite(firedMs)) return true; + const { findings } = judgeThread({ + shell: input.shell, + trigger: REJUDGE_TRIGGER[alert.kind], + nowMs: firedMs, + stuckMinutes: input.stuckMinutes, + ...(input.remoteTaskCount === undefined ? {} : { remoteTaskCount: input.remoteTaskCount }), + ...(input.childActivityMs === undefined ? {} : { childActivityMs: input.childActivityMs }), + }); + const prefix = `${alert.environmentId}:${alert.threadId}:${alert.kind}:`; + return findings.some( + (finding) => + finding.kind === alert.kind && + alert.id === `${prefix}${finding.signature}:${alert.occurredAt}`, + ); +} + const NEXT_STEP: Record = { "lane-ended": "Check the thread: restart, repair or replace it, or settle it if its work is done. It sends no further alert for this run.", diff --git a/docs/operations/watcher-thread.md b/docs/operations/watcher-thread.md index 7bcf43d0dc57d..6a241e490e752 100644 --- a/docs/operations/watcher-thread.md +++ b/docs/operations/watcher-thread.md @@ -10,7 +10,7 @@ One thread, by default Lead's on the laptop, hears everything that needs attenti | `stuck` | An active run has had no new activity for `watcherStuckMinutes` (default 20; 0 turns it off), or a thread waits on background work that no longer exists. A parent whose child threads are active is not stuck. | | `needs-person` | An agent-created lane raises a pending question or approval, or its inbox is assigned to a founder or set to `action`. | -Each event reports once per thread and kind until its state changes: a new run, a different message, or the condition clearing. Delegated children are skipped; their parent already hears their result. The watcher thread never reports about itself. +Each event reports once per thread and kind until its state changes: a new run, a different message, or the condition clearing. Delivery judges each event again, as of when it fired, and drops one that is no longer true: the lane restarted, settled, registered a wait, merged its pull request or got its answer. A dropped event stays reported, so the same state coming back does not alert again until its run or message changes. While the watcher is busy (running, queued or waiting), every alert to it, including model and usage-limit alerts, waits up to 10 minutes instead of queueing behind that turn, so it is judged when the watcher can act. A held alert is looked at again on a later flush, about every 30 to 60 seconds. Delegated children are skipped; their parent already hears their result. The watcher thread never reports about itself. ## Moving the watcher