From 37a8a4c13ae5324f717b7a90b0701d2b338aebb6 Mon Sep 17 00:00:00 2001 From: nullStack65 <112618179+nullStack65@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:19:52 +0000 Subject: [PATCH 1/3] fix(server): stop no-longer-true events from waking the watcher A lane-ended, stuck, promise or needs-person alert was judged once, when it fired, and then queued into the watcher thread behind whatever turn it was running. By the time the watcher read it the lane had often restarted, settled, registered a wait or had its question answered, and the coordinator spent a full-context turn learning nothing. Delivery now judges each thread event again, as of the time it fired, and drops it when that finding no longer holds. While the watcher is mid-turn, watcher deliveries are held (rechecked every 30 s, at most 10 minutes) so the check runs when the watcher can act, instead of when the message is queued. Co-Authored-By: Claude Opus 5.5 --- .../ModelAlertReactor.test.ts | 17 ++ .../src/orchestration-v2/ModelAlertReactor.ts | 145 +++++++++++++----- .../ThreadEventAlerts.test.ts | 86 +++++++++++ .../src/orchestration-v2/ThreadEventAlerts.ts | 39 +++++ docs/operations/watcher-thread.md | 2 +- 5 files changed, 253 insertions(+), 36 deletions(-) diff --git a/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts b/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts index 3566598da64ac..336f8456eb751 100644 --- a/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts +++ b/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts @@ -7,6 +7,9 @@ import { afterDeliveryAttempt, afterDrop, afterHold, + BUSY_WATCHER_MAX_HOLD_MS, + BUSY_WATCHER_RECHECK_MS, + busyWatcherHold, deliverAlertToWatcher, INCIDENT_GROUP_INTERVAL_MS, INCIDENT_GROUP_WAIT_MS, @@ -481,6 +484,20 @@ it("a held item waits without failing, and a dropped one leaves without counting assert.deepEqual(dropped.incidentSentAt, {}); }); +it.each([ + ["an idle watcher", "idle", 0, null], + ["a watcher that cannot be read", null, 0, null], + ["a watcher mid-turn", "running", 0, BUSY_WATCHER_RECHECK_MS], + ["a watcher with a queued turn", "queued", 0, BUSY_WATCHER_RECHECK_MS], + ["a watcher waiting on its own background work", "waiting", 0, null], + ["a watcher busy near the cap", "running", BUSY_WATCHER_MAX_HOLD_MS - 1_000, 1_000], + ["a watcher busy past the cap", "running", BUSY_WATCHER_MAX_HOLD_MS, null], +] as const)("busy watcher hold: %s", (_name, status, ageMs, waitMs) => { + const createdAt = 1_000_000; + const nowMs = createdAt + ageMs; + assert.equal(busyWatcherHold(status, createdAt, nowMs), waitMs === null ? null : nowMs + waitMs); +}); + it("a model alert survives a newer run that fell back too, unlike a thread event", () => { const shell = { latestRunId: "run:2", diff --git a/apps/server/src/orchestration-v2/ModelAlertReactor.ts b/apps/server/src/orchestration-v2/ModelAlertReactor.ts index a4909326c6526..d0709a772a72c 100644 --- a/apps/server/src/orchestration-v2/ModelAlertReactor.ts +++ b/apps/server/src/orchestration-v2/ModelAlertReactor.ts @@ -48,6 +48,7 @@ import { ThreadEventAlert, type ThreadEventKind, threadEventMessage, + threadEventStillHolds, type ThreadEventTrigger, type ThreadFinding, } from "./ThreadEventAlerts.ts"; @@ -453,6 +454,27 @@ export function afterHold( }; } +const BUSY_WATCHER_STATUSES = new Set(["preparing", "queued", "starting", "running"]); +/** Matches the flush interval, so a held alert is looked at again on the next flush. */ +export const BUSY_WATCHER_RECHECK_MS = 30_000; +export const BUSY_WATCHER_MAX_HOLD_MS = 10 * 60 * 1000; + +/** + * While the watcher is mid-turn, a sent alert only queues behind that turn and is judged stale or + * not when it was sent. Holding it until the turn ends lets the delivery check drop what went stale + * meanwhile. Bounded, so a watcher that never goes idle still hears everything. + */ +export function busyWatcherHold( + watcherStatus: string | null, + createdAt: number, + nowMs: number, +): number | null { + if (watcherStatus === null || !BUSY_WATCHER_STATUSES.has(watcherStatus)) return null; + const deadline = createdAt + BUSY_WATCHER_MAX_HOLD_MS; + if (nowMs >= deadline) return null; + return Math.min(nowMs + BUSY_WATCHER_RECHECK_MS, deadline); +} + /** * Records a thread that stopped at a usage limit. The server resumes it at the reset, so Lead * hears nothing unless it does not come back; otherwise every thread of one provider account @@ -762,7 +784,80 @@ export const make = Effect.gen(function* () { alert.type === "model-degraded" ? { providerInstanceId: alert.providerInstanceId, newerRunSupersedes: false } : undefined; - return (yield* stillCurrent(alert.threadId, alert.runId, about)) ? alert : null; + if (!(yield* stillCurrent(alert.threadId, alert.runId, about))) return null; + if (alert.type === "thread-event") { + const shell = yield* projections.getThreadShell(ThreadId.make(alert.threadId)); + if (shell === null) return null; + const holds = threadEventStillHolds({ + alert, + shell, + stuckMinutes: (yield* settings.getSettings).watcherStuckMinutes, + ...(yield* judgeContext(shell, alert.kind === "stuck" ? "sweep" : "run-ended")), + }); + if (!holds) return null; + } + return alert; + }); + + /** What the shell does not carry: open delegated tasks elsewhere, and children's last activity. */ + const judgeContext = Effect.fnUntraced(function* ( + shell: OrchestrationV2ThreadShell, + trigger: ThreadEventTrigger, + remoteTasks?: ReadonlyMap, + ) { + const remoteTaskCount = + trigger === "request" || trigger === "inbox" + ? 0 + : remoteTasks !== undefined + ? (remoteTasks.get(shell.id) ?? 0) + : (yield* projections.getOpenRemoteDelegatedTasks).filter( + (task) => task.parentThreadId === shell.id, + ).length; + // A parent that waits on children is not stuck while they work. + const childActivityMs = + trigger === "sweep" + ? Math.max( + 0, + ...(yield* Effect.forEach( + (shell.pendingBackgroundTasks ?? []).flatMap((task) => + task.kind === "subagent" && task.childThreadId !== undefined + ? [task.childThreadId] + : [], + ), + (childId) => + projections + .getThreadShell(childId) + .pipe( + Effect.map((child) => + child === null ? 0 : DateTime.toEpochMillis(child.updatedAt), + ), + ), + )), + ) + : null; + return { remoteTaskCount, childActivityMs }; + }); + + /** The watcher thread's run status, or null when it cannot be read. */ + const watcherStatus = Effect.fnUntraced(function* ( + watcher: ModelAlertWatcher, + here: EnvironmentId, + ) { + if (watcher.environmentId === here) { + return (yield* projections.getThreadShell(watcher.threadId))?.status ?? null; + } + return yield* peers + .call({ + environmentId: watcher.environmentId, + tool: "t3_thread_read", + arguments: { threadId: watcher.threadId, limit: 1, runLimit: 1 }, + limits: { runtimeMode: "full-access", interactionMode: "default" }, + success: WatcherThreadStatus, + }) + .pipe( + Effect.map(({ thread }) => thread.status), + Effect.orElseSucceed(() => null), + ); }); /** @@ -826,7 +921,15 @@ export const make = Effect.gen(function* () { const deliver = Effect.fnUntraced(function* (entry: Delivery) { const current = yield* settings.getSettings; const alert = yield* currentAlert(entry.alert); - if (alert === null) return { _tag: "dropped" } satisfies DeliveryResult; + if (alert === null) { + yield* Effect.logInfo("watcher alert no longer true; dropped without waking anyone", { + alertId: entry.alert.id, + type: entry.alert.type, + kind: entry.alert.type === "thread-event" ? entry.alert.kind : null, + destination: entry.destination, + }); + return { _tag: "dropped" } satisfies DeliveryResult; + } if (entry.destination === "webhook") { const url = current.modelAlertWebhookUrl.trim(); if (url === "") return { _tag: "sent" } satisfies DeliveryResult; @@ -840,7 +943,10 @@ export const make = Effect.gen(function* () { const watcher = current.modelAlertWatcher; if (watcher === null) return { _tag: "sent" } satisfies DeliveryResult; const here = yield* environment.getEnvironmentId; - const until = yield* holdUntil(watcher, here, alert, yield* Clock.currentTimeMillis); + const nowMs = yield* Clock.currentTimeMillis; + const until = + (yield* holdUntil(watcher, here, alert, nowMs)) ?? + busyWatcherHold(yield* watcherStatus(watcher, here), entry.createdAt, nowMs); if (until !== null) return { _tag: "held", until } satisfies DeliveryResult; yield* deliverToWatcher(alert, watcher, here); return { _tag: "sent" } satisfies DeliveryResult; @@ -1029,44 +1135,13 @@ export const make = Effect.gen(function* () { if (watcher === null && current.modelAlertWebhookUrl.trim() === "") return; const shell = yield* projections.getThreadShell(threadId); if (shell === null || shell.deletedAt !== null) return; - const remoteTaskCount = - trigger === "request" || trigger === "inbox" - ? 0 - : remoteTasks !== undefined - ? (remoteTasks.get(threadId) ?? 0) - : (yield* projections.getOpenRemoteDelegatedTasks).filter( - (task) => task.parentThreadId === threadId, - ).length; - // A parent that waits on children is not stuck while they work. - const childActivityMs = - trigger === "sweep" - ? Math.max( - 0, - ...(yield* Effect.forEach( - (shell.pendingBackgroundTasks ?? []).flatMap((task) => - task.kind === "subagent" && task.childThreadId !== undefined - ? [task.childThreadId] - : [], - ), - (childId) => - projections - .getThreadShell(childId) - .pipe( - Effect.map((child) => - child === null ? 0 : DateTime.toEpochMillis(child.updatedAt), - ), - ), - )), - ) - : null; const nowMs = yield* Clock.currentTimeMillis; const judged = judgeThread({ shell, trigger, nowMs, stuckMinutes: current.watcherStuckMinutes, - remoteTaskCount, - childActivityMs, + ...(yield* judgeContext(shell, trigger, remoteTasks)), usageLimitReported, }); const label = (yield* environment.getDescriptor).label; diff --git a/apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts b/apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts index 80abe7f06abf2..c244dd53eef80 100644 --- a/apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts +++ b/apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts @@ -7,6 +7,7 @@ import { promisesToContinue, threadEventAlert, threadEventMessage, + threadEventStillHolds, type ThreadEventTrigger, type WatchedShell, } from "./ThreadEventAlerts.ts"; @@ -303,6 +304,91 @@ it("a recurrence of the same state is a new message", () => { assert.notEqual(make("2026-11-19T05:00:00.000Z").id, make("2026-11-19T05:30:00.000Z").id); }); +const fired = (overrides: Partial, trigger: ThreadEventTrigger) => { + const shell = shellWith(overrides); + const finding = judge(shell, trigger).findings[0]!; + return threadEventAlert({ + environmentId: "env-r720" as never, + environmentLabel: "primary-r720", + shell, + finding, + occurredAt: DateTime.formatIso(DateTime.makeUnsafe(NOW)), + }); +}; +const stillHolds = ( + alert: ReturnType, + overrides: Partial, + base: Partial, +) => + threadEventStillHolds({ alert, shell: shellWith({ ...base, ...overrides }), stuckMinutes: 20 }); + +const failed = { status: "failed" } as never; +const openWithPr = { status: "completed", pullRequests: [openPullRequest] } as never; +const quietRun = { + activeRunId: "run:8", + status: "running", + activityRunStatus: "running", + updatedAt: at(25), +} as never; +const asking = { + pendingRuntimeRequest: { id: "request:9", kind: "approval", userOnly: false }, +} as never; +const promised = { status: "idle", latestVisibleMessage: assistantSays(FLEET_PROMISE) } as never; + +it.each([ + ["lane-ended, nothing changed", failed, "run-ended", {}, true], + ["lane-ended, the lane restarted itself", failed, "run-ended", quietRun, false], + [ + "lane-ended, a newer run already finished", + failed, + "run-ended", + { latestRunId: "run:8", status: "completed" }, + false, + ], + ["lane-ended, the lane was settled", failed, "run-ended", { settledOverride: "settled" }, false], + ["lane-ended, the lane was archived", failed, "run-ended", { archivedAt: at(1) }, false], + [ + "lane-ended with an open PR, a PR watch registered since", + openWithPr, + "run-ended", + { pendingBackgroundTasks: [{ taskId: "pull-request-watch:97", kind: "monitor" }] }, + false, + ], + [ + "lane-ended with an open PR, the PR merged since", + openWithPr, + "run-ended", + { pullRequests: [{ ...(openPullRequest as object), snapshot: { state: "merged" } }] }, + false, + ], + ["stuck, nothing changed", quietRun, "sweep", {}, true], + ["stuck, new activity since", quietRun, "sweep", { updatedAt: at(-1) }, false], + [ + "needs-person, the question was answered", + asking, + "request", + { pendingRuntimeRequest: null }, + false, + ], + [ + "promise-without-wake, a real wait registered since", + promised, + "settled", + { pendingBackgroundTasks: [{ taskId: "b1", kind: "command" }] }, + false, + ], +] as const)("at delivery, %s -> still holds %s", (_name, base, trigger, now, expected) => { + const alert = fired(base, trigger); + assert.equal(stillHolds(alert, now as never, base), expected); +}); + +it("a thread event still holds when delivery waited past the recent-run window", () => { + // Judged at the time it fired, not now: a watcher offline for two hours still hears it. + const alert = fired(failed, "run-ended"); + assert.equal(stillHolds(alert, {}, failed), true); + assert.equal(stillHolds({ ...alert, occurredAt: "not a time" }, quietRun, failed), true); +}); + it("each trigger evaluates only the kinds it can change", () => { assert.deepEqual(judge(shellWith(), "run-ended").evaluated, [ "lane-ended", diff --git a/apps/server/src/orchestration-v2/ThreadEventAlerts.ts b/apps/server/src/orchestration-v2/ThreadEventAlerts.ts index 502c4b2bc3141..7baf71500809b 100644 --- a/apps/server/src/orchestration-v2/ThreadEventAlerts.ts +++ b/apps/server/src/orchestration-v2/ThreadEventAlerts.ts @@ -326,6 +326,45 @@ export function threadEventAlert(input: { }; } +/** The trigger that judges each kind again at delivery: any trigger whose kinds include it. */ +const REJUDGE_TRIGGER: Record = { + "lane-ended": "run-ended", + "promise-without-wake": "settled", + stuck: "sweep", + "needs-person": "request", +}; + +/** + * Whether a queued thread event is still true of the thread as it stands now, judged at the time it + * fired so the recent-run window does not expire while delivery waits. A lane that restarted, + * settled, registered a wait, merged its pull request or got its answer since then is no news. + */ +export function threadEventStillHolds(input: { + readonly alert: ThreadEventAlert; + readonly shell: WatchedShell; + readonly stuckMinutes: number; + readonly remoteTaskCount?: number; + readonly childActivityMs?: number | null; +}): boolean { + const { alert } = input; + const firedMs = Date.parse(alert.occurredAt); + if (!Number.isFinite(firedMs)) return true; + const { findings } = judgeThread({ + shell: input.shell, + trigger: REJUDGE_TRIGGER[alert.kind], + nowMs: firedMs, + stuckMinutes: input.stuckMinutes, + ...(input.remoteTaskCount === undefined ? {} : { remoteTaskCount: input.remoteTaskCount }), + ...(input.childActivityMs === undefined ? {} : { childActivityMs: input.childActivityMs }), + }); + const prefix = `${alert.environmentId}:${alert.threadId}:${alert.kind}:`; + return findings.some( + (finding) => + finding.kind === alert.kind && + alert.id === `${prefix}${finding.signature}:${alert.occurredAt}`, + ); +} + const NEXT_STEP: Record = { "lane-ended": "Check the thread: restart, repair or replace it, or settle it if its work is done. It sends no further alert for this run.", diff --git a/docs/operations/watcher-thread.md b/docs/operations/watcher-thread.md index 7bcf43d0dc57d..02210f467f8ba 100644 --- a/docs/operations/watcher-thread.md +++ b/docs/operations/watcher-thread.md @@ -10,7 +10,7 @@ One thread, by default Lead's on the laptop, hears everything that needs attenti | `stuck` | An active run has had no new activity for `watcherStuckMinutes` (default 20; 0 turns it off), or a thread waits on background work that no longer exists. A parent whose child threads are active is not stuck. | | `needs-person` | An agent-created lane raises a pending question or approval, or its inbox is assigned to a founder or set to `action`. | -Each event reports once per thread and kind until its state changes: a new run, a different message, or the condition clearing. Delegated children are skipped; their parent already hears their result. The watcher thread never reports about itself. +Each event reports once per thread and kind until its state changes: a new run, a different message, or the condition clearing. Delivery judges each event again, as of when it fired, and drops one that is no longer true: the lane restarted or was replaced, settled, registered a wait, merged its pull request or got its answer. While the watcher is mid-turn, alerts wait (up to 10 minutes) instead of queueing behind that turn, so they are judged when it can act on them. Delegated children are skipped; their parent already hears their result. The watcher thread never reports about itself. ## Moving the watcher From bb20080430c84feefe1a500f05853eb284197204 Mon Sep 17 00:00:00 2001 From: nullStack65 <112618179+nullStack65@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:33:23 +0000 Subject: [PATCH 2/3] Watcher hold: never block on a failed watcher read; fix tests for the hold A peer read that dies (not just fails) now delivers as if the watcher were idle. The delegated-completion tests now expect the second local alert to wait for the watcher's queued turn, and the remote stub answers the status read. Co-Authored-By: Claude Opus 5.5 --- .../DelegatedCompletionDelivery.test.ts | 10 ++++++++++ apps/server/src/orchestration-v2/ModelAlertReactor.ts | 3 ++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts index fce4fb9354ec6..0433de375232c 100644 --- a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts +++ b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts @@ -1432,6 +1432,9 @@ const makeModelAlertReactor = (input: { Layer.mock(PeerMcpClient.PeerMcpClient)({ call: (request) => Effect.gen(function* () { + // The busy-watcher check reads the remote watcher first; it finds it idle. + if (request.tool === "t3_thread_read") + return { thread: { status: "idle" } } as never; yield* Ref.update(sends, (all) => [ ...all, { @@ -5088,6 +5091,13 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => { now, }), ); + // The first alert queued a turn on the watcher, so the next one waits for that turn, + // up to the hold cap, instead of queueing behind it. + assert.deepEqual( + (yield* alerts).map((message) => message.text.includes(`/${topLevelThreadId})`)), + [false], + ); + yield* TestClock.adjust("11 minutes"); assert.deepEqual( (yield* alerts).map((message) => message.text.includes(`/${topLevelThreadId})`)), [false, true], diff --git a/apps/server/src/orchestration-v2/ModelAlertReactor.ts b/apps/server/src/orchestration-v2/ModelAlertReactor.ts index d0709a772a72c..d029ca4142b35 100644 --- a/apps/server/src/orchestration-v2/ModelAlertReactor.ts +++ b/apps/server/src/orchestration-v2/ModelAlertReactor.ts @@ -856,7 +856,8 @@ export const make = Effect.gen(function* () { }) .pipe( Effect.map(({ thread }) => thread.status), - Effect.orElseSucceed(() => null), + // A read that fails or dies must never block the alert; deliver as if idle. + Effect.catchCause(() => Effect.succeed(null)), ); }); From 357fdab3216564d9cbfb20c534835e6da33ecb94 Mon Sep 17 00:00:00 2001 From: nullStack65 <112618179+nullStack65@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:42:02 +0000 Subject: [PATCH 3/3] Watcher hold: hold for a waiting watcher, read it once per flush - A watcher whose run is waiting also queues a sent alert, so it is held too. - One watcher read per flush is shared by every due alert and by the usage-limit hold. - currentAlert re-judges with the shared REJUDGE_TRIGGER map. - Keep rows for needs-person, promise-without-wake and lane-ended with an open PR. - Doc states the hold covers every alert kind and the 30-60 s recheck. Co-Authored-By: Claude Opus 5.5 --- .../ModelAlertReactor.test.ts | 2 +- .../src/orchestration-v2/ModelAlertReactor.ts | 107 ++++++++++-------- .../ThreadEventAlerts.test.ts | 3 + .../src/orchestration-v2/ThreadEventAlerts.ts | 2 +- docs/operations/watcher-thread.md | 2 +- 5 files changed, 65 insertions(+), 51 deletions(-) diff --git a/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts b/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts index 336f8456eb751..f1d781547c70c 100644 --- a/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts +++ b/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts @@ -489,7 +489,7 @@ it.each([ ["a watcher that cannot be read", null, 0, null], ["a watcher mid-turn", "running", 0, BUSY_WATCHER_RECHECK_MS], ["a watcher with a queued turn", "queued", 0, BUSY_WATCHER_RECHECK_MS], - ["a watcher waiting on its own background work", "waiting", 0, null], + ["a watcher waiting on its own background work", "waiting", 0, BUSY_WATCHER_RECHECK_MS], ["a watcher busy near the cap", "running", BUSY_WATCHER_MAX_HOLD_MS - 1_000, 1_000], ["a watcher busy past the cap", "running", BUSY_WATCHER_MAX_HOLD_MS, null], ] as const)("busy watcher hold: %s", (_name, status, ageMs, waitMs) => { diff --git a/apps/server/src/orchestration-v2/ModelAlertReactor.ts b/apps/server/src/orchestration-v2/ModelAlertReactor.ts index d029ca4142b35..fd23f6a7628f8 100644 --- a/apps/server/src/orchestration-v2/ModelAlertReactor.ts +++ b/apps/server/src/orchestration-v2/ModelAlertReactor.ts @@ -48,6 +48,7 @@ import { ThreadEventAlert, type ThreadEventKind, threadEventMessage, + REJUDGE_TRIGGER, threadEventStillHolds, type ThreadEventTrigger, type ThreadFinding, @@ -454,7 +455,13 @@ export function afterHold( }; } -const BUSY_WATCHER_STATUSES = new Set(["preparing", "queued", "starting", "running"]); +const BUSY_WATCHER_STATUSES = new Set([ + "preparing", + "queued", + "starting", + "running", + "waiting", +]); /** Matches the flush interval, so a held alert is looked at again on the next flush. */ export const BUSY_WATCHER_RECHECK_MS = 30_000; export const BUSY_WATCHER_MAX_HOLD_MS = 10 * 60 * 1000; @@ -724,6 +731,13 @@ export const make = Effect.gen(function* () { here, ); + type WatcherView = { + readonly status: string; + readonly providerInstanceId: string | null; + /** Null when unknown, as for a remote watcher. */ + readonly usageLimited: boolean | null; + readonly resetAt: string | null; + }; const WatcherThreadStatus = Schema.Struct({ thread: Schema.Struct({ status: Schema.String, providerInstanceId: Schema.String }), }); @@ -792,7 +806,7 @@ export const make = Effect.gen(function* () { alert, shell, stuckMinutes: (yield* settings.getSettings).watcherStuckMinutes, - ...(yield* judgeContext(shell, alert.kind === "stuck" ? "sweep" : "run-ended")), + ...(yield* judgeContext(shell, REJUDGE_TRIGGER[alert.kind])), }); if (!holds) return null; } @@ -838,13 +852,20 @@ export const make = Effect.gen(function* () { return { remoteTaskCount, childActivityMs }; }); - /** The watcher thread's run status, or null when it cannot be read. */ - const watcherStatus = Effect.fnUntraced(function* ( + /** The watcher thread as one delivery needs it, or null when it cannot be read. */ + const readWatcher = Effect.fnUntraced(function* ( watcher: ModelAlertWatcher, here: EnvironmentId, ) { if (watcher.environmentId === here) { - return (yield* projections.getThreadShell(watcher.threadId))?.status ?? null; + const shell = yield* projections.getThreadShell(watcher.threadId); + if (shell === null) return null; + return { + status: shell.status as string, + providerInstanceId: shell.providerInstanceId as string | null, + usageLimited: shell.lastErrorClass === "usage_limit", + resetAt: shell.usageLimitResetAt ?? null, + } satisfies WatcherView; } return yield* peers .call({ @@ -855,7 +876,13 @@ export const make = Effect.gen(function* () { success: WatcherThreadStatus, }) .pipe( - Effect.map(({ thread }) => thread.status), + // The read result carries no error class or reset time. + Effect.map(({ thread }): WatcherView => ({ + status: thread.status, + providerInstanceId: thread.providerInstanceId, + usageLimited: null, + resetAt: null, + })), // A read that fails or dies must never block the alert; deliver as if idle. Effect.catchCause(() => Effect.succeed(null)), ); @@ -865,61 +892,35 @@ export const make = Effect.gen(function* () { * When the watcher's own thread is stopped at the same account limit, the time to look again. * Anything sent now would only queue a turn that fails the same way. */ - const holdUntil = Effect.fnUntraced(function* ( - watcher: ModelAlertWatcher, - here: EnvironmentId, - alert: WatcherAlert, - nowMs: number, - ) { + const holdUntil = (view: WatcherView | null, alert: WatcherAlert, nowMs: number) => { const providerInstanceId = alert.type === "usage-limit-incident" || (alert.type === "model-degraded" && alert.reasons.includes("usage_limit")) ? alert.providerInstanceId : null; - if (providerInstanceId === null) return null; - let down = false; - let resetAt: string | null = alert.type === "usage-limit-incident" ? alert.resetAt : null; - if (watcher.environmentId === here) { - const shell = yield* projections.getThreadShell(watcher.threadId); - down = - shell !== null && - shell.status === "failed" && - shell.providerInstanceId === providerInstanceId && - shell.lastErrorClass === "usage_limit"; - resetAt = shell?.usageLimitResetAt ?? resetAt; - } else { - // The read result carries no error class, so a remote watcher counts as down only while - // the reset this alert reports is still ahead. - down = yield* peers - .call({ - environmentId: watcher.environmentId, - tool: "t3_thread_read", - arguments: { threadId: watcher.threadId, limit: 1, runLimit: 1 }, - limits: { runtimeMode: "full-access", interactionMode: "default" }, - success: WatcherThreadStatus, - }) - .pipe( - Effect.map( - ({ thread }) => - thread.status === "failed" && thread.providerInstanceId === providerInstanceId, - ), - // An unreadable watcher is the delivery's problem, not a reason to hold. - Effect.orElseSucceed(() => false), - ); - } + if (providerInstanceId === null || view === null) return null; + // A remote watcher counts as down only while the reset this alert reports is still ahead. + const down = + view.status === "failed" && + view.providerInstanceId === providerInstanceId && + view.usageLimited !== false; + const resetAt = view.resetAt ?? (alert.type === "usage-limit-incident" ? alert.resetAt : null); if (!down) return null; // Past the reset the limit is over: whatever keeps the watcher down, a message may restart it. const resetMs = resetAt === null ? Number.NaN : Date.parse(resetAt); if (!Number.isFinite(resetMs) || resetMs <= nowMs) return null; return Math.min(resetMs, nowMs + HOLD_RECHECK_MS); - }); + }; type DeliveryResult = | { readonly _tag: "sent" } | { readonly _tag: "dropped" } | { readonly _tag: "held"; readonly until: number }; - const deliver = Effect.fnUntraced(function* (entry: Delivery) { + const deliver = Effect.fnUntraced(function* ( + entry: Delivery, + watcherView: Effect.Effect, + ) { const current = yield* settings.getSettings; const alert = yield* currentAlert(entry.alert); if (alert === null) { @@ -945,9 +946,10 @@ export const make = Effect.gen(function* () { if (watcher === null) return { _tag: "sent" } satisfies DeliveryResult; const here = yield* environment.getEnvironmentId; const nowMs = yield* Clock.currentTimeMillis; + const view = yield* watcherView; const until = - (yield* holdUntil(watcher, here, alert, nowMs)) ?? - busyWatcherHold(yield* watcherStatus(watcher, here), entry.createdAt, nowMs); + holdUntil(view, alert, nowMs) ?? + busyWatcherHold(view?.status ?? null, entry.createdAt, nowMs); if (until !== null) return { _tag: "held", until } satisfies DeliveryResult; yield* deliverToWatcher(alert, watcher, here); return { _tag: "sent" } satisfies DeliveryResult; @@ -956,11 +958,20 @@ export const make = Effect.gen(function* () { const flush: ModelAlertReactor["Service"]["flush"] = Effect.gen(function* () { const nowMs = yield* Clock.currentTimeMillis; const due = (yield* Ref.get(state)).outbox.filter((entry) => entry.nextAttemptAt <= nowMs); + if (due.length === 0) return; + // One read of the watcher per flush, shared by every alert due in it. + const watcherView = yield* Effect.cached( + Effect.gen(function* () { + const watcher = (yield* settings.getSettings).modelAlertWatcher; + if (watcher === null) return null; + return yield* readWatcher(watcher, yield* environment.getEnvironmentId); + }).pipe(Effect.catchCause(() => Effect.succeed(null))), + ); yield* Effect.forEach( due, (entry) => Effect.gen(function* () { - const result = yield* deliver(entry).pipe( + const result = yield* deliver(entry, watcherView).pipe( Effect.timeout("30 seconds"), Effect.catchCause((cause) => Effect.logWarning("delivering a model alert failed; it will be retried", { diff --git a/apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts b/apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts index c244dd53eef80..24fb597c21ce8 100644 --- a/apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts +++ b/apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts @@ -361,6 +361,7 @@ it.each([ { pullRequests: [{ ...(openPullRequest as object), snapshot: { state: "merged" } }] }, false, ], + ["lane-ended with an open PR, nothing changed", openWithPr, "run-ended", {}, true], ["stuck, nothing changed", quietRun, "sweep", {}, true], ["stuck, new activity since", quietRun, "sweep", { updatedAt: at(-1) }, false], [ @@ -370,6 +371,8 @@ it.each([ { pendingRuntimeRequest: null }, false, ], + ["needs-person, still unanswered", asking, "request", {}, true], + ["promise-without-wake, nothing changed", promised, "settled", {}, true], [ "promise-without-wake, a real wait registered since", promised, diff --git a/apps/server/src/orchestration-v2/ThreadEventAlerts.ts b/apps/server/src/orchestration-v2/ThreadEventAlerts.ts index 7baf71500809b..aa61f31970dc4 100644 --- a/apps/server/src/orchestration-v2/ThreadEventAlerts.ts +++ b/apps/server/src/orchestration-v2/ThreadEventAlerts.ts @@ -327,7 +327,7 @@ export function threadEventAlert(input: { } /** The trigger that judges each kind again at delivery: any trigger whose kinds include it. */ -const REJUDGE_TRIGGER: Record = { +export const REJUDGE_TRIGGER: Record = { "lane-ended": "run-ended", "promise-without-wake": "settled", stuck: "sweep", diff --git a/docs/operations/watcher-thread.md b/docs/operations/watcher-thread.md index 02210f467f8ba..6a241e490e752 100644 --- a/docs/operations/watcher-thread.md +++ b/docs/operations/watcher-thread.md @@ -10,7 +10,7 @@ One thread, by default Lead's on the laptop, hears everything that needs attenti | `stuck` | An active run has had no new activity for `watcherStuckMinutes` (default 20; 0 turns it off), or a thread waits on background work that no longer exists. A parent whose child threads are active is not stuck. | | `needs-person` | An agent-created lane raises a pending question or approval, or its inbox is assigned to a founder or set to `action`. | -Each event reports once per thread and kind until its state changes: a new run, a different message, or the condition clearing. Delivery judges each event again, as of when it fired, and drops one that is no longer true: the lane restarted or was replaced, settled, registered a wait, merged its pull request or got its answer. While the watcher is mid-turn, alerts wait (up to 10 minutes) instead of queueing behind that turn, so they are judged when it can act on them. Delegated children are skipped; their parent already hears their result. The watcher thread never reports about itself. +Each event reports once per thread and kind until its state changes: a new run, a different message, or the condition clearing. Delivery judges each event again, as of when it fired, and drops one that is no longer true: the lane restarted, settled, registered a wait, merged its pull request or got its answer. A dropped event stays reported, so the same state coming back does not alert again until its run or message changes. While the watcher is busy (running, queued or waiting), every alert to it, including model and usage-limit alerts, waits up to 10 minutes instead of queueing behind that turn, so it is judged when the watcher can act. A held alert is looked at again on a later flush, about every 30 to 60 seconds. Delegated children are skipped; their parent already hears their result. The watcher thread never reports about itself. ## Moving the watcher