From 921217ab21630afed0624538e251d0d56dee03be Mon Sep 17 00:00:00 2001 From: nullStack65 <112618179+nullStack65@users.noreply.github.com> Date: Thu, 19 Nov 2026 05:54:36 +0000 Subject: [PATCH 1/3] feat(server): wake the watcher thread when a lane ends, stalls, promises or needs a person The model-alert watcher now also hears four deterministic thread events, judged server-side from the thread shell: lane-ended, promise-without-wake, stuck and needs-person. They share the model alert outbox, retry, cross-environment forwarding, dedupe (per thread and kind until state changes) and webhook. Adds watcherStuckMinutes (default 20) and docs on moving the watcher to an always-on Lead. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/closura-arc-ci.yml | 1 + .../ModelAlertReactor.test.ts | 139 ++++++ .../src/orchestration-v2/ModelAlertReactor.ts | 447 +++++++++++++----- .../ThreadEventAlerts.test.ts | 308 ++++++++++++ .../src/orchestration-v2/ThreadEventAlerts.ts | 352 ++++++++++++++ docs/operations/watcher-thread.md | 17 + packages/contracts/src/settings.ts | 22 +- 7 files changed, 1176 insertions(+), 110 deletions(-) create mode 100644 apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts create mode 100644 apps/server/src/orchestration-v2/ThreadEventAlerts.ts create mode 100644 docs/operations/watcher-thread.md diff --git a/.github/workflows/closura-arc-ci.yml b/.github/workflows/closura-arc-ci.yml index bb5a8f791935..f416c6c5c1ed 100644 --- a/.github/workflows/closura-arc-ci.yml +++ b/.github/workflows/closura-arc-ci.yml @@ -159,6 +159,7 @@ jobs: apps/server/src/orchestration-v2/ThreadLaunchService.test.ts \ apps/server/src/orchestration-v2/SubagentProjection.test.ts \ apps/server/src/orchestration-v2/ModelAlertReactor.test.ts \ + apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts \ apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts \ packages/shared/src/threadPlacement.test.ts \ apps/server/src/peer/RemoteDelegation.test.ts \ diff --git a/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts b/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts index 11277dc3d395..3744fb10861b 100644 --- a/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts +++ b/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts @@ -1,14 +1,19 @@ import { assert, it } from "@effect/vitest"; import { EnvironmentId, type OrchestrationV2Run, ThreadId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; import { afterDeliveryAttempt, + deliverAlertToWatcher, EMPTY_MODEL_ALERT_STATE, type ModelAlert, modelAlertDestinations, modelAlertFor, recordModelAlert, + recordThreadEvents, + watcherAlertMessage, } from "./ModelAlertReactor.ts"; +import type { ThreadEventAlert, ThreadEventKind, ThreadFinding } from "./ThreadEventAlerts.ts"; const alertFor = (runId: string, overrides: Partial = {}): ModelAlert => ({ type: "model-degraded", @@ -149,3 +154,137 @@ it.each([ expected, ); }); + +const eventAlert = (finding: ThreadFinding): ThreadEventAlert => ({ + type: "thread-event", + kind: finding.kind, + id: `env:thread:fleet:${finding.kind}:${finding.signature}`, + environmentId: "env", + environmentLabel: "primary-r720", + threadId: "thread:fleet", + threadTitle: "Fleet overnight", + threadLink: "[Fleet overnight](t3-thread://v2/env/thread:fleet)", + runId: "run:1", + detail: finding.detail, + lastAssistantLine: "A background watch will wake this thread.", + lastError: null, + openPullRequests: [], + occurredAt: "2026-11-19T03:00:00.000Z", +}); + +const finding = (kind: ThreadEventKind, signature: string): ThreadFinding => ({ + kind, + signature, + detail: `${kind} detail`, +}); + +it("reports a thread event once per kind and signature until its state changes", () => { + let state = EMPTY_MODEL_ALERT_STATE; + const enqueued: Array = []; + const step = ( + evaluated: ReadonlyArray, + findings: ReadonlyArray, + ) => { + const result = recordThreadEvents( + state, + { threadId: "thread:fleet", evaluated, findings }, + eventAlert, + ["watcher"], + 0, + ); + state = result.state; + enqueued.push(result.enqueued); + }; + step(["stuck"], [finding("stuck", "run:1")]); + // The same run still stuck on the next sweep says nothing. + step(["stuck"], [finding("stuck", "run:1")]); + // Another kind on the same thread is its own event. + step(["lane-ended"], [finding("lane-ended", "run:1")]); + // A trigger that does not evaluate a kind leaves its dedupe alone. + step(["needs-person"], []); + step(["stuck"], [finding("stuck", "run:1")]); + // Recovery clears it. The same signature queues nothing while its first delivery still waits; + // a new run reports at once. + step(["stuck"], []); + step(["stuck"], [finding("stuck", "run:1")]); + step(["stuck"], [finding("stuck", "run:2")]); + assert.deepEqual(enqueued, [true, false, true, false, false, false, false, true]); + assert.deepEqual( + state.outbox.map((entry) => entry.alert.id), + [ + "env:thread:fleet:stuck:run:1", + "env:thread:fleet:lane-ended:run:1", + "env:thread:fleet:stuck:run:2", + ], + ); +}); + +it("queues one delivery per destination for a thread event, beside model alerts", () => { + const { state, enqueued } = recordThreadEvents( + EMPTY_MODEL_ALERT_STATE, + { + threadId: "thread:fleet", + evaluated: ["needs-person"], + findings: [finding("needs-person", "request:1")], + }, + eventAlert, + ["watcher", "webhook"], + 0, + ); + assert.equal(enqueued, true); + assert.deepEqual( + state.outbox.map((entry) => [entry.alert.type, entry.destination]), + [ + ["thread-event", "watcher"], + ["thread-event", "webhook"], + ], + ); + const withModel = recordModelAlert( + state, + { alert: alertFor("run:1"), healthy: false, threadId: "thread:fleet" }, + ["watcher"], + 0, + ); + assert.deepEqual( + withModel.state.outbox.map((entry) => entry.alert.type), + ["thread-event", "thread-event", "model-degraded"], + ); + assert.include(watcherAlertMessage(alertFor("run:1")), "Model alert from"); + assert.include(watcherAlertMessage(eventAlert(finding("stuck", "run:1"))), "Thread event from"); +}); + +it.effect("forwards a thread event to a watcher on another environment with one retry key", () => + Effect.gen(function* () { + const dispatched: Array = []; + const called: Array<{ + readonly arguments?: Record; + readonly environmentId: string; + }> = []; + const deps = { + dispatch: (command: unknown) => Effect.sync(() => void dispatched.push(command)), + call: (input: never) => Effect.sync(() => void called.push(input)), + sha256Hex: (identity: string) => Effect.succeed(`digest(${identity.length})`), + } as never; + const alert = eventAlert(finding("promise-without-wake", "message:1")); + const laptop = { environmentId: EnvironmentId.make("laptop"), threadId: ThreadId.make("lead") }; + yield* deliverAlertToWatcher(deps, alert, laptop, EnvironmentId.make("r720")); + yield* deliverAlertToWatcher(deps, alert, laptop, EnvironmentId.make("r720")); + assert.equal(dispatched.length, 0); + assert.equal(called.length, 2); + assert.equal(called[0]!.environmentId, "laptop"); + assert.equal(called[0]!.arguments?.threadId, "lead"); + assert.equal(called[0]!.arguments?.mode, "queue"); + assert.equal(called[0]!.arguments?.message, watcherAlertMessage(alert)); + assert.match(String(called[0]!.arguments?.clientRequestId), /^thread-event:/); + assert.equal(called[0]!.arguments?.clientRequestId, called[1]!.arguments?.clientRequestId); + + // The same alert on the watcher's own machine queues into the thread directly. + yield* deliverAlertToWatcher(deps, alert, laptop, EnvironmentId.make("laptop")); + assert.equal(called.length, 2); + assert.equal(dispatched.length, 1); + assert.equal((dispatched[0] as { threadId: string }).threadId, "lead"); + assert.deepEqual((dispatched[0] as { dispatchMode: unknown }).dispatchMode, { + type: "queue_after_active", + }); + }), +); diff --git a/apps/server/src/orchestration-v2/ModelAlertReactor.ts b/apps/server/src/orchestration-v2/ModelAlertReactor.ts index 52c99fad07cf..71e7822885ef 100644 --- a/apps/server/src/orchestration-v2/ModelAlertReactor.ts +++ b/apps/server/src/orchestration-v2/ModelAlertReactor.ts @@ -1,6 +1,7 @@ import { CommandId, type EnvironmentId, + isListedThreadLineage, MessageId, type ModelAlertWatcher, type OrchestrationV2DomainEvent, @@ -8,6 +9,7 @@ import { type OrchestrationV2ThreadProjection, OrchestrationV2WorkerAlertReason, OrchestratorMcpThreadSendResult, + type ThreadId, } from "@t3tools/contracts"; import { runModelSignals, workerAlert } from "@t3tools/provider-core/server/subagentProjection"; import { writeFileStringAtomically } from "@t3tools/shared/atomicWrite"; @@ -38,6 +40,15 @@ import { forkParked } from "../serverActivation.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as Orchestrator from "./Orchestrator.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; +import { + judgeThread, + threadEventAlert, + ThreadEventAlert, + type ThreadEventKind, + threadEventMessage, + type ThreadEventTrigger, + type ThreadFinding, +} from "./ThreadEventAlerts.ts"; export const ModelAlert = Schema.Struct({ type: Schema.Literal("model-degraded"), @@ -59,12 +70,16 @@ export const ModelAlert = Schema.Struct({ }); export type ModelAlert = typeof ModelAlert.Type; +/** Everything the watcher thread hears: model alerts and deterministic thread events. */ +export const WatcherAlert = Schema.Union([ModelAlert, ThreadEventAlert]); +export type WatcherAlert = typeof WatcherAlert.Type; + const Destination = Schema.Literals(["watcher", "webhook"]); type Destination = typeof Destination.Type; const Delivery = Schema.Struct({ destination: Destination, - alert: ModelAlert, + alert: WatcherAlert, attempts: Schema.Int, nextAttemptAt: Schema.Number, createdAt: Schema.Number, @@ -75,13 +90,23 @@ export const ModelAlertState = Schema.Struct({ version: Schema.Literal(1), /** The alert signature each thread last reported; a healthy run clears it. */ degraded: Schema.Record(Schema.String, Schema.String), + /** The signature of each thread event last reported, by thread and kind; a changed state clears it. */ + reported: Schema.Record(Schema.String, Schema.String).pipe( + Schema.withDecodingDefault(Effect.succeed({})), + ), outbox: Schema.Array(Delivery), }); export type ModelAlertState = typeof ModelAlertState.Type; -export const EMPTY_MODEL_ALERT_STATE: ModelAlertState = { version: 1, degraded: {}, outbox: [] }; +export const EMPTY_MODEL_ALERT_STATE: ModelAlertState = { + version: 1, + degraded: {}, + reported: {}, + outbox: [], +}; const MAX_DEGRADED_THREADS = 1_000; +const MAX_REPORTED = 2_000; const MAX_OUTBOX = 200; const OUTBOX_TTL_MS = 7 * 24 * 60 * 60 * 1000; const RETRY_BASE_MS = 15_000; @@ -175,12 +200,28 @@ export function recordModelAlert( if (state.degraded[alert.threadId] === next) return { state, enqueued: false }; const { [alert.threadId]: _previous, ...rest } = state.degraded; const kept = Object.entries(rest).slice(-(MAX_DEGRADED_THREADS - 1)); + const queued = enqueueAlert(state.outbox, alert, destinations, nowMs); + return { + state: { + ...state, + degraded: { ...Object.fromEntries(kept), [alert.threadId]: next }, + outbox: queued.outbox, + }, + enqueued: queued.enqueued, + }; +} + +/** The outbox with the alert queued once per destination; an alert already queued is not repeated. */ +function enqueueAlert( + outbox: ModelAlertState["outbox"], + alert: WatcherAlert, + destinations: ReadonlyArray, + nowMs: number, +): { readonly outbox: ModelAlertState["outbox"]; readonly enqueued: boolean } { const added = destinations .filter( (destination) => - !state.outbox.some( - (entry) => entry.alert.id === alert.id && entry.destination === destination, - ), + !outbox.some((entry) => entry.alert.id === alert.id && entry.destination === destination), ) .map((destination): Delivery => ({ destination, @@ -190,15 +231,57 @@ export function recordModelAlert( createdAt: nowMs, })); return { - state: { - ...state, - degraded: { ...Object.fromEntries(kept), [alert.threadId]: next }, - outbox: [...pruneOutbox(state.outbox, nowMs), ...added].slice(-MAX_OUTBOX), - }, + outbox: [...pruneOutbox(outbox, nowMs), ...added].slice(-MAX_OUTBOX), enqueued: added.length > 0, }; } +const reportedKey = (threadId: string, kind: ThreadEventKind) => `${threadId}\n${kind}`; + +/** + * Records what a trigger found true of a thread. A kind reports once per signature: it stays quiet + * until the thread stops matching it or matches it with a different signature, such as a new run. + */ +export function recordThreadEvents( + state: ModelAlertState, + judged: { + readonly threadId: string; + readonly evaluated: ReadonlyArray; + readonly findings: ReadonlyArray; + }, + toAlert: (finding: ThreadFinding) => ThreadEventAlert, + destinations: ReadonlyArray, + nowMs: number, +): { readonly state: ModelAlertState; readonly enqueued: boolean } { + let reported = state.reported; + let outbox = state.outbox; + let enqueued = false; + for (const kind of judged.evaluated) { + const key = reportedKey(judged.threadId, kind); + const kindFindings = judged.findings.filter((finding) => finding.kind === kind); + if (kindFindings.length === 0) { + if (reported[key] !== undefined) { + const { [key]: _cleared, ...rest } = reported; + reported = rest; + } + continue; + } + // One finding per kind reports at a time; the first (most specific) one wins. + const finding = kindFindings.find((candidate) => reported[key] !== candidate.signature); + if (finding === undefined) continue; + const { [key]: _previous, ...rest } = reported; + reported = { + ...Object.fromEntries(Object.entries(rest).slice(-(MAX_REPORTED - 1))), + [key]: finding.signature, + }; + const queued = enqueueAlert(outbox, toAlert(finding), destinations, nowMs); + outbox = queued.outbox; + enqueued ||= queued.enqueued; + } + if (reported === state.reported && outbox === state.outbox) return { state, enqueued }; + return { state: { ...state, reported, outbox }, enqueued }; +} + const pruneOutbox = (outbox: ReadonlyArray, nowMs: number) => outbox.filter((entry) => nowMs - entry.createdAt < OUTBOX_TTL_MS); @@ -247,6 +330,50 @@ function modelAlertMessage(alert: ModelAlert): string { ].join("\n\n"); } +export const watcherAlertMessage = (alert: WatcherAlert) => + alert.type === "thread-event" ? threadEventMessage(alert) : modelAlertMessage(alert); + +/** + * Puts an alert in the watcher thread, here or on the linked machine that owns it. The retry key is + * one per alert and watcher, so a retry after a lost answer sends once. + */ +export const deliverAlertToWatcher = Effect.fnUntraced(function* ( + deps: { + readonly dispatch: Orchestrator.OrchestratorV2["Service"]["dispatch"]; + readonly call: PeerMcpClient.PeerMcpClient["Service"]["call"]; + readonly sha256Hex: (identity: string) => Effect.Effect; + }, + alert: WatcherAlert, + watcher: ModelAlertWatcher, + here: EnvironmentId, +) { + const message = watcherAlertMessage(alert); + // Run ids can be long; a retry key must stay under the 256 characters the peer accepts. + const identity = `${alert.id}\n${watcher.environmentId}\n${watcher.threadId}`; + const key = `${alert.type === "thread-event" ? "thread-event" : "model-alert"}:${yield* deps.sha256Hex(identity)}`; + if (watcher.environmentId === here) { + yield* deps.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(`server:${key}`), + threadId: watcher.threadId, + messageId: MessageId.make(`message:${key}`), + text: message, + attachments: [], + dispatchMode: { type: "queue_after_active" }, + createdBy: "agent", + creationSource: "server", + }); + return; + } + yield* deps.call({ + environmentId: watcher.environmentId, + tool: "t3_thread_send", + arguments: { threadId: watcher.threadId, message, mode: "queue", clientRequestId: key }, + limits: { runtimeMode: "full-access", interactionMode: "default" }, + success: OrchestratorMcpThreadSendResult, + }); +}); + const ModelAlertStateJson = Schema.fromJsonString(ModelAlertState); const decodeState = Schema.decodeUnknownOption(ModelAlertStateJson); const encodeState = Schema.encodeEffect(ModelAlertStateJson); @@ -259,6 +386,8 @@ export class ModelAlertReactor extends Context.Service< readonly handle: (event: OrchestrationV2DomainEvent) => Effect.Effect; /** Attempts every queued delivery that is due. */ readonly flush: Effect.Effect; + /** Judges every live lane for the time-based events: stuck. */ + readonly sweep: Effect.Effect; } >()("t3/orchestration-v2/ModelAlertReactor") {} @@ -309,41 +438,17 @@ export const make = Effect.gen(function* () { }), ); - const deliverToWatcher = Effect.fnUntraced(function* ( - alert: ModelAlert, - watcher: ModelAlertWatcher, - here: EnvironmentId, - ) { - const message = modelAlertMessage(alert); - // Run ids can be long; a retry key must stay under the 256 characters the peer accepts. - const identity = `${alert.id}\n${watcher.environmentId}\n${watcher.threadId}`; - const digest = yield* crypto + const sha256Hex = (identity: string) => + crypto .digest(Digest.sha256, new TextEncoder().encode(identity)) - .pipe(Effect.orDie); - const key = `model-alert:${Hex.encode(digest)}`; - if (watcher.environmentId === here) { - yield* engine.dispatch({ - type: "message.dispatch", - commandId: CommandId.make(`server:${key}`), - threadId: watcher.threadId, - messageId: MessageId.make(`message:${key}`), - text: message, - attachments: [], - dispatchMode: { type: "queue_after_active" }, - createdBy: "agent", - creationSource: "server", - }); - return; - } - // The watcher's machine keeps one retry key per alert, so a retry after a lost answer sends once. - yield* peers.call({ - environmentId: watcher.environmentId, - tool: "t3_thread_send", - arguments: { threadId: watcher.threadId, message, mode: "queue", clientRequestId: key }, - limits: { runtimeMode: "full-access", interactionMode: "default" }, - success: OrchestratorMcpThreadSendResult, - }); - }); + .pipe(Effect.map(Hex.encode), Effect.orDie); + const deliverToWatcher = (alert: WatcherAlert, watcher: ModelAlertWatcher, here: EnvironmentId) => + deliverAlertToWatcher( + { dispatch: engine.dispatch, call: peers.call, sha256Hex }, + alert, + watcher, + here, + ); const deliver = Effect.fnUntraced(function* (entry: Delivery) { const current = yield* settings.getSettings; @@ -391,70 +496,198 @@ export const make = Effect.gen(function* () { ); }); - const handle: ModelAlertReactor["Service"]["handle"] = Effect.fnUntraced( - function* (event) { - if (event.type !== "run.updated" || !TERMINAL_RUN_STATUSES.includes(event.payload.status)) { - return; - } - const run = event.payload; - const records = yield* projections.getThreadRecords( - event.threadId, - ["providerTurns", "turnItems"], - { - providerTurnAttemptIds: run.activeAttemptId === null ? [] : [run.activeAttemptId], - turnItemRunIds: [run.id], - turnItemTypes: ["system_notice", "error"], - }, + /** Queues a run's model alert, and says whether it reported a usage limit. */ + const handleModelAlert = Effect.fnUntraced(function* ( + event: Extract, + ) { + const run = event.payload; + const records = yield* projections.getThreadRecords( + event.threadId, + ["providerTurns", "turnItems"], + { + providerTurnAttemptIds: run.activeAttemptId === null ? [] : [run.activeAttemptId], + turnItemRunIds: [run.id], + turnItemTypes: ["system_notice", "error"], + }, + ); + const here = yield* environment.getEnvironmentId; + const judged = modelAlertFor({ + environmentId: here, + environmentLabel: "", + thread: records.thread, + run, + projection: records, + occurredAt: DateTime.formatIso(yield* DateTime.now), + }); + if (judged.alert === undefined && !judged.healthy) return { usageLimited: false }; + const outcome = + judged.alert === undefined + ? judged + : { + ...judged, + alert: { + ...judged.alert, + environmentLabel: (yield* environment.getDescriptor).label, + }, + }; + const current = yield* settings.getSettings; + const destinations = modelAlertDestinations({ + watcher: current.modelAlertWatcher, + webhookUrl: current.modelAlertWebhookUrl, + here, + lineage: records.thread.lineage, + }); + const nowMs = yield* Clock.currentTimeMillis; + const enqueued = yield* update((current) => { + const result = recordModelAlert( + current, + { ...outcome, threadId: event.threadId }, + destinations, + nowMs, ); - const here = yield* environment.getEnvironmentId; - const judged = modelAlertFor({ - environmentId: here, - environmentLabel: "", - thread: records.thread, - run, - projection: records, - occurredAt: DateTime.formatIso(yield* DateTime.now), - }); - if (judged.alert === undefined && !judged.healthy) return; - const outcome = - judged.alert === undefined - ? judged - : { - ...judged, - alert: { - ...judged.alert, - environmentLabel: (yield* environment.getDescriptor).label, - }, - }; - const current = yield* settings.getSettings; - const destinations = modelAlertDestinations({ - watcher: current.modelAlertWatcher, - webhookUrl: current.modelAlertWebhookUrl, - here, - lineage: records.thread.lineage, - }); - const nowMs = yield* Clock.currentTimeMillis; - const enqueued = yield* update((current) => { - const result = recordModelAlert( - current, - { ...outcome, threadId: event.threadId }, - destinations, - nowMs, - ); - return [result.enqueued, result.state]; - }); - if (enqueued) yield* Queue.offer(wake, undefined); - }, - (effect, event) => - effect.pipe( - Effect.catchCause((cause) => - Effect.logWarning("judging a run for a model alert failed", { - threadId: event.threadId, - cause, + return [result.enqueued, result.state]; + }); + if (enqueued) yield* Queue.offer(wake, undefined); + return { usageLimited: judged.alert?.reasons.includes("usage_limit") === true }; + }); + + /** Judges one thread for the kinds a trigger can change, and queues what is newly true. */ + const handleThreadEvents = Effect.fnUntraced(function* ( + threadId: ThreadId, + trigger: ThreadEventTrigger, + usageLimitReported: boolean, + ) { + const current = yield* settings.getSettings; + const here = yield* environment.getEnvironmentId; + const watcher = current.modelAlertWatcher; + // The watcher's own turns would report to itself forever. + if (watcher?.environmentId === here && watcher.threadId === threadId) return; + if (watcher === null && current.modelAlertWebhookUrl.trim() === "") return; + const shell = yield* projections.getThreadShell(threadId); + if (shell === null || shell.deletedAt !== null) return; + const remoteTaskCount = + trigger === "request" || trigger === "inbox" + ? 0 + : (yield* projections.getOpenRemoteDelegatedTasks).filter( + (task) => task.parentThreadId === threadId, + ).length; + // A parent that waits on children is not stuck while they work. + const childActivityMs = + trigger === "sweep" + ? Math.max( + 0, + ...(yield* Effect.forEach( + (shell.pendingBackgroundTasks ?? []).flatMap((task) => + task.kind === "subagent" && task.childThreadId !== undefined + ? [task.childThreadId] + : [], + ), + (childId) => + projections + .getThreadShell(childId) + .pipe( + Effect.map((child) => + child === null ? 0 : DateTime.toEpochMillis(child.updatedAt), + ), + ), + )), + ) + : null; + const nowMs = yield* Clock.currentTimeMillis; + const judged = judgeThread({ + shell, + trigger, + nowMs, + stuckMinutes: current.watcherStuckMinutes, + remoteTaskCount, + childActivityMs, + usageLimitReported, + }); + const label = (yield* environment.getDescriptor).label; + const occurredAt = DateTime.formatIso(yield* DateTime.now); + const destinations = modelAlertDestinations({ + watcher, + webhookUrl: current.modelAlertWebhookUrl, + here, + lineage: shell.lineage, + }); + const enqueued = yield* update((state) => { + const result = recordThreadEvents( + state, + { threadId, evaluated: judged.evaluated, findings: judged.findings }, + (finding) => + threadEventAlert({ + environmentId: here, + environmentLabel: label, + shell, + finding, + occurredAt, }), - ), + destinations, + nowMs, + ); + return [result.enqueued, result.state]; + }); + if (enqueued) yield* Queue.offer(wake, undefined); + }); + + const threadTrigger = (event: OrchestrationV2DomainEvent): ThreadEventTrigger | undefined => { + switch (event.type) { + case "run.updated": + return TERMINAL_RUN_STATUSES.includes(event.payload.status) ? "run-ended" : undefined; + case "thread.settled": + return "settled"; + case "thread.metadata-updated": + return "inbox"; + case "runtime-request.updated": + return "request"; + default: + return undefined; + } + }; + + const handle: ModelAlertReactor["Service"]["handle"] = Effect.fnUntraced(function* (event) { + const trigger = threadTrigger(event); + if (trigger === undefined) return; + const modelAlert = + event.type === "run.updated" + ? yield* handleModelAlert(event).pipe( + Effect.catchCause((cause) => + Effect.logWarning("judging a run for a model alert failed", { + threadId: event.threadId, + cause, + }).pipe(Effect.as({ usageLimited: false })), + ), + ) + : { usageLimited: false }; + yield* handleThreadEvents(event.threadId, trigger, modelAlert.usageLimited).pipe( + Effect.catchCause((cause) => + Effect.logWarning("judging a thread for a watcher event failed", { + threadId: event.threadId, + trigger, + cause, + }), ), - ); + ); + }); + + /** Looks at every thread that may be live for the time-based kinds no event announces. */ + const sweep: ModelAlertReactor["Service"]["sweep"] = Effect.gen(function* () { + const index = yield* projections.listActiveShellIndex(); + yield* Effect.forEach( + index.filter((entry) => entry.mayBeLive && isListedThreadLineage(entry.lineage)), + (entry) => + handleThreadEvents(entry.id, "sweep", false).pipe( + Effect.catchCause((cause) => + Effect.logWarning("sweeping a thread for a watcher event failed", { + threadId: entry.id, + cause, + }), + ), + ), + { discard: true }, + ); + }).pipe(Effect.catchCause((cause) => Effect.logWarning("watcher sweep failed", { cause }))); const start: ModelAlertReactor["Service"]["start"] = Effect.fn("ModelAlertReactor.start")( function* () { @@ -476,10 +709,12 @@ export const make = Effect.gen(function* () { ), ), ); + // Stuck lanes emit no event, so the sweep is the one timer here. + yield* forkParked(sweep.pipe(Effect.repeat(Schedule.spaced("60 seconds")))); }, ); - return ModelAlertReactor.of({ start, handle, flush }); + return ModelAlertReactor.of({ start, handle, flush, sweep }); }); export const layer = Layer.effect(ModelAlertReactor, make); diff --git a/apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts b/apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts new file mode 100644 index 000000000000..02e0553e7935 --- /dev/null +++ b/apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts @@ -0,0 +1,308 @@ +import { assert, it } from "@effect/vitest"; +import * as DateTime from "effect/DateTime"; + +import { + judgeThread, + lastLine, + promisesToContinue, + threadEventAlert, + threadEventMessage, + type ThreadEventTrigger, + type WatchedShell, +} from "./ThreadEventAlerts.ts"; + +const NOW = Date.parse("2026-11-19T05:00:00.000Z"); +const at = (minutesAgo: number) => DateTime.makeUnsafe(NOW - minutesAgo * 60_000); + +// The Fleet rollout lane said this, settled with no tracked task, and its R720 build died unnoticed. +const FLEET_PROMISE = + "The desktop build is running on R720. A background watch will wake this thread when it finishes, and I'll install it on ASUS after that. I'll settle this thread only once all three machines are confirmed."; + +const shellWith = (overrides: Partial = {}): WatchedShell => + ({ + id: "thread:lane", + title: "Fleet rollout", + createdBy: "agent", + lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: null }, + status: "completed", + activeRunId: null, + latestRunId: "run:7", + latestRunCompletedAt: at(2), + activityRunStatus: null, + blockedOnThreadWait: false, + pendingRuntimeRequest: null, + pendingBackgroundTasks: [], + pullRequests: [], + latestVisibleMessage: null, + lastError: null, + settledOverride: null, + archivedAt: null, + snoozedUntil: null, + limitRecovery: null, + updatedAt: at(2), + ...overrides, + }) as unknown as WatchedShell; + +const judge = ( + shell: WatchedShell, + trigger: ThreadEventTrigger, + extra: Partial[0]> = {}, +) => judgeThread({ shell, trigger, nowMs: NOW, stuckMinutes: 20, ...extra }); + +const openPullRequest = { + host: "github.com", + repository: "nullStack65/t3code", + number: 97, + url: "https://github.com/nullStack65/t3code/pull/97", + source: "agent", + linkedAt: "2026-11-19T04:00:00.000Z", + snapshot: null, + stack: null, +} as never; + +const assistantSays = (text: string) => + ({ id: "message:1", role: "assistant", text, updatedAt: at(2) }) as never; + +it.each([ + ["a failed run", { status: "failed" }, ["lane-ended"]], + ["an interrupted run", { status: "interrupted" }, ["lane-ended"]], + ["a completed run with nothing open", { status: "completed" }, []], + [ + "a completed run with an open pull request and no task", + { status: "completed", pullRequests: [openPullRequest] }, + ["lane-ended"], + ], + [ + "a completed run with an open pull request and a tracked task", + { + status: "completed", + pullRequests: [openPullRequest], + pendingBackgroundTasks: [{ taskId: "pull-request-watch:97", kind: "monitor" }], + }, + [], + ], + ["a failed run that ended long ago", { status: "failed", latestRunCompletedAt: at(120) }, []], + ["a failed run on a settled lane", { status: "failed", settledOverride: "settled" }, []], + [ + "a failed run on a delegated child", + { + status: "failed", + lineage: { parentThreadId: "thread:lead", relationshipToParent: "subagent" }, + }, + [], + ], + ["a run still going", { status: "failed", activeRunId: "run:8" }, []], +] as const)("lane-ended: %s -> %j", (_name, overrides, expected) => { + const { findings } = judge(shellWith(overrides as never), "run-ended"); + assert.deepEqual( + findings.filter((finding) => finding.kind === "lane-ended").map((finding) => finding.kind), + expected, + ); +}); + +it("lane-ended stays quiet for a failure the model alert already reported as a usage limit", () => { + const { findings } = judge(shellWith({ status: "failed" } as never), "run-ended", { + usageLimitReported: true, + }); + assert.deepEqual(findings, []); +}); + +it("promise-without-wake: flags the Fleet lane's real message when nothing is tracked", () => { + const { findings } = judge( + shellWith({ latestVisibleMessage: assistantSays(FLEET_PROMISE) } as never), + "run-ended", + ); + assert.deepEqual( + findings.map((finding) => finding.kind), + ["promise-without-wake"], + ); + assert.equal(findings[0]?.signature, "message:1"); +}); + +it.each([ + [ + "a tracked background command", + { pendingBackgroundTasks: [{ taskId: "b1", kind: "command" }] }, + 0, + ], + ["a delegated task on another machine", {}, 1], + ["a pull request watch", { pullRequests: [{ ...(openPullRequest as object), watch: {} }] }, 0], + ["an armed usage-limit resume", { limitRecovery: { autoResume: true } }, 0], + ["a pending question", { pendingRuntimeRequest: { id: "request:1", kind: "user_input" } }, 0], +] as const)("promise-without-wake: %s is a wake handle", (_name, overrides, remoteTaskCount) => { + const { findings } = judge( + shellWith({ latestVisibleMessage: assistantSays(FLEET_PROMISE), ...overrides } as never), + "run-ended", + { remoteTaskCount }, + ); + assert.deepEqual(findings, []); +}); + +it("promise-without-wake: judges a settling thread even when its last run ended long ago", () => { + // A settled lane is not `watched`, so the settle trigger sees the shell as it was just before. + const { findings } = judge( + shellWith({ + latestVisibleMessage: assistantSays(FLEET_PROMISE), + latestRunCompletedAt: at(300), + } as never), + "settled", + ); + assert.deepEqual( + findings.map((finding) => finding.kind), + ["promise-without-wake"], + ); +}); + +it.each([ + [FLEET_PROMISE, true], + ["I'll install the desktop build after the R720 build finishes.", true], + ["Keeping this lane open until the rollout is verified.", true], + ["I'll be woken when the command ends.", true], + ["The rollout is live on all three machines. Nothing left to do.", false], + ["I installed it after the build finished.", false], +] as const)("promise phrases: %j -> %s", (text, expected) => { + assert.equal(promisesToContinue(text), expected); +}); + +it("stuck: an active run with no new item for the configured minutes", () => { + const active = { activeRunId: "run:8", status: "running", activityRunStatus: "running" }; + const quiet = judge(shellWith({ ...active, updatedAt: at(25) } as never), "sweep"); + assert.deepEqual( + quiet.findings.map((finding) => [finding.kind, finding.signature]), + [["stuck", "run:8"]], + ); + assert.deepEqual( + judge(shellWith({ ...active, updatedAt: at(5) } as never), "sweep").findings, + [], + ); + // The threshold is configurable and zero turns the check off. + assert.deepEqual( + judge(shellWith({ ...active, updatedAt: at(5) } as never), "sweep", { stuckMinutes: 3 }) + .findings.length, + 1, + ); + assert.deepEqual( + judge(shellWith({ ...active, updatedAt: at(500) } as never), "sweep", { stuckMinutes: 0 }) + .findings, + [], + ); + // A parent whose child is still working is not stuck. + assert.deepEqual( + judge(shellWith({ ...active, updatedAt: at(25) } as never), "sweep", { + childActivityMs: NOW - 60_000, + }).findings, + [], + ); +}); + +it("stuck: a thread waiting on background work that no longer exists", () => { + const waiting = { status: "waiting", activityRunStatus: "waiting", updatedAt: at(30) }; + const orphan = judge(shellWith(waiting as never), "sweep"); + assert.deepEqual( + orphan.findings.map((finding) => [finding.kind, finding.signature]), + [["stuck", "run:7:orphan-wait"]], + ); + const tracked = judge( + shellWith({ + ...waiting, + pendingBackgroundTasks: [{ taskId: "bdejsjjol", kind: "command", awaited: true }], + } as never), + "sweep", + ); + assert.deepEqual(tracked.findings, []); +}); + +it("needs-person: an agent lane raising a question or flipping to a founder", () => { + const question = judge( + shellWith({ + pendingRuntimeRequest: { id: "request:9", kind: "approval", userOnly: false }, + } as never), + "request", + ); + assert.deepEqual( + question.findings.map((finding) => [finding.kind, finding.signature]), + [["needs-person", "request:request:9"]], + ); + const flipped = judge( + shellWith({ + inbox: { + owner: "kameron", + assignee: { kind: "human", id: "kameron" }, + audience: "kameron", + attentionKind: "action", + reason: "decision", + decisionRequest: { + founder: "kameron", + kind: "decision", + question: "Spend $20 on the build host?", + requestedAt: "2026-11-19T04:59:00.000Z", + }, + }, + } as never), + "inbox", + ); + assert.equal(flipped.findings.length, 1); + assert.match(flipped.findings[0]!.detail, /assigned to kameron.*Spend \$20/); + // An agent-only background lane, and a thread a founder opened themselves, do not alert. + assert.deepEqual( + judge( + shellWith({ + inbox: { + owner: null, + assignee: null, + audience: "agent-only", + attentionKind: "background", + }, + } as never), + "inbox", + ).findings, + [], + ); + assert.deepEqual( + judge( + shellWith({ + createdBy: "user", + pendingRuntimeRequest: { id: "request:9", kind: "approval" }, + } as never), + "request", + ).findings, + [], + ); +}); + +it("each trigger evaluates only the kinds it can change", () => { + assert.deepEqual(judge(shellWith(), "run-ended").evaluated, [ + "lane-ended", + "promise-without-wake", + ]); + assert.deepEqual(judge(shellWith(), "sweep").evaluated, ["stuck"]); + assert.deepEqual(judge(shellWith(), "inbox").evaluated, ["needs-person"]); +}); + +it("the wake carries environment, thread link, title, kind, run, last line and open pull requests", () => { + const shell = shellWith({ + status: "completed", + pullRequests: [openPullRequest], + latestVisibleMessage: assistantSays(`${"x".repeat(400)}\n${FLEET_PROMISE}`), + } as never); + const finding = judge(shell, "run-ended").findings[0]!; + const alert = threadEventAlert({ + environmentId: "env-r720" as never, + environmentLabel: "primary-r720", + shell, + finding, + occurredAt: "2026-11-19T05:00:00.000Z", + }); + assert.equal(alert.runId, "run:7"); + assert.equal(alert.threadTitle, "Fleet rollout"); + assert.deepEqual(alert.openPullRequests, ["https://github.com/nullStack65/t3code/pull/97"]); + const message = threadEventMessage(alert); + assert.include(message, "primary-r720"); + assert.include(message, "env-r720"); + assert.include(message, "Fleet rollout"); + assert.include(message, "Event: lane-ended."); + assert.include(message, "Run run:7."); + assert.include(message, "pull/97"); + assert.include(message, "Last assistant line:"); + assert.isAtMost(lastLine("a".repeat(1000))!.length, 240); +}); diff --git a/apps/server/src/orchestration-v2/ThreadEventAlerts.ts b/apps/server/src/orchestration-v2/ThreadEventAlerts.ts new file mode 100644 index 000000000000..c4fbc68500f2 --- /dev/null +++ b/apps/server/src/orchestration-v2/ThreadEventAlerts.ts @@ -0,0 +1,352 @@ +import { + type EnvironmentId, + isListedThreadLineage, + type OrchestrationV2ThreadShell, +} from "@t3tools/contracts"; +import { visibleThreadPullRequests } from "@t3tools/shared/threadPullRequests"; +import { formatThreadLink } from "@t3tools/shared/threadLinks"; +import * as DateTime from "effect/DateTime"; +import * as Schema from "effect/Schema"; + +/** + * Deterministic thread events for the watcher thread, judged from a thread's shell with no agent + * judgment. Delivery, retry and dedupe live in ModelAlertReactor, which carries these beside its + * model alerts. + */ +export const ThreadEventKind = Schema.Literals([ + "lane-ended", + "promise-without-wake", + "stuck", + "needs-person", +]); +export type ThreadEventKind = typeof ThreadEventKind.Type; + +export const ThreadEventAlert = Schema.Struct({ + type: Schema.Literal("thread-event"), + kind: ThreadEventKind, + id: Schema.String, + environmentId: Schema.String, + environmentLabel: Schema.String, + threadId: Schema.String, + threadTitle: Schema.String, + threadLink: Schema.String, + runId: Schema.NullOr(Schema.String), + /** What happened, in a sentence: how the run ended, what is missing, what is waiting. */ + detail: Schema.String, + lastAssistantLine: Schema.NullOr(Schema.String), + lastError: Schema.NullOr(Schema.String), + openPullRequests: Schema.Array(Schema.String), + occurredAt: Schema.String, +}); +export type ThreadEventAlert = typeof ThreadEventAlert.Type; + +/** What made the server look at a thread; each trigger judges only the kinds it can change. */ +export type ThreadEventTrigger = "run-ended" | "settled" | "inbox" | "request" | "sweep"; + +const KINDS_BY_TRIGGER: Record> = { + "run-ended": ["lane-ended", "promise-without-wake"], + settled: ["promise-without-wake"], + inbox: ["needs-person"], + request: ["needs-person"], + sweep: ["stuck"], +}; + +export type WatchedShell = Pick< + OrchestrationV2ThreadShell, + | "id" + | "title" + | "createdBy" + | "lineage" + | "inbox" + | "status" + | "activeRunId" + | "latestRunId" + | "latestRunCompletedAt" + | "activityRunStatus" + | "blockedOnThreadWait" + | "pendingRuntimeRequest" + | "pendingBackgroundTasks" + | "pullRequests" + | "latestVisibleMessage" + | "lastError" + | "settledOverride" + | "archivedAt" + | "snoozedUntil" + | "limitRecovery" + | "updatedAt" +>; + +export interface ThreadFinding { + readonly kind: ThreadEventKind; + readonly detail: string; + /** The state the finding describes; the same one reports once, a different one reports again. */ + readonly signature: string; +} + +/** A run that ended longer ago than this is history, not news, whatever woke the judge. */ +const RECENT_RUN_END_MS = 15 * 60 * 1000; + +const timeMs = (value: DateTime.Utc | null | undefined) => + value === null || value === undefined ? null : DateTime.toEpochMillis(value); + +const isLive = (shell: WatchedShell) => + shell.activeRunId !== null || (shell.activityRunStatus ?? null) !== null; + +const openPullRequests = (shell: WatchedShell) => + visibleThreadPullRequests(shell.pullRequests ?? []).filter( + (link) => link.snapshot === null || link.snapshot.state === "open", + ); + +/** A pending task, a watching PR, a delegated task or a snooze that will bring the thread back. */ +export function hasWakeHandle(shell: WatchedShell, remoteTaskCount: number, nowMs: number) { + if ((shell.pendingBackgroundTasks ?? []).length > 0) return true; + if (remoteTaskCount > 0) return true; + if ((shell.pullRequests ?? []).some((link) => link.watch !== undefined)) return true; + if (shell.limitRecovery?.autoResume === true || shell.limitRecovery?.snooze === true) return true; + const snoozedUntil = timeMs(shell.snoozedUntil); + return snoozedUntil !== null && snoozedUntil > nowMs; +} + +/** + * Phrases an agent uses to say something will wake it or that it will carry on later. An explicit + * list on purpose: a false alarm costs a Lead turn, and the agent can always avoid the phrase by + * registering a real wait. + */ +const PROMISE_PHRASES: ReadonlyArray = [ + /\bwill (?:wake|notify|ping|resume) (?:this|the|me|it|my)\b/i, + /\bwill (?:be )?woken\b/i, + /\bI(?:'ll| will) be (?:woken|notified|pinged)\b/i, + /\b(?:watch|monitor|task|job|timer|check)\b[^.\n]{0,40}\bwill (?:wake|resume|notify)\b/i, + /\bI(?:'ll| will) (?:install|merge|continue|resume|finish|roll(?: it)? out|deploy|pick (?:this|it) (?:back )?up|check|come back)\b[^.\n]{0,80}\b(?:after|once|when|as soon as|later)\b/i, + /\b(?:keeping|keep|leaving|leave|holding|hold) (?:this|the) (?:lane|thread) open\b/i, + /\bwill (?:continue|resume|carry on) (?:when|once|after|later)\b/i, +]; + +export const promisesToContinue = (text: string) => + PROMISE_PHRASES.some((phrase) => phrase.test(text)); + +/** The last non-empty line of a message, trimmed to keep a wake short. */ +export function lastLine(text: string, max = 240): string | null { + const line = text + .split("\n") + .map((candidate) => candidate.trim()) + .findLast((candidate) => candidate.length > 0); + if (line === undefined) return null; + return line.length <= max ? line : `${line.slice(0, max - 1)}…`; +} + +const IDLE_STATUSES = new Set(["idle", "completed"]); + +/** + * Which of the trigger's kinds are true of the thread right now. `evaluated` lists every kind the + * trigger can judge, so a kind that is no longer true can reset its dedupe. + */ +export function judgeThread(input: { + readonly shell: WatchedShell; + readonly trigger: ThreadEventTrigger; + readonly nowMs: number; + readonly stuckMinutes: number; + /** Open delegated tasks on a linked environment, which the shell does not list. */ + readonly remoteTaskCount?: number; + /** The latest activity among the children this thread waits on. */ + readonly childActivityMs?: number | null; + /** A model alert for the run already reported a usage limit, so a failure adds nothing. */ + readonly usageLimitReported?: boolean; +}): { readonly evaluated: ReadonlyArray; readonly findings: ThreadFinding[] } { + const { shell, nowMs } = input; + const evaluated = KINDS_BY_TRIGGER[input.trigger]; + const findings: ThreadFinding[] = []; + // Only a top-level lane that nobody has archived or settled is worth waking the watcher for. + const watched = + isListedThreadLineage(shell.lineage) && + shell.archivedAt === null && + shell.settledOverride !== "settled"; + const remoteTaskCount = input.remoteTaskCount ?? 0; + const tasks = shell.pendingBackgroundTasks ?? []; + + if (evaluated.includes("lane-ended") && watched && !isLive(shell)) { + const endedMs = timeMs(shell.latestRunCompletedAt); + const recent = endedMs !== null && nowMs - endedMs <= RECENT_RUN_END_MS; + const prs = openPullRequests(shell); + if (recent && shell.latestRunId !== null) { + if ( + (shell.status === "failed" && input.usageLimitReported !== true) || + shell.status === "interrupted" + ) { + findings.push({ + kind: "lane-ended", + detail: `its last run ${shell.status}`, + signature: shell.latestRunId, + }); + } else if ( + shell.status === "completed" && + prs.length > 0 && + !hasWakeHandle(shell, remoteTaskCount, nowMs) + ) { + findings.push({ + kind: "lane-ended", + detail: `its last run completed with ${prs.length} open pull request${prs.length === 1 ? "" : "s"} and nothing tracked that will wake it`, + signature: shell.latestRunId, + }); + } + } + } + + if (evaluated.includes("promise-without-wake") && watched && !isLive(shell)) { + const message = shell.latestVisibleMessage; + const endedMs = timeMs(shell.latestRunCompletedAt); + const recent = + input.trigger === "settled" || (endedMs !== null && nowMs - endedMs <= RECENT_RUN_END_MS); + if ( + recent && + IDLE_STATUSES.has(shell.status) && + message?.role === "assistant" && + promisesToContinue(message.text) && + !hasWakeHandle(shell, remoteTaskCount, nowMs) && + shell.pendingRuntimeRequest === null && + shell.inbox?.decisionRequest === undefined + ) { + findings.push({ + kind: "promise-without-wake", + detail: + "its last message says it will be woken or will continue later, but nothing tracked will wake it", + signature: message.id, + }); + } + } + + if (evaluated.includes("stuck") && watched && input.stuckMinutes > 0) { + const activityMs = Math.max(timeMs(shell.updatedAt) ?? 0, input.childActivityMs ?? 0); + const quietMinutes = Math.floor((nowMs - activityMs) / 60_000); + const quiet = activityMs > 0 && quietMinutes >= input.stuckMinutes; + const waiting = shell.activityRunStatus === "waiting" || shell.status === "waiting"; + if (quiet && waiting) { + // A wait the shell still shows, with nothing behind it that could end it. + if ( + tasks.length === 0 && + remoteTaskCount === 0 && + shell.pendingRuntimeRequest === null && + !(shell.pullRequests ?? []).some((link) => link.watch !== undefined) + ) { + findings.push({ + kind: "stuck", + detail: `it has waited ${quietMinutes} minutes on background work that no longer exists`, + signature: `${shell.latestRunId ?? shell.id}:orphan-wait`, + }); + } + } else if ( + quiet && + shell.activeRunId !== null && + shell.blockedOnThreadWait !== true && + shell.pendingRuntimeRequest === null + ) { + findings.push({ + kind: "stuck", + detail: `its run is active with no new activity for ${quietMinutes} minutes`, + signature: shell.activeRunId, + }); + } + } + + if (evaluated.includes("needs-person") && watched && shell.createdBy === "agent") { + const request = shell.pendingRuntimeRequest; + const inbox = shell.inbox; + if (request !== null) { + findings.push({ + kind: "needs-person", + detail: `it is waiting on ${request.userOnly === true ? "a secret or link only the user can give" : `a ${request.kind.replaceAll("_", " ")}`}`, + signature: `request:${request.id}`, + }); + } + const founderAssignee = inbox?.assignee?.kind === "human" ? inbox.assignee.id : null; + if (inbox !== undefined && (founderAssignee !== null || inbox.attentionKind === "action")) { + const ask = inbox.decisionRequest; + findings.push({ + kind: "needs-person", + detail: [ + founderAssignee === null + ? "its inbox attention is set to action" + : `its inbox is assigned to ${founderAssignee}`, + ask === undefined ? null : `with an open ${ask.kind}: ${ask.question}`, + ask === undefined && inbox.reason !== undefined ? `for ${inbox.reason}` : null, + ] + .filter((part) => part !== null) + .join(" "), + signature: `inbox:${founderAssignee ?? ""}:${inbox.attentionKind}:${ask?.requestedAt ?? ""}`, + }); + } + } + + return { evaluated, findings }; +} + +export function threadEventAlert(input: { + readonly environmentId: EnvironmentId; + readonly environmentLabel: string; + readonly shell: Pick< + WatchedShell, + | "id" + | "title" + | "latestRunId" + | "latestVisibleMessage" + | "lastError" + | "pullRequests" + | "activeRunId" + >; + readonly finding: ThreadFinding; + readonly occurredAt: string; +}): ThreadEventAlert { + const { shell, finding } = input; + const message = shell.latestVisibleMessage; + return { + type: "thread-event", + kind: finding.kind, + id: `${input.environmentId}:${shell.id}:${finding.kind}:${finding.signature}`, + environmentId: input.environmentId, + environmentLabel: input.environmentLabel, + threadId: shell.id, + threadTitle: shell.title, + threadLink: formatThreadLink( + { environmentId: input.environmentId, threadId: shell.id }, + shell.title, + ), + runId: shell.activeRunId ?? shell.latestRunId, + detail: finding.detail, + lastAssistantLine: message?.role === "assistant" ? lastLine(message.text) : null, + lastError: + shell.lastError === null || shell.lastError === undefined ? null : lastLine(shell.lastError), + openPullRequests: openPullRequests({ pullRequests: shell.pullRequests } as WatchedShell) + .map((link) => link.url) + .slice(0, 5), + occurredAt: input.occurredAt, + }; +} + +const NEXT_STEP: Record = { + "lane-ended": + "Check the thread: restart, repair or replace it, or settle it if its work is done. It sends no further alert for this run.", + "promise-without-wake": + "Tell the thread to register a real wait (a tracked background command, a PR watch or a delegate) and carry on, or finish the work yourself.", + stuck: + "Read the thread. Interrupt and restart it, or replace it. It sends no further alert for this run.", + "needs-person": + "Try to clear it yourself first: answer from what you know, approve what Kameron already authorized, or delegate it. Raise it to a founder only if it needs a passkey, a phone prompt, physical presence or new spend.", +}; + +export function threadEventMessage(alert: ThreadEventAlert): string { + return [ + `Thread event from ${alert.environmentLabel} (environment ${alert.environmentId}): ${alert.threadLink} ${alert.detail}.`, + [ + `Event: ${alert.kind}.`, + alert.runId === null ? null : `Run ${alert.runId}.`, + alert.openPullRequests.length === 0 + ? "No open pull requests." + : `Open pull requests: ${alert.openPullRequests.join(", ")}.`, + alert.lastAssistantLine === null ? null : `Last assistant line: "${alert.lastAssistantLine}"`, + alert.lastError === null ? null : `Last error: ${alert.lastError}`, + ] + .filter((part) => part !== null) + .join(" "), + NEXT_STEP[alert.kind], + ].join("\n\n"); +} diff --git a/docs/operations/watcher-thread.md b/docs/operations/watcher-thread.md new file mode 100644 index 000000000000..7bcf43d0dc57 --- /dev/null +++ b/docs/operations/watcher-thread.md @@ -0,0 +1,17 @@ +# Watcher thread + +One thread, by default Lead's on the laptop, hears everything that needs attention from every linked environment. The server decides each event from thread state with no agent judgment, queues it durably, and retries until the watcher's machine answers. + +| Event | Fires when | +| ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `model-degraded` | A run ran another model than selected, hit a provider fallback, or hit a usage limit. | +| `lane-ended` | A top-level lane's run ended `failed` or `interrupted`, or `completed` with open pull requests and nothing tracked to wake it. | +| `promise-without-wake` | A lane went idle or settled after saying it will be woken or will continue later, with no background task, pull request watch, delegated task or armed resume. The phrase list is `PROMISE_PHRASES` in `ThreadEventAlerts.ts`. | +| `stuck` | An active run has had no new activity for `watcherStuckMinutes` (default 20; 0 turns it off), or a thread waits on background work that no longer exists. A parent whose child threads are active is not stuck. | +| `needs-person` | An agent-created lane raises a pending question or approval, or its inbox is assigned to a founder or set to `action`. | + +Each event reports once per thread and kind until its state changes: a new run, a different message, or the condition clearing. Delegated children are skipped; their parent already hears their result. The watcher thread never reports about itself. + +## Moving the watcher + +The watcher is the `modelAlertWatcher` server setting (`{ environmentId, threadId }`, or `null` for none). Its Closura default is in `packages/contracts/src/settings.ts` (`DEFAULT_MODEL_ALERT_WATCHER`). To move Lead to an always-on machine such as the R720, change that one default and ship a build, or set `modelAlertWatcher` in each machine's settings to override it. `modelAlertWebhookUrl` additionally POSTs every alert as JSON. diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 099abf6484ea..c31e13fffc98 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -1168,7 +1168,8 @@ export const ModelAlertWatcher = Schema.Struct({ }); export type ModelAlertWatcher = typeof ModelAlertWatcher.Type; -// Closura: Lead's thread on the laptop hears every machine's alerts. +// Closura: Lead's thread on the laptop hears every machine's alerts and thread events. Moving Lead +// to an always-on machine means changing this one value (or `modelAlertWatcher` on each machine). const DEFAULT_MODEL_ALERT_WATCHER: ModelAlertWatcher = { environmentId: EnvironmentId.make("2f360b2e-0386-498a-b9cd-62999f3525ba"), threadId: ThreadId.make("0ff5603d-8291-4f56-adaf-5132c62016cf"), @@ -1559,14 +1560,24 @@ export const ServerSettings = Schema.Struct({ Schema.withDecodingDefault(Effect.succeed(true)), ), /** - * Where a run that ran another model than selected, hit a provider safeguard fallback or a usage - * limit is reported, once per thread until the problem changes. Null reports to no thread. + * The watcher thread. It hears a run that ran another model than selected, hit a provider + * safeguard fallback or a usage limit, and each lane that ended, promised to continue with + * nothing to wake it, got stuck or needs a person. Each reports once per thread until its state + * changes. Null reports to no thread. */ modelAlertWatcher: Schema.NullOr(ModelAlertWatcher).pipe( Schema.withDecodingDefault(Effect.succeed(DEFAULT_MODEL_ALERT_WATCHER)), ), - /** Also POSTs each model alert as JSON here. Empty posts nothing. */ + /** Also POSTs each alert and thread event as JSON here. Empty posts nothing. */ modelAlertWebhookUrl: Schema.String.pipe(Schema.withDecodingDefault(Effect.succeed(""))), + /** + * Minutes an active run may go without a new item before the watcher hears it is stuck. Zero + * turns the check off. + */ + watcherStuckMinutes: Schema.Int.check(Schema.isBetween({ minimum: 0, maximum: 1_440 })).pipe( + // Closura: twenty minutes without any activity is long enough to be worth a look. + Schema.withDecodingDefault(Effect.succeed(20)), + ), /** Beta: Claude and Codex predict the next prompt after completed turns. Codex follows a change at once; open Claude threads after they restart. */ promptSuggestionsEnabled: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), sidebarAutoSettleKeepPullRequests: Schema.Boolean.pipe( @@ -2013,6 +2024,9 @@ export const ServerSettingsPatch = Schema.Struct({ automaticPlacementByDefault: Schema.optionalKey(Schema.Boolean), modelAlertWatcher: Schema.optionalKey(Schema.NullOr(ModelAlertWatcher)), modelAlertWebhookUrl: Schema.optionalKey(TrimmedString), + watcherStuckMinutes: Schema.optionalKey( + Schema.Int.check(Schema.isBetween({ minimum: 0, maximum: 1_440 })), + ), promptSuggestionsEnabled: Schema.optionalKey(Schema.Boolean), backgroundActivity: Schema.optionalKey( Schema.Struct({ From 227b12e067c0e74fdbe1753f9373835f8bae90c0 Mon Sep 17 00:00:00 2001 From: nullStack65 <112618179+nullStack65@users.noreply.github.com> Date: Thu, 19 Nov 2026 06:19:52 +0000 Subject: [PATCH 2/3] fix(server): watcher events only for agent lanes, one message per occurrence, per-signature dedupe, one task scan per sweep Review of #98: - lane-ended, promise-without-wake and stuck now apply only to agent-created lanes, so a person stopping or leaving their own thread no longer wakes Lead. - The alert id includes the time, so a recurrence of the same state is a new message instead of a replayed command. - A kind holding several findings (a question and an inbox flip) remembers each signature, so they stop alternating on every trigger. - The 60 second sweep reads open delegated tasks once, not once per thread. Co-Authored-By: Claude Sonnet 5.5 --- .../ModelAlertReactor.test.ts | 44 ++++++++++++++---- .../src/orchestration-v2/ModelAlertReactor.ts | 46 +++++++++++++------ .../ThreadEventAlerts.test.ts | 33 +++++++++++++ .../src/orchestration-v2/ThreadEventAlerts.ts | 10 ++-- 4 files changed, 105 insertions(+), 28 deletions(-) diff --git a/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts b/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts index 3744fb10861b..5a84289a1842 100644 --- a/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts +++ b/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts @@ -155,10 +155,12 @@ it.each([ ); }); +// Like threadEventAlert, each occurrence gets its own id. +let occurrence = 0; const eventAlert = (finding: ThreadFinding): ThreadEventAlert => ({ type: "thread-event", kind: finding.kind, - id: `env:thread:fleet:${finding.kind}:${finding.signature}`, + id: `env:thread:fleet:${finding.kind}:${finding.signature}:${(occurrence += 1)}`, environmentId: "env", environmentLabel: "primary-r720", threadId: "thread:fleet", @@ -203,22 +205,44 @@ it("reports a thread event once per kind and signature until its state changes", // A trigger that does not evaluate a kind leaves its dedupe alone. step(["needs-person"], []); step(["stuck"], [finding("stuck", "run:1")]); - // Recovery clears it. The same signature queues nothing while its first delivery still waits; - // a new run reports at once. + // Recovery clears it, so the same state reports again as a new occurrence. step(["stuck"], []); step(["stuck"], [finding("stuck", "run:1")]); step(["stuck"], [finding("stuck", "run:2")]); - assert.deepEqual(enqueued, [true, false, true, false, false, false, false, true]); + assert.deepEqual(enqueued, [true, false, true, false, false, false, true, true]); assert.deepEqual( - state.outbox.map((entry) => entry.alert.id), - [ - "env:thread:fleet:stuck:run:1", - "env:thread:fleet:lane-ended:run:1", - "env:thread:fleet:stuck:run:2", - ], + state.outbox.map((entry) => (entry.alert.type === "thread-event" ? entry.alert.kind : "")), + ["stuck", "lane-ended", "stuck", "stuck"], ); }); +it("a thread holding a question and an inbox flip reports each once, not on every trigger", () => { + const both = [finding("needs-person", "request:1"), finding("needs-person", "inbox:kameron")]; + let state = EMPTY_MODEL_ALERT_STATE; + const enqueued: Array = []; + const step = (findings: ReadonlyArray) => { + const result = recordThreadEvents( + state, + { threadId: "thread:fleet", evaluated: ["needs-person"], findings }, + eventAlert, + ["watcher"], + 0, + ); + state = result.state; + enqueued.push(result.enqueued); + }; + step(both); + step(both); + step(both); + // The question is answered; the inbox flip is still open and was already reported. + step([both[1]!]); + // Then the flip clears and returns: a new occurrence. + step([]); + step([both[1]!]); + assert.deepEqual(enqueued, [true, false, false, false, false, true]); + assert.equal(state.outbox.length, 3); +}); + it("queues one delivery per destination for a thread event, beside model alerts", () => { const { state, enqueued } = recordThreadEvents( EMPTY_MODEL_ALERT_STATE, diff --git a/apps/server/src/orchestration-v2/ModelAlertReactor.ts b/apps/server/src/orchestration-v2/ModelAlertReactor.ts index 71e7822885ef..1e461f9def0d 100644 --- a/apps/server/src/orchestration-v2/ModelAlertReactor.ts +++ b/apps/server/src/orchestration-v2/ModelAlertReactor.ts @@ -266,17 +266,24 @@ export function recordThreadEvents( } continue; } - // One finding per kind reports at a time; the first (most specific) one wins. - const finding = kindFindings.find((candidate) => reported[key] !== candidate.signature); - if (finding === undefined) continue; - const { [key]: _previous, ...rest } = reported; - reported = { - ...Object.fromEntries(Object.entries(rest).slice(-(MAX_REPORTED - 1))), - [key]: finding.signature, - }; - const queued = enqueueAlert(outbox, toAlert(finding), destinations, nowMs); - outbox = queued.outbox; - enqueued ||= queued.enqueued; + // A kind can hold several findings at once (a question and an inbox flip), so it remembers + // each signature it reported. One that stops being true is forgotten and can report again. + const known = (reported[key] ?? "").split("\n").filter((signature) => signature !== ""); + const current = kindFindings.map((finding) => finding.signature); + const stored = current.join("\n"); + if (reported[key] !== stored) { + const { [key]: _previous, ...rest } = reported; + reported = { + ...Object.fromEntries(Object.entries(rest).slice(-(MAX_REPORTED - 1))), + [key]: stored, + }; + } + for (const finding of kindFindings) { + if (known.includes(finding.signature)) continue; + const queued = enqueueAlert(outbox, toAlert(finding), destinations, nowMs); + outbox = queued.outbox; + enqueued ||= queued.enqueued; + } } if (reported === state.reported && outbox === state.outbox) return { state, enqueued }; return { state: { ...state, reported, outbox }, enqueued }; @@ -556,6 +563,8 @@ export const make = Effect.gen(function* () { threadId: ThreadId, trigger: ThreadEventTrigger, usageLimitReported: boolean, + /** Open delegated tasks by parent thread, when the caller read them once for many threads. */ + remoteTasks?: ReadonlyMap, ) { const current = yield* settings.getSettings; const here = yield* environment.getEnvironmentId; @@ -568,9 +577,11 @@ export const make = Effect.gen(function* () { const remoteTaskCount = trigger === "request" || trigger === "inbox" ? 0 - : (yield* projections.getOpenRemoteDelegatedTasks).filter( - (task) => task.parentThreadId === threadId, - ).length; + : remoteTasks !== undefined + ? (remoteTasks.get(threadId) ?? 0) + : (yield* projections.getOpenRemoteDelegatedTasks).filter( + (task) => task.parentThreadId === threadId, + ).length; // A parent that waits on children is not stuck while they work. const childActivityMs = trigger === "sweep" @@ -674,10 +685,15 @@ export const make = Effect.gen(function* () { /** Looks at every thread that may be live for the time-based kinds no event announces. */ const sweep: ModelAlertReactor["Service"]["sweep"] = Effect.gen(function* () { const index = yield* projections.listActiveShellIndex(); + // One scan of the delegated tasks for the whole sweep, not one per thread. + const remoteTasks = new Map(); + for (const task of yield* projections.getOpenRemoteDelegatedTasks) { + remoteTasks.set(task.parentThreadId, (remoteTasks.get(task.parentThreadId) ?? 0) + 1); + } yield* Effect.forEach( index.filter((entry) => entry.mayBeLive && isListedThreadLineage(entry.lineage)), (entry) => - handleThreadEvents(entry.id, "sweep", false).pipe( + handleThreadEvents(entry.id, "sweep", false, remoteTasks).pipe( Effect.catchCause((cause) => Effect.logWarning("sweeping a thread for a watcher event failed", { threadId: entry.id, diff --git a/apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts b/apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts index 02e0553e7935..5dfb585dfd96 100644 --- a/apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts +++ b/apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts @@ -100,6 +100,25 @@ it.each([ ); }); +it.each(["failed", "interrupted"] as const)( + "a %s run on a thread a person created is not a lane event", + (status) => { + const { findings } = judge(shellWith({ status, createdBy: "user" } as never), "run-ended"); + assert.deepEqual(findings, []); + const stuck = judge( + shellWith({ + createdBy: "user", + status: "running", + activeRunId: "run:8", + activityRunStatus: "running", + updatedAt: at(90), + } as never), + "sweep", + ); + assert.deepEqual(stuck.findings, []); + }, +); + it("lane-ended stays quiet for a failure the model alert already reported as a usage limit", () => { const { findings } = judge(shellWith({ status: "failed" } as never), "run-ended", { usageLimitReported: true, @@ -270,6 +289,20 @@ it("needs-person: an agent lane raising a question or flipping to a founder", () ); }); +it("a recurrence of the same state is a new message", () => { + const shell = shellWith({ status: "failed" } as never); + const finding = judge(shell, "run-ended").findings[0]!; + const make = (occurredAt: string) => + threadEventAlert({ + environmentId: "env-r720" as never, + environmentLabel: "primary-r720", + shell, + finding, + occurredAt, + }); + assert.notEqual(make("2026-11-19T05:00:00.000Z").id, make("2026-11-19T05:30:00.000Z").id); +}); + it("each trigger evaluates only the kinds it can change", () => { assert.deepEqual(judge(shellWith(), "run-ended").evaluated, [ "lane-ended", diff --git a/apps/server/src/orchestration-v2/ThreadEventAlerts.ts b/apps/server/src/orchestration-v2/ThreadEventAlerts.ts index c4fbc68500f2..502c4b2bc314 100644 --- a/apps/server/src/orchestration-v2/ThreadEventAlerts.ts +++ b/apps/server/src/orchestration-v2/ThreadEventAlerts.ts @@ -156,8 +156,10 @@ export function judgeThread(input: { const { shell, nowMs } = input; const evaluated = KINDS_BY_TRIGGER[input.trigger]; const findings: ThreadFinding[] = []; - // Only a top-level lane that nobody has archived or settled is worth waking the watcher for. + // Only a top-level lane an agent runs, that nobody has archived or settled, is worth waking the + // watcher for. A person's own thread ending, stopping or going quiet is not news. const watched = + shell.createdBy === "agent" && isListedThreadLineage(shell.lineage) && shell.archivedAt === null && shell.settledOverride !== "settled"; @@ -248,7 +250,7 @@ export function judgeThread(input: { } } - if (evaluated.includes("needs-person") && watched && shell.createdBy === "agent") { + if (evaluated.includes("needs-person") && watched) { const request = shell.pendingRuntimeRequest; const inbox = shell.inbox; if (request !== null) { @@ -301,7 +303,9 @@ export function threadEventAlert(input: { return { type: "thread-event", kind: finding.kind, - id: `${input.environmentId}:${shell.id}:${finding.kind}:${finding.signature}`, + // The time makes each occurrence its own message: the delivery key hashes this id, and a + // recurrence of the same state must not be taken for a replay of the first. + id: `${input.environmentId}:${shell.id}:${finding.kind}:${finding.signature}:${input.occurredAt}`, environmentId: input.environmentId, environmentLabel: input.environmentLabel, threadId: shell.id, From 2979b3f1c67b93ef4f7d3b480fc3003bb6480179 Mon Sep 17 00:00:00 2001 From: nullStack65 <112618179+nullStack65@users.noreply.github.com> Date: Thu, 19 Nov 2026 06:28:43 +0000 Subject: [PATCH 3/3] fix(server): type the lane-ended test's expected kinds as a mutable array Found by the R720 typecheck of the stacked branch; the same line is in #98. Co-Authored-By: Claude Sonnet 5.5 --- apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts b/apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts index 5dfb585dfd96..80abe7f06abf 100644 --- a/apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts +++ b/apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts @@ -96,7 +96,7 @@ it.each([ const { findings } = judge(shellWith(overrides as never), "run-ended"); assert.deepEqual( findings.filter((finding) => finding.kind === "lane-ended").map((finding) => finding.kind), - expected, + [...expected], ); });