diff --git a/.github/workflows/closura-arc-ci.yml b/.github/workflows/closura-arc-ci.yml index bb5a8f791935..f416c6c5c1ed 100644 --- a/.github/workflows/closura-arc-ci.yml +++ b/.github/workflows/closura-arc-ci.yml @@ -159,6 +159,7 @@ jobs: apps/server/src/orchestration-v2/ThreadLaunchService.test.ts \ apps/server/src/orchestration-v2/SubagentProjection.test.ts \ apps/server/src/orchestration-v2/ModelAlertReactor.test.ts \ + apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts \ apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts \ packages/shared/src/threadPlacement.test.ts \ apps/server/src/peer/RemoteDelegation.test.ts \ diff --git a/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts b/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts index 11277dc3d395..5a84289a1842 100644 --- a/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts +++ b/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts @@ -1,14 +1,19 @@ import { assert, it } from "@effect/vitest"; import { EnvironmentId, type OrchestrationV2Run, ThreadId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; import { afterDeliveryAttempt, + deliverAlertToWatcher, EMPTY_MODEL_ALERT_STATE, type ModelAlert, modelAlertDestinations, modelAlertFor, recordModelAlert, + recordThreadEvents, + watcherAlertMessage, } from "./ModelAlertReactor.ts"; +import type { ThreadEventAlert, ThreadEventKind, ThreadFinding } from "./ThreadEventAlerts.ts"; const alertFor = (runId: string, overrides: Partial = {}): ModelAlert => ({ type: "model-degraded", @@ -149,3 +154,161 @@ it.each([ expected, ); }); + +// Like threadEventAlert, each occurrence gets its own id. +let occurrence = 0; +const eventAlert = (finding: ThreadFinding): ThreadEventAlert => ({ + type: "thread-event", + kind: finding.kind, + id: `env:thread:fleet:${finding.kind}:${finding.signature}:${(occurrence += 1)}`, + environmentId: "env", + environmentLabel: "primary-r720", + threadId: "thread:fleet", + threadTitle: "Fleet overnight", + threadLink: "[Fleet overnight](t3-thread://v2/env/thread:fleet)", + runId: "run:1", + detail: finding.detail, + lastAssistantLine: "A background watch will wake this thread.", + lastError: null, + openPullRequests: [], + occurredAt: "2026-11-19T03:00:00.000Z", +}); + +const finding = (kind: ThreadEventKind, signature: string): ThreadFinding => ({ + kind, + signature, + detail: `${kind} detail`, +}); + +it("reports a thread event once per kind and signature until its state changes", () => { + let state = EMPTY_MODEL_ALERT_STATE; + const enqueued: Array = []; + const step = ( + evaluated: ReadonlyArray, + findings: ReadonlyArray, + ) => { + const result = recordThreadEvents( + state, + { threadId: "thread:fleet", evaluated, findings }, + eventAlert, + ["watcher"], + 0, + ); + state = result.state; + enqueued.push(result.enqueued); + }; + step(["stuck"], [finding("stuck", "run:1")]); + // The same run still stuck on the next sweep says nothing. + step(["stuck"], [finding("stuck", "run:1")]); + // Another kind on the same thread is its own event. + step(["lane-ended"], [finding("lane-ended", "run:1")]); + // A trigger that does not evaluate a kind leaves its dedupe alone. + step(["needs-person"], []); + step(["stuck"], [finding("stuck", "run:1")]); + // Recovery clears it, so the same state reports again as a new occurrence. + step(["stuck"], []); + step(["stuck"], [finding("stuck", "run:1")]); + step(["stuck"], [finding("stuck", "run:2")]); + assert.deepEqual(enqueued, [true, false, true, false, false, false, true, true]); + assert.deepEqual( + state.outbox.map((entry) => (entry.alert.type === "thread-event" ? entry.alert.kind : "")), + ["stuck", "lane-ended", "stuck", "stuck"], + ); +}); + +it("a thread holding a question and an inbox flip reports each once, not on every trigger", () => { + const both = [finding("needs-person", "request:1"), finding("needs-person", "inbox:kameron")]; + let state = EMPTY_MODEL_ALERT_STATE; + const enqueued: Array = []; + const step = (findings: ReadonlyArray) => { + const result = recordThreadEvents( + state, + { threadId: "thread:fleet", evaluated: ["needs-person"], findings }, + eventAlert, + ["watcher"], + 0, + ); + state = result.state; + enqueued.push(result.enqueued); + }; + step(both); + step(both); + step(both); + // The question is answered; the inbox flip is still open and was already reported. + step([both[1]!]); + // Then the flip clears and returns: a new occurrence. + step([]); + step([both[1]!]); + assert.deepEqual(enqueued, [true, false, false, false, false, true]); + assert.equal(state.outbox.length, 3); +}); + +it("queues one delivery per destination for a thread event, beside model alerts", () => { + const { state, enqueued } = recordThreadEvents( + EMPTY_MODEL_ALERT_STATE, + { + threadId: "thread:fleet", + evaluated: ["needs-person"], + findings: [finding("needs-person", "request:1")], + }, + eventAlert, + ["watcher", "webhook"], + 0, + ); + assert.equal(enqueued, true); + assert.deepEqual( + state.outbox.map((entry) => [entry.alert.type, entry.destination]), + [ + ["thread-event", "watcher"], + ["thread-event", "webhook"], + ], + ); + const withModel = recordModelAlert( + state, + { alert: alertFor("run:1"), healthy: false, threadId: "thread:fleet" }, + ["watcher"], + 0, + ); + assert.deepEqual( + withModel.state.outbox.map((entry) => entry.alert.type), + ["thread-event", "thread-event", "model-degraded"], + ); + assert.include(watcherAlertMessage(alertFor("run:1")), "Model alert from"); + assert.include(watcherAlertMessage(eventAlert(finding("stuck", "run:1"))), "Thread event from"); +}); + +it.effect("forwards a thread event to a watcher on another environment with one retry key", () => + Effect.gen(function* () { + const dispatched: Array = []; + const called: Array<{ + readonly arguments?: Record; + readonly environmentId: string; + }> = []; + const deps = { + dispatch: (command: unknown) => Effect.sync(() => void dispatched.push(command)), + call: (input: never) => Effect.sync(() => void called.push(input)), + sha256Hex: (identity: string) => Effect.succeed(`digest(${identity.length})`), + } as never; + const alert = eventAlert(finding("promise-without-wake", "message:1")); + const laptop = { environmentId: EnvironmentId.make("laptop"), threadId: ThreadId.make("lead") }; + yield* deliverAlertToWatcher(deps, alert, laptop, EnvironmentId.make("r720")); + yield* deliverAlertToWatcher(deps, alert, laptop, EnvironmentId.make("r720")); + assert.equal(dispatched.length, 0); + assert.equal(called.length, 2); + assert.equal(called[0]!.environmentId, "laptop"); + assert.equal(called[0]!.arguments?.threadId, "lead"); + assert.equal(called[0]!.arguments?.mode, "queue"); + assert.equal(called[0]!.arguments?.message, watcherAlertMessage(alert)); + assert.match(String(called[0]!.arguments?.clientRequestId), /^thread-event:/); + assert.equal(called[0]!.arguments?.clientRequestId, called[1]!.arguments?.clientRequestId); + + // The same alert on the watcher's own machine queues into the thread directly. + yield* deliverAlertToWatcher(deps, alert, laptop, EnvironmentId.make("laptop")); + assert.equal(called.length, 2); + assert.equal(dispatched.length, 1); + assert.equal((dispatched[0] as { threadId: string }).threadId, "lead"); + assert.deepEqual((dispatched[0] as { dispatchMode: unknown }).dispatchMode, { + type: "queue_after_active", + }); + }), +); diff --git a/apps/server/src/orchestration-v2/ModelAlertReactor.ts b/apps/server/src/orchestration-v2/ModelAlertReactor.ts index 52c99fad07cf..1e461f9def0d 100644 --- a/apps/server/src/orchestration-v2/ModelAlertReactor.ts +++ b/apps/server/src/orchestration-v2/ModelAlertReactor.ts @@ -1,6 +1,7 @@ import { CommandId, type EnvironmentId, + isListedThreadLineage, MessageId, type ModelAlertWatcher, type OrchestrationV2DomainEvent, @@ -8,6 +9,7 @@ import { type OrchestrationV2ThreadProjection, OrchestrationV2WorkerAlertReason, OrchestratorMcpThreadSendResult, + type ThreadId, } from "@t3tools/contracts"; import { runModelSignals, workerAlert } from "@t3tools/provider-core/server/subagentProjection"; import { writeFileStringAtomically } from "@t3tools/shared/atomicWrite"; @@ -38,6 +40,15 @@ import { forkParked } from "../serverActivation.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as Orchestrator from "./Orchestrator.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; +import { + judgeThread, + threadEventAlert, + ThreadEventAlert, + type ThreadEventKind, + threadEventMessage, + type ThreadEventTrigger, + type ThreadFinding, +} from "./ThreadEventAlerts.ts"; export const ModelAlert = Schema.Struct({ type: Schema.Literal("model-degraded"), @@ -59,12 +70,16 @@ export const ModelAlert = Schema.Struct({ }); export type ModelAlert = typeof ModelAlert.Type; +/** Everything the watcher thread hears: model alerts and deterministic thread events. */ +export const WatcherAlert = Schema.Union([ModelAlert, ThreadEventAlert]); +export type WatcherAlert = typeof WatcherAlert.Type; + const Destination = Schema.Literals(["watcher", "webhook"]); type Destination = typeof Destination.Type; const Delivery = Schema.Struct({ destination: Destination, - alert: ModelAlert, + alert: WatcherAlert, attempts: Schema.Int, nextAttemptAt: Schema.Number, createdAt: Schema.Number, @@ -75,13 +90,23 @@ export const ModelAlertState = Schema.Struct({ version: Schema.Literal(1), /** The alert signature each thread last reported; a healthy run clears it. */ degraded: Schema.Record(Schema.String, Schema.String), + /** The signature of each thread event last reported, by thread and kind; a changed state clears it. */ + reported: Schema.Record(Schema.String, Schema.String).pipe( + Schema.withDecodingDefault(Effect.succeed({})), + ), outbox: Schema.Array(Delivery), }); export type ModelAlertState = typeof ModelAlertState.Type; -export const EMPTY_MODEL_ALERT_STATE: ModelAlertState = { version: 1, degraded: {}, outbox: [] }; +export const EMPTY_MODEL_ALERT_STATE: ModelAlertState = { + version: 1, + degraded: {}, + reported: {}, + outbox: [], +}; const MAX_DEGRADED_THREADS = 1_000; +const MAX_REPORTED = 2_000; const MAX_OUTBOX = 200; const OUTBOX_TTL_MS = 7 * 24 * 60 * 60 * 1000; const RETRY_BASE_MS = 15_000; @@ -175,12 +200,28 @@ export function recordModelAlert( if (state.degraded[alert.threadId] === next) return { state, enqueued: false }; const { [alert.threadId]: _previous, ...rest } = state.degraded; const kept = Object.entries(rest).slice(-(MAX_DEGRADED_THREADS - 1)); + const queued = enqueueAlert(state.outbox, alert, destinations, nowMs); + return { + state: { + ...state, + degraded: { ...Object.fromEntries(kept), [alert.threadId]: next }, + outbox: queued.outbox, + }, + enqueued: queued.enqueued, + }; +} + +/** The outbox with the alert queued once per destination; an alert already queued is not repeated. */ +function enqueueAlert( + outbox: ModelAlertState["outbox"], + alert: WatcherAlert, + destinations: ReadonlyArray, + nowMs: number, +): { readonly outbox: ModelAlertState["outbox"]; readonly enqueued: boolean } { const added = destinations .filter( (destination) => - !state.outbox.some( - (entry) => entry.alert.id === alert.id && entry.destination === destination, - ), + !outbox.some((entry) => entry.alert.id === alert.id && entry.destination === destination), ) .map((destination): Delivery => ({ destination, @@ -190,15 +231,64 @@ export function recordModelAlert( createdAt: nowMs, })); return { - state: { - ...state, - degraded: { ...Object.fromEntries(kept), [alert.threadId]: next }, - outbox: [...pruneOutbox(state.outbox, nowMs), ...added].slice(-MAX_OUTBOX), - }, + outbox: [...pruneOutbox(outbox, nowMs), ...added].slice(-MAX_OUTBOX), enqueued: added.length > 0, }; } +const reportedKey = (threadId: string, kind: ThreadEventKind) => `${threadId}\n${kind}`; + +/** + * Records what a trigger found true of a thread. A kind reports once per signature: it stays quiet + * until the thread stops matching it or matches it with a different signature, such as a new run. + */ +export function recordThreadEvents( + state: ModelAlertState, + judged: { + readonly threadId: string; + readonly evaluated: ReadonlyArray; + readonly findings: ReadonlyArray; + }, + toAlert: (finding: ThreadFinding) => ThreadEventAlert, + destinations: ReadonlyArray, + nowMs: number, +): { readonly state: ModelAlertState; readonly enqueued: boolean } { + let reported = state.reported; + let outbox = state.outbox; + let enqueued = false; + for (const kind of judged.evaluated) { + const key = reportedKey(judged.threadId, kind); + const kindFindings = judged.findings.filter((finding) => finding.kind === kind); + if (kindFindings.length === 0) { + if (reported[key] !== undefined) { + const { [key]: _cleared, ...rest } = reported; + reported = rest; + } + continue; + } + // A kind can hold several findings at once (a question and an inbox flip), so it remembers + // each signature it reported. One that stops being true is forgotten and can report again. + const known = (reported[key] ?? "").split("\n").filter((signature) => signature !== ""); + const current = kindFindings.map((finding) => finding.signature); + const stored = current.join("\n"); + if (reported[key] !== stored) { + const { [key]: _previous, ...rest } = reported; + reported = { + ...Object.fromEntries(Object.entries(rest).slice(-(MAX_REPORTED - 1))), + [key]: stored, + }; + } + for (const finding of kindFindings) { + if (known.includes(finding.signature)) continue; + const queued = enqueueAlert(outbox, toAlert(finding), destinations, nowMs); + outbox = queued.outbox; + enqueued ||= queued.enqueued; + } + } + if (reported === state.reported && outbox === state.outbox) return { state, enqueued }; + return { state: { ...state, reported, outbox }, enqueued }; +} + const pruneOutbox = (outbox: ReadonlyArray, nowMs: number) => outbox.filter((entry) => nowMs - entry.createdAt < OUTBOX_TTL_MS); @@ -247,6 +337,50 @@ function modelAlertMessage(alert: ModelAlert): string { ].join("\n\n"); } +export const watcherAlertMessage = (alert: WatcherAlert) => + alert.type === "thread-event" ? threadEventMessage(alert) : modelAlertMessage(alert); + +/** + * Puts an alert in the watcher thread, here or on the linked machine that owns it. The retry key is + * one per alert and watcher, so a retry after a lost answer sends once. + */ +export const deliverAlertToWatcher = Effect.fnUntraced(function* ( + deps: { + readonly dispatch: Orchestrator.OrchestratorV2["Service"]["dispatch"]; + readonly call: PeerMcpClient.PeerMcpClient["Service"]["call"]; + readonly sha256Hex: (identity: string) => Effect.Effect; + }, + alert: WatcherAlert, + watcher: ModelAlertWatcher, + here: EnvironmentId, +) { + const message = watcherAlertMessage(alert); + // Run ids can be long; a retry key must stay under the 256 characters the peer accepts. + const identity = `${alert.id}\n${watcher.environmentId}\n${watcher.threadId}`; + const key = `${alert.type === "thread-event" ? "thread-event" : "model-alert"}:${yield* deps.sha256Hex(identity)}`; + if (watcher.environmentId === here) { + yield* deps.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(`server:${key}`), + threadId: watcher.threadId, + messageId: MessageId.make(`message:${key}`), + text: message, + attachments: [], + dispatchMode: { type: "queue_after_active" }, + createdBy: "agent", + creationSource: "server", + }); + return; + } + yield* deps.call({ + environmentId: watcher.environmentId, + tool: "t3_thread_send", + arguments: { threadId: watcher.threadId, message, mode: "queue", clientRequestId: key }, + limits: { runtimeMode: "full-access", interactionMode: "default" }, + success: OrchestratorMcpThreadSendResult, + }); +}); + const ModelAlertStateJson = Schema.fromJsonString(ModelAlertState); const decodeState = Schema.decodeUnknownOption(ModelAlertStateJson); const encodeState = Schema.encodeEffect(ModelAlertStateJson); @@ -259,6 +393,8 @@ export class ModelAlertReactor extends Context.Service< readonly handle: (event: OrchestrationV2DomainEvent) => Effect.Effect; /** Attempts every queued delivery that is due. */ readonly flush: Effect.Effect; + /** Judges every live lane for the time-based events: stuck. */ + readonly sweep: Effect.Effect; } >()("t3/orchestration-v2/ModelAlertReactor") {} @@ -309,41 +445,17 @@ export const make = Effect.gen(function* () { }), ); - const deliverToWatcher = Effect.fnUntraced(function* ( - alert: ModelAlert, - watcher: ModelAlertWatcher, - here: EnvironmentId, - ) { - const message = modelAlertMessage(alert); - // Run ids can be long; a retry key must stay under the 256 characters the peer accepts. - const identity = `${alert.id}\n${watcher.environmentId}\n${watcher.threadId}`; - const digest = yield* crypto + const sha256Hex = (identity: string) => + crypto .digest(Digest.sha256, new TextEncoder().encode(identity)) - .pipe(Effect.orDie); - const key = `model-alert:${Hex.encode(digest)}`; - if (watcher.environmentId === here) { - yield* engine.dispatch({ - type: "message.dispatch", - commandId: CommandId.make(`server:${key}`), - threadId: watcher.threadId, - messageId: MessageId.make(`message:${key}`), - text: message, - attachments: [], - dispatchMode: { type: "queue_after_active" }, - createdBy: "agent", - creationSource: "server", - }); - return; - } - // The watcher's machine keeps one retry key per alert, so a retry after a lost answer sends once. - yield* peers.call({ - environmentId: watcher.environmentId, - tool: "t3_thread_send", - arguments: { threadId: watcher.threadId, message, mode: "queue", clientRequestId: key }, - limits: { runtimeMode: "full-access", interactionMode: "default" }, - success: OrchestratorMcpThreadSendResult, - }); - }); + .pipe(Effect.map(Hex.encode), Effect.orDie); + const deliverToWatcher = (alert: WatcherAlert, watcher: ModelAlertWatcher, here: EnvironmentId) => + deliverAlertToWatcher( + { dispatch: engine.dispatch, call: peers.call, sha256Hex }, + alert, + watcher, + here, + ); const deliver = Effect.fnUntraced(function* (entry: Delivery) { const current = yield* settings.getSettings; @@ -391,70 +503,207 @@ export const make = Effect.gen(function* () { ); }); - const handle: ModelAlertReactor["Service"]["handle"] = Effect.fnUntraced( - function* (event) { - if (event.type !== "run.updated" || !TERMINAL_RUN_STATUSES.includes(event.payload.status)) { - return; - } - const run = event.payload; - const records = yield* projections.getThreadRecords( - event.threadId, - ["providerTurns", "turnItems"], - { - providerTurnAttemptIds: run.activeAttemptId === null ? [] : [run.activeAttemptId], - turnItemRunIds: [run.id], - turnItemTypes: ["system_notice", "error"], - }, + /** Queues a run's model alert, and says whether it reported a usage limit. */ + const handleModelAlert = Effect.fnUntraced(function* ( + event: Extract, + ) { + const run = event.payload; + const records = yield* projections.getThreadRecords( + event.threadId, + ["providerTurns", "turnItems"], + { + providerTurnAttemptIds: run.activeAttemptId === null ? [] : [run.activeAttemptId], + turnItemRunIds: [run.id], + turnItemTypes: ["system_notice", "error"], + }, + ); + const here = yield* environment.getEnvironmentId; + const judged = modelAlertFor({ + environmentId: here, + environmentLabel: "", + thread: records.thread, + run, + projection: records, + occurredAt: DateTime.formatIso(yield* DateTime.now), + }); + if (judged.alert === undefined && !judged.healthy) return { usageLimited: false }; + const outcome = + judged.alert === undefined + ? judged + : { + ...judged, + alert: { + ...judged.alert, + environmentLabel: (yield* environment.getDescriptor).label, + }, + }; + const current = yield* settings.getSettings; + const destinations = modelAlertDestinations({ + watcher: current.modelAlertWatcher, + webhookUrl: current.modelAlertWebhookUrl, + here, + lineage: records.thread.lineage, + }); + const nowMs = yield* Clock.currentTimeMillis; + const enqueued = yield* update((current) => { + const result = recordModelAlert( + current, + { ...outcome, threadId: event.threadId }, + destinations, + nowMs, ); - const here = yield* environment.getEnvironmentId; - const judged = modelAlertFor({ - environmentId: here, - environmentLabel: "", - thread: records.thread, - run, - projection: records, - occurredAt: DateTime.formatIso(yield* DateTime.now), - }); - if (judged.alert === undefined && !judged.healthy) return; - const outcome = - judged.alert === undefined - ? judged - : { - ...judged, - alert: { - ...judged.alert, - environmentLabel: (yield* environment.getDescriptor).label, - }, - }; - const current = yield* settings.getSettings; - const destinations = modelAlertDestinations({ - watcher: current.modelAlertWatcher, - webhookUrl: current.modelAlertWebhookUrl, - here, - lineage: records.thread.lineage, - }); - const nowMs = yield* Clock.currentTimeMillis; - const enqueued = yield* update((current) => { - const result = recordModelAlert( - current, - { ...outcome, threadId: event.threadId }, - destinations, - nowMs, - ); - return [result.enqueued, result.state]; - }); - if (enqueued) yield* Queue.offer(wake, undefined); - }, - (effect, event) => - effect.pipe( - Effect.catchCause((cause) => - Effect.logWarning("judging a run for a model alert failed", { - threadId: event.threadId, - cause, + return [result.enqueued, result.state]; + }); + if (enqueued) yield* Queue.offer(wake, undefined); + return { usageLimited: judged.alert?.reasons.includes("usage_limit") === true }; + }); + + /** Judges one thread for the kinds a trigger can change, and queues what is newly true. */ + const handleThreadEvents = Effect.fnUntraced(function* ( + threadId: ThreadId, + trigger: ThreadEventTrigger, + usageLimitReported: boolean, + /** Open delegated tasks by parent thread, when the caller read them once for many threads. */ + remoteTasks?: ReadonlyMap, + ) { + const current = yield* settings.getSettings; + const here = yield* environment.getEnvironmentId; + const watcher = current.modelAlertWatcher; + // The watcher's own turns would report to itself forever. + if (watcher?.environmentId === here && watcher.threadId === threadId) return; + if (watcher === null && current.modelAlertWebhookUrl.trim() === "") return; + const shell = yield* projections.getThreadShell(threadId); + if (shell === null || shell.deletedAt !== null) return; + const remoteTaskCount = + trigger === "request" || trigger === "inbox" + ? 0 + : remoteTasks !== undefined + ? (remoteTasks.get(threadId) ?? 0) + : (yield* projections.getOpenRemoteDelegatedTasks).filter( + (task) => task.parentThreadId === threadId, + ).length; + // A parent that waits on children is not stuck while they work. + const childActivityMs = + trigger === "sweep" + ? Math.max( + 0, + ...(yield* Effect.forEach( + (shell.pendingBackgroundTasks ?? []).flatMap((task) => + task.kind === "subagent" && task.childThreadId !== undefined + ? [task.childThreadId] + : [], + ), + (childId) => + projections + .getThreadShell(childId) + .pipe( + Effect.map((child) => + child === null ? 0 : DateTime.toEpochMillis(child.updatedAt), + ), + ), + )), + ) + : null; + const nowMs = yield* Clock.currentTimeMillis; + const judged = judgeThread({ + shell, + trigger, + nowMs, + stuckMinutes: current.watcherStuckMinutes, + remoteTaskCount, + childActivityMs, + usageLimitReported, + }); + const label = (yield* environment.getDescriptor).label; + const occurredAt = DateTime.formatIso(yield* DateTime.now); + const destinations = modelAlertDestinations({ + watcher, + webhookUrl: current.modelAlertWebhookUrl, + here, + lineage: shell.lineage, + }); + const enqueued = yield* update((state) => { + const result = recordThreadEvents( + state, + { threadId, evaluated: judged.evaluated, findings: judged.findings }, + (finding) => + threadEventAlert({ + environmentId: here, + environmentLabel: label, + shell, + finding, + occurredAt, }), - ), + destinations, + nowMs, + ); + return [result.enqueued, result.state]; + }); + if (enqueued) yield* Queue.offer(wake, undefined); + }); + + const threadTrigger = (event: OrchestrationV2DomainEvent): ThreadEventTrigger | undefined => { + switch (event.type) { + case "run.updated": + return TERMINAL_RUN_STATUSES.includes(event.payload.status) ? "run-ended" : undefined; + case "thread.settled": + return "settled"; + case "thread.metadata-updated": + return "inbox"; + case "runtime-request.updated": + return "request"; + default: + return undefined; + } + }; + + const handle: ModelAlertReactor["Service"]["handle"] = Effect.fnUntraced(function* (event) { + const trigger = threadTrigger(event); + if (trigger === undefined) return; + const modelAlert = + event.type === "run.updated" + ? yield* handleModelAlert(event).pipe( + Effect.catchCause((cause) => + Effect.logWarning("judging a run for a model alert failed", { + threadId: event.threadId, + cause, + }).pipe(Effect.as({ usageLimited: false })), + ), + ) + : { usageLimited: false }; + yield* handleThreadEvents(event.threadId, trigger, modelAlert.usageLimited).pipe( + Effect.catchCause((cause) => + Effect.logWarning("judging a thread for a watcher event failed", { + threadId: event.threadId, + trigger, + cause, + }), ), - ); + ); + }); + + /** Looks at every thread that may be live for the time-based kinds no event announces. */ + const sweep: ModelAlertReactor["Service"]["sweep"] = Effect.gen(function* () { + const index = yield* projections.listActiveShellIndex(); + // One scan of the delegated tasks for the whole sweep, not one per thread. + const remoteTasks = new Map(); + for (const task of yield* projections.getOpenRemoteDelegatedTasks) { + remoteTasks.set(task.parentThreadId, (remoteTasks.get(task.parentThreadId) ?? 0) + 1); + } + yield* Effect.forEach( + index.filter((entry) => entry.mayBeLive && isListedThreadLineage(entry.lineage)), + (entry) => + handleThreadEvents(entry.id, "sweep", false, remoteTasks).pipe( + Effect.catchCause((cause) => + Effect.logWarning("sweeping a thread for a watcher event failed", { + threadId: entry.id, + cause, + }), + ), + ), + { discard: true }, + ); + }).pipe(Effect.catchCause((cause) => Effect.logWarning("watcher sweep failed", { cause }))); const start: ModelAlertReactor["Service"]["start"] = Effect.fn("ModelAlertReactor.start")( function* () { @@ -476,10 +725,12 @@ export const make = Effect.gen(function* () { ), ), ); + // Stuck lanes emit no event, so the sweep is the one timer here. + yield* forkParked(sweep.pipe(Effect.repeat(Schedule.spaced("60 seconds")))); }, ); - return ModelAlertReactor.of({ start, handle, flush }); + return ModelAlertReactor.of({ start, handle, flush, sweep }); }); export const layer = Layer.effect(ModelAlertReactor, make); diff --git a/apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts b/apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts new file mode 100644 index 000000000000..80abe7f06abf --- /dev/null +++ b/apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts @@ -0,0 +1,341 @@ +import { assert, it } from "@effect/vitest"; +import * as DateTime from "effect/DateTime"; + +import { + judgeThread, + lastLine, + promisesToContinue, + threadEventAlert, + threadEventMessage, + type ThreadEventTrigger, + type WatchedShell, +} from "./ThreadEventAlerts.ts"; + +const NOW = Date.parse("2026-11-19T05:00:00.000Z"); +const at = (minutesAgo: number) => DateTime.makeUnsafe(NOW - minutesAgo * 60_000); + +// The Fleet rollout lane said this, settled with no tracked task, and its R720 build died unnoticed. +const FLEET_PROMISE = + "The desktop build is running on R720. A background watch will wake this thread when it finishes, and I'll install it on ASUS after that. I'll settle this thread only once all three machines are confirmed."; + +const shellWith = (overrides: Partial = {}): WatchedShell => + ({ + id: "thread:lane", + title: "Fleet rollout", + createdBy: "agent", + lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: null }, + status: "completed", + activeRunId: null, + latestRunId: "run:7", + latestRunCompletedAt: at(2), + activityRunStatus: null, + blockedOnThreadWait: false, + pendingRuntimeRequest: null, + pendingBackgroundTasks: [], + pullRequests: [], + latestVisibleMessage: null, + lastError: null, + settledOverride: null, + archivedAt: null, + snoozedUntil: null, + limitRecovery: null, + updatedAt: at(2), + ...overrides, + }) as unknown as WatchedShell; + +const judge = ( + shell: WatchedShell, + trigger: ThreadEventTrigger, + extra: Partial[0]> = {}, +) => judgeThread({ shell, trigger, nowMs: NOW, stuckMinutes: 20, ...extra }); + +const openPullRequest = { + host: "github.com", + repository: "nullStack65/t3code", + number: 97, + url: "https://github.com/nullStack65/t3code/pull/97", + source: "agent", + linkedAt: "2026-11-19T04:00:00.000Z", + snapshot: null, + stack: null, +} as never; + +const assistantSays = (text: string) => + ({ id: "message:1", role: "assistant", text, updatedAt: at(2) }) as never; + +it.each([ + ["a failed run", { status: "failed" }, ["lane-ended"]], + ["an interrupted run", { status: "interrupted" }, ["lane-ended"]], + ["a completed run with nothing open", { status: "completed" }, []], + [ + "a completed run with an open pull request and no task", + { status: "completed", pullRequests: [openPullRequest] }, + ["lane-ended"], + ], + [ + "a completed run with an open pull request and a tracked task", + { + status: "completed", + pullRequests: [openPullRequest], + pendingBackgroundTasks: [{ taskId: "pull-request-watch:97", kind: "monitor" }], + }, + [], + ], + ["a failed run that ended long ago", { status: "failed", latestRunCompletedAt: at(120) }, []], + ["a failed run on a settled lane", { status: "failed", settledOverride: "settled" }, []], + [ + "a failed run on a delegated child", + { + status: "failed", + lineage: { parentThreadId: "thread:lead", relationshipToParent: "subagent" }, + }, + [], + ], + ["a run still going", { status: "failed", activeRunId: "run:8" }, []], +] as const)("lane-ended: %s -> %j", (_name, overrides, expected) => { + const { findings } = judge(shellWith(overrides as never), "run-ended"); + assert.deepEqual( + findings.filter((finding) => finding.kind === "lane-ended").map((finding) => finding.kind), + [...expected], + ); +}); + +it.each(["failed", "interrupted"] as const)( + "a %s run on a thread a person created is not a lane event", + (status) => { + const { findings } = judge(shellWith({ status, createdBy: "user" } as never), "run-ended"); + assert.deepEqual(findings, []); + const stuck = judge( + shellWith({ + createdBy: "user", + status: "running", + activeRunId: "run:8", + activityRunStatus: "running", + updatedAt: at(90), + } as never), + "sweep", + ); + assert.deepEqual(stuck.findings, []); + }, +); + +it("lane-ended stays quiet for a failure the model alert already reported as a usage limit", () => { + const { findings } = judge(shellWith({ status: "failed" } as never), "run-ended", { + usageLimitReported: true, + }); + assert.deepEqual(findings, []); +}); + +it("promise-without-wake: flags the Fleet lane's real message when nothing is tracked", () => { + const { findings } = judge( + shellWith({ latestVisibleMessage: assistantSays(FLEET_PROMISE) } as never), + "run-ended", + ); + assert.deepEqual( + findings.map((finding) => finding.kind), + ["promise-without-wake"], + ); + assert.equal(findings[0]?.signature, "message:1"); +}); + +it.each([ + [ + "a tracked background command", + { pendingBackgroundTasks: [{ taskId: "b1", kind: "command" }] }, + 0, + ], + ["a delegated task on another machine", {}, 1], + ["a pull request watch", { pullRequests: [{ ...(openPullRequest as object), watch: {} }] }, 0], + ["an armed usage-limit resume", { limitRecovery: { autoResume: true } }, 0], + ["a pending question", { pendingRuntimeRequest: { id: "request:1", kind: "user_input" } }, 0], +] as const)("promise-without-wake: %s is a wake handle", (_name, overrides, remoteTaskCount) => { + const { findings } = judge( + shellWith({ latestVisibleMessage: assistantSays(FLEET_PROMISE), ...overrides } as never), + "run-ended", + { remoteTaskCount }, + ); + assert.deepEqual(findings, []); +}); + +it("promise-without-wake: judges a settling thread even when its last run ended long ago", () => { + // A settled lane is not `watched`, so the settle trigger sees the shell as it was just before. + const { findings } = judge( + shellWith({ + latestVisibleMessage: assistantSays(FLEET_PROMISE), + latestRunCompletedAt: at(300), + } as never), + "settled", + ); + assert.deepEqual( + findings.map((finding) => finding.kind), + ["promise-without-wake"], + ); +}); + +it.each([ + [FLEET_PROMISE, true], + ["I'll install the desktop build after the R720 build finishes.", true], + ["Keeping this lane open until the rollout is verified.", true], + ["I'll be woken when the command ends.", true], + ["The rollout is live on all three machines. Nothing left to do.", false], + ["I installed it after the build finished.", false], +] as const)("promise phrases: %j -> %s", (text, expected) => { + assert.equal(promisesToContinue(text), expected); +}); + +it("stuck: an active run with no new item for the configured minutes", () => { + const active = { activeRunId: "run:8", status: "running", activityRunStatus: "running" }; + const quiet = judge(shellWith({ ...active, updatedAt: at(25) } as never), "sweep"); + assert.deepEqual( + quiet.findings.map((finding) => [finding.kind, finding.signature]), + [["stuck", "run:8"]], + ); + assert.deepEqual( + judge(shellWith({ ...active, updatedAt: at(5) } as never), "sweep").findings, + [], + ); + // The threshold is configurable and zero turns the check off. + assert.deepEqual( + judge(shellWith({ ...active, updatedAt: at(5) } as never), "sweep", { stuckMinutes: 3 }) + .findings.length, + 1, + ); + assert.deepEqual( + judge(shellWith({ ...active, updatedAt: at(500) } as never), "sweep", { stuckMinutes: 0 }) + .findings, + [], + ); + // A parent whose child is still working is not stuck. + assert.deepEqual( + judge(shellWith({ ...active, updatedAt: at(25) } as never), "sweep", { + childActivityMs: NOW - 60_000, + }).findings, + [], + ); +}); + +it("stuck: a thread waiting on background work that no longer exists", () => { + const waiting = { status: "waiting", activityRunStatus: "waiting", updatedAt: at(30) }; + const orphan = judge(shellWith(waiting as never), "sweep"); + assert.deepEqual( + orphan.findings.map((finding) => [finding.kind, finding.signature]), + [["stuck", "run:7:orphan-wait"]], + ); + const tracked = judge( + shellWith({ + ...waiting, + pendingBackgroundTasks: [{ taskId: "bdejsjjol", kind: "command", awaited: true }], + } as never), + "sweep", + ); + assert.deepEqual(tracked.findings, []); +}); + +it("needs-person: an agent lane raising a question or flipping to a founder", () => { + const question = judge( + shellWith({ + pendingRuntimeRequest: { id: "request:9", kind: "approval", userOnly: false }, + } as never), + "request", + ); + assert.deepEqual( + question.findings.map((finding) => [finding.kind, finding.signature]), + [["needs-person", "request:request:9"]], + ); + const flipped = judge( + shellWith({ + inbox: { + owner: "kameron", + assignee: { kind: "human", id: "kameron" }, + audience: "kameron", + attentionKind: "action", + reason: "decision", + decisionRequest: { + founder: "kameron", + kind: "decision", + question: "Spend $20 on the build host?", + requestedAt: "2026-11-19T04:59:00.000Z", + }, + }, + } as never), + "inbox", + ); + assert.equal(flipped.findings.length, 1); + assert.match(flipped.findings[0]!.detail, /assigned to kameron.*Spend \$20/); + // An agent-only background lane, and a thread a founder opened themselves, do not alert. + assert.deepEqual( + judge( + shellWith({ + inbox: { + owner: null, + assignee: null, + audience: "agent-only", + attentionKind: "background", + }, + } as never), + "inbox", + ).findings, + [], + ); + assert.deepEqual( + judge( + shellWith({ + createdBy: "user", + pendingRuntimeRequest: { id: "request:9", kind: "approval" }, + } as never), + "request", + ).findings, + [], + ); +}); + +it("a recurrence of the same state is a new message", () => { + const shell = shellWith({ status: "failed" } as never); + const finding = judge(shell, "run-ended").findings[0]!; + const make = (occurredAt: string) => + threadEventAlert({ + environmentId: "env-r720" as never, + environmentLabel: "primary-r720", + shell, + finding, + occurredAt, + }); + assert.notEqual(make("2026-11-19T05:00:00.000Z").id, make("2026-11-19T05:30:00.000Z").id); +}); + +it("each trigger evaluates only the kinds it can change", () => { + assert.deepEqual(judge(shellWith(), "run-ended").evaluated, [ + "lane-ended", + "promise-without-wake", + ]); + assert.deepEqual(judge(shellWith(), "sweep").evaluated, ["stuck"]); + assert.deepEqual(judge(shellWith(), "inbox").evaluated, ["needs-person"]); +}); + +it("the wake carries environment, thread link, title, kind, run, last line and open pull requests", () => { + const shell = shellWith({ + status: "completed", + pullRequests: [openPullRequest], + latestVisibleMessage: assistantSays(`${"x".repeat(400)}\n${FLEET_PROMISE}`), + } as never); + const finding = judge(shell, "run-ended").findings[0]!; + const alert = threadEventAlert({ + environmentId: "env-r720" as never, + environmentLabel: "primary-r720", + shell, + finding, + occurredAt: "2026-11-19T05:00:00.000Z", + }); + assert.equal(alert.runId, "run:7"); + assert.equal(alert.threadTitle, "Fleet rollout"); + assert.deepEqual(alert.openPullRequests, ["https://github.com/nullStack65/t3code/pull/97"]); + const message = threadEventMessage(alert); + assert.include(message, "primary-r720"); + assert.include(message, "env-r720"); + assert.include(message, "Fleet rollout"); + assert.include(message, "Event: lane-ended."); + assert.include(message, "Run run:7."); + assert.include(message, "pull/97"); + assert.include(message, "Last assistant line:"); + assert.isAtMost(lastLine("a".repeat(1000))!.length, 240); +}); diff --git a/apps/server/src/orchestration-v2/ThreadEventAlerts.ts b/apps/server/src/orchestration-v2/ThreadEventAlerts.ts new file mode 100644 index 000000000000..502c4b2bc314 --- /dev/null +++ b/apps/server/src/orchestration-v2/ThreadEventAlerts.ts @@ -0,0 +1,356 @@ +import { + type EnvironmentId, + isListedThreadLineage, + type OrchestrationV2ThreadShell, +} from "@t3tools/contracts"; +import { visibleThreadPullRequests } from "@t3tools/shared/threadPullRequests"; +import { formatThreadLink } from "@t3tools/shared/threadLinks"; +import * as DateTime from "effect/DateTime"; +import * as Schema from "effect/Schema"; + +/** + * Deterministic thread events for the watcher thread, judged from a thread's shell with no agent + * judgment. Delivery, retry and dedupe live in ModelAlertReactor, which carries these beside its + * model alerts. + */ +export const ThreadEventKind = Schema.Literals([ + "lane-ended", + "promise-without-wake", + "stuck", + "needs-person", +]); +export type ThreadEventKind = typeof ThreadEventKind.Type; + +export const ThreadEventAlert = Schema.Struct({ + type: Schema.Literal("thread-event"), + kind: ThreadEventKind, + id: Schema.String, + environmentId: Schema.String, + environmentLabel: Schema.String, + threadId: Schema.String, + threadTitle: Schema.String, + threadLink: Schema.String, + runId: Schema.NullOr(Schema.String), + /** What happened, in a sentence: how the run ended, what is missing, what is waiting. */ + detail: Schema.String, + lastAssistantLine: Schema.NullOr(Schema.String), + lastError: Schema.NullOr(Schema.String), + openPullRequests: Schema.Array(Schema.String), + occurredAt: Schema.String, +}); +export type ThreadEventAlert = typeof ThreadEventAlert.Type; + +/** What made the server look at a thread; each trigger judges only the kinds it can change. */ +export type ThreadEventTrigger = "run-ended" | "settled" | "inbox" | "request" | "sweep"; + +const KINDS_BY_TRIGGER: Record> = { + "run-ended": ["lane-ended", "promise-without-wake"], + settled: ["promise-without-wake"], + inbox: ["needs-person"], + request: ["needs-person"], + sweep: ["stuck"], +}; + +export type WatchedShell = Pick< + OrchestrationV2ThreadShell, + | "id" + | "title" + | "createdBy" + | "lineage" + | "inbox" + | "status" + | "activeRunId" + | "latestRunId" + | "latestRunCompletedAt" + | "activityRunStatus" + | "blockedOnThreadWait" + | "pendingRuntimeRequest" + | "pendingBackgroundTasks" + | "pullRequests" + | "latestVisibleMessage" + | "lastError" + | "settledOverride" + | "archivedAt" + | "snoozedUntil" + | "limitRecovery" + | "updatedAt" +>; + +export interface ThreadFinding { + readonly kind: ThreadEventKind; + readonly detail: string; + /** The state the finding describes; the same one reports once, a different one reports again. */ + readonly signature: string; +} + +/** A run that ended longer ago than this is history, not news, whatever woke the judge. */ +const RECENT_RUN_END_MS = 15 * 60 * 1000; + +const timeMs = (value: DateTime.Utc | null | undefined) => + value === null || value === undefined ? null : DateTime.toEpochMillis(value); + +const isLive = (shell: WatchedShell) => + shell.activeRunId !== null || (shell.activityRunStatus ?? null) !== null; + +const openPullRequests = (shell: WatchedShell) => + visibleThreadPullRequests(shell.pullRequests ?? []).filter( + (link) => link.snapshot === null || link.snapshot.state === "open", + ); + +/** A pending task, a watching PR, a delegated task or a snooze that will bring the thread back. */ +export function hasWakeHandle(shell: WatchedShell, remoteTaskCount: number, nowMs: number) { + if ((shell.pendingBackgroundTasks ?? []).length > 0) return true; + if (remoteTaskCount > 0) return true; + if ((shell.pullRequests ?? []).some((link) => link.watch !== undefined)) return true; + if (shell.limitRecovery?.autoResume === true || shell.limitRecovery?.snooze === true) return true; + const snoozedUntil = timeMs(shell.snoozedUntil); + return snoozedUntil !== null && snoozedUntil > nowMs; +} + +/** + * Phrases an agent uses to say something will wake it or that it will carry on later. An explicit + * list on purpose: a false alarm costs a Lead turn, and the agent can always avoid the phrase by + * registering a real wait. + */ +const PROMISE_PHRASES: ReadonlyArray = [ + /\bwill (?:wake|notify|ping|resume) (?:this|the|me|it|my)\b/i, + /\bwill (?:be )?woken\b/i, + /\bI(?:'ll| will) be (?:woken|notified|pinged)\b/i, + /\b(?:watch|monitor|task|job|timer|check)\b[^.\n]{0,40}\bwill (?:wake|resume|notify)\b/i, + /\bI(?:'ll| will) (?:install|merge|continue|resume|finish|roll(?: it)? out|deploy|pick (?:this|it) (?:back )?up|check|come back)\b[^.\n]{0,80}\b(?:after|once|when|as soon as|later)\b/i, + /\b(?:keeping|keep|leaving|leave|holding|hold) (?:this|the) (?:lane|thread) open\b/i, + /\bwill (?:continue|resume|carry on) (?:when|once|after|later)\b/i, +]; + +export const promisesToContinue = (text: string) => + PROMISE_PHRASES.some((phrase) => phrase.test(text)); + +/** The last non-empty line of a message, trimmed to keep a wake short. */ +export function lastLine(text: string, max = 240): string | null { + const line = text + .split("\n") + .map((candidate) => candidate.trim()) + .findLast((candidate) => candidate.length > 0); + if (line === undefined) return null; + return line.length <= max ? line : `${line.slice(0, max - 1)}…`; +} + +const IDLE_STATUSES = new Set(["idle", "completed"]); + +/** + * Which of the trigger's kinds are true of the thread right now. `evaluated` lists every kind the + * trigger can judge, so a kind that is no longer true can reset its dedupe. + */ +export function judgeThread(input: { + readonly shell: WatchedShell; + readonly trigger: ThreadEventTrigger; + readonly nowMs: number; + readonly stuckMinutes: number; + /** Open delegated tasks on a linked environment, which the shell does not list. */ + readonly remoteTaskCount?: number; + /** The latest activity among the children this thread waits on. */ + readonly childActivityMs?: number | null; + /** A model alert for the run already reported a usage limit, so a failure adds nothing. */ + readonly usageLimitReported?: boolean; +}): { readonly evaluated: ReadonlyArray; readonly findings: ThreadFinding[] } { + const { shell, nowMs } = input; + const evaluated = KINDS_BY_TRIGGER[input.trigger]; + const findings: ThreadFinding[] = []; + // Only a top-level lane an agent runs, that nobody has archived or settled, is worth waking the + // watcher for. A person's own thread ending, stopping or going quiet is not news. + const watched = + shell.createdBy === "agent" && + isListedThreadLineage(shell.lineage) && + shell.archivedAt === null && + shell.settledOverride !== "settled"; + const remoteTaskCount = input.remoteTaskCount ?? 0; + const tasks = shell.pendingBackgroundTasks ?? []; + + if (evaluated.includes("lane-ended") && watched && !isLive(shell)) { + const endedMs = timeMs(shell.latestRunCompletedAt); + const recent = endedMs !== null && nowMs - endedMs <= RECENT_RUN_END_MS; + const prs = openPullRequests(shell); + if (recent && shell.latestRunId !== null) { + if ( + (shell.status === "failed" && input.usageLimitReported !== true) || + shell.status === "interrupted" + ) { + findings.push({ + kind: "lane-ended", + detail: `its last run ${shell.status}`, + signature: shell.latestRunId, + }); + } else if ( + shell.status === "completed" && + prs.length > 0 && + !hasWakeHandle(shell, remoteTaskCount, nowMs) + ) { + findings.push({ + kind: "lane-ended", + detail: `its last run completed with ${prs.length} open pull request${prs.length === 1 ? "" : "s"} and nothing tracked that will wake it`, + signature: shell.latestRunId, + }); + } + } + } + + if (evaluated.includes("promise-without-wake") && watched && !isLive(shell)) { + const message = shell.latestVisibleMessage; + const endedMs = timeMs(shell.latestRunCompletedAt); + const recent = + input.trigger === "settled" || (endedMs !== null && nowMs - endedMs <= RECENT_RUN_END_MS); + if ( + recent && + IDLE_STATUSES.has(shell.status) && + message?.role === "assistant" && + promisesToContinue(message.text) && + !hasWakeHandle(shell, remoteTaskCount, nowMs) && + shell.pendingRuntimeRequest === null && + shell.inbox?.decisionRequest === undefined + ) { + findings.push({ + kind: "promise-without-wake", + detail: + "its last message says it will be woken or will continue later, but nothing tracked will wake it", + signature: message.id, + }); + } + } + + if (evaluated.includes("stuck") && watched && input.stuckMinutes > 0) { + const activityMs = Math.max(timeMs(shell.updatedAt) ?? 0, input.childActivityMs ?? 0); + const quietMinutes = Math.floor((nowMs - activityMs) / 60_000); + const quiet = activityMs > 0 && quietMinutes >= input.stuckMinutes; + const waiting = shell.activityRunStatus === "waiting" || shell.status === "waiting"; + if (quiet && waiting) { + // A wait the shell still shows, with nothing behind it that could end it. + if ( + tasks.length === 0 && + remoteTaskCount === 0 && + shell.pendingRuntimeRequest === null && + !(shell.pullRequests ?? []).some((link) => link.watch !== undefined) + ) { + findings.push({ + kind: "stuck", + detail: `it has waited ${quietMinutes} minutes on background work that no longer exists`, + signature: `${shell.latestRunId ?? shell.id}:orphan-wait`, + }); + } + } else if ( + quiet && + shell.activeRunId !== null && + shell.blockedOnThreadWait !== true && + shell.pendingRuntimeRequest === null + ) { + findings.push({ + kind: "stuck", + detail: `its run is active with no new activity for ${quietMinutes} minutes`, + signature: shell.activeRunId, + }); + } + } + + if (evaluated.includes("needs-person") && watched) { + const request = shell.pendingRuntimeRequest; + const inbox = shell.inbox; + if (request !== null) { + findings.push({ + kind: "needs-person", + detail: `it is waiting on ${request.userOnly === true ? "a secret or link only the user can give" : `a ${request.kind.replaceAll("_", " ")}`}`, + signature: `request:${request.id}`, + }); + } + const founderAssignee = inbox?.assignee?.kind === "human" ? inbox.assignee.id : null; + if (inbox !== undefined && (founderAssignee !== null || inbox.attentionKind === "action")) { + const ask = inbox.decisionRequest; + findings.push({ + kind: "needs-person", + detail: [ + founderAssignee === null + ? "its inbox attention is set to action" + : `its inbox is assigned to ${founderAssignee}`, + ask === undefined ? null : `with an open ${ask.kind}: ${ask.question}`, + ask === undefined && inbox.reason !== undefined ? `for ${inbox.reason}` : null, + ] + .filter((part) => part !== null) + .join(" "), + signature: `inbox:${founderAssignee ?? ""}:${inbox.attentionKind}:${ask?.requestedAt ?? ""}`, + }); + } + } + + return { evaluated, findings }; +} + +export function threadEventAlert(input: { + readonly environmentId: EnvironmentId; + readonly environmentLabel: string; + readonly shell: Pick< + WatchedShell, + | "id" + | "title" + | "latestRunId" + | "latestVisibleMessage" + | "lastError" + | "pullRequests" + | "activeRunId" + >; + readonly finding: ThreadFinding; + readonly occurredAt: string; +}): ThreadEventAlert { + const { shell, finding } = input; + const message = shell.latestVisibleMessage; + return { + type: "thread-event", + kind: finding.kind, + // The time makes each occurrence its own message: the delivery key hashes this id, and a + // recurrence of the same state must not be taken for a replay of the first. + id: `${input.environmentId}:${shell.id}:${finding.kind}:${finding.signature}:${input.occurredAt}`, + environmentId: input.environmentId, + environmentLabel: input.environmentLabel, + threadId: shell.id, + threadTitle: shell.title, + threadLink: formatThreadLink( + { environmentId: input.environmentId, threadId: shell.id }, + shell.title, + ), + runId: shell.activeRunId ?? shell.latestRunId, + detail: finding.detail, + lastAssistantLine: message?.role === "assistant" ? lastLine(message.text) : null, + lastError: + shell.lastError === null || shell.lastError === undefined ? null : lastLine(shell.lastError), + openPullRequests: openPullRequests({ pullRequests: shell.pullRequests } as WatchedShell) + .map((link) => link.url) + .slice(0, 5), + occurredAt: input.occurredAt, + }; +} + +const NEXT_STEP: Record = { + "lane-ended": + "Check the thread: restart, repair or replace it, or settle it if its work is done. It sends no further alert for this run.", + "promise-without-wake": + "Tell the thread to register a real wait (a tracked background command, a PR watch or a delegate) and carry on, or finish the work yourself.", + stuck: + "Read the thread. Interrupt and restart it, or replace it. It sends no further alert for this run.", + "needs-person": + "Try to clear it yourself first: answer from what you know, approve what Kameron already authorized, or delegate it. Raise it to a founder only if it needs a passkey, a phone prompt, physical presence or new spend.", +}; + +export function threadEventMessage(alert: ThreadEventAlert): string { + return [ + `Thread event from ${alert.environmentLabel} (environment ${alert.environmentId}): ${alert.threadLink} ${alert.detail}.`, + [ + `Event: ${alert.kind}.`, + alert.runId === null ? null : `Run ${alert.runId}.`, + alert.openPullRequests.length === 0 + ? "No open pull requests." + : `Open pull requests: ${alert.openPullRequests.join(", ")}.`, + alert.lastAssistantLine === null ? null : `Last assistant line: "${alert.lastAssistantLine}"`, + alert.lastError === null ? null : `Last error: ${alert.lastError}`, + ] + .filter((part) => part !== null) + .join(" "), + NEXT_STEP[alert.kind], + ].join("\n\n"); +} diff --git a/docs/operations/watcher-thread.md b/docs/operations/watcher-thread.md new file mode 100644 index 000000000000..7bcf43d0dc57 --- /dev/null +++ b/docs/operations/watcher-thread.md @@ -0,0 +1,17 @@ +# Watcher thread + +One thread, by default Lead's on the laptop, hears everything that needs attention from every linked environment. The server decides each event from thread state with no agent judgment, queues it durably, and retries until the watcher's machine answers. + +| Event | Fires when | +| ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `model-degraded` | A run ran another model than selected, hit a provider fallback, or hit a usage limit. | +| `lane-ended` | A top-level lane's run ended `failed` or `interrupted`, or `completed` with open pull requests and nothing tracked to wake it. | +| `promise-without-wake` | A lane went idle or settled after saying it will be woken or will continue later, with no background task, pull request watch, delegated task or armed resume. The phrase list is `PROMISE_PHRASES` in `ThreadEventAlerts.ts`. | +| `stuck` | An active run has had no new activity for `watcherStuckMinutes` (default 20; 0 turns it off), or a thread waits on background work that no longer exists. A parent whose child threads are active is not stuck. | +| `needs-person` | An agent-created lane raises a pending question or approval, or its inbox is assigned to a founder or set to `action`. | + +Each event reports once per thread and kind until its state changes: a new run, a different message, or the condition clearing. Delegated children are skipped; their parent already hears their result. The watcher thread never reports about itself. + +## Moving the watcher + +The watcher is the `modelAlertWatcher` server setting (`{ environmentId, threadId }`, or `null` for none). Its Closura default is in `packages/contracts/src/settings.ts` (`DEFAULT_MODEL_ALERT_WATCHER`). To move Lead to an always-on machine such as the R720, change that one default and ship a build, or set `modelAlertWatcher` in each machine's settings to override it. `modelAlertWebhookUrl` additionally POSTs every alert as JSON. diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 099abf6484ea..c31e13fffc98 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -1168,7 +1168,8 @@ export const ModelAlertWatcher = Schema.Struct({ }); export type ModelAlertWatcher = typeof ModelAlertWatcher.Type; -// Closura: Lead's thread on the laptop hears every machine's alerts. +// Closura: Lead's thread on the laptop hears every machine's alerts and thread events. Moving Lead +// to an always-on machine means changing this one value (or `modelAlertWatcher` on each machine). const DEFAULT_MODEL_ALERT_WATCHER: ModelAlertWatcher = { environmentId: EnvironmentId.make("2f360b2e-0386-498a-b9cd-62999f3525ba"), threadId: ThreadId.make("0ff5603d-8291-4f56-adaf-5132c62016cf"), @@ -1559,14 +1560,24 @@ export const ServerSettings = Schema.Struct({ Schema.withDecodingDefault(Effect.succeed(true)), ), /** - * Where a run that ran another model than selected, hit a provider safeguard fallback or a usage - * limit is reported, once per thread until the problem changes. Null reports to no thread. + * The watcher thread. It hears a run that ran another model than selected, hit a provider + * safeguard fallback or a usage limit, and each lane that ended, promised to continue with + * nothing to wake it, got stuck or needs a person. Each reports once per thread until its state + * changes. Null reports to no thread. */ modelAlertWatcher: Schema.NullOr(ModelAlertWatcher).pipe( Schema.withDecodingDefault(Effect.succeed(DEFAULT_MODEL_ALERT_WATCHER)), ), - /** Also POSTs each model alert as JSON here. Empty posts nothing. */ + /** Also POSTs each alert and thread event as JSON here. Empty posts nothing. */ modelAlertWebhookUrl: Schema.String.pipe(Schema.withDecodingDefault(Effect.succeed(""))), + /** + * Minutes an active run may go without a new item before the watcher hears it is stuck. Zero + * turns the check off. + */ + watcherStuckMinutes: Schema.Int.check(Schema.isBetween({ minimum: 0, maximum: 1_440 })).pipe( + // Closura: twenty minutes without any activity is long enough to be worth a look. + Schema.withDecodingDefault(Effect.succeed(20)), + ), /** Beta: Claude and Codex predict the next prompt after completed turns. Codex follows a change at once; open Claude threads after they restart. */ promptSuggestionsEnabled: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), sidebarAutoSettleKeepPullRequests: Schema.Boolean.pipe( @@ -2013,6 +2024,9 @@ export const ServerSettingsPatch = Schema.Struct({ automaticPlacementByDefault: Schema.optionalKey(Schema.Boolean), modelAlertWatcher: Schema.optionalKey(Schema.NullOr(ModelAlertWatcher)), modelAlertWebhookUrl: Schema.optionalKey(TrimmedString), + watcherStuckMinutes: Schema.optionalKey( + Schema.Int.check(Schema.isBetween({ minimum: 0, maximum: 1_440 })), + ), promptSuggestionsEnabled: Schema.optionalKey(Schema.Boolean), backgroundActivity: Schema.optionalKey( Schema.Struct({