diff --git a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts index 0433de375232c..d40891b9a117a 100644 --- a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts +++ b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts @@ -5105,6 +5105,105 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => { }), ); + it.effect("a lane's thread events reach the thread that launched it and never Lead", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const eventSink = yield* EventSink.EventSinkV2; + const now = yield* DateTime.now; + const projectId = ProjectId.make("project:lane-routing"); + yield* ProjectService.ProjectService.pipe( + Effect.flatMap((projects) => + projects.create({ + commandId: CommandId.make("command:lane-routing-project"), + projectId, + title: "Lane routing", + workspaceRoot: "/workspace/lane-routing", + }), + ), + ); + const createThread = ( + name: string, + launch?: { readonly inboxOrigin: { readonly kind: "agent"; readonly id: string } }, + ) => + orchestrator + .dispatch({ + type: "thread.create", + createdBy: launch === undefined ? "user" : "agent", + creationSource: launch === undefined ? "web" : "mcp", + ...(launch === undefined ? {} : { inboxOrigin: launch.inboxOrigin }), + commandId: CommandId.make(`command:lane-routing:${name}`), + threadId: ThreadId.make(`thread:lane-routing:${name}`), + projectId, + title: name, + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + }) + .pipe(Effect.as(ThreadId.make(`thread:lane-routing:${name}`))); + const lead = yield* createThread("Chief of Staff"); + const cto = yield* createThread("CTO"); + const archivedLauncher = yield* createThread("Archived launcher"); + yield* orchestrator.dispatch({ + type: "thread.archive", + commandId: CommandId.make("command:lane-routing:archive"), + threadId: archivedLauncher, + }); + const launchedByCto = yield* createThread("Launched by CTO", { + inboxOrigin: { kind: "agent", id: cto }, + }); + const launchedByLead = yield* createThread("Launched by Lead", { + inboxOrigin: { kind: "agent", id: lead }, + }); + const orphans = [ + yield* createThread("Launcher archived", { + inboxOrigin: { kind: "agent", id: archivedLauncher }, + }), + yield* createThread("Launcher never existed", { + inboxOrigin: { kind: "agent", id: "thread:lane-routing:missing" }, + }), + yield* createThread("Launcher unknown", { inboxOrigin: { kind: "agent", id: "agent" } }), + ]; + const { reactor } = yield* makeModelAlertReactor({ + watcher: { environmentId: hereEnvironmentId, threadId: lead }, + }); + for (const lane of [launchedByCto, launchedByLead, ...orphans]) { + const failed = runEvent({ + threadId: lane, + runId: RunId.make(`run:${lane}`), + ordinal: 1, + status: "failed", + now, + }); + yield* eventSink.write({ + commandId: CommandId.make(`command:${lane}:failed`), + events: [failed], + }); + yield* reactor.handle(failed); + } + yield* reactor.flush; + yield* TestClock.adjust("11 minutes"); + yield* reactor.flush; + const threadEventsIn = (threadId: ThreadId) => + orchestrator + .getThreadProjection(threadId) + .pipe( + Effect.map((projection) => + projection.messages.filter((message) => message.text.startsWith("Thread event from")), + ), + ); + const toCto = yield* threadEventsIn(cto); + assert.equal(toCto.length, 1); + assert.include(toCto[0]?.text, `/${launchedByCto})`); + assert.include(toCto[0]?.text, "its last run failed"); + const toLead = yield* threadEventsIn(lead); + assert.equal(toLead.length, 1); + assert.include(toLead[0]?.text, `/${launchedByLead})`); + assert.deepEqual(yield* threadEventsIn(archivedLauncher), []); + }), + ); + it.effect("sends a model alert over the link to the watcher's machine and retries it", () => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; diff --git a/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts b/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts index d41d0fa06b293..58fe1d3ff5feb 100644 --- a/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts +++ b/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts @@ -24,6 +24,7 @@ import { recordUsageLimit, RESUME_GRACE_MS, recordThreadEvents, + threadEventDestinations, watcherAlertMessage, } from "./ModelAlertReactor.ts"; import { ProviderExhaustion } from "./ProviderExhaustion.ts"; @@ -295,6 +296,44 @@ it("queues one delivery per destination for a thread event, beside model alerts" assert.include(watcherAlertMessage(eventAlert(finding("stuck", "run:1"))), "Thread event from"); }); +it("a thread event with nowhere to go stays unreported until a destination exists", () => { + const judged = { + threadId: "thread:fleet", + evaluated: ["lane-ended"] as const, + findings: [finding("lane-ended", "run:1")], + }; + const dropped = recordThreadEvents(EMPTY_MODEL_ALERT_STATE, judged, eventAlert, [], 0); + assert.equal(dropped.enqueued, false); + assert.deepEqual(dropped.state.outbox, []); + assert.deepEqual(dropped.state.reported, {}); + // The same event again, still with nowhere to go, changes nothing. + const again = recordThreadEvents(dropped.state, judged, eventAlert, [], 0); + assert.equal(again.enqueued, false); + assert.deepEqual(again.state.outbox, []); + assert.deepEqual(again.state.reported, {}); + // A webhook configured later receives the event that was dropped. + const delivered = recordThreadEvents(dropped.state, judged, eventAlert, ["webhook"], 0); + assert.equal(delivered.enqueued, true); + assert.deepEqual( + delivered.state.outbox.map((entry) => entry.destination), + ["webhook"], + ); + // And once delivered it stays quiet. + const quiet = recordThreadEvents(delivered.state, judged, eventAlert, ["webhook"], 0); + assert.equal(quiet.enqueued, false); +}); + +it("a thread event with no launching thread to hear it goes only to a configured webhook", () => { + const cto = { environmentId: EnvironmentId.make("laptop"), threadId: ThreadId.make("cto") }; + assert.deepEqual(threadEventDestinations(cto, ""), ["watcher"]); + assert.deepEqual(threadEventDestinations(cto, "https://hooks.example/alerts"), [ + "watcher", + "webhook", + ]); + assert.deepEqual(threadEventDestinations(null, "https://hooks.example/alerts"), ["webhook"]); + assert.deepEqual(threadEventDestinations(null, " "), []); +}); + it.effect("forwards a thread event to a watcher on another environment with one retry key", () => Effect.gen(function* () { const dispatched: Array = []; diff --git a/apps/server/src/orchestration-v2/ModelAlertReactor.ts b/apps/server/src/orchestration-v2/ModelAlertReactor.ts index b7b62364d2da7..3f1633580d835 100644 --- a/apps/server/src/orchestration-v2/ModelAlertReactor.ts +++ b/apps/server/src/orchestration-v2/ModelAlertReactor.ts @@ -45,6 +45,7 @@ import * as ProjectionStore from "./ProjectionStore.ts"; import { judgeThread, threadEventAlert, + launchingThreadId, ThreadEventAlert, type ThreadEventKind, threadEventMessage, @@ -238,6 +239,14 @@ export function modelAlertDestinations(input: { ]; } +export const threadEventDestinations = ( + recipient: ModelAlertWatcher | null, + webhookUrl: string, +): ReadonlyArray => [ + ...(recipient === null ? [] : (["watcher"] as const)), + ...(webhookUrl.trim() === "" ? [] : (["webhook"] as const)), +]; + const signature = (alert: ModelAlert) => `${alert.reasons.join(",")}|${alert.executedModel ?? ""}|${alert.fallbackModels.join(",")}`; @@ -337,21 +346,28 @@ export function recordThreadEvents( // A kind can hold several findings at once (a question and an inbox flip), so it remembers // each signature it reported. One that stops being true is forgotten and can report again. const known = (reported[key] ?? "").split("\n").filter((signature) => signature !== ""); - const current = kindFindings.map((finding) => finding.signature); - const stored = current.join("\n"); - if (reported[key] !== stored) { - const { [key]: _previous, ...rest } = reported; - reported = { - ...Object.fromEntries(Object.entries(rest).slice(-(MAX_REPORTED - 1))), - [key]: stored, - }; - } + const delivered: Array = []; for (const finding of kindFindings) { - if (known.includes(finding.signature)) continue; + if (known.includes(finding.signature)) { + delivered.push(finding.signature); + continue; + } const queued = enqueueAlert(outbox, toAlert(finding), destinations, nowMs); outbox = queued.outbox; + if (queued.enqueued) delivered.push(finding.signature); enqueued ||= queued.enqueued; } + // A finding with nowhere to go stays unreported, so a later launcher or webhook still hears it. + const stored = delivered.join("\n"); + if (reported[key] !== stored) { + const { [key]: _previous, ...rest } = reported; + reported = stored + ? { + ...Object.fromEntries(Object.entries(rest).slice(-(MAX_REPORTED - 1))), + [key]: stored, + } + : rest; + } } if (reported === state.reported && outbox === state.outbox) return { state, enqueued }; return { state: { ...state, reported, outbox }, enqueued }; @@ -1060,15 +1076,19 @@ export const make = Effect.gen(function* () { return { _tag: "sent" } satisfies DeliveryResult; } const watcher = current.modelAlertWatcher; - if (watcher === null) return { _tag: "sent" } satisfies DeliveryResult; + const target = alert.type === "thread-event" ? (alert.recipient ?? null) : watcher; + if (target === null) return { _tag: "sent" } satisfies DeliveryResult; const here = yield* environment.getEnvironmentId; const nowMs = yield* Clock.currentTimeMillis; - const view = yield* watcherView; + const view = + target.environmentId === watcher?.environmentId && target.threadId === watcher.threadId + ? yield* watcherView + : yield* readWatcher(target, here).pipe(Effect.catchCause(() => Effect.succeed(null))); const until = holdUntil(view, alert, nowMs) ?? busyWatcherHold(view?.status ?? null, entry.createdAt, nowMs); if (until !== null) return { _tag: "held", until } satisfies DeliveryResult; - yield* deliverToWatcher(alert, watcher, here); + yield* deliverToWatcher(alert, target, here); return { _tag: "sent" } satisfies DeliveryResult; }); @@ -1248,6 +1268,19 @@ export const make = Effect.gen(function* () { return { usageLimited: judged.alert?.reasons.includes("usage_limit") === true }; }); + const launchingThread = Effect.fnUntraced(function* ( + shell: OrchestrationV2ThreadShell, + here: EnvironmentId, + ) { + const threadId = launchingThreadId(shell.inbox); + if (threadId === null || threadId === shell.id) return null; + const launcher = yield* projections.getThreadShell(ThreadId.make(threadId)); + if (launcher === null || launcher.deletedAt !== null || launcher.archivedAt !== null) { + return null; + } + return { environmentId: here, threadId: launcher.id } satisfies ModelAlertWatcher; + }); + /** Judges one thread for the kinds a trigger can change, and queues what is newly true. */ const handleThreadEvents = Effect.fnUntraced(function* ( threadId: ThreadId, @@ -1275,12 +1308,14 @@ export const make = Effect.gen(function* () { }); const label = (yield* environment.getDescriptor).label; const occurredAt = DateTime.formatIso(yield* DateTime.now); - const destinations = modelAlertDestinations({ - watcher, - webhookUrl: current.modelAlertWebhookUrl, - here, - lineage: shell.lineage, - }); + const recipient = yield* launchingThread(shell, here); + const destinations = threadEventDestinations(recipient, current.modelAlertWebhookUrl); + if (recipient === null && destinations.length === 0 && judged.findings.length > 0) { + yield* Effect.logWarning("a thread event has no launching thread and no webhook", { + threadId, + kinds: judged.findings.map((finding) => finding.kind), + }); + } const enqueued = yield* update((state) => { const result = recordThreadEvents( state, @@ -1292,6 +1327,7 @@ export const make = Effect.gen(function* () { shell, finding, occurredAt, + recipient, }), destinations, nowMs, diff --git a/apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts b/apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts index 24fb597c21ce8..54001d28fd39c 100644 --- a/apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts +++ b/apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts @@ -300,6 +300,7 @@ it("a recurrence of the same state is a new message", () => { shell, finding, occurredAt, + recipient: null, }); assert.notEqual(make("2026-11-19T05:00:00.000Z").id, make("2026-11-19T05:30:00.000Z").id); }); @@ -313,6 +314,7 @@ const fired = (overrides: Partial, trigger: ThreadEventTrigger) => shell, finding, occurredAt: DateTime.formatIso(DateTime.makeUnsafe(NOW)), + recipient: null, }); }; const stillHolds = ( @@ -414,6 +416,7 @@ it("the wake carries environment, thread link, title, kind, run, last line and o shell, finding, occurredAt: "2026-11-19T05:00:00.000Z", + recipient: null, }); assert.equal(alert.runId, "run:7"); assert.equal(alert.threadTitle, "Fleet rollout"); diff --git a/apps/server/src/orchestration-v2/ThreadEventAlerts.ts b/apps/server/src/orchestration-v2/ThreadEventAlerts.ts index aa61f31970dc4..c99d7f1985027 100644 --- a/apps/server/src/orchestration-v2/ThreadEventAlerts.ts +++ b/apps/server/src/orchestration-v2/ThreadEventAlerts.ts @@ -1,6 +1,7 @@ import { type EnvironmentId, isListedThreadLineage, + ModelAlertWatcher, type OrchestrationV2ThreadShell, } from "@t3tools/contracts"; import { visibleThreadPullRequests } from "@t3tools/shared/threadPullRequests"; @@ -37,6 +38,7 @@ export const ThreadEventAlert = Schema.Struct({ lastError: Schema.NullOr(Schema.String), openPullRequests: Schema.Array(Schema.String), occurredAt: Schema.String, + recipient: Schema.optionalKey(ModelAlertWatcher), }); export type ThreadEventAlert = typeof ThreadEventAlert.Type; @@ -282,6 +284,9 @@ export function judgeThread(input: { return { evaluated, findings }; } +export const launchingThreadId = (inbox: WatchedShell["inbox"]) => + [inbox?.origin, inbox?.requestedBy].find((actor) => actor?.kind === "agent")?.id ?? null; + export function threadEventAlert(input: { readonly environmentId: EnvironmentId; readonly environmentLabel: string; @@ -297,6 +302,7 @@ export function threadEventAlert(input: { >; readonly finding: ThreadFinding; readonly occurredAt: string; + readonly recipient: ModelAlertWatcher | null; }): ThreadEventAlert { const { shell, finding } = input; const message = shell.latestVisibleMessage; @@ -323,6 +329,7 @@ export function threadEventAlert(input: { .map((link) => link.url) .slice(0, 5), occurredAt: input.occurredAt, + ...(input.recipient === null ? {} : { recipient: input.recipient }), }; } diff --git a/docs/operations/watcher-thread.md b/docs/operations/watcher-thread.md index 6a241e490e752..416ff2e016f86 100644 --- a/docs/operations/watcher-thread.md +++ b/docs/operations/watcher-thread.md @@ -1,6 +1,6 @@ # Watcher thread -One thread, by default Lead's on the laptop, hears everything that needs attention from every linked environment. The server decides each event from thread state with no agent judgment, queues it durably, and retries until the watcher's machine answers. +One thread, by default Lead's on the laptop, hears model, usage-limit and link alerts from every linked environment. The thread events below go instead to the thread that launched the lane, read from the lane's `inbox.origin`, else `inbox.requestedBy`. When that thread is archived, deleted or unknown, a thread event reaches no thread, only `modelAlertWebhookUrl` when set, so a founder-facing watcher never hears about a lane it did not launch. The server decides each event from thread state with no agent judgment, queues it durably, and retries until the receiving machine answers. | Event | Fires when | | ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |