From e29e697c0bd35fd23cf500b813aeb2ae31637ee7 Mon Sep 17 00:00:00 2001 From: Kameron Smith Date: Thu, 19 Nov 2026 20:09:01 -0500 Subject: [PATCH 1/2] fix(server): lane events go to the thread that launched the lane Lane-ended, stuck, promise-without-wake and needs-person events went to the one configured watcher, Lead's founder-facing thread, for every lane on every machine. They now go to the lane's launching thread, read from inbox.origin or else inbox.requestedBy. When that thread is archived, deleted or unknown, only the webhook hears it, never the watcher. Model, usage-limit and link alerts still go to the watcher. Model: claude-opus-5-5 Task: fix Agent: Claude Code Co-Authored-By: Claude Opus 5.5 --- .../DelegatedCompletionDelivery.test.ts | 99 +++++++++++++++++++ .../ModelAlertReactor.test.ts | 12 +++ .../src/orchestration-v2/ModelAlertReactor.ts | 41 ++++++-- .../ThreadEventAlerts.test.ts | 3 + .../src/orchestration-v2/ThreadEventAlerts.ts | 7 ++ docs/operations/watcher-thread.md | 2 +- 6 files changed, 154 insertions(+), 10 deletions(-) diff --git a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts index 0433de375232c..d40891b9a117a 100644 --- a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts +++ b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts @@ -5105,6 +5105,105 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => { }), ); + it.effect("a lane's thread events reach the thread that launched it and never Lead", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const eventSink = yield* EventSink.EventSinkV2; + const now = yield* DateTime.now; + const projectId = ProjectId.make("project:lane-routing"); + yield* ProjectService.ProjectService.pipe( + Effect.flatMap((projects) => + projects.create({ + commandId: CommandId.make("command:lane-routing-project"), + projectId, + title: "Lane routing", + workspaceRoot: "/workspace/lane-routing", + }), + ), + ); + const createThread = ( + name: string, + launch?: { readonly inboxOrigin: { readonly kind: "agent"; readonly id: string } }, + ) => + orchestrator + .dispatch({ + type: "thread.create", + createdBy: launch === undefined ? "user" : "agent", + creationSource: launch === undefined ? "web" : "mcp", + ...(launch === undefined ? {} : { inboxOrigin: launch.inboxOrigin }), + commandId: CommandId.make(`command:lane-routing:${name}`), + threadId: ThreadId.make(`thread:lane-routing:${name}`), + projectId, + title: name, + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + }) + .pipe(Effect.as(ThreadId.make(`thread:lane-routing:${name}`))); + const lead = yield* createThread("Chief of Staff"); + const cto = yield* createThread("CTO"); + const archivedLauncher = yield* createThread("Archived launcher"); + yield* orchestrator.dispatch({ + type: "thread.archive", + commandId: CommandId.make("command:lane-routing:archive"), + threadId: archivedLauncher, + }); + const launchedByCto = yield* createThread("Launched by CTO", { + inboxOrigin: { kind: "agent", id: cto }, + }); + const launchedByLead = yield* createThread("Launched by Lead", { + inboxOrigin: { kind: "agent", id: lead }, + }); + const orphans = [ + yield* createThread("Launcher archived", { + inboxOrigin: { kind: "agent", id: archivedLauncher }, + }), + yield* createThread("Launcher never existed", { + inboxOrigin: { kind: "agent", id: "thread:lane-routing:missing" }, + }), + yield* createThread("Launcher unknown", { inboxOrigin: { kind: "agent", id: "agent" } }), + ]; + const { reactor } = yield* makeModelAlertReactor({ + watcher: { environmentId: hereEnvironmentId, threadId: lead }, + }); + for (const lane of [launchedByCto, launchedByLead, ...orphans]) { + const failed = runEvent({ + threadId: lane, + runId: RunId.make(`run:${lane}`), + ordinal: 1, + status: "failed", + now, + }); + yield* eventSink.write({ + commandId: CommandId.make(`command:${lane}:failed`), + events: [failed], + }); + yield* reactor.handle(failed); + } + yield* reactor.flush; + yield* TestClock.adjust("11 minutes"); + yield* reactor.flush; + const threadEventsIn = (threadId: ThreadId) => + orchestrator + .getThreadProjection(threadId) + .pipe( + Effect.map((projection) => + projection.messages.filter((message) => message.text.startsWith("Thread event from")), + ), + ); + const toCto = yield* threadEventsIn(cto); + assert.equal(toCto.length, 1); + assert.include(toCto[0]?.text, `/${launchedByCto})`); + assert.include(toCto[0]?.text, "its last run failed"); + const toLead = yield* threadEventsIn(lead); + assert.equal(toLead.length, 1); + assert.include(toLead[0]?.text, `/${launchedByLead})`); + assert.deepEqual(yield* threadEventsIn(archivedLauncher), []); + }), + ); + it.effect("sends a model alert over the link to the watcher's machine and retries it", () => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; diff --git a/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts b/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts index d41d0fa06b293..72c0084816a4c 100644 --- a/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts +++ b/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts @@ -24,6 +24,7 @@ import { recordUsageLimit, RESUME_GRACE_MS, recordThreadEvents, + threadEventDestinations, watcherAlertMessage, } from "./ModelAlertReactor.ts"; import { ProviderExhaustion } from "./ProviderExhaustion.ts"; @@ -295,6 +296,17 @@ it("queues one delivery per destination for a thread event, beside model alerts" assert.include(watcherAlertMessage(eventAlert(finding("stuck", "run:1"))), "Thread event from"); }); +it("a thread event with no launching thread to hear it goes only to a configured webhook", () => { + const cto = { environmentId: EnvironmentId.make("laptop"), threadId: ThreadId.make("cto") }; + assert.deepEqual(threadEventDestinations(cto, ""), ["watcher"]); + assert.deepEqual(threadEventDestinations(cto, "https://hooks.example/alerts"), [ + "watcher", + "webhook", + ]); + assert.deepEqual(threadEventDestinations(null, "https://hooks.example/alerts"), ["webhook"]); + assert.deepEqual(threadEventDestinations(null, " "), []); +}); + it.effect("forwards a thread event to a watcher on another environment with one retry key", () => Effect.gen(function* () { const dispatched: Array = []; diff --git a/apps/server/src/orchestration-v2/ModelAlertReactor.ts b/apps/server/src/orchestration-v2/ModelAlertReactor.ts index b7b62364d2da7..0322f613b2a19 100644 --- a/apps/server/src/orchestration-v2/ModelAlertReactor.ts +++ b/apps/server/src/orchestration-v2/ModelAlertReactor.ts @@ -45,6 +45,7 @@ import * as ProjectionStore from "./ProjectionStore.ts"; import { judgeThread, threadEventAlert, + launchingThreadId, ThreadEventAlert, type ThreadEventKind, threadEventMessage, @@ -238,6 +239,14 @@ export function modelAlertDestinations(input: { ]; } +export const threadEventDestinations = ( + recipient: ModelAlertWatcher | null, + webhookUrl: string, +): ReadonlyArray => [ + ...(recipient === null ? [] : (["watcher"] as const)), + ...(webhookUrl.trim() === "" ? [] : (["webhook"] as const)), +]; + const signature = (alert: ModelAlert) => `${alert.reasons.join(",")}|${alert.executedModel ?? ""}|${alert.fallbackModels.join(",")}`; @@ -1060,15 +1069,19 @@ export const make = Effect.gen(function* () { return { _tag: "sent" } satisfies DeliveryResult; } const watcher = current.modelAlertWatcher; - if (watcher === null) return { _tag: "sent" } satisfies DeliveryResult; + const target = alert.type === "thread-event" ? (alert.recipient ?? null) : watcher; + if (target === null) return { _tag: "sent" } satisfies DeliveryResult; const here = yield* environment.getEnvironmentId; const nowMs = yield* Clock.currentTimeMillis; - const view = yield* watcherView; + const view = + target.environmentId === watcher?.environmentId && target.threadId === watcher.threadId + ? yield* watcherView + : yield* readWatcher(target, here).pipe(Effect.catchCause(() => Effect.succeed(null))); const until = holdUntil(view, alert, nowMs) ?? busyWatcherHold(view?.status ?? null, entry.createdAt, nowMs); if (until !== null) return { _tag: "held", until } satisfies DeliveryResult; - yield* deliverToWatcher(alert, watcher, here); + yield* deliverToWatcher(alert, target, here); return { _tag: "sent" } satisfies DeliveryResult; }); @@ -1248,6 +1261,19 @@ export const make = Effect.gen(function* () { return { usageLimited: judged.alert?.reasons.includes("usage_limit") === true }; }); + const launchingThread = Effect.fnUntraced(function* ( + shell: OrchestrationV2ThreadShell, + here: EnvironmentId, + ) { + const threadId = launchingThreadId(shell.inbox); + if (threadId === null || threadId === shell.id) return null; + const launcher = yield* projections.getThreadShell(ThreadId.make(threadId)); + if (launcher === null || launcher.deletedAt !== null || launcher.archivedAt !== null) { + return null; + } + return { environmentId: here, threadId: launcher.id } satisfies ModelAlertWatcher; + }); + /** Judges one thread for the kinds a trigger can change, and queues what is newly true. */ const handleThreadEvents = Effect.fnUntraced(function* ( threadId: ThreadId, @@ -1275,12 +1301,8 @@ export const make = Effect.gen(function* () { }); const label = (yield* environment.getDescriptor).label; const occurredAt = DateTime.formatIso(yield* DateTime.now); - const destinations = modelAlertDestinations({ - watcher, - webhookUrl: current.modelAlertWebhookUrl, - here, - lineage: shell.lineage, - }); + const recipient = yield* launchingThread(shell, here); + const destinations = threadEventDestinations(recipient, current.modelAlertWebhookUrl); const enqueued = yield* update((state) => { const result = recordThreadEvents( state, @@ -1292,6 +1314,7 @@ export const make = Effect.gen(function* () { shell, finding, occurredAt, + recipient, }), destinations, nowMs, diff --git a/apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts b/apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts index 24fb597c21ce8..54001d28fd39c 100644 --- a/apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts +++ b/apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts @@ -300,6 +300,7 @@ it("a recurrence of the same state is a new message", () => { shell, finding, occurredAt, + recipient: null, }); assert.notEqual(make("2026-11-19T05:00:00.000Z").id, make("2026-11-19T05:30:00.000Z").id); }); @@ -313,6 +314,7 @@ const fired = (overrides: Partial, trigger: ThreadEventTrigger) => shell, finding, occurredAt: DateTime.formatIso(DateTime.makeUnsafe(NOW)), + recipient: null, }); }; const stillHolds = ( @@ -414,6 +416,7 @@ it("the wake carries environment, thread link, title, kind, run, last line and o shell, finding, occurredAt: "2026-11-19T05:00:00.000Z", + recipient: null, }); assert.equal(alert.runId, "run:7"); assert.equal(alert.threadTitle, "Fleet rollout"); diff --git a/apps/server/src/orchestration-v2/ThreadEventAlerts.ts b/apps/server/src/orchestration-v2/ThreadEventAlerts.ts index aa61f31970dc4..c99d7f1985027 100644 --- a/apps/server/src/orchestration-v2/ThreadEventAlerts.ts +++ b/apps/server/src/orchestration-v2/ThreadEventAlerts.ts @@ -1,6 +1,7 @@ import { type EnvironmentId, isListedThreadLineage, + ModelAlertWatcher, type OrchestrationV2ThreadShell, } from "@t3tools/contracts"; import { visibleThreadPullRequests } from "@t3tools/shared/threadPullRequests"; @@ -37,6 +38,7 @@ export const ThreadEventAlert = Schema.Struct({ lastError: Schema.NullOr(Schema.String), openPullRequests: Schema.Array(Schema.String), occurredAt: Schema.String, + recipient: Schema.optionalKey(ModelAlertWatcher), }); export type ThreadEventAlert = typeof ThreadEventAlert.Type; @@ -282,6 +284,9 @@ export function judgeThread(input: { return { evaluated, findings }; } +export const launchingThreadId = (inbox: WatchedShell["inbox"]) => + [inbox?.origin, inbox?.requestedBy].find((actor) => actor?.kind === "agent")?.id ?? null; + export function threadEventAlert(input: { readonly environmentId: EnvironmentId; readonly environmentLabel: string; @@ -297,6 +302,7 @@ export function threadEventAlert(input: { >; readonly finding: ThreadFinding; readonly occurredAt: string; + readonly recipient: ModelAlertWatcher | null; }): ThreadEventAlert { const { shell, finding } = input; const message = shell.latestVisibleMessage; @@ -323,6 +329,7 @@ export function threadEventAlert(input: { .map((link) => link.url) .slice(0, 5), occurredAt: input.occurredAt, + ...(input.recipient === null ? {} : { recipient: input.recipient }), }; } diff --git a/docs/operations/watcher-thread.md b/docs/operations/watcher-thread.md index 6a241e490e752..416ff2e016f86 100644 --- a/docs/operations/watcher-thread.md +++ b/docs/operations/watcher-thread.md @@ -1,6 +1,6 @@ # Watcher thread -One thread, by default Lead's on the laptop, hears everything that needs attention from every linked environment. The server decides each event from thread state with no agent judgment, queues it durably, and retries until the watcher's machine answers. +One thread, by default Lead's on the laptop, hears model, usage-limit and link alerts from every linked environment. The thread events below go instead to the thread that launched the lane, read from the lane's `inbox.origin`, else `inbox.requestedBy`. When that thread is archived, deleted or unknown, a thread event reaches no thread, only `modelAlertWebhookUrl` when set, so a founder-facing watcher never hears about a lane it did not launch. The server decides each event from thread state with no agent judgment, queues it durably, and retries until the receiving machine answers. | Event | Fires when | | ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | From 7819b63c11dfa4d633fa3944bb122c83bc3c1452 Mon Sep 17 00:00:00 2001 From: nullStack65 <112618179+nullStack65@users.noreply.github.com> Date: Fri, 20 Nov 2026 03:51:22 +0000 Subject: [PATCH 2/2] fix(server): an undelivered lane event stays unreported instead of being dropped A lane whose launching thread cannot be resolved, and a webhook left at its default empty value, queued nothing and was still marked reported, so the event never reached anyone and never reported again. The signature is now kept only for what was actually queued, and the drop is logged. A launcher or webhook that appears later still hears it. Jeff is not legally attached to anything, and an unresolved launcher still never falls back to a founder-facing thread. Model: grok-4.7 Task: fix Agent: CTO build lane (CLO-916) Lane: magpie Harness: t3/claude-code Model-family: Grok Model-generation: 4.7 Skills-SHA: unknown (source unavailable) Co-Authored-By: Claude Code --- .../ModelAlertReactor.test.ts | 27 +++++++++++++++ .../src/orchestration-v2/ModelAlertReactor.ts | 33 +++++++++++++------ 2 files changed, 50 insertions(+), 10 deletions(-) diff --git a/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts b/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts index 72c0084816a4c..58fe1d3ff5feb 100644 --- a/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts +++ b/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts @@ -296,6 +296,33 @@ it("queues one delivery per destination for a thread event, beside model alerts" assert.include(watcherAlertMessage(eventAlert(finding("stuck", "run:1"))), "Thread event from"); }); +it("a thread event with nowhere to go stays unreported until a destination exists", () => { + const judged = { + threadId: "thread:fleet", + evaluated: ["lane-ended"] as const, + findings: [finding("lane-ended", "run:1")], + }; + const dropped = recordThreadEvents(EMPTY_MODEL_ALERT_STATE, judged, eventAlert, [], 0); + assert.equal(dropped.enqueued, false); + assert.deepEqual(dropped.state.outbox, []); + assert.deepEqual(dropped.state.reported, {}); + // The same event again, still with nowhere to go, changes nothing. + const again = recordThreadEvents(dropped.state, judged, eventAlert, [], 0); + assert.equal(again.enqueued, false); + assert.deepEqual(again.state.outbox, []); + assert.deepEqual(again.state.reported, {}); + // A webhook configured later receives the event that was dropped. + const delivered = recordThreadEvents(dropped.state, judged, eventAlert, ["webhook"], 0); + assert.equal(delivered.enqueued, true); + assert.deepEqual( + delivered.state.outbox.map((entry) => entry.destination), + ["webhook"], + ); + // And once delivered it stays quiet. + const quiet = recordThreadEvents(delivered.state, judged, eventAlert, ["webhook"], 0); + assert.equal(quiet.enqueued, false); +}); + it("a thread event with no launching thread to hear it goes only to a configured webhook", () => { const cto = { environmentId: EnvironmentId.make("laptop"), threadId: ThreadId.make("cto") }; assert.deepEqual(threadEventDestinations(cto, ""), ["watcher"]); diff --git a/apps/server/src/orchestration-v2/ModelAlertReactor.ts b/apps/server/src/orchestration-v2/ModelAlertReactor.ts index 0322f613b2a19..3f1633580d835 100644 --- a/apps/server/src/orchestration-v2/ModelAlertReactor.ts +++ b/apps/server/src/orchestration-v2/ModelAlertReactor.ts @@ -346,21 +346,28 @@ export function recordThreadEvents( // A kind can hold several findings at once (a question and an inbox flip), so it remembers // each signature it reported. One that stops being true is forgotten and can report again. const known = (reported[key] ?? "").split("\n").filter((signature) => signature !== ""); - const current = kindFindings.map((finding) => finding.signature); - const stored = current.join("\n"); - if (reported[key] !== stored) { - const { [key]: _previous, ...rest } = reported; - reported = { - ...Object.fromEntries(Object.entries(rest).slice(-(MAX_REPORTED - 1))), - [key]: stored, - }; - } + const delivered: Array = []; for (const finding of kindFindings) { - if (known.includes(finding.signature)) continue; + if (known.includes(finding.signature)) { + delivered.push(finding.signature); + continue; + } const queued = enqueueAlert(outbox, toAlert(finding), destinations, nowMs); outbox = queued.outbox; + if (queued.enqueued) delivered.push(finding.signature); enqueued ||= queued.enqueued; } + // A finding with nowhere to go stays unreported, so a later launcher or webhook still hears it. + const stored = delivered.join("\n"); + if (reported[key] !== stored) { + const { [key]: _previous, ...rest } = reported; + reported = stored + ? { + ...Object.fromEntries(Object.entries(rest).slice(-(MAX_REPORTED - 1))), + [key]: stored, + } + : rest; + } } if (reported === state.reported && outbox === state.outbox) return { state, enqueued }; return { state: { ...state, reported, outbox }, enqueued }; @@ -1303,6 +1310,12 @@ export const make = Effect.gen(function* () { const occurredAt = DateTime.formatIso(yield* DateTime.now); const recipient = yield* launchingThread(shell, here); const destinations = threadEventDestinations(recipient, current.modelAlertWebhookUrl); + if (recipient === null && destinations.length === 0 && judged.findings.length > 0) { + yield* Effect.logWarning("a thread event has no launching thread and no webhook", { + threadId, + kinds: judged.findings.map((finding) => finding.kind), + }); + } const enqueued = yield* update((state) => { const result = recordThreadEvents( state,