Skip to content

Commit 12cf9bc

Browse files
nullStack65kameron-smith-closuraclaude
authored
fix(server): lane events go to the thread that launched the lane (#113)
* fix(server): lane events go to the thread that launched the lane Lane-ended, stuck, promise-without-wake and needs-person events went to the one configured watcher, Lead's founder-facing thread, for every lane on every machine. They now go to the lane's launching thread, read from inbox.origin or else inbox.requestedBy. When that thread is archived, deleted or unknown, only the webhook hears it, never the watcher. Model, usage-limit and link alerts still go to the watcher. Model: claude-opus-5-5 Task: fix Agent: Claude Code Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix(server): an undelivered lane event stays unreported instead of being dropped A lane whose launching thread cannot be resolved, and a webhook left at its default empty value, queued nothing and was still marked reported, so the event never reached anyone and never reported again. The signature is now kept only for what was actually queued, and the drop is logged. A launcher or webhook that appears later still hears it. Jeff is not legally attached to anything, and an unresolved launcher still never falls back to a founder-facing thread. Model: grok-4.7 Task: fix Agent: CTO build lane (CLO-916) Lane: magpie Harness: t3/claude-code Model-family: Grok Model-generation: 4.7 Skills-SHA: unknown (source unavailable) Co-Authored-By: Claude Code <noreply@anthropic.com> --------- Co-authored-by: Kameron Smith <kameron.smith@closura.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 55adff8 commit 12cf9bc

6 files changed

Lines changed: 204 additions & 20 deletions

File tree

‎apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5105,6 +5105,105 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => {
51055105
}),
51065106
);
51075107

5108+
it.effect("a lane's thread events reach the thread that launched it and never Lead", () =>
5109+
Effect.gen(function* () {
5110+
const orchestrator = yield* Orchestrator.OrchestratorV2;
5111+
const eventSink = yield* EventSink.EventSinkV2;
5112+
const now = yield* DateTime.now;
5113+
const projectId = ProjectId.make("project:lane-routing");
5114+
yield* ProjectService.ProjectService.pipe(
5115+
Effect.flatMap((projects) =>
5116+
projects.create({
5117+
commandId: CommandId.make("command:lane-routing-project"),
5118+
projectId,
5119+
title: "Lane routing",
5120+
workspaceRoot: "/workspace/lane-routing",
5121+
}),
5122+
),
5123+
);
5124+
const createThread = (
5125+
name: string,
5126+
launch?: { readonly inboxOrigin: { readonly kind: "agent"; readonly id: string } },
5127+
) =>
5128+
orchestrator
5129+
.dispatch({
5130+
type: "thread.create",
5131+
createdBy: launch === undefined ? "user" : "agent",
5132+
creationSource: launch === undefined ? "web" : "mcp",
5133+
...(launch === undefined ? {} : { inboxOrigin: launch.inboxOrigin }),
5134+
commandId: CommandId.make(`command:lane-routing:${name}`),
5135+
threadId: ThreadId.make(`thread:lane-routing:${name}`),
5136+
projectId,
5137+
title: name,
5138+
modelSelection,
5139+
runtimeMode: "full-access",
5140+
interactionMode: "default",
5141+
branch: null,
5142+
worktreePath: null,
5143+
})
5144+
.pipe(Effect.as(ThreadId.make(`thread:lane-routing:${name}`)));
5145+
const lead = yield* createThread("Chief of Staff");
5146+
const cto = yield* createThread("CTO");
5147+
const archivedLauncher = yield* createThread("Archived launcher");
5148+
yield* orchestrator.dispatch({
5149+
type: "thread.archive",
5150+
commandId: CommandId.make("command:lane-routing:archive"),
5151+
threadId: archivedLauncher,
5152+
});
5153+
const launchedByCto = yield* createThread("Launched by CTO", {
5154+
inboxOrigin: { kind: "agent", id: cto },
5155+
});
5156+
const launchedByLead = yield* createThread("Launched by Lead", {
5157+
inboxOrigin: { kind: "agent", id: lead },
5158+
});
5159+
const orphans = [
5160+
yield* createThread("Launcher archived", {
5161+
inboxOrigin: { kind: "agent", id: archivedLauncher },
5162+
}),
5163+
yield* createThread("Launcher never existed", {
5164+
inboxOrigin: { kind: "agent", id: "thread:lane-routing:missing" },
5165+
}),
5166+
yield* createThread("Launcher unknown", { inboxOrigin: { kind: "agent", id: "agent" } }),
5167+
];
5168+
const { reactor } = yield* makeModelAlertReactor({
5169+
watcher: { environmentId: hereEnvironmentId, threadId: lead },
5170+
});
5171+
for (const lane of [launchedByCto, launchedByLead, ...orphans]) {
5172+
const failed = runEvent({
5173+
threadId: lane,
5174+
runId: RunId.make(`run:${lane}`),
5175+
ordinal: 1,
5176+
status: "failed",
5177+
now,
5178+
});
5179+
yield* eventSink.write({
5180+
commandId: CommandId.make(`command:${lane}:failed`),
5181+
events: [failed],
5182+
});
5183+
yield* reactor.handle(failed);
5184+
}
5185+
yield* reactor.flush;
5186+
yield* TestClock.adjust("11 minutes");
5187+
yield* reactor.flush;
5188+
const threadEventsIn = (threadId: ThreadId) =>
5189+
orchestrator
5190+
.getThreadProjection(threadId)
5191+
.pipe(
5192+
Effect.map((projection) =>
5193+
projection.messages.filter((message) => message.text.startsWith("Thread event from")),
5194+
),
5195+
);
5196+
const toCto = yield* threadEventsIn(cto);
5197+
assert.equal(toCto.length, 1);
5198+
assert.include(toCto[0]?.text, `/${launchedByCto})`);
5199+
assert.include(toCto[0]?.text, "its last run failed");
5200+
const toLead = yield* threadEventsIn(lead);
5201+
assert.equal(toLead.length, 1);
5202+
assert.include(toLead[0]?.text, `/${launchedByLead})`);
5203+
assert.deepEqual(yield* threadEventsIn(archivedLauncher), []);
5204+
}),
5205+
);
5206+
51085207
it.effect("sends a model alert over the link to the watcher's machine and retries it", () =>
51095208
Effect.gen(function* () {
51105209
const orchestrator = yield* Orchestrator.OrchestratorV2;

‎apps/server/src/orchestration-v2/ModelAlertReactor.test.ts‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import {
2424
recordUsageLimit,
2525
RESUME_GRACE_MS,
2626
recordThreadEvents,
27+
threadEventDestinations,
2728
watcherAlertMessage,
2829
} from "./ModelAlertReactor.ts";
2930
import { ProviderExhaustion } from "./ProviderExhaustion.ts";
@@ -295,6 +296,44 @@ it("queues one delivery per destination for a thread event, beside model alerts"
295296
assert.include(watcherAlertMessage(eventAlert(finding("stuck", "run:1"))), "Thread event from");
296297
});
297298

299+
it("a thread event with nowhere to go stays unreported until a destination exists", () => {
300+
const judged = {
301+
threadId: "thread:fleet",
302+
evaluated: ["lane-ended"] as const,
303+
findings: [finding("lane-ended", "run:1")],
304+
};
305+
const dropped = recordThreadEvents(EMPTY_MODEL_ALERT_STATE, judged, eventAlert, [], 0);
306+
assert.equal(dropped.enqueued, false);
307+
assert.deepEqual(dropped.state.outbox, []);
308+
assert.deepEqual(dropped.state.reported, {});
309+
// The same event again, still with nowhere to go, changes nothing.
310+
const again = recordThreadEvents(dropped.state, judged, eventAlert, [], 0);
311+
assert.equal(again.enqueued, false);
312+
assert.deepEqual(again.state.outbox, []);
313+
assert.deepEqual(again.state.reported, {});
314+
// A webhook configured later receives the event that was dropped.
315+
const delivered = recordThreadEvents(dropped.state, judged, eventAlert, ["webhook"], 0);
316+
assert.equal(delivered.enqueued, true);
317+
assert.deepEqual(
318+
delivered.state.outbox.map((entry) => entry.destination),
319+
["webhook"],
320+
);
321+
// And once delivered it stays quiet.
322+
const quiet = recordThreadEvents(delivered.state, judged, eventAlert, ["webhook"], 0);
323+
assert.equal(quiet.enqueued, false);
324+
});
325+
326+
it("a thread event with no launching thread to hear it goes only to a configured webhook", () => {
327+
const cto = { environmentId: EnvironmentId.make("laptop"), threadId: ThreadId.make("cto") };
328+
assert.deepEqual(threadEventDestinations(cto, ""), ["watcher"]);
329+
assert.deepEqual(threadEventDestinations(cto, "https://hooks.example/alerts"), [
330+
"watcher",
331+
"webhook",
332+
]);
333+
assert.deepEqual(threadEventDestinations(null, "https://hooks.example/alerts"), ["webhook"]);
334+
assert.deepEqual(threadEventDestinations(null, " "), []);
335+
});
336+
298337
it.effect("forwards a thread event to a watcher on another environment with one retry key", () =>
299338
Effect.gen(function* () {
300339
const dispatched: Array<unknown> = [];

‎apps/server/src/orchestration-v2/ModelAlertReactor.ts‎

Lines changed: 55 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ import * as ProjectionStore from "./ProjectionStore.ts";
4545
import {
4646
judgeThread,
4747
threadEventAlert,
48+
launchingThreadId,
4849
ThreadEventAlert,
4950
type ThreadEventKind,
5051
threadEventMessage,
@@ -238,6 +239,14 @@ export function modelAlertDestinations(input: {
238239
];
239240
}
240241

242+
export const threadEventDestinations = (
243+
recipient: ModelAlertWatcher | null,
244+
webhookUrl: string,
245+
): ReadonlyArray<Destination> => [
246+
...(recipient === null ? [] : (["watcher"] as const)),
247+
...(webhookUrl.trim() === "" ? [] : (["webhook"] as const)),
248+
];
249+
241250
const signature = (alert: ModelAlert) =>
242251
`${alert.reasons.join(",")}|${alert.executedModel ?? ""}|${alert.fallbackModels.join(",")}`;
243252

@@ -337,21 +346,28 @@ export function recordThreadEvents(
337346
// A kind can hold several findings at once (a question and an inbox flip), so it remembers
338347
// each signature it reported. One that stops being true is forgotten and can report again.
339348
const known = (reported[key] ?? "").split("\n").filter((signature) => signature !== "");
340-
const current = kindFindings.map((finding) => finding.signature);
341-
const stored = current.join("\n");
342-
if (reported[key] !== stored) {
343-
const { [key]: _previous, ...rest } = reported;
344-
reported = {
345-
...Object.fromEntries(Object.entries(rest).slice(-(MAX_REPORTED - 1))),
346-
[key]: stored,
347-
};
348-
}
349+
const delivered: Array<string> = [];
349350
for (const finding of kindFindings) {
350-
if (known.includes(finding.signature)) continue;
351+
if (known.includes(finding.signature)) {
352+
delivered.push(finding.signature);
353+
continue;
354+
}
351355
const queued = enqueueAlert(outbox, toAlert(finding), destinations, nowMs);
352356
outbox = queued.outbox;
357+
if (queued.enqueued) delivered.push(finding.signature);
353358
enqueued ||= queued.enqueued;
354359
}
360+
// A finding with nowhere to go stays unreported, so a later launcher or webhook still hears it.
361+
const stored = delivered.join("\n");
362+
if (reported[key] !== stored) {
363+
const { [key]: _previous, ...rest } = reported;
364+
reported = stored
365+
? {
366+
...Object.fromEntries(Object.entries(rest).slice(-(MAX_REPORTED - 1))),
367+
[key]: stored,
368+
}
369+
: rest;
370+
}
355371
}
356372
if (reported === state.reported && outbox === state.outbox) return { state, enqueued };
357373
return { state: { ...state, reported, outbox }, enqueued };
@@ -1060,15 +1076,19 @@ export const make = Effect.gen(function* () {
10601076
return { _tag: "sent" } satisfies DeliveryResult;
10611077
}
10621078
const watcher = current.modelAlertWatcher;
1063-
if (watcher === null) return { _tag: "sent" } satisfies DeliveryResult;
1079+
const target = alert.type === "thread-event" ? (alert.recipient ?? null) : watcher;
1080+
if (target === null) return { _tag: "sent" } satisfies DeliveryResult;
10641081
const here = yield* environment.getEnvironmentId;
10651082
const nowMs = yield* Clock.currentTimeMillis;
1066-
const view = yield* watcherView;
1083+
const view =
1084+
target.environmentId === watcher?.environmentId && target.threadId === watcher.threadId
1085+
? yield* watcherView
1086+
: yield* readWatcher(target, here).pipe(Effect.catchCause(() => Effect.succeed(null)));
10671087
const until =
10681088
holdUntil(view, alert, nowMs) ??
10691089
busyWatcherHold(view?.status ?? null, entry.createdAt, nowMs);
10701090
if (until !== null) return { _tag: "held", until } satisfies DeliveryResult;
1071-
yield* deliverToWatcher(alert, watcher, here);
1091+
yield* deliverToWatcher(alert, target, here);
10721092
return { _tag: "sent" } satisfies DeliveryResult;
10731093
});
10741094

@@ -1248,6 +1268,19 @@ export const make = Effect.gen(function* () {
12481268
return { usageLimited: judged.alert?.reasons.includes("usage_limit") === true };
12491269
});
12501270

1271+
const launchingThread = Effect.fnUntraced(function* (
1272+
shell: OrchestrationV2ThreadShell,
1273+
here: EnvironmentId,
1274+
) {
1275+
const threadId = launchingThreadId(shell.inbox);
1276+
if (threadId === null || threadId === shell.id) return null;
1277+
const launcher = yield* projections.getThreadShell(ThreadId.make(threadId));
1278+
if (launcher === null || launcher.deletedAt !== null || launcher.archivedAt !== null) {
1279+
return null;
1280+
}
1281+
return { environmentId: here, threadId: launcher.id } satisfies ModelAlertWatcher;
1282+
});
1283+
12511284
/** Judges one thread for the kinds a trigger can change, and queues what is newly true. */
12521285
const handleThreadEvents = Effect.fnUntraced(function* (
12531286
threadId: ThreadId,
@@ -1275,12 +1308,14 @@ export const make = Effect.gen(function* () {
12751308
});
12761309
const label = (yield* environment.getDescriptor).label;
12771310
const occurredAt = DateTime.formatIso(yield* DateTime.now);
1278-
const destinations = modelAlertDestinations({
1279-
watcher,
1280-
webhookUrl: current.modelAlertWebhookUrl,
1281-
here,
1282-
lineage: shell.lineage,
1283-
});
1311+
const recipient = yield* launchingThread(shell, here);
1312+
const destinations = threadEventDestinations(recipient, current.modelAlertWebhookUrl);
1313+
if (recipient === null && destinations.length === 0 && judged.findings.length > 0) {
1314+
yield* Effect.logWarning("a thread event has no launching thread and no webhook", {
1315+
threadId,
1316+
kinds: judged.findings.map((finding) => finding.kind),
1317+
});
1318+
}
12841319
const enqueued = yield* update((state) => {
12851320
const result = recordThreadEvents(
12861321
state,
@@ -1292,6 +1327,7 @@ export const make = Effect.gen(function* () {
12921327
shell,
12931328
finding,
12941329
occurredAt,
1330+
recipient,
12951331
}),
12961332
destinations,
12971333
nowMs,

‎apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -300,6 +300,7 @@ it("a recurrence of the same state is a new message", () => {
300300
shell,
301301
finding,
302302
occurredAt,
303+
recipient: null,
303304
});
304305
assert.notEqual(make("2026-11-19T05:00:00.000Z").id, make("2026-11-19T05:30:00.000Z").id);
305306
});
@@ -313,6 +314,7 @@ const fired = (overrides: Partial<WatchedShell>, trigger: ThreadEventTrigger) =>
313314
shell,
314315
finding,
315316
occurredAt: DateTime.formatIso(DateTime.makeUnsafe(NOW)),
317+
recipient: null,
316318
});
317319
};
318320
const stillHolds = (
@@ -414,6 +416,7 @@ it("the wake carries environment, thread link, title, kind, run, last line and o
414416
shell,
415417
finding,
416418
occurredAt: "2026-11-19T05:00:00.000Z",
419+
recipient: null,
417420
});
418421
assert.equal(alert.runId, "run:7");
419422
assert.equal(alert.threadTitle, "Fleet rollout");

‎apps/server/src/orchestration-v2/ThreadEventAlerts.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import {
22
type EnvironmentId,
33
isListedThreadLineage,
4+
ModelAlertWatcher,
45
type OrchestrationV2ThreadShell,
56
} from "@t3tools/contracts";
67
import { visibleThreadPullRequests } from "@t3tools/shared/threadPullRequests";
@@ -37,6 +38,7 @@ export const ThreadEventAlert = Schema.Struct({
3738
lastError: Schema.NullOr(Schema.String),
3839
openPullRequests: Schema.Array(Schema.String),
3940
occurredAt: Schema.String,
41+
recipient: Schema.optionalKey(ModelAlertWatcher),
4042
});
4143
export type ThreadEventAlert = typeof ThreadEventAlert.Type;
4244

@@ -282,6 +284,9 @@ export function judgeThread(input: {
282284
return { evaluated, findings };
283285
}
284286

287+
export const launchingThreadId = (inbox: WatchedShell["inbox"]) =>
288+
[inbox?.origin, inbox?.requestedBy].find((actor) => actor?.kind === "agent")?.id ?? null;
289+
285290
export function threadEventAlert(input: {
286291
readonly environmentId: EnvironmentId;
287292
readonly environmentLabel: string;
@@ -297,6 +302,7 @@ export function threadEventAlert(input: {
297302
>;
298303
readonly finding: ThreadFinding;
299304
readonly occurredAt: string;
305+
readonly recipient: ModelAlertWatcher | null;
300306
}): ThreadEventAlert {
301307
const { shell, finding } = input;
302308
const message = shell.latestVisibleMessage;
@@ -323,6 +329,7 @@ export function threadEventAlert(input: {
323329
.map((link) => link.url)
324330
.slice(0, 5),
325331
occurredAt: input.occurredAt,
332+
...(input.recipient === null ? {} : { recipient: input.recipient }),
326333
};
327334
}
328335

‎docs/operations/watcher-thread.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# Watcher thread
22

3-
One thread, by default Lead's on the laptop, hears everything that needs attention from every linked environment. The server decides each event from thread state with no agent judgment, queues it durably, and retries until the watcher's machine answers.
3+
One thread, by default Lead's on the laptop, hears model, usage-limit and link alerts from every linked environment. The thread events below go instead to the thread that launched the lane, read from the lane's `inbox.origin`, else `inbox.requestedBy`. When that thread is archived, deleted or unknown, a thread event reaches no thread, only `modelAlertWebhookUrl` when set, so a founder-facing watcher never hears about a lane it did not launch. The server decides each event from thread state with no agent judgment, queues it durably, and retries until the receiving machine answers.
44

55
| Event | Fires when |
66
| ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |

0 commit comments

Comments
 (0)