Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -5105,6 +5105,105 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => {
}),
);

it.effect("a lane's thread events reach the thread that launched it and never Lead", () =>
Effect.gen(function* () {
const orchestrator = yield* Orchestrator.OrchestratorV2;
const eventSink = yield* EventSink.EventSinkV2;
const now = yield* DateTime.now;
const projectId = ProjectId.make("project:lane-routing");
yield* ProjectService.ProjectService.pipe(
Effect.flatMap((projects) =>
projects.create({
commandId: CommandId.make("command:lane-routing-project"),
projectId,
title: "Lane routing",
workspaceRoot: "/workspace/lane-routing",
}),
),
);
const createThread = (
name: string,
launch?: { readonly inboxOrigin: { readonly kind: "agent"; readonly id: string } },
) =>
orchestrator
.dispatch({
type: "thread.create",
createdBy: launch === undefined ? "user" : "agent",
creationSource: launch === undefined ? "web" : "mcp",
...(launch === undefined ? {} : { inboxOrigin: launch.inboxOrigin }),
commandId: CommandId.make(`command:lane-routing:${name}`),
threadId: ThreadId.make(`thread:lane-routing:${name}`),
projectId,
title: name,
modelSelection,
runtimeMode: "full-access",
interactionMode: "default",
branch: null,
worktreePath: null,
})
.pipe(Effect.as(ThreadId.make(`thread:lane-routing:${name}`)));
const lead = yield* createThread("Chief of Staff");
const cto = yield* createThread("CTO");
const archivedLauncher = yield* createThread("Archived launcher");
yield* orchestrator.dispatch({
type: "thread.archive",
commandId: CommandId.make("command:lane-routing:archive"),
threadId: archivedLauncher,
});
const launchedByCto = yield* createThread("Launched by CTO", {
inboxOrigin: { kind: "agent", id: cto },
});
const launchedByLead = yield* createThread("Launched by Lead", {
inboxOrigin: { kind: "agent", id: lead },
});
const orphans = [
yield* createThread("Launcher archived", {
inboxOrigin: { kind: "agent", id: archivedLauncher },
}),
yield* createThread("Launcher never existed", {
inboxOrigin: { kind: "agent", id: "thread:lane-routing:missing" },
}),
yield* createThread("Launcher unknown", { inboxOrigin: { kind: "agent", id: "agent" } }),
];
const { reactor } = yield* makeModelAlertReactor({
watcher: { environmentId: hereEnvironmentId, threadId: lead },
});
for (const lane of [launchedByCto, launchedByLead, ...orphans]) {
const failed = runEvent({
threadId: lane,
runId: RunId.make(`run:${lane}`),
ordinal: 1,
status: "failed",
now,
});
yield* eventSink.write({
commandId: CommandId.make(`command:${lane}:failed`),
events: [failed],
});
yield* reactor.handle(failed);
}
yield* reactor.flush;
yield* TestClock.adjust("11 minutes");
yield* reactor.flush;
const threadEventsIn = (threadId: ThreadId) =>
orchestrator
.getThreadProjection(threadId)
.pipe(
Effect.map((projection) =>
projection.messages.filter((message) => message.text.startsWith("Thread event from")),
),
);
const toCto = yield* threadEventsIn(cto);
assert.equal(toCto.length, 1);
assert.include(toCto[0]?.text, `/${launchedByCto})`);
assert.include(toCto[0]?.text, "its last run failed");
const toLead = yield* threadEventsIn(lead);
assert.equal(toLead.length, 1);
assert.include(toLead[0]?.text, `/${launchedByLead})`);
assert.deepEqual(yield* threadEventsIn(archivedLauncher), []);
}),
);

it.effect("sends a model alert over the link to the watcher's machine and retries it", () =>
Effect.gen(function* () {
const orchestrator = yield* Orchestrator.OrchestratorV2;
Expand Down
39 changes: 39 additions & 0 deletions apps/server/src/orchestration-v2/ModelAlertReactor.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import {
recordUsageLimit,
RESUME_GRACE_MS,
recordThreadEvents,
threadEventDestinations,
watcherAlertMessage,
} from "./ModelAlertReactor.ts";
import { ProviderExhaustion } from "./ProviderExhaustion.ts";
Expand Down Expand Up @@ -295,6 +296,44 @@ it("queues one delivery per destination for a thread event, beside model alerts"
assert.include(watcherAlertMessage(eventAlert(finding("stuck", "run:1"))), "Thread event from");
});

it("a thread event with nowhere to go stays unreported until a destination exists", () => {
const judged = {
threadId: "thread:fleet",
evaluated: ["lane-ended"] as const,
findings: [finding("lane-ended", "run:1")],
};
const dropped = recordThreadEvents(EMPTY_MODEL_ALERT_STATE, judged, eventAlert, [], 0);
assert.equal(dropped.enqueued, false);
assert.deepEqual(dropped.state.outbox, []);
assert.deepEqual(dropped.state.reported, {});
// The same event again, still with nowhere to go, changes nothing.
const again = recordThreadEvents(dropped.state, judged, eventAlert, [], 0);
assert.equal(again.enqueued, false);
assert.deepEqual(again.state.outbox, []);
assert.deepEqual(again.state.reported, {});
// A webhook configured later receives the event that was dropped.
const delivered = recordThreadEvents(dropped.state, judged, eventAlert, ["webhook"], 0);
assert.equal(delivered.enqueued, true);
assert.deepEqual(
delivered.state.outbox.map((entry) => entry.destination),
["webhook"],
);
// And once delivered it stays quiet.
const quiet = recordThreadEvents(delivered.state, judged, eventAlert, ["webhook"], 0);
assert.equal(quiet.enqueued, false);
});

it("a thread event with no launching thread to hear it goes only to a configured webhook", () => {
const cto = { environmentId: EnvironmentId.make("laptop"), threadId: ThreadId.make("cto") };
assert.deepEqual(threadEventDestinations(cto, ""), ["watcher"]);
assert.deepEqual(threadEventDestinations(cto, "https://hooks.example/alerts"), [
"watcher",
"webhook",
]);
assert.deepEqual(threadEventDestinations(null, "https://hooks.example/alerts"), ["webhook"]);
assert.deepEqual(threadEventDestinations(null, " "), []);
});

it.effect("forwards a thread event to a watcher on another environment with one retry key", () =>
Effect.gen(function* () {
const dispatched: Array<unknown> = [];
Expand Down
74 changes: 55 additions & 19 deletions apps/server/src/orchestration-v2/ModelAlertReactor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ import * as ProjectionStore from "./ProjectionStore.ts";
import {
judgeThread,
threadEventAlert,
launchingThreadId,
ThreadEventAlert,
type ThreadEventKind,
threadEventMessage,
Expand Down Expand Up @@ -238,6 +239,14 @@ export function modelAlertDestinations(input: {
];
}

export const threadEventDestinations = (
recipient: ModelAlertWatcher | null,
webhookUrl: string,
): ReadonlyArray<Destination> => [
...(recipient === null ? [] : (["watcher"] as const)),
...(webhookUrl.trim() === "" ? [] : (["webhook"] as const)),
];

const signature = (alert: ModelAlert) =>
`${alert.reasons.join(",")}|${alert.executedModel ?? ""}|${alert.fallbackModels.join(",")}`;

Expand Down Expand Up @@ -337,21 +346,28 @@ export function recordThreadEvents(
// A kind can hold several findings at once (a question and an inbox flip), so it remembers
// each signature it reported. One that stops being true is forgotten and can report again.
const known = (reported[key] ?? "").split("\n").filter((signature) => signature !== "");
const current = kindFindings.map((finding) => finding.signature);
const stored = current.join("\n");
if (reported[key] !== stored) {
const { [key]: _previous, ...rest } = reported;
reported = {
...Object.fromEntries(Object.entries(rest).slice(-(MAX_REPORTED - 1))),
[key]: stored,
};
}
const delivered: Array<string> = [];
for (const finding of kindFindings) {
if (known.includes(finding.signature)) continue;
if (known.includes(finding.signature)) {
delivered.push(finding.signature);
continue;
}
const queued = enqueueAlert(outbox, toAlert(finding), destinations, nowMs);
outbox = queued.outbox;
if (queued.enqueued) delivered.push(finding.signature);
enqueued ||= queued.enqueued;
}
// A finding with nowhere to go stays unreported, so a later launcher or webhook still hears it.
const stored = delivered.join("\n");
if (reported[key] !== stored) {
const { [key]: _previous, ...rest } = reported;
reported = stored
? {
...Object.fromEntries(Object.entries(rest).slice(-(MAX_REPORTED - 1))),
[key]: stored,
}
: rest;
}
}
if (reported === state.reported && outbox === state.outbox) return { state, enqueued };
return { state: { ...state, reported, outbox }, enqueued };
Expand Down Expand Up @@ -1060,15 +1076,19 @@ export const make = Effect.gen(function* () {
return { _tag: "sent" } satisfies DeliveryResult;
}
const watcher = current.modelAlertWatcher;
if (watcher === null) return { _tag: "sent" } satisfies DeliveryResult;
const target = alert.type === "thread-event" ? (alert.recipient ?? null) : watcher;
if (target === null) return { _tag: "sent" } satisfies DeliveryResult;
const here = yield* environment.getEnvironmentId;
const nowMs = yield* Clock.currentTimeMillis;
const view = yield* watcherView;
const view =
target.environmentId === watcher?.environmentId && target.threadId === watcher.threadId
? yield* watcherView
: yield* readWatcher(target, here).pipe(Effect.catchCause(() => Effect.succeed(null)));
const until =
holdUntil(view, alert, nowMs) ??
busyWatcherHold(view?.status ?? null, entry.createdAt, nowMs);
if (until !== null) return { _tag: "held", until } satisfies DeliveryResult;
yield* deliverToWatcher(alert, watcher, here);
yield* deliverToWatcher(alert, target, here);
return { _tag: "sent" } satisfies DeliveryResult;
});

Expand Down Expand Up @@ -1248,6 +1268,19 @@ export const make = Effect.gen(function* () {
return { usageLimited: judged.alert?.reasons.includes("usage_limit") === true };
});

const launchingThread = Effect.fnUntraced(function* (
shell: OrchestrationV2ThreadShell,
here: EnvironmentId,
) {
const threadId = launchingThreadId(shell.inbox);
if (threadId === null || threadId === shell.id) return null;
const launcher = yield* projections.getThreadShell(ThreadId.make(threadId));
if (launcher === null || launcher.deletedAt !== null || launcher.archivedAt !== null) {
return null;
}
return { environmentId: here, threadId: launcher.id } satisfies ModelAlertWatcher;
});

/** Judges one thread for the kinds a trigger can change, and queues what is newly true. */
const handleThreadEvents = Effect.fnUntraced(function* (
threadId: ThreadId,
Expand Down Expand Up @@ -1275,12 +1308,14 @@ export const make = Effect.gen(function* () {
});
const label = (yield* environment.getDescriptor).label;
const occurredAt = DateTime.formatIso(yield* DateTime.now);
const destinations = modelAlertDestinations({
watcher,
webhookUrl: current.modelAlertWebhookUrl,
here,
lineage: shell.lineage,
});
const recipient = yield* launchingThread(shell, here);
const destinations = threadEventDestinations(recipient, current.modelAlertWebhookUrl);
if (recipient === null && destinations.length === 0 && judged.findings.length > 0) {
yield* Effect.logWarning("a thread event has no launching thread and no webhook", {
threadId,
kinds: judged.findings.map((finding) => finding.kind),
});
}
const enqueued = yield* update((state) => {
const result = recordThreadEvents(
state,
Expand All @@ -1292,6 +1327,7 @@ export const make = Effect.gen(function* () {
shell,
finding,
occurredAt,
recipient,
}),
destinations,
nowMs,
Expand Down
3 changes: 3 additions & 0 deletions apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -300,6 +300,7 @@ it("a recurrence of the same state is a new message", () => {
shell,
finding,
occurredAt,
recipient: null,
});
assert.notEqual(make("2026-11-19T05:00:00.000Z").id, make("2026-11-19T05:30:00.000Z").id);
});
Expand All @@ -313,6 +314,7 @@ const fired = (overrides: Partial<WatchedShell>, trigger: ThreadEventTrigger) =>
shell,
finding,
occurredAt: DateTime.formatIso(DateTime.makeUnsafe(NOW)),
recipient: null,
});
};
const stillHolds = (
Expand Down Expand Up @@ -414,6 +416,7 @@ it("the wake carries environment, thread link, title, kind, run, last line and o
shell,
finding,
occurredAt: "2026-11-19T05:00:00.000Z",
recipient: null,
});
assert.equal(alert.runId, "run:7");
assert.equal(alert.threadTitle, "Fleet rollout");
Expand Down
7 changes: 7 additions & 0 deletions apps/server/src/orchestration-v2/ThreadEventAlerts.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import {
type EnvironmentId,
isListedThreadLineage,
ModelAlertWatcher,
type OrchestrationV2ThreadShell,
} from "@t3tools/contracts";
import { visibleThreadPullRequests } from "@t3tools/shared/threadPullRequests";
Expand Down Expand Up @@ -37,6 +38,7 @@ export const ThreadEventAlert = Schema.Struct({
lastError: Schema.NullOr(Schema.String),
openPullRequests: Schema.Array(Schema.String),
occurredAt: Schema.String,
recipient: Schema.optionalKey(ModelAlertWatcher),
});
export type ThreadEventAlert = typeof ThreadEventAlert.Type;

Expand Down Expand Up @@ -282,6 +284,9 @@ export function judgeThread(input: {
return { evaluated, findings };
}

export const launchingThreadId = (inbox: WatchedShell["inbox"]) =>
[inbox?.origin, inbox?.requestedBy].find((actor) => actor?.kind === "agent")?.id ?? null;

export function threadEventAlert(input: {
readonly environmentId: EnvironmentId;
readonly environmentLabel: string;
Expand All @@ -297,6 +302,7 @@ export function threadEventAlert(input: {
>;
readonly finding: ThreadFinding;
readonly occurredAt: string;
readonly recipient: ModelAlertWatcher | null;
}): ThreadEventAlert {
const { shell, finding } = input;
const message = shell.latestVisibleMessage;
Expand All @@ -323,6 +329,7 @@ export function threadEventAlert(input: {
.map((link) => link.url)
.slice(0, 5),
occurredAt: input.occurredAt,
...(input.recipient === null ? {} : { recipient: input.recipient }),
};
}

Expand Down
2 changes: 1 addition & 1 deletion docs/operations/watcher-thread.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Watcher thread

One thread, by default Lead's on the laptop, hears everything that needs attention from every linked environment. The server decides each event from thread state with no agent judgment, queues it durably, and retries until the watcher's machine answers.
One thread, by default Lead's on the laptop, hears model, usage-limit and link alerts from every linked environment. The thread events below go instead to the thread that launched the lane, read from the lane's `inbox.origin`, else `inbox.requestedBy`. When that thread is archived, deleted or unknown, a thread event reaches no thread, only `modelAlertWebhookUrl` when set, so a founder-facing watcher never hears about a lane it did not launch. The server decides each event from thread state with no agent judgment, queues it durably, and retries until the receiving machine answers.

| Event | Fires when |
| ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
Expand Down
Loading