Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/closura-arc-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,7 @@ jobs:
apps/server/src/orchestration-v2/ThreadLaunchService.test.ts \
apps/server/src/orchestration-v2/SubagentProjection.test.ts \
apps/server/src/orchestration-v2/ModelAlertReactor.test.ts \
apps/server/src/orchestration-v2/ThreadEventAlerts.test.ts \
apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts \
packages/shared/src/threadPlacement.test.ts \
apps/server/src/peer/RemoteDelegation.test.ts \
Expand Down
163 changes: 163 additions & 0 deletions apps/server/src/orchestration-v2/ModelAlertReactor.test.ts
Original file line number Diff line number Diff line change
@@ -1,14 +1,19 @@
import { assert, it } from "@effect/vitest";
import { EnvironmentId, type OrchestrationV2Run, ThreadId } from "@t3tools/contracts";
import * as Effect from "effect/Effect";

import {
afterDeliveryAttempt,
deliverAlertToWatcher,
EMPTY_MODEL_ALERT_STATE,
type ModelAlert,
modelAlertDestinations,
modelAlertFor,
recordModelAlert,
recordThreadEvents,
watcherAlertMessage,
} from "./ModelAlertReactor.ts";
import type { ThreadEventAlert, ThreadEventKind, ThreadFinding } from "./ThreadEventAlerts.ts";

const alertFor = (runId: string, overrides: Partial<ModelAlert> = {}): ModelAlert => ({
type: "model-degraded",
Expand Down Expand Up @@ -149,3 +154,161 @@ it.each([
expected,
);
});

// Like threadEventAlert, each occurrence gets its own id.
let occurrence = 0;
const eventAlert = (finding: ThreadFinding): ThreadEventAlert => ({
type: "thread-event",
kind: finding.kind,
id: `env:thread:fleet:${finding.kind}:${finding.signature}:${(occurrence += 1)}`,
environmentId: "env",
environmentLabel: "primary-r720",
threadId: "thread:fleet",
threadTitle: "Fleet overnight",
threadLink: "[Fleet overnight](t3-thread://v2/env/thread:fleet)",
runId: "run:1",
detail: finding.detail,
lastAssistantLine: "A background watch will wake this thread.",
lastError: null,
openPullRequests: [],
occurredAt: "2026-11-19T03:00:00.000Z",
});

const finding = (kind: ThreadEventKind, signature: string): ThreadFinding => ({
kind,
signature,
detail: `${kind} detail`,
});

it("reports a thread event once per kind and signature until its state changes", () => {
let state = EMPTY_MODEL_ALERT_STATE;
const enqueued: Array<boolean> = [];
const step = (
evaluated: ReadonlyArray<ThreadEventKind>,
findings: ReadonlyArray<ThreadFinding>,
) => {
const result = recordThreadEvents(
state,
{ threadId: "thread:fleet", evaluated, findings },
eventAlert,
["watcher"],
0,
);
state = result.state;
enqueued.push(result.enqueued);
};
step(["stuck"], [finding("stuck", "run:1")]);
// The same run still stuck on the next sweep says nothing.
step(["stuck"], [finding("stuck", "run:1")]);
// Another kind on the same thread is its own event.
step(["lane-ended"], [finding("lane-ended", "run:1")]);
// A trigger that does not evaluate a kind leaves its dedupe alone.
step(["needs-person"], []);
step(["stuck"], [finding("stuck", "run:1")]);
// Recovery clears it, so the same state reports again as a new occurrence.
step(["stuck"], []);
step(["stuck"], [finding("stuck", "run:1")]);
step(["stuck"], [finding("stuck", "run:2")]);
assert.deepEqual(enqueued, [true, false, true, false, false, false, true, true]);
assert.deepEqual(
state.outbox.map((entry) => (entry.alert.type === "thread-event" ? entry.alert.kind : "")),
["stuck", "lane-ended", "stuck", "stuck"],
);
});

it("a thread holding a question and an inbox flip reports each once, not on every trigger", () => {
const both = [finding("needs-person", "request:1"), finding("needs-person", "inbox:kameron")];
let state = EMPTY_MODEL_ALERT_STATE;
const enqueued: Array<boolean> = [];
const step = (findings: ReadonlyArray<ThreadFinding>) => {
const result = recordThreadEvents(
state,
{ threadId: "thread:fleet", evaluated: ["needs-person"], findings },
eventAlert,
["watcher"],
0,
);
state = result.state;
enqueued.push(result.enqueued);
};
step(both);
step(both);
step(both);
// The question is answered; the inbox flip is still open and was already reported.
step([both[1]!]);
// Then the flip clears and returns: a new occurrence.
step([]);
step([both[1]!]);
assert.deepEqual(enqueued, [true, false, false, false, false, true]);
assert.equal(state.outbox.length, 3);
});

it("queues one delivery per destination for a thread event, beside model alerts", () => {
const { state, enqueued } = recordThreadEvents(
EMPTY_MODEL_ALERT_STATE,
{
threadId: "thread:fleet",
evaluated: ["needs-person"],
findings: [finding("needs-person", "request:1")],
},
eventAlert,
["watcher", "webhook"],
0,
);
assert.equal(enqueued, true);
assert.deepEqual(
state.outbox.map((entry) => [entry.alert.type, entry.destination]),
[
["thread-event", "watcher"],
["thread-event", "webhook"],
],
);
const withModel = recordModelAlert(
state,
{ alert: alertFor("run:1"), healthy: false, threadId: "thread:fleet" },
["watcher"],
0,
);
assert.deepEqual(
withModel.state.outbox.map((entry) => entry.alert.type),
["thread-event", "thread-event", "model-degraded"],
);
assert.include(watcherAlertMessage(alertFor("run:1")), "Model alert from");
assert.include(watcherAlertMessage(eventAlert(finding("stuck", "run:1"))), "Thread event from");
});

it.effect("forwards a thread event to a watcher on another environment with one retry key", () =>
Effect.gen(function* () {
const dispatched: Array<unknown> = [];
const called: Array<{
readonly arguments?: Record<string, unknown>;
readonly environmentId: string;
}> = [];
const deps = {
dispatch: (command: unknown) => Effect.sync(() => void dispatched.push(command)),
call: (input: never) => Effect.sync(() => void called.push(input)),
sha256Hex: (identity: string) => Effect.succeed(`digest(${identity.length})`),
} as never;
const alert = eventAlert(finding("promise-without-wake", "message:1"));
const laptop = { environmentId: EnvironmentId.make("laptop"), threadId: ThreadId.make("lead") };
yield* deliverAlertToWatcher(deps, alert, laptop, EnvironmentId.make("r720"));
yield* deliverAlertToWatcher(deps, alert, laptop, EnvironmentId.make("r720"));
assert.equal(dispatched.length, 0);
assert.equal(called.length, 2);
assert.equal(called[0]!.environmentId, "laptop");
assert.equal(called[0]!.arguments?.threadId, "lead");
assert.equal(called[0]!.arguments?.mode, "queue");
assert.equal(called[0]!.arguments?.message, watcherAlertMessage(alert));
assert.match(String(called[0]!.arguments?.clientRequestId), /^thread-event:/);
assert.equal(called[0]!.arguments?.clientRequestId, called[1]!.arguments?.clientRequestId);

// The same alert on the watcher's own machine queues into the thread directly.
yield* deliverAlertToWatcher(deps, alert, laptop, EnvironmentId.make("laptop"));
assert.equal(called.length, 2);
assert.equal(dispatched.length, 1);
assert.equal((dispatched[0] as { threadId: string }).threadId, "lead");
assert.deepEqual((dispatched[0] as { dispatchMode: unknown }).dispatchMode, {
type: "queue_after_active",
});
}),
);
Loading
Loading