Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions apps/server/src/mcp/OrchestratorMcpService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1521,6 +1521,7 @@ const make = Effect.gen(function* () {
projectId: parent.thread.projectId,
commandId,
instanceId: target.modelSelection.instanceId,
launch: true,
})
.pipe(Effect.mapError(threadManagementFailure));

Expand Down
306 changes: 305 additions & 1 deletion apps/server/src/orchestration-v2/ModelAlertReactor.test.ts
Original file line number Diff line number Diff line change
@@ -1,18 +1,28 @@
import { assert, it } from "@effect/vitest";
import { EnvironmentId, type OrchestrationV2Run, ThreadId } from "@t3tools/contracts";
import * as DateTime from "effect/DateTime";
import * as Effect from "effect/Effect";

import {
afterDeliveryAttempt,
afterDrop,
afterHold,
deliverAlertToWatcher,
INCIDENT_GROUP_INTERVAL_MS,
INCIDENT_GROUP_WAIT_MS,
EMPTY_MODEL_ALERT_STATE,
type ModelAlert,
modelAlertDestinations,
modelAlertFor,
overdueResumes,
recordModelAlert,
recordUsageLimit,
RESUME_GRACE_MS,
recordThreadEvents,
watcherAlertMessage,
} from "./ModelAlertReactor.ts";
import { ProviderExhaustion } from "./ProviderExhaustion.ts";
import { alertIsStale, expectsAutoResume, replacedBySibling } from "./UsageLimitIncidents.ts";
import type { ThreadEventAlert, ThreadEventKind, ThreadFinding } from "./ThreadEventAlerts.ts";

const alertFor = (runId: string, overrides: Partial<ModelAlert> = {}): ModelAlert => ({
Expand Down Expand Up @@ -62,7 +72,10 @@ it("alerts a thread that falls back every run once, until it recovers or changes
step(alertFor("run:6"));
assert.deepEqual(enqueued, [true, false, false, false, false, true, false, true]);
assert.deepEqual(
state.outbox.map((entry) => [entry.alert.runId, entry.destination]),
state.outbox.map((entry) => [
entry.alert.type === "model-degraded" ? entry.alert.runId : entry.alert.id,
entry.destination,
]),
[
["run:1", "watcher"],
["run:1", "webhook"],
Expand Down Expand Up @@ -312,3 +325,294 @@ it.effect("forwards a thread event to a watcher on another environment with one
});
}),
);

const limitInput = (
index: number,
overrides: Partial<Parameters<typeof recordUsageLimit>[1]> = {},
) => ({
member: {
threadId: `thread:${index}`,
threadTitle: `Lane ${index}`,
threadLink: `[Lane ${index}](t3-thread://v2/env/thread:${index})`,
runId: `run:${index}`,
},
environmentId: "env",
environmentLabel: "primary-r720",
providerInstanceId: "claudeAgent",
resetAt: "2026-11-19T09:00:00.000Z",
resumeFailed: false,
willResume: false,
signature: `usage_limit:limit:run:${index}`,
occurredAt: "2026-11-19T04:00:00.000Z",
...overrides,
});

const stormOf = (
count: number,
overrides: Partial<Parameters<typeof recordUsageLimit>[1]> = {},
destinations: ReadonlyArray<"watcher" | "webhook"> = ["watcher"],
) => {
let state = EMPTY_MODEL_ALERT_STATE;
for (let index = 0; index < count; index += 1) {
state = recordUsageLimit(
state,
limitInput(index, overrides),
destinations,
1_000 + index,
).state;
}
return state;
};

it("20 threads hitting one usage limit queue exactly one Lead-facing item", () => {
const state = stormOf(20);
assert.equal(state.outbox.length, 1);
const [entry] = state.outbox;
assert.equal(entry!.destination, "watcher");
assert.equal(entry!.alert.type, "usage-limit-incident");
if (entry!.alert.type !== "usage-limit-incident") return;
assert.equal(entry!.alert.members.length, 20);
assert.equal(entry!.alert.resetAt, "2026-11-19T09:00:00.000Z");
// The first member waits for the rest of the storm.
assert.equal(entry!.nextAttemptAt, 1_000 + INCIDENT_GROUP_WAIT_MS);
const message = watcherAlertMessage(entry!.alert);
assert.include(message, "20 threads stopped");
assert.include(message, "resets at 2026-11-19T09:00:00.000Z");
assert.include(message, "[Lane 19]");
assert.notInclude(message, "Model alert from");
});

it("a repeat on the same run, or another run of an already listed thread, adds no item", () => {
let state = stormOf(3);
state = recordUsageLimit(state, limitInput(1), ["watcher"], 5_000).state;
state = recordUsageLimit(
state,
limitInput(1, {
signature: "usage_limit:limit:run:99",
member: { ...limitInput(1).member, runId: "run:99" },
}),
["watcher"],
6_000,
).state;
assert.equal(state.outbox.length, 1);
const alert = state.outbox[0]!.alert;
assert.equal(alert.type === "usage-limit-incident" && alert.members.length, 3);
});

it("a limit with auto-resume ahead sends Lead nothing, and a thread that comes back clears", () => {
let state = stormOf(20, { willResume: true });
assert.equal(state.outbox.length, 0);
assert.equal(Object.keys(state.resuming).length, 20);
state = recordModelAlert(
state,
{ alert: undefined, healthy: true, threadId: "thread:3" },
["watcher"],
2_000,
).state;
assert.equal(Object.keys(state.resuming).length, 19);
assert.equal(state.resuming["thread:3"], undefined);
});

it("threads that do not come back after the reset make one separate incident", () => {
let state = stormOf(20, { willResume: true });
const nowMs = Date.parse("2026-11-19T09:00:00.000Z") + RESUME_GRACE_MS + 1;
assert.deepEqual(overdueResumes(state, nowMs - 2).length, 0);
const overdue = overdueResumes(state, nowMs);
assert.equal(overdue.length, 20);
for (const [threadId, entry] of overdue) {
const index = Number(threadId.split(":")[1]);
const { [threadId]: _done, ...resuming } = state.resuming;
state = recordUsageLimit(
{ ...state, resuming },
limitInput(index, {
resumeFailed: true,
signature: `usage_limit:resume-missed:${entry.runId}`,
}),
["watcher"],
nowMs,
).state;
}
assert.equal(state.outbox.length, 1);
const alert = state.outbox[0]!.alert;
assert.equal(alert.type === "usage-limit-incident" && alert.resumeFailed, true);
assert.include(watcherAlertMessage(alert), "did not come back after the limit reset");
});

it("members joining while a digest is sent wait for the next one, and stragglers wait a group interval", () => {
const state = stormOf(2);
const sent = state.outbox[0]!;
const joined = recordUsageLimit(state, limitInput(2), ["watcher"], 40_000).state;
assert.equal(joined.outbox.length, 1);
const afterSend = afterDeliveryAttempt(joined, sent, true, 41_000);
assert.equal(afterSend.outbox.length, 1);
const rest = afterSend.outbox[0]!;
assert.equal(rest.alert.type === "usage-limit-incident" && rest.alert.members.length, 1);
assert.equal(rest.nextAttemptAt, 41_000 + INCIDENT_GROUP_INTERVAL_MS);
// Once nothing joined, the sent digest is gone and a later thread starts a delayed new item.
const quiet = afterDeliveryAttempt(state, sent, true, 41_000);
assert.equal(quiet.outbox.length, 0);
const later = recordUsageLimit(quiet, limitInput(7), ["watcher"], 50_000).state;
assert.equal(later.outbox.length, 1);
assert.equal(later.outbox[0]!.nextAttemptAt, 41_000 + INCIDENT_GROUP_INTERVAL_MS);
assert.notEqual(later.outbox[0]!.alert.id, sent.alert.id);
// The leftover digest has its own id, so its delivery key is not a replay of the first.
assert.notEqual(rest.alert.id, sent.alert.id);
});

it("a held item waits without failing, and a dropped one leaves without counting as sent", () => {
const state = stormOf(2);
const entry = state.outbox[0]!;
const held = afterHold(state, entry, 999_000);
assert.deepEqual([held.outbox[0]!.attempts, held.outbox[0]!.nextAttemptAt], [0, 999_000]);
// Members still merge into a held item, so Lead gets one digest when it is back.
const merged = recordUsageLimit(held, limitInput(5), ["watcher"], 70_000).state;
assert.equal(merged.outbox.length, 1);
assert.equal(merged.outbox[0]!.nextAttemptAt, 999_000);
// The snapshot was judged stale, but a thread joined after it: that one still reaches Lead.
const dropped = afterDrop(merged, entry, 80_000);
assert.equal(dropped.outbox.length, 1);
const rest = dropped.outbox[0]!.alert;
assert.deepEqual(
rest.type === "usage-limit-incident" && rest.members.map((member) => member.threadId),
["thread:5"],
);
assert.notEqual(rest.id, entry.alert.id);
assert.equal(afterDrop(held, entry, 80_000).outbox.length, 0);
assert.deepEqual(dropped.incidentSentAt, {});
});

it.each([
["the same run", { latestRunId: "run:1", activeRunId: null, status: "failed" }, false],
[
"a newer run that is running",
{ latestRunId: "run:2", activeRunId: "run:2", status: "running" },
true,
],
[
"a newer run that completed",
{ latestRunId: "run:2", activeRunId: null, status: "completed" },
true,
],
[
"a newer run that failed again",
{ latestRunId: "run:2", activeRunId: null, status: "failed" },
false,
],
["an archived thread", { latestRunId: "run:1", status: "failed", archivedAt: "x" }, true],
] as const)("alert staleness at delivery: %s -> %s", (_name, shell, stale) => {
assert.equal(
alertIsStale(
{ activeRunId: null, archivedAt: null, deletedAt: null, ...shell } as never,
"run:1",
),
stale,
);
assert.equal(alertIsStale(null, "run:1"), true);
});

it("auto-resume is expected only for a first limit on a resumable thread with a known reset", () => {
const shell = {
usageLimitResetAt: "2026-11-19T09:00:00.000Z",
archivedAt: null,
settledOverride: null,
pendingRuntimeRequest: null,
snoozedUntil: null,
} as never;
const nowMs = Date.parse("2026-11-19T04:00:00.000Z");
const expects = (overrides: object) =>
expectsAutoResume({ autoResumeEnabled: true, runWasResume: false, shell, nowMs, ...overrides });
assert.equal(expects({}), true);
assert.equal(expects({ autoResumeEnabled: false }), false);
assert.equal(expects({ runWasResume: true }), false);
assert.equal(expects({ shell: { ...(shell as object), usageLimitResetAt: null } }), false);
assert.equal(expects({ shell: { ...(shell as object), settledOverride: "settled" } }), false);
// The worker skips a child of a closed parent and waits out a snooze, so neither resumes in time.
assert.equal(expects({ parentClosed: true }), false);
const snoozedTo = (iso: string) => ({
shell: { ...(shell as object), snoozedUntil: DateTime.makeUnsafe(iso) },
graceMs: RESUME_GRACE_MS,
});
assert.equal(expects(snoozedTo("2026-11-19T09:10:00.000Z")), true);
assert.equal(expects(snoozedTo("2026-11-19T12:00:00.000Z")), false);
});

it.effect("delivers a 20-thread storm to Lead as one message", () =>
Effect.gen(function* () {
const state = stormOf(20, {}, ["watcher", "webhook"]);
const dispatched: Array<unknown> = [];
const deps = {
dispatch: (command: unknown) => Effect.sync(() => void dispatched.push(command)),
call: () => Effect.void,
sha256Hex: (identity: string) => Effect.succeed(`digest(${identity.length})`),
} as never;
const lead = { environmentId: EnvironmentId.make("env"), threadId: ThreadId.make("lead") };
const due = state.outbox.filter((entry) => entry.destination === "watcher");
for (const entry of due) {
yield* deliverAlertToWatcher(deps, entry.alert, lead, EnvironmentId.make("env"));
}
assert.equal(dispatched.length, 1);
assert.include((dispatched[0] as { text: string }).text, "20 threads stopped");
}),
);

it("a model alert is stale once its thread is settled or runs on another provider", () => {
const base = {
latestRunId: "run:1",
activeRunId: null,
status: "failed",
archivedAt: null,
deletedAt: null,
settledOverride: null,
providerInstanceId: "magpie-primary-codex",
} as never;
const about = { providerInstanceId: "magpie-primary-codex" };
assert.equal(alertIsStale(base, "run:1", about), false);
assert.equal(
alertIsStale({ ...(base as object), settledOverride: "settled" } as never, "run:1", about),
true,
);
assert.equal(
alertIsStale(
{ ...(base as object), providerInstanceId: "claudeAgent" } as never,
"run:1",
about,
),
true,
);
// Thread events are not about a provider, and a settle is what some of them report.
assert.equal(
alertIsStale({ ...(base as object), settledOverride: "settled" } as never, "run:1"),
false,
);
});

it("a delegated child is replaced when its parent started a later one with the same task title", () => {
const sibling = (overrides: object) =>
({
title: " review pr 98 ",
status: "running",
activeRunId: "run:2",
archivedAt: null,
deletedAt: null,
...overrides,
}) as never;
const child = { title: "Review PR 98" };
assert.equal(replacedBySibling(child, [sibling({})]), true);
assert.equal(replacedBySibling(child, [sibling({ title: "Build ASUS" })]), false);
assert.equal(replacedBySibling(child, []), false);
// Templated siblings that also failed, or are gone, replace nothing.
assert.equal(replacedBySibling(child, [sibling({ status: "failed", activeRunId: null })]), false);
assert.equal(replacedBySibling(child, [sibling({ archivedAt: "x" })]), false);
});

it("a provider out of usage is tracked until its reset, and a healthy run clears it", () => {
ProviderExhaustion.clear();
assert.equal(ProviderExhaustion.until("magpie-primary-codex", 1_000), null);
ProviderExhaustion.mark("magpie-primary-codex", 5_000);
assert.equal(ProviderExhaustion.until("magpie-primary-codex", 1_000), 5_000);
assert.equal(ProviderExhaustion.until("claudeAgent", 1_000), null);
assert.equal(ProviderExhaustion.until("magpie-primary-codex", 5_000), null);
ProviderExhaustion.mark("magpie-primary-codex", 9_000);
ProviderExhaustion.clear("magpie-primary-codex");
assert.equal(ProviderExhaustion.until("magpie-primary-codex", 1_000), null);
});
Loading