Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -1432,6 +1432,9 @@ const makeModelAlertReactor = (input: {
Layer.mock(PeerMcpClient.PeerMcpClient)({
call: (request) =>
Effect.gen(function* () {
// The busy-watcher check reads the remote watcher first; it finds it idle.
if (request.tool === "t3_thread_read")
return { thread: { status: "idle" } } as never;
yield* Ref.update(sends, (all) => [
...all,
{
Expand Down Expand Up @@ -5088,6 +5091,13 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => {
now,
}),
);
// The first alert queued a turn on the watcher, so the next one waits for that turn,
// up to the hold cap, instead of queueing behind it.
assert.deepEqual(
(yield* alerts).map((message) => message.text.includes(`/${topLevelThreadId})`)),
[false],
);
yield* TestClock.adjust("11 minutes");
assert.deepEqual(
(yield* alerts).map((message) => message.text.includes(`/${topLevelThreadId})`)),
[false, true],
Expand Down
17 changes: 17 additions & 0 deletions apps/server/src/orchestration-v2/ModelAlertReactor.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@ import {
afterDeliveryAttempt,
afterDrop,
afterHold,
BUSY_WATCHER_MAX_HOLD_MS,
BUSY_WATCHER_RECHECK_MS,
busyWatcherHold,
deliverAlertToWatcher,
INCIDENT_GROUP_INTERVAL_MS,
INCIDENT_GROUP_WAIT_MS,
Expand Down Expand Up @@ -481,6 +484,20 @@ it("a held item waits without failing, and a dropped one leaves without counting
assert.deepEqual(dropped.incidentSentAt, {});
});

it.each([
["an idle watcher", "idle", 0, null],
["a watcher that cannot be read", null, 0, null],
["a watcher mid-turn", "running", 0, BUSY_WATCHER_RECHECK_MS],
["a watcher with a queued turn", "queued", 0, BUSY_WATCHER_RECHECK_MS],
["a watcher waiting on its own background work", "waiting", 0, BUSY_WATCHER_RECHECK_MS],
["a watcher busy near the cap", "running", BUSY_WATCHER_MAX_HOLD_MS - 1_000, 1_000],
["a watcher busy past the cap", "running", BUSY_WATCHER_MAX_HOLD_MS, null],
] as const)("busy watcher hold: %s", (_name, status, ageMs, waitMs) => {
const createdAt = 1_000_000;
const nowMs = createdAt + ageMs;
assert.equal(busyWatcherHold(status, createdAt, nowMs), waitMs === null ? null : nowMs + waitMs);
});

it("a model alert survives a newer run that fell back too, unlike a thread event", () => {
const shell = {
latestRunId: "run:2",
Expand Down
237 changes: 162 additions & 75 deletions apps/server/src/orchestration-v2/ModelAlertReactor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ import {
ThreadEventAlert,
type ThreadEventKind,
threadEventMessage,
REJUDGE_TRIGGER,
threadEventStillHolds,
type ThreadEventTrigger,
type ThreadFinding,
} from "./ThreadEventAlerts.ts";
Expand Down Expand Up @@ -453,6 +455,33 @@ export function afterHold(
};
}

const BUSY_WATCHER_STATUSES = new Set<string>([
"preparing",
"queued",
"starting",
"running",
"waiting",
]);
/** Matches the flush interval, so a held alert is looked at again on the next flush. */
export const BUSY_WATCHER_RECHECK_MS = 30_000;
export const BUSY_WATCHER_MAX_HOLD_MS = 10 * 60 * 1000;

/**
* While the watcher is mid-turn, a sent alert only queues behind that turn and is judged stale or
* not when it was sent. Holding it until the turn ends lets the delivery check drop what went stale
* meanwhile. Bounded, so a watcher that never goes idle still hears everything.
*/
export function busyWatcherHold(
watcherStatus: string | null,
createdAt: number,
nowMs: number,
): number | null {
if (watcherStatus === null || !BUSY_WATCHER_STATUSES.has(watcherStatus)) return null;
const deadline = createdAt + BUSY_WATCHER_MAX_HOLD_MS;
if (nowMs >= deadline) return null;
return Math.min(nowMs + BUSY_WATCHER_RECHECK_MS, deadline);
}

/**
* Records a thread that stopped at a usage limit. The server resumes it at the reset, so Lead
* hears nothing unless it does not come back; otherwise every thread of one provider account
Expand Down Expand Up @@ -702,6 +731,13 @@ export const make = Effect.gen(function* () {
here,
);

type WatcherView = {
readonly status: string;
readonly providerInstanceId: string | null;
/** Null when unknown, as for a remote watcher. */
readonly usageLimited: boolean | null;
readonly resetAt: string | null;
};
const WatcherThreadStatus = Schema.Struct({
thread: Schema.Struct({ status: Schema.String, providerInstanceId: Schema.String }),
});
Expand Down Expand Up @@ -762,71 +798,140 @@ export const make = Effect.gen(function* () {
alert.type === "model-degraded"
? { providerInstanceId: alert.providerInstanceId, newerRunSupersedes: false }
: undefined;
return (yield* stillCurrent(alert.threadId, alert.runId, about)) ? alert : null;
if (!(yield* stillCurrent(alert.threadId, alert.runId, about))) return null;
if (alert.type === "thread-event") {
const shell = yield* projections.getThreadShell(ThreadId.make(alert.threadId));
if (shell === null) return null;
const holds = threadEventStillHolds({
alert,
shell,
stuckMinutes: (yield* settings.getSettings).watcherStuckMinutes,
...(yield* judgeContext(shell, REJUDGE_TRIGGER[alert.kind])),
});
if (!holds) return null;
}
return alert;
});

/** What the shell does not carry: open delegated tasks elsewhere, and children's last activity. */
const judgeContext = Effect.fnUntraced(function* (
shell: OrchestrationV2ThreadShell,
trigger: ThreadEventTrigger,
remoteTasks?: ReadonlyMap<string, number>,
) {
const remoteTaskCount =
trigger === "request" || trigger === "inbox"
? 0
: remoteTasks !== undefined
? (remoteTasks.get(shell.id) ?? 0)
: (yield* projections.getOpenRemoteDelegatedTasks).filter(
(task) => task.parentThreadId === shell.id,
).length;
// A parent that waits on children is not stuck while they work.
const childActivityMs =
trigger === "sweep"
? Math.max(
0,
...(yield* Effect.forEach(
(shell.pendingBackgroundTasks ?? []).flatMap((task) =>
task.kind === "subagent" && task.childThreadId !== undefined
? [task.childThreadId]
: [],
),
(childId) =>
projections
.getThreadShell(childId)
.pipe(
Effect.map((child) =>
child === null ? 0 : DateTime.toEpochMillis(child.updatedAt),
),
),
)),
)
: null;
return { remoteTaskCount, childActivityMs };
});

/** The watcher thread as one delivery needs it, or null when it cannot be read. */
const readWatcher = Effect.fnUntraced(function* (
watcher: ModelAlertWatcher,
here: EnvironmentId,
) {
if (watcher.environmentId === here) {
const shell = yield* projections.getThreadShell(watcher.threadId);
if (shell === null) return null;
return {
status: shell.status as string,
providerInstanceId: shell.providerInstanceId as string | null,
usageLimited: shell.lastErrorClass === "usage_limit",
resetAt: shell.usageLimitResetAt ?? null,
} satisfies WatcherView;
}
return yield* peers
.call({
environmentId: watcher.environmentId,
tool: "t3_thread_read",
arguments: { threadId: watcher.threadId, limit: 1, runLimit: 1 },
limits: { runtimeMode: "full-access", interactionMode: "default" },
success: WatcherThreadStatus,
})
.pipe(
// The read result carries no error class or reset time.
Effect.map(({ thread }): WatcherView => ({
status: thread.status,
providerInstanceId: thread.providerInstanceId,
usageLimited: null,
resetAt: null,
})),
// A read that fails or dies must never block the alert; deliver as if idle.
Effect.catchCause(() => Effect.succeed(null)),
);
});

/**
* When the watcher's own thread is stopped at the same account limit, the time to look again.
* Anything sent now would only queue a turn that fails the same way.
*/
const holdUntil = Effect.fnUntraced(function* (
watcher: ModelAlertWatcher,
here: EnvironmentId,
alert: WatcherAlert,
nowMs: number,
) {
const holdUntil = (view: WatcherView | null, alert: WatcherAlert, nowMs: number) => {
const providerInstanceId =
alert.type === "usage-limit-incident" ||
(alert.type === "model-degraded" && alert.reasons.includes("usage_limit"))
? alert.providerInstanceId
: null;
if (providerInstanceId === null) return null;
let down = false;
let resetAt: string | null = alert.type === "usage-limit-incident" ? alert.resetAt : null;
if (watcher.environmentId === here) {
const shell = yield* projections.getThreadShell(watcher.threadId);
down =
shell !== null &&
shell.status === "failed" &&
shell.providerInstanceId === providerInstanceId &&
shell.lastErrorClass === "usage_limit";
resetAt = shell?.usageLimitResetAt ?? resetAt;
} else {
// The read result carries no error class, so a remote watcher counts as down only while
// the reset this alert reports is still ahead.
down = yield* peers
.call({
environmentId: watcher.environmentId,
tool: "t3_thread_read",
arguments: { threadId: watcher.threadId, limit: 1, runLimit: 1 },
limits: { runtimeMode: "full-access", interactionMode: "default" },
success: WatcherThreadStatus,
})
.pipe(
Effect.map(
({ thread }) =>
thread.status === "failed" && thread.providerInstanceId === providerInstanceId,
),
// An unreadable watcher is the delivery's problem, not a reason to hold.
Effect.orElseSucceed(() => false),
);
}
if (providerInstanceId === null || view === null) return null;
// A remote watcher counts as down only while the reset this alert reports is still ahead.
const down =
view.status === "failed" &&
view.providerInstanceId === providerInstanceId &&
view.usageLimited !== false;
const resetAt = view.resetAt ?? (alert.type === "usage-limit-incident" ? alert.resetAt : null);
if (!down) return null;
// Past the reset the limit is over: whatever keeps the watcher down, a message may restart it.
const resetMs = resetAt === null ? Number.NaN : Date.parse(resetAt);
if (!Number.isFinite(resetMs) || resetMs <= nowMs) return null;
return Math.min(resetMs, nowMs + HOLD_RECHECK_MS);
});
};

type DeliveryResult =
| { readonly _tag: "sent" }
| { readonly _tag: "dropped" }
| { readonly _tag: "held"; readonly until: number };

const deliver = Effect.fnUntraced(function* (entry: Delivery) {
const deliver = Effect.fnUntraced(function* (
entry: Delivery,
watcherView: Effect.Effect<WatcherView | null>,
) {
const current = yield* settings.getSettings;
const alert = yield* currentAlert(entry.alert);
if (alert === null) return { _tag: "dropped" } satisfies DeliveryResult;
if (alert === null) {
yield* Effect.logInfo("watcher alert no longer true; dropped without waking anyone", {
alertId: entry.alert.id,
type: entry.alert.type,
kind: entry.alert.type === "thread-event" ? entry.alert.kind : null,
destination: entry.destination,
});
return { _tag: "dropped" } satisfies DeliveryResult;
}
if (entry.destination === "webhook") {
const url = current.modelAlertWebhookUrl.trim();
if (url === "") return { _tag: "sent" } satisfies DeliveryResult;
Expand All @@ -840,7 +945,11 @@ export const make = Effect.gen(function* () {
const watcher = current.modelAlertWatcher;
if (watcher === null) return { _tag: "sent" } satisfies DeliveryResult;
const here = yield* environment.getEnvironmentId;
const until = yield* holdUntil(watcher, here, alert, yield* Clock.currentTimeMillis);
const nowMs = yield* Clock.currentTimeMillis;
const view = yield* watcherView;
const until =
holdUntil(view, alert, nowMs) ??
busyWatcherHold(view?.status ?? null, entry.createdAt, nowMs);
if (until !== null) return { _tag: "held", until } satisfies DeliveryResult;
yield* deliverToWatcher(alert, watcher, here);
return { _tag: "sent" } satisfies DeliveryResult;
Expand All @@ -849,11 +958,20 @@ export const make = Effect.gen(function* () {
const flush: ModelAlertReactor["Service"]["flush"] = Effect.gen(function* () {
const nowMs = yield* Clock.currentTimeMillis;
const due = (yield* Ref.get(state)).outbox.filter((entry) => entry.nextAttemptAt <= nowMs);
if (due.length === 0) return;
// One read of the watcher per flush, shared by every alert due in it.
const watcherView = yield* Effect.cached(
Effect.gen(function* () {
const watcher = (yield* settings.getSettings).modelAlertWatcher;
if (watcher === null) return null;
return yield* readWatcher(watcher, yield* environment.getEnvironmentId);
}).pipe(Effect.catchCause(() => Effect.succeed(null))),
);
yield* Effect.forEach(
due,
(entry) =>
Effect.gen(function* () {
const result = yield* deliver(entry).pipe(
const result = yield* deliver(entry, watcherView).pipe(
Effect.timeout("30 seconds"),
Effect.catchCause((cause) =>
Effect.logWarning("delivering a model alert failed; it will be retried", {
Expand Down Expand Up @@ -1029,44 +1147,13 @@ export const make = Effect.gen(function* () {
if (watcher === null && current.modelAlertWebhookUrl.trim() === "") return;
const shell = yield* projections.getThreadShell(threadId);
if (shell === null || shell.deletedAt !== null) return;
const remoteTaskCount =
trigger === "request" || trigger === "inbox"
? 0
: remoteTasks !== undefined
? (remoteTasks.get(threadId) ?? 0)
: (yield* projections.getOpenRemoteDelegatedTasks).filter(
(task) => task.parentThreadId === threadId,
).length;
// A parent that waits on children is not stuck while they work.
const childActivityMs =
trigger === "sweep"
? Math.max(
0,
...(yield* Effect.forEach(
(shell.pendingBackgroundTasks ?? []).flatMap((task) =>
task.kind === "subagent" && task.childThreadId !== undefined
? [task.childThreadId]
: [],
),
(childId) =>
projections
.getThreadShell(childId)
.pipe(
Effect.map((child) =>
child === null ? 0 : DateTime.toEpochMillis(child.updatedAt),
),
),
)),
)
: null;
const nowMs = yield* Clock.currentTimeMillis;
const judged = judgeThread({
shell,
trigger,
nowMs,
stuckMinutes: current.watcherStuckMinutes,
remoteTaskCount,
childActivityMs,
...(yield* judgeContext(shell, trigger, remoteTasks)),
usageLimitReported,
});
const label = (yield* environment.getDescriptor).label;
Expand Down
Loading
Loading