diff --git a/.github/workflows/closura-arc-ci.yml b/.github/workflows/closura-arc-ci.yml index f1da93204077..8186a2a564dc 100644 --- a/.github/workflows/closura-arc-ci.yml +++ b/.github/workflows/closura-arc-ci.yml @@ -153,6 +153,8 @@ jobs: packages/client-runtime/src/state/threadAttribution.test.ts \ apps/server/src/orchestration-v2/ThreadLaunchService.test.ts \ apps/server/src/orchestration-v2/SubagentProjection.test.ts \ + apps/server/src/orchestration-v2/ModelAlertReactor.test.ts \ + apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts \ packages/shared/src/threadPlacement.test.ts \ apps/server/src/peer/RemoteDelegation.test.ts \ apps/server/src/mcp/OrchestratorMcpService.test.ts \ diff --git a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts index b3771fc668da..fce4fb9354ec 100644 --- a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts +++ b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts @@ -9,6 +9,7 @@ import { assert, it, vi } from "@effect/vitest"; import { CheckpointId, CommandId, + EnvironmentId, EventId, MessageId, type ModelSelection, @@ -19,6 +20,8 @@ import { ProviderInstanceId, ProviderSessionId, ProviderThreadId, + ProviderTurnId, + RunAttemptId, RunId, RuntimeRequestId, ThreadId, @@ -29,6 +32,8 @@ import * as Effect from "effect/Effect"; import * as Deferred from "effect/Deferred"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; +import * as Ref from "effect/Ref"; +import * as HttpClient from "effect/http/HttpClient"; import * as Stream from "effect/Stream"; import * as Tracer from "effect/Tracer"; import * as TestClock from "effect/testing/TestClock"; @@ -46,7 +51,11 @@ import * as VcsDriverRegistry from "../vcs/VcsDriverRegistry.ts"; import * as VcsProcess from "../vcs/VcsProcess.ts"; import * as WorkspacePaths from "../workspace/WorkspacePaths.ts"; import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts"; +import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; +import * as PeerMcpClient from "../peer/PeerMcpClient.ts"; import * as EventSink from "./EventSink.ts"; +import * as ModelAlertReactor from "./ModelAlertReactor.ts"; +import * as ProjectionStore from "./ProjectionStore.ts"; import * as Orchestrator from "./Orchestrator.ts"; import * as ProviderRuntimeRecovery from "./ProviderRuntimeRecoveryService.ts"; import { continueRestartedRun } from "./RestartContinuation.ts"; @@ -125,6 +134,7 @@ const layerTest = Layer.mergeAll( RuntimeLayer.layerEventSink, ThreadCommandExecutor.layer, EffectOutbox.layer, + ProjectionStore.layer, ).pipe( Layer.provideMerge(RuntimeLayer.layerProjectService), Layer.provide( @@ -1398,6 +1408,100 @@ const runEvent = (input: { }, }); +const hereEnvironmentId = EnvironmentId.make("environment:model-alert-here"); + +/** A model alert reactor here whose peer link records each send, failing the first `failSends`. */ +const makeModelAlertReactor = (input: { + readonly watcher: { readonly environmentId: EnvironmentId; readonly threadId: ThreadId }; + readonly failSends?: number; +}) => + Effect.gen(function* () { + const sends = yield* Ref.make>>([]); + const failures = yield* Ref.make(input.failSends ?? 0); + const reactor = yield* ModelAlertReactor.make.pipe( + Effect.provide( + Layer.mergeAll( + ServerSettings.layerTest({ modelAlertWatcher: input.watcher }), + Layer.mock(ServerEnvironment.ServerEnvironment)({ + getEnvironmentId: Effect.succeed(hereEnvironmentId), + getDescriptor: Effect.succeed({ + environmentId: hereEnvironmentId, + label: "test-box", + } as never), + }), + Layer.mock(PeerMcpClient.PeerMcpClient)({ + call: (request) => + Effect.gen(function* () { + yield* Ref.update(sends, (all) => [ + ...all, + { + environmentId: request.environmentId, + tool: request.tool, + ...request.arguments, + }, + ]); + if ((yield* Ref.getAndUpdate(failures, (left) => left - 1)) > 0) { + return yield* Effect.die("the watcher's machine is asleep"); + } + return {} as never; + }), + }), + Layer.succeed( + HttpClient.HttpClient, + HttpClient.make(() => Effect.die("no webhook is set")), + ), + layerServerConfig, + ).pipe(Layer.provideMerge(Layer.merge(NodeServices.layer, NodeCrypto.layer))), + ), + ); + return { reactor, sends: Ref.get(sends) }; + }); + +/** Finishes a run whose provider reported running `executedModel`. */ +const finishRunOnModel = (input: { + readonly threadId: ThreadId; + readonly runId: RunId; + readonly ordinal: number; + readonly executedModel: string; + readonly now: DateTime.Utc; +}) => + Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const attemptId = RunAttemptId.make(`attempt:${input.runId}`); + const run = runEvent({ ...input, status: "completed" }); + const finished = { + ...run, + id: EventId.make(`event:${input.runId}:on-model`), + payload: { ...run.payload, activeAttemptId: attemptId }, + }; + yield* eventSink.write({ + commandId: CommandId.make(`command:${input.runId}:on-model`), + events: [ + { + id: EventId.make(`event:${input.runId}:provider-turn`), + type: "provider-turn.updated", + threadId: input.threadId, + runId: input.runId, + occurredAt: input.now, + payload: { + id: ProviderTurnId.make(`provider-turn:${input.runId}`), + providerThreadId: ProviderThreadId.make(`provider-thread:${input.runId}`), + nodeId: NodeId.make(`node:${input.runId}`), + runAttemptId: attemptId, + nativeTurnRef: null, + ordinal: 1, + status: "completed", + startedAt: input.now, + completedAt: input.now, + model: input.executedModel, + }, + }, + finished, + ], + }); + return finished; + }); + const seedRestartCancelledChild = (input: { readonly parentThreadId: ThreadId; readonly projectId: ProjectId; @@ -4871,6 +4975,191 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => { ); // Running does not prove prompt consumption in Claude; re-offer cut deliveries even when resuming the turn. + it.effect("alerts the watcher and the parent once when a child ran another model", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const now = yield* DateTime.now; + const projectId = ProjectId.make("project:model-alert"); + const threadId = ThreadId.make("thread:model-alert-parent"); + const runId = RunId.make("run:model-alert-parent"); + const rootNodeId = NodeId.make("node:model-alert-parent-root"); + yield* seedParentWithTerminalTask({ + threadId, + projectId, + runId, + rootNodeId, + taskId: NodeId.make("node:model-alert-parent-settled"), + deliveryState: "delivered", + now, + }); + const child = yield* seedRestartCancelledChild({ + parentThreadId: threadId, + projectId, + parentRunId: runId, + rootNodeId, + name: "model-alert-child", + completionWake: "always", + continuationPending: false, + runStatus: "completed", + now, + }); + const finished = yield* finishRunOnModel({ + threadId: child.childThreadId, + runId: child.childRunId, + ordinal: 1, + executedModel: "gpt-5.4-mini", + now, + }); + yield* orchestrator.recoverDelegatedTask(child.childThreadId); + const task = (yield* orchestrator.getThreadProjection(threadId)).subagents.find( + (row) => row.id === child.taskId, + ); + assert.deepEqual(task?.workerAlert?.reasons, ["model_substituted"]); + + const watcherThreadId = ThreadId.make("thread:model-alert-watcher"); + yield* orchestrator.dispatch({ + type: "thread.create", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make("command:model-alert-watcher"), + threadId: watcherThreadId, + projectId, + title: "Lead", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + }); + const { reactor } = yield* makeModelAlertReactor({ + watcher: { environmentId: hereEnvironmentId, threadId: watcherThreadId }, + }); + const alerts = Effect.gen(function* () { + yield* reactor.flush; + return (yield* orchestrator.getThreadProjection(watcherThreadId)).messages.filter( + (message) => message.text.startsWith("Model alert"), + ); + }); + yield* reactor.handle(finished); + // A replayed event and the child's next run on the same fallback stay quiet. + yield* reactor.handle(finished); + yield* reactor.handle( + yield* finishRunOnModel({ + threadId: child.childThreadId, + runId: RunId.make("run:model-alert-child:2"), + ordinal: 2, + executedModel: "gpt-5.4-mini", + now, + }), + ); + const [alert, ...more] = yield* alerts; + assert.deepEqual(more, []); + for (const part of [ + "Model alert from test-box (environment environment:model-alert-here)", + `(t3-thread://v2/environment:model-alert-here/${child.childThreadId})`, + "it ran gpt-5.4-mini instead of gpt-5.4", + `Run ${child.childRunId} ended completed. Reasons: model_substituted.`, + ]) { + assert.include(alert?.text, part); + } + + // A top-level thread has no parent to tell, so only the watcher hears it. + const topLevelThreadId = ThreadId.make("thread:model-alert-top-level"); + yield* orchestrator.dispatch({ + type: "thread.create", + createdBy: "agent", + creationSource: "mcp", + commandId: CommandId.make("command:model-alert-top-level"), + threadId: topLevelThreadId, + projectId, + title: "Fleet overnight", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + }); + yield* reactor.handle( + yield* finishRunOnModel({ + threadId: topLevelThreadId, + runId: RunId.make("run:model-alert-top-level"), + ordinal: 1, + executedModel: "gpt-5.4-mini", + now, + }), + ); + assert.deepEqual( + (yield* alerts).map((message) => message.text.includes(`/${topLevelThreadId})`)), + [false, true], + ); + }), + ); + + it.effect("sends a model alert over the link to the watcher's machine and retries it", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const now = yield* DateTime.now; + const projectId = ProjectId.make("project:model-alert-remote"); + const threadId = ThreadId.make("thread:model-alert-remote"); + yield* ProjectService.ProjectService.pipe( + Effect.flatMap((projects) => + projects.create({ + commandId: CommandId.make("command:model-alert-remote-project"), + projectId, + title: "Model alert remote", + workspaceRoot: "/workspace/model-alert-remote", + }), + ), + ); + yield* orchestrator.dispatch({ + type: "thread.create", + createdBy: "agent", + creationSource: "mcp", + commandId: CommandId.make("command:model-alert-remote-thread"), + threadId, + projectId, + title: "On the box", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + }); + const laptop = EnvironmentId.make("environment:laptop"); + const lead = ThreadId.make("thread:lead"); + const { reactor, sends } = yield* makeModelAlertReactor({ + watcher: { environmentId: laptop, threadId: lead }, + failSends: 1, + }); + yield* reactor.handle( + yield* finishRunOnModel({ + threadId, + runId: RunId.make("run:model-alert-remote"), + ordinal: 1, + executedModel: "gpt-5.4-mini", + now, + }), + ); + yield* reactor.flush; + // Not due yet: the failed send waits before its retry. + yield* reactor.flush; + yield* TestClock.adjust("15 seconds"); + yield* reactor.flush; + yield* TestClock.adjust("1 hour"); + yield* reactor.flush; + const sent = yield* sends; + assert.equal(sent.length, 2); + assert.deepEqual(sent[0], sent[1]); + assert.deepInclude(sent[0], { + environmentId: laptop, + tool: "t3_thread_send", + threadId: lead, + mode: "queue", + }); + assert.match(String(sent[0]?.clientRequestId), /^model-alert:[0-9a-f]{64}$/); + }), + ); + it.effect("re-offers a cut delivery even when a continuation resumes its turn", () => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; diff --git a/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts b/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts new file mode 100644 index 000000000000..11277dc3d395 --- /dev/null +++ b/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts @@ -0,0 +1,151 @@ +import { assert, it } from "@effect/vitest"; +import { EnvironmentId, type OrchestrationV2Run, ThreadId } from "@t3tools/contracts"; + +import { + afterDeliveryAttempt, + EMPTY_MODEL_ALERT_STATE, + type ModelAlert, + modelAlertDestinations, + modelAlertFor, + recordModelAlert, +} from "./ModelAlertReactor.ts"; + +const alertFor = (runId: string, overrides: Partial = {}): ModelAlert => ({ + type: "model-degraded", + id: `env:${runId}`, + environmentId: "env", + environmentLabel: "primary-r720", + threadId: "thread:fleet", + threadTitle: "Fleet overnight", + threadLink: "[Fleet overnight](t3-thread://v2/env/thread:fleet)", + parentThreadId: null, + runId, + runStatus: "completed", + providerInstanceId: "claudeAgent", + selectedModel: "claude-opus-5-5", + executedModel: "claude-opus-4-8", + fallbackModels: ["claude-opus-4-8"], + reasons: ["model_substituted", "provider_fallback"], + occurredAt: "2026-11-19T03:00:00.000Z", + ...overrides, +}); + +const record = ( + state: typeof EMPTY_MODEL_ALERT_STATE, + alert: ModelAlert | undefined, + healthy = false, +) => + recordModelAlert(state, { alert, healthy, threadId: "thread:fleet" }, ["watcher", "webhook"], 0); + +it("alerts a thread that falls back every run once, until it recovers or changes", () => { + let state = EMPTY_MODEL_ALERT_STATE; + const enqueued: Array = []; + const step = (alert: ModelAlert | undefined, healthy = false) => { + const result = record(state, alert, healthy); + state = result.state; + enqueued.push(result.enqueued); + }; + step(alertFor("run:1")); + step(alertFor("run:2")); + step(alertFor("run:3")); + // A run with no model report says nothing either way. + step(undefined); + step(alertFor("run:4")); + // A different problem alerts again. + step(alertFor("run:5", { reasons: ["usage_limit"], executedModel: null, fallbackModels: [] })); + step(undefined, true); + step(alertFor("run:6")); + assert.deepEqual(enqueued, [true, false, false, false, false, true, false, true]); + assert.deepEqual( + state.outbox.map((entry) => [entry.alert.runId, entry.destination]), + [ + ["run:1", "watcher"], + ["run:1", "webhook"], + ["run:5", "watcher"], + ["run:5", "webhook"], + ["run:6", "watcher"], + ["run:6", "webhook"], + ], + ); +}); + +it("keeps a failed delivery with a growing delay and drops a delivered one", () => { + const { state } = recordModelAlert( + EMPTY_MODEL_ALERT_STATE, + { alert: alertFor("run:1"), healthy: false, threadId: "thread:fleet" }, + ["watcher", "webhook"], + 0, + ); + const [watcher, webhook] = state.outbox; + const once = afterDeliveryAttempt(state, watcher!, false, 1_000); + const twice = afterDeliveryAttempt(once, once.outbox[0]!, false, 20_000); + assert.deepEqual( + twice.outbox.map((entry) => [entry.destination, entry.attempts, entry.nextAttemptAt]), + [ + ["watcher", 2, 50_000], + ["webhook", 0, 0], + ], + ); + const delivered = afterDeliveryAttempt(twice, webhook!, true, 21_000); + assert.deepEqual( + delivered.outbox.map((entry) => entry.destination), + ["watcher"], + ); + // A week later an undeliverable alert is dropped. + assert.deepEqual( + afterDeliveryAttempt(delivered, webhook!, true, 8 * 24 * 60 * 60 * 1000).outbox, + [], + ); +}); + +it.each([ + ["completed", true], + ["failed", false], + ["interrupted", false], +] as const)("a %s run with no model report is healthy: %s", (status, healthy) => { + const outcome = modelAlertFor({ + environmentId: EnvironmentId.make("env"), + environmentLabel: "box", + thread: { + id: ThreadId.make("thread:codex"), + title: "Codex", + lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: null }, + } as never, + run: { + id: "run:codex", + status, + activeAttemptId: null, + modelSelection: { instanceId: "codex", model: "gpt-6.1-sol" }, + } as unknown as OrchestrationV2Run, + projection: { turnItems: [], providerTurns: [] }, + occurredAt: "2026-11-19T03:00:00.000Z", + }); + assert.deepEqual(outcome, { alert: undefined, healthy }); +}); + +it.each([ + ["a top-level thread", null, null, "", ["watcher"]], + ["a delegated child of the watcher", "subagent", "thread:lead", "", []], + ["a fork of the watcher", "fork", "thread:lead", "", ["watcher"]], + [ + "a child of another thread", + "subagent", + "thread:other", + "https://hooks.example/x", + ["watcher", "webhook"], + ], +] as const)("%s alerts %j", (_name, relationshipToParent, parentThreadId, webhookUrl, expected) => { + const here = EnvironmentId.make("env"); + assert.deepEqual( + modelAlertDestinations({ + watcher: { environmentId: here, threadId: ThreadId.make("thread:lead") }, + webhookUrl, + here, + lineage: { + relationshipToParent, + parentThreadId: parentThreadId === null ? null : ThreadId.make(parentThreadId), + }, + }), + expected, + ); +}); diff --git a/apps/server/src/orchestration-v2/ModelAlertReactor.ts b/apps/server/src/orchestration-v2/ModelAlertReactor.ts new file mode 100644 index 000000000000..52c99fad07cf --- /dev/null +++ b/apps/server/src/orchestration-v2/ModelAlertReactor.ts @@ -0,0 +1,485 @@ +import { + CommandId, + type EnvironmentId, + MessageId, + type ModelAlertWatcher, + type OrchestrationV2DomainEvent, + type OrchestrationV2Run, + type OrchestrationV2ThreadProjection, + OrchestrationV2WorkerAlertReason, + OrchestratorMcpThreadSendResult, +} from "@t3tools/contracts"; +import { runModelSignals, workerAlert } from "@t3tools/provider-core/server/subagentProjection"; +import { writeFileStringAtomically } from "@t3tools/shared/atomicWrite"; +import { formatThreadLink } from "@t3tools/shared/threadLinks"; +import * as Clock from "effect/Clock"; +import * as Context from "effect/Context"; +import * as Digest from "effect/crypto/Digest"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Hex from "effect/encoding/Hex"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as Queue from "effect/Queue"; +import * as Ref from "effect/Ref"; +import * as Schedule from "effect/Schedule"; +import * as Schema from "effect/Schema"; +import type * as Scope from "effect/Scope"; +import * as Semaphore from "effect/Semaphore"; +import * as Stream from "effect/Stream"; +import * as HttpClient from "effect/http/HttpClient"; +import * as HttpClientRequest from "effect/http/HttpClientRequest"; + +import * as ServerConfig from "../config.ts"; +import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; +import * as PeerMcpClient from "../peer/PeerMcpClient.ts"; +import { forkParked } from "../serverActivation.ts"; +import * as ServerSettings from "../serverSettings.ts"; +import * as Orchestrator from "./Orchestrator.ts"; +import * as ProjectionStore from "./ProjectionStore.ts"; + +export const ModelAlert = Schema.Struct({ + type: Schema.Literal("model-degraded"), + id: Schema.String, + environmentId: Schema.String, + environmentLabel: Schema.String, + threadId: Schema.String, + threadTitle: Schema.String, + threadLink: Schema.String, + parentThreadId: Schema.NullOr(Schema.String), + runId: Schema.String, + runStatus: Schema.String, + providerInstanceId: Schema.String, + selectedModel: Schema.String, + executedModel: Schema.NullOr(Schema.String), + fallbackModels: Schema.Array(Schema.String), + reasons: Schema.Array(OrchestrationV2WorkerAlertReason), + occurredAt: Schema.String, +}); +export type ModelAlert = typeof ModelAlert.Type; + +const Destination = Schema.Literals(["watcher", "webhook"]); +type Destination = typeof Destination.Type; + +const Delivery = Schema.Struct({ + destination: Destination, + alert: ModelAlert, + attempts: Schema.Int, + nextAttemptAt: Schema.Number, + createdAt: Schema.Number, +}); +type Delivery = typeof Delivery.Type; + +export const ModelAlertState = Schema.Struct({ + version: Schema.Literal(1), + /** The alert signature each thread last reported; a healthy run clears it. */ + degraded: Schema.Record(Schema.String, Schema.String), + outbox: Schema.Array(Delivery), +}); +export type ModelAlertState = typeof ModelAlertState.Type; + +export const EMPTY_MODEL_ALERT_STATE: ModelAlertState = { version: 1, degraded: {}, outbox: [] }; + +const MAX_DEGRADED_THREADS = 1_000; +const MAX_OUTBOX = 200; +const OUTBOX_TTL_MS = 7 * 24 * 60 * 60 * 1000; +const RETRY_BASE_MS = 15_000; +const RETRY_MAX_MS = 15 * 60 * 1000; + +/** A run's model alert, or whether it finished cleanly and so shows the thread healthy again. */ +export function modelAlertFor(input: { + readonly environmentId: EnvironmentId; + readonly environmentLabel: string; + readonly thread: Pick; + readonly run: OrchestrationV2Run; + readonly projection: Pick; + readonly occurredAt: string; +}): { readonly alert: ModelAlert | undefined; readonly healthy: boolean } { + const { run, thread } = input; + const signals = runModelSignals(input.projection, run); + // The task-withholding heuristic is for a parent judging its worker, not for a model alert. + const reasons = workerAlert({ ...signals, finalText: undefined })?.reasons; + if (reasons === undefined) { + return { alert: undefined, healthy: run.status === "completed" }; + } + return { + healthy: false, + alert: { + type: "model-degraded", + id: `${input.environmentId}:${run.id}`, + environmentId: input.environmentId, + environmentLabel: input.environmentLabel, + threadId: thread.id, + threadTitle: thread.title, + threadLink: formatThreadLink( + { environmentId: input.environmentId, threadId: thread.id }, + thread.title, + ), + parentThreadId: thread.lineage.parentThreadId, + runId: run.id, + runStatus: run.status, + providerInstanceId: run.providerInstanceId, + selectedModel: run.modelSelection.model, + executedModel: signals.executedModel ?? null, + fallbackModels: [...new Set(signals.fallbackModels)], + reasons, + occurredAt: input.occurredAt, + }, + }; +} + +/** Where a thread's alert goes. A delegated child's parent hears it with its result already. */ +export function modelAlertDestinations(input: { + readonly watcher: ModelAlertWatcher | null; + readonly webhookUrl: string; + readonly here: EnvironmentId; + readonly lineage: Pick< + OrchestrationV2ThreadProjection["thread"]["lineage"], + "parentThreadId" | "relationshipToParent" + >; +}): ReadonlyArray { + const { watcher, lineage } = input; + const watcherIsParent = + watcher?.environmentId === input.here && + lineage.relationshipToParent === "subagent" && + watcher.threadId === lineage.parentThreadId; + return [ + ...(watcher === null || watcherIsParent ? [] : (["watcher"] as const)), + ...(input.webhookUrl.trim() === "" ? [] : (["webhook"] as const)), + ]; +} + +const signature = (alert: ModelAlert) => + `${alert.reasons.join(",")}|${alert.executedModel ?? ""}|${alert.fallbackModels.join(",")}`; + +/** + * Records a run's outcome. A thread alerts once per signature: later runs with the same problem + * stay quiet until a healthy run or a different problem. + */ +export function recordModelAlert( + state: ModelAlertState, + outcome: ReturnType & { readonly threadId: string }, + destinations: ReadonlyArray, + nowMs: number, +): { readonly state: ModelAlertState; readonly enqueued: boolean } { + const { alert } = outcome; + if (alert === undefined) { + if (!outcome.healthy || state.degraded[outcome.threadId] === undefined) { + return { state, enqueued: false }; + } + const { [outcome.threadId]: _cleared, ...degraded } = state.degraded; + return { state: { ...state, degraded }, enqueued: false }; + } + const next = signature(alert); + if (state.degraded[alert.threadId] === next) return { state, enqueued: false }; + const { [alert.threadId]: _previous, ...rest } = state.degraded; + const kept = Object.entries(rest).slice(-(MAX_DEGRADED_THREADS - 1)); + const added = destinations + .filter( + (destination) => + !state.outbox.some( + (entry) => entry.alert.id === alert.id && entry.destination === destination, + ), + ) + .map((destination): Delivery => ({ + destination, + alert, + attempts: 0, + nextAttemptAt: nowMs, + createdAt: nowMs, + })); + return { + state: { + ...state, + degraded: { ...Object.fromEntries(kept), [alert.threadId]: next }, + outbox: [...pruneOutbox(state.outbox, nowMs), ...added].slice(-MAX_OUTBOX), + }, + enqueued: added.length > 0, + }; +} + +const pruneOutbox = (outbox: ReadonlyArray, nowMs: number) => + outbox.filter((entry) => nowMs - entry.createdAt < OUTBOX_TTL_MS); + +const sameDelivery = (left: Delivery, right: Delivery) => + left.alert.id === right.alert.id && left.destination === right.destination; + +/** The state after one delivery attempt: a delivered entry leaves, a failed one waits longer. */ +export function afterDeliveryAttempt( + state: ModelAlertState, + entry: Delivery, + delivered: boolean, + nowMs: number, +): ModelAlertState { + return { + ...state, + outbox: pruneOutbox(state.outbox, nowMs).flatMap((candidate) => { + if (!sameDelivery(candidate, entry)) return [candidate]; + if (delivered) return []; + const attempts = candidate.attempts + 1; + return [ + { + ...candidate, + attempts, + nextAttemptAt: nowMs + Math.min(RETRY_BASE_MS * 2 ** (attempts - 1), RETRY_MAX_MS), + }, + ]; + }), + }; +} + +const REASON_TEXT: Record string> = { + model_substituted: (alert) => + `it ran ${alert.executedModel ?? "another model"} instead of ${alert.selectedModel}`, + provider_fallback: (alert) => + `a provider safeguard switched it to ${alert.fallbackModels.join(", ")}`, + usage_limit: () => "it hit a provider usage limit", + withheld_task: () => "its final message withholds the task", +}; + +function modelAlertMessage(alert: ModelAlert): string { + const why = alert.reasons.map((reason) => REASON_TEXT[reason](alert)).join("; "); + return [ + `Model alert from ${alert.environmentLabel} (environment ${alert.environmentId}): ${alert.threadLink} ${why}.`, + `Selected model: ${alert.selectedModel}. Executed model: ${alert.executedModel ?? "not reported"}. Provider: ${alert.providerInstanceId}. Run ${alert.runId} ended ${alert.runStatus}. Reasons: ${alert.reasons.join(", ")}.`, + "Fix it: switch the thread's model or provider, or replace it. That thread sends no further alert until the problem changes.", + ].join("\n\n"); +} + +const ModelAlertStateJson = Schema.fromJsonString(ModelAlertState); +const decodeState = Schema.decodeUnknownOption(ModelAlertStateJson); +const encodeState = Schema.encodeEffect(ModelAlertStateJson); + +export class ModelAlertReactor extends Context.Service< + ModelAlertReactor, + { + readonly start: () => Effect.Effect; + /** Judges a domain event; a finished run may queue an alert. */ + readonly handle: (event: OrchestrationV2DomainEvent) => Effect.Effect; + /** Attempts every queued delivery that is due. */ + readonly flush: Effect.Effect; + } +>()("t3/orchestration-v2/ModelAlertReactor") {} + +const TERMINAL_RUN_STATUSES: ReadonlyArray = [ + "completed", + "failed", + "interrupted", +]; + +export const make = Effect.gen(function* () { + const engine = yield* Orchestrator.OrchestratorV2; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const settings = yield* ServerSettings.ServerSettingsService; + const environment = yield* ServerEnvironment.ServerEnvironment; + const peers = yield* PeerMcpClient.PeerMcpClient; + const crypto = yield* Digest.Digest; + const http = (yield* HttpClient.HttpClient).pipe(HttpClient.filterStatusOk); + const config = yield* ServerConfig.ServerConfig; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const filePath = path.join(config.stateDir, "model-alerts.json"); + + const loaded = yield* fs.readFileString(filePath).pipe( + Effect.map((text) => decodeState(text)), + Effect.map((state) => (state._tag === "Some" ? state.value : EMPTY_MODEL_ALERT_STATE)), + Effect.orElseSucceed(() => EMPTY_MODEL_ALERT_STATE), + ); + const state = yield* Ref.make(loaded); + const lock = yield* Semaphore.make(1); + const wake = yield* Queue.dropping(1); + + const update = (change: (current: ModelAlertState) => readonly [A, ModelAlertState]) => + lock.withPermits(1)( + Effect.gen(function* () { + const current = yield* Ref.get(state); + const [result, next] = change(current); + if (next === current) return result; + yield* Ref.set(state, next); + yield* encodeState(next).pipe( + Effect.flatMap((contents) => writeFileStringAtomically({ filePath, contents })), + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + Effect.catchCause((cause) => + Effect.logWarning("saving model alerts failed", { filePath, cause }), + ), + ); + return result; + }), + ); + + const deliverToWatcher = Effect.fnUntraced(function* ( + alert: ModelAlert, + watcher: ModelAlertWatcher, + here: EnvironmentId, + ) { + const message = modelAlertMessage(alert); + // Run ids can be long; a retry key must stay under the 256 characters the peer accepts. + const identity = `${alert.id}\n${watcher.environmentId}\n${watcher.threadId}`; + const digest = yield* crypto + .digest(Digest.sha256, new TextEncoder().encode(identity)) + .pipe(Effect.orDie); + const key = `model-alert:${Hex.encode(digest)}`; + if (watcher.environmentId === here) { + yield* engine.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(`server:${key}`), + threadId: watcher.threadId, + messageId: MessageId.make(`message:${key}`), + text: message, + attachments: [], + dispatchMode: { type: "queue_after_active" }, + createdBy: "agent", + creationSource: "server", + }); + return; + } + // The watcher's machine keeps one retry key per alert, so a retry after a lost answer sends once. + yield* peers.call({ + environmentId: watcher.environmentId, + tool: "t3_thread_send", + arguments: { threadId: watcher.threadId, message, mode: "queue", clientRequestId: key }, + limits: { runtimeMode: "full-access", interactionMode: "default" }, + success: OrchestratorMcpThreadSendResult, + }); + }); + + const deliver = Effect.fnUntraced(function* (entry: Delivery) { + const current = yield* settings.getSettings; + if (entry.destination === "webhook") { + const url = current.modelAlertWebhookUrl.trim(); + if (url === "") return; + // A webhook URL can embed a credential, so its failure names only the host. + const host = URL.canParse(url) ? new URL(url).host : "an unparsable URL"; + yield* http + .execute(HttpClientRequest.post(url).pipe(HttpClientRequest.bodyJsonUnsafe(entry.alert))) + .pipe(Effect.catchCause(() => Effect.fail(`model alert webhook POST to ${host} failed`))); + return; + } + const watcher = current.modelAlertWatcher; + if (watcher === null) return; + yield* deliverToWatcher(entry.alert, watcher, yield* environment.getEnvironmentId); + }); + + const flush: ModelAlertReactor["Service"]["flush"] = Effect.gen(function* () { + const nowMs = yield* Clock.currentTimeMillis; + const due = (yield* Ref.get(state)).outbox.filter((entry) => entry.nextAttemptAt <= nowMs); + yield* Effect.forEach( + due, + (entry) => + Effect.gen(function* () { + const delivered = yield* deliver(entry).pipe( + Effect.timeout("30 seconds"), + Effect.as(true), + Effect.catchCause((cause) => + Effect.logWarning("delivering a model alert failed; it will be retried", { + alertId: entry.alert.id, + destination: entry.destination, + attempts: entry.attempts + 1, + cause, + }).pipe(Effect.as(false)), + ), + ); + const attemptedAt = yield* Clock.currentTimeMillis; + yield* update((current) => [ + undefined, + afterDeliveryAttempt(current, entry, delivered, attemptedAt), + ]); + }), + { concurrency: 4, discard: true }, + ); + }); + + const handle: ModelAlertReactor["Service"]["handle"] = Effect.fnUntraced( + function* (event) { + if (event.type !== "run.updated" || !TERMINAL_RUN_STATUSES.includes(event.payload.status)) { + return; + } + const run = event.payload; + const records = yield* projections.getThreadRecords( + event.threadId, + ["providerTurns", "turnItems"], + { + providerTurnAttemptIds: run.activeAttemptId === null ? [] : [run.activeAttemptId], + turnItemRunIds: [run.id], + turnItemTypes: ["system_notice", "error"], + }, + ); + const here = yield* environment.getEnvironmentId; + const judged = modelAlertFor({ + environmentId: here, + environmentLabel: "", + thread: records.thread, + run, + projection: records, + occurredAt: DateTime.formatIso(yield* DateTime.now), + }); + if (judged.alert === undefined && !judged.healthy) return; + const outcome = + judged.alert === undefined + ? judged + : { + ...judged, + alert: { + ...judged.alert, + environmentLabel: (yield* environment.getDescriptor).label, + }, + }; + const current = yield* settings.getSettings; + const destinations = modelAlertDestinations({ + watcher: current.modelAlertWatcher, + webhookUrl: current.modelAlertWebhookUrl, + here, + lineage: records.thread.lineage, + }); + const nowMs = yield* Clock.currentTimeMillis; + const enqueued = yield* update((current) => { + const result = recordModelAlert( + current, + { ...outcome, threadId: event.threadId }, + destinations, + nowMs, + ); + return [result.enqueued, result.state]; + }); + if (enqueued) yield* Queue.offer(wake, undefined); + }, + (effect, event) => + effect.pipe( + Effect.catchCause((cause) => + Effect.logWarning("judging a run for a model alert failed", { + threadId: event.threadId, + cause, + }), + ), + ), + ); + + const start: ModelAlertReactor["Service"]["start"] = Effect.fn("ModelAlertReactor.start")( + function* () { + yield* forkParked( + Stream.runForEach(engine.streamDomainEvents, handle).pipe( + Effect.tapCause((cause) => + Effect.logWarning("model alert event stream failed; resubscribing", { cause }), + ), + Effect.retry(Schedule.spaced("5 seconds")), + ), + ); + // Retries wait out a sleeping or restarting watcher machine; a new alert goes at once. + yield* forkParked( + Effect.forever( + Queue.take(wake).pipe( + Effect.timeoutOption("30 seconds"), + Effect.andThen(flush), + Effect.catchCause((cause) => Effect.logWarning("model alert flush failed", { cause })), + ), + ), + ); + }, + ); + + return ModelAlertReactor.of({ start, handle, flush }); +}); + +export const layer = Layer.effect(ModelAlertReactor, make); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index efc7b50c8ee5..e209bbc5fa8d 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -8,6 +8,7 @@ import * as StorageCleanup from "./storageCleanup.ts"; import * as PullRequestSyncReactor from "./orchestration-v2/PullRequestSyncReactor.ts"; import * as PullRequestWatchReactor from "./orchestration-v2/PullRequestWatchReactor.ts"; import * as DelegatedTaskQuestionReactor from "./orchestration-v2/DelegatedTaskQuestionReactor.ts"; +import * as ModelAlertReactor from "./orchestration-v2/ModelAlertReactor.ts"; // Effect wraps this raw Node HTTP server after its startup and write-error guards attach. // @effect-diagnostics nodeBuiltinImport:off import * as NodeHttp from "node:http"; @@ -862,12 +863,17 @@ const layerMakeRoutes = Layer.mergeAll( Layer.effectDiscard( RemoteDelegation.RemoteDelegation.pipe(Effect.flatMap((remote) => remote.start())), ), + // Tells the watcher thread, here or on a linked machine, when a run here ran a fallback model. + Layer.effectDiscard( + ModelAlertReactor.ModelAlertReactor.pipe(Effect.flatMap((reactor) => reactor.start())), + ), ).pipe( Layer.provide( PullRequestReviewGuideService.layer.pipe(Layer.provide(PullRequestReviewGuides.layer)), ), // delegate_task to a linked environment, from /mcp and the follower above. Layer.provide(RemoteDelegation.layer.pipe(Layer.provide(ProjectionStoreV2.layer))), + Layer.provide(ModelAlertReactor.layer.pipe(Layer.provide(ProjectionStoreV2.layer))), // Links agents ask for: shown in a thread from /mcp, answered over WebSocket. Layer.provide(PeerLinkRequests.layer), // Share the service so HTTP/WS clients single-flight reads and invalidate each other's cached patches. diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 7450fd6202d0..099abf6484ea 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -1161,6 +1161,19 @@ export const BackgroundActivitySettings = Schema.Struct({ }).pipe(Schema.withDecodingDefault(Effect.succeed({}))); export type BackgroundActivitySettings = typeof BackgroundActivitySettings.Type; +/** A thread on some environment that hears every model alert from this one. */ +export const ModelAlertWatcher = Schema.Struct({ + environmentId: EnvironmentId, + threadId: ThreadId, +}); +export type ModelAlertWatcher = typeof ModelAlertWatcher.Type; + +// Closura: Lead's thread on the laptop hears every machine's alerts. +const DEFAULT_MODEL_ALERT_WATCHER: ModelAlertWatcher = { + environmentId: EnvironmentId.make("2f360b2e-0386-498a-b9cd-62999f3525ba"), + threadId: ThreadId.make("0ff5603d-8291-4f56-adaf-5132c62016cf"), +}; + export const ResponseStreamingMode = Schema.Literals(["turn", "paragraph"]); export type ResponseStreamingMode = typeof ResponseStreamingMode.Type; @@ -1545,6 +1558,15 @@ export const ServerSettings = Schema.Struct({ automaticPlacementByDefault: Schema.Boolean.pipe( Schema.withDecodingDefault(Effect.succeed(true)), ), + /** + * Where a run that ran another model than selected, hit a provider safeguard fallback or a usage + * limit is reported, once per thread until the problem changes. Null reports to no thread. + */ + modelAlertWatcher: Schema.NullOr(ModelAlertWatcher).pipe( + Schema.withDecodingDefault(Effect.succeed(DEFAULT_MODEL_ALERT_WATCHER)), + ), + /** Also POSTs each model alert as JSON here. Empty posts nothing. */ + modelAlertWebhookUrl: Schema.String.pipe(Schema.withDecodingDefault(Effect.succeed(""))), /** Beta: Claude and Codex predict the next prompt after completed turns. Codex follows a change at once; open Claude threads after they restart. */ promptSuggestionsEnabled: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), sidebarAutoSettleKeepPullRequests: Schema.Boolean.pipe( @@ -1989,6 +2011,8 @@ export const ServerSettingsPatch = Schema.Struct({ automaticPlacementExcludedEnvironmentIds: Schema.optionalKey(Schema.Array(EnvironmentId)), machinePlacementRole: Schema.optionalKey(MachinePlacementRole), automaticPlacementByDefault: Schema.optionalKey(Schema.Boolean), + modelAlertWatcher: Schema.optionalKey(Schema.NullOr(ModelAlertWatcher)), + modelAlertWebhookUrl: Schema.optionalKey(TrimmedString), promptSuggestionsEnabled: Schema.optionalKey(Schema.Boolean), backgroundActivity: Schema.optionalKey( Schema.Struct({ diff --git a/packages/provider-core/src/server/subagentProjection.ts b/packages/provider-core/src/server/subagentProjection.ts index 9df8fa8273fa..64f05ee7de6a 100644 --- a/packages/provider-core/src/server/subagentProjection.ts +++ b/packages/provider-core/src/server/subagentProjection.ts @@ -258,6 +258,33 @@ export function workerAlert(input: { }; } +/** What a run's provider reported about its model: what ran, safeguard fallbacks, a usage limit. */ +export function runModelSignals( + projection: Pick, + run: OrchestrationV2Run, +) { + const executedModel = latestProviderTurnForAttempt( + projection.providerTurns, + run.activeAttemptId, + )?.model; + const failure = + run.status === "failed" + ? projection.turnItems + .filter((item) => item.runId === run.id && item.type === "error") + .toSorted((left, right) => right.ordinal - left.ordinal)[0] + : undefined; + return { + requestedModel: executedModel === undefined ? undefined : run.modelSelection.model, + executedModel, + fallbackModels: projection.turnItems.flatMap((item) => + item.runId === run.id && item.type === "system_notice" && item.fallbackModel !== undefined + ? [item.fallbackModel] + : [], + ), + usageLimited: failure?.type === "error" && failure.failure.class === "usage_limit", + }; +} + /** Run signal of the notice that carries a provider's stop reason on a completed run. */ export const PROVIDER_STOP_REASON_SIGNAL = "provider-stop-reason"; @@ -327,19 +354,8 @@ export function subagentResultForRun( (run.status === "completed" ? withStopNotice(projection, run, "Child task completed without an assistant result.") : `Child task ended with status ${run.status}.`); - const executedModel = latestProviderTurnForAttempt( - projection.providerTurns, - run.activeAttemptId, - )?.model; const alert = workerAlert({ - requestedModel: executedModel === undefined ? undefined : run.modelSelection.model, - executedModel, - fallbackModels: projection.turnItems.flatMap((item) => - item.runId === run.id && item.type === "system_notice" && item.fallbackModel !== undefined - ? [item.fallbackModel] - : [], - ), - usageLimited: failure?.type === "error" && failure.failure.class === "usage_limit", + ...runModelSignals(projection, run), finalText: message?.text ?? turnItem?.text, }); return {