diff --git a/apps/server/integration/OrchestrationEngineHarness.integration.ts b/apps/server/integration/OrchestrationEngineHarness.integration.ts index 751c71cabebcc..4c48ec4e7e736 100644 --- a/apps/server/integration/OrchestrationEngineHarness.integration.ts +++ b/apps/server/integration/OrchestrationEngineHarness.integration.ts @@ -68,6 +68,7 @@ import { } from "../src/orchestration/Services/OrchestrationEngine.ts"; import { ThreadDeletionReactor } from "../src/orchestration/Services/ThreadDeletionReactor.ts"; import * as ThreadSettlementReactor from "../src/orchestration/ThreadSettlementReactor.ts"; +import * as SessionSpendGuardReactor from "../src/orchestration/SessionSpendGuardReactor.ts"; import * as PullRequestSyncReactor from "../src/orchestration/PullRequestSyncReactor.ts"; import * as ThreadPullRequestReactor from "../src/orchestration/ThreadPullRequestReactor.ts"; import { OrchestrationReactor } from "../src/orchestration/Services/OrchestrationReactor.ts"; @@ -421,6 +422,12 @@ export const makeOrchestrationIntegrationHarness = ( drain: Effect.void, }), ), + Layer.provideMerge( + Layer.succeed(SessionSpendGuardReactor.SessionSpendGuardReactor, { + start: () => Effect.void, + drain: Effect.void, + }), + ), Layer.provideMerge( Layer.succeed(PullRequestSyncReactor.PullRequestSyncReactor, { start: () => Effect.void, diff --git a/apps/server/src/orchestration/Layers/OrchestrationReactor.test.ts b/apps/server/src/orchestration/Layers/OrchestrationReactor.test.ts index d25c442ac8b23..6b06359e47ad0 100644 --- a/apps/server/src/orchestration/Layers/OrchestrationReactor.test.ts +++ b/apps/server/src/orchestration/Layers/OrchestrationReactor.test.ts @@ -10,6 +10,7 @@ import { ProviderCommandReactor } from "../Services/ProviderCommandReactor.ts"; import { ProviderRuntimeIngestionService } from "../Services/ProviderRuntimeIngestion.ts"; import { ThreadDeletionReactor } from "../Services/ThreadDeletionReactor.ts"; import * as ThreadSettlementReactor from "../ThreadSettlementReactor.ts"; +import * as SessionSpendGuardReactor from "../SessionSpendGuardReactor.ts"; import * as PullRequestSyncReactor from "../PullRequestSyncReactor.ts"; import * as ThreadPullRequestReactor from "../ThreadPullRequestReactor.ts"; import { OrchestrationReactor } from "../Services/OrchestrationReactor.ts"; @@ -95,6 +96,15 @@ describe("OrchestrationReactor", () => { drain: Effect.void, }), ), + Layer.provideMerge( + Layer.succeed(SessionSpendGuardReactor.SessionSpendGuardReactor, { + start: () => { + started.push("session-spend-guard-reactor"); + return Effect.void; + }, + drain: Effect.void, + }), + ), Layer.provideMerge( Layer.succeed(PullRequestSyncReactor.PullRequestSyncReactor, { start: () => { @@ -128,6 +138,7 @@ describe("OrchestrationReactor", () => { "thread-deletion-reactor", "thread-pull-request-reactor", "thread-settlement-reactor", + "session-spend-guard-reactor", "pull-request-sync-reactor", "agent-awareness-relay", "storage-cleanup", diff --git a/apps/server/src/orchestration/Layers/OrchestrationReactor.ts b/apps/server/src/orchestration/Layers/OrchestrationReactor.ts index da5cc337e4208..18845b485b7f3 100644 --- a/apps/server/src/orchestration/Layers/OrchestrationReactor.ts +++ b/apps/server/src/orchestration/Layers/OrchestrationReactor.ts @@ -10,6 +10,7 @@ import { ProviderCommandReactor } from "../Services/ProviderCommandReactor.ts"; import { ProviderRuntimeIngestionService } from "../Services/ProviderRuntimeIngestion.ts"; import { ThreadDeletionReactor } from "../Services/ThreadDeletionReactor.ts"; import * as ThreadSettlementReactor from "../ThreadSettlementReactor.ts"; +import * as SessionSpendGuardReactor from "../SessionSpendGuardReactor.ts"; import * as PullRequestSyncReactor from "../PullRequestSyncReactor.ts"; import * as ThreadPullRequestReactor from "../ThreadPullRequestReactor.ts"; import * as AgentAwarenessRelay from "../../relay/AgentAwarenessRelay.ts"; @@ -21,6 +22,7 @@ export const makeOrchestrationReactor = Effect.gen(function* () { const checkpointReactor = yield* CheckpointReactor; const threadDeletionReactor = yield* ThreadDeletionReactor; const threadSettlementReactor = yield* ThreadSettlementReactor.ThreadSettlementReactor; + const sessionSpendGuardReactor = yield* SessionSpendGuardReactor.SessionSpendGuardReactor; const pullRequestSyncReactor = yield* PullRequestSyncReactor.PullRequestSyncReactor; const threadPullRequestReactor = yield* ThreadPullRequestReactor.ThreadPullRequestReactor; const agentAwarenessRelay = yield* AgentAwarenessRelay.AgentAwarenessRelay; @@ -33,6 +35,7 @@ export const makeOrchestrationReactor = Effect.gen(function* () { yield* threadDeletionReactor.start(); yield* threadPullRequestReactor.start(); yield* threadSettlementReactor.start(); + yield* sessionSpendGuardReactor.start(); yield* pullRequestSyncReactor.start(); yield* agentAwarenessRelay.start(); yield* storageCleanup.start(); diff --git a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts index 0c80c49b15508..e3a1a1d454560 100644 --- a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts +++ b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts @@ -1469,6 +1469,31 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { `, }); + const listThreadActivityRowsByThreadAndKindsUnbounded = SqlSchema.findAll({ + Request: ThreadActivityKindsLookupInput, + Result: ProjectionThreadActivityDbRowSchema, + execute: ({ threadId, activityKinds }) => + sql` + SELECT + activity_id AS "activityId", + thread_id AS "threadId", + turn_id AS "turnId", + tone, + kind, + summary, + payload_json AS "payload", + sequence, + created_at AS "createdAt" + FROM projection_thread_activities + WHERE thread_id = ${threadId} + AND ${sql.in("kind", activityKinds)} + ORDER BY + sequence ASC, + created_at ASC, + activity_id ASC + `, + }); + const getUserInputActivityRow = SqlSchema.findOneOption({ Request: Schema.Struct({ threadId: ThreadId, requestId: ApprovalRequestId }), Result: ProjectionThreadActivityDbRowSchema, @@ -3442,7 +3467,9 @@ pending_approval_requests AS ( : listThreadActivityRowsByThreadWindow({ threadId, ...bounds }) : activityRead.query.activityKinds.length === 0 ? Effect.succeed([]) - : listThreadActivityRowsByThreadAndKinds({ + : (activityRead.query.unboundedActivities === true + ? listThreadActivityRowsByThreadAndKindsUnbounded + : listThreadActivityRowsByThreadAndKinds)({ threadId, activityKinds: activityRead.query.activityKinds, }) diff --git a/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts b/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts index 49b8a4126f68a..ad193fccedc56 100644 --- a/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts +++ b/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts @@ -516,7 +516,17 @@ describe("ProviderCommandReactor", () => { generateThreadTitle, }), ), - Layer.provideMerge(ServerSettingsService.layerTest(input?.serverSettingsOverrides ?? {})), + Layer.provideMerge( + ServerSettingsService.layerTest({ + ...input?.serverSettingsOverrides, + // These fixtures are imported threads with no scope, which the guard + // refuses. Leave it off unless a test opts back into enforcement. + sessionSpendGuard: { + mode: "off", + ...input?.serverSettingsOverrides?.sessionSpendGuard, + }, + }), + ), Layer.provideMerge(serverEnvironmentIdentityTestLayer), Layer.provideMerge(SqlitePersistenceMemory), Layer.provideMerge(ServerConfig.layerTest(process.cwd(), baseDir)), @@ -878,6 +888,309 @@ describe("ProviderCommandReactor", () => { }), ); + effectIt.effect("refuses an automated turn on an imported thread with no execution scope", () => + Effect.gen(function* () { + const harness = yield* Effect.promise(() => + createHarness({ + serverSettingsOverrides: { sessionSpendGuard: { mode: "enforce" } }, + }), + ); + const threadId = ThreadId.make("thread-imported"); + const createdAt = "2026-01-01T00:00:00.000Z"; + yield* harness.engine.dispatch({ + type: "thread.create", + commandId: CommandId.make("cmd-imported-thread"), + threadId, + projectId: asProjectId("project-1"), + title: "Imported session", + modelSelection: { + instanceId: ProviderInstanceId.make("codex"), + model: "gpt-5-codex", + }, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "approval-required", + branch: null, + worktreePath: null, + historyImport: true, + createdAt, + }); + yield* harness.engine.dispatch({ + type: "thread.turn.start", + commandId: CommandId.make("cmd-imported-turn"), + threadId, + message: { + messageId: MessageId.make("message-imported-turn"), + role: "user", + text: "continue the imported session", + attachments: [], + }, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "approval-required", + createdAt, + }); + yield* Effect.promise(() => harness.drain()); + const thread = (yield* Effect.promise(() => harness.readModel())).threads.find( + (entry) => entry.id === threadId, + ); + expect(thread?.activities).toContainEqual( + expect.objectContaining({ + kind: "provider.turn.start.failed", + summary: "Turn refused by the session spend guard", + }), + ); + expect(harness.sendTurn).not.toHaveBeenCalled(); + expect(harness.startSession).not.toHaveBeenCalled(); + }), + ); + + effectIt.effect( + "refuses the next user turn at the session ceiling even after a linked pull request", + () => + Effect.gen(function* () { + const harness = yield* Effect.promise(() => + createHarness({ + serverSettingsOverrides: { + sessionSpendGuard: { + mode: "enforce", + sessionCeilingUsd: 1, + idleStopUsd: 50, + idleFlagUsd: 25, + }, + usagePriceOverrides: { + "gpt-5-codex": { + inputCostPerMillionTokens: 1_000_000, + outputCostPerMillionTokens: 0, + cacheReadCostPerMillionTokens: 0, + }, + }, + }, + }), + ); + const threadId = ThreadId.make("thread-crown"); + const modelSelection = { + instanceId: ProviderInstanceId.make("codex"), + model: "gpt-5-codex", + }; + NodeFS.mkdirSync("/tmp/provider-project", { recursive: true }); + const workspacePath = NodeFS.mkdtempSync( + NodePath.join(NodeOS.tmpdir(), "crown-coordinator-"), + ); + yield* harness.engine.dispatch({ + type: "thread.create", + commandId: CommandId.make("cmd-crown-thread"), + threadId, + projectId: asProjectId("project-1"), + title: "Manager work", + modelSelection, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "approval-required", + branch: null, + worktreePath: workspacePath, + executionScope: "coordinator", + createdAt: "2099-01-01T00:00:00.000Z", + }); + yield* harness.engine.dispatch({ + type: "thread.pull-request.link", + commandId: CommandId.make("cmd-crown-pr"), + threadId, + host: "github.com", + repository: "nullStack65/t3code", + number: 1, + url: "https://github.com/nullStack65/t3code/pull/1", + source: "created", + }); + const nudges = [ + ["message-crown-status", "Status? Move all work forward.", "2099-01-01T00:00:01.000Z"], + ["message-crown-callback", "Here is another handoff.", "2099-01-01T00:00:02.000Z"], + ] as const; + for (const [messageId, text, createdAt] of nudges) { + yield* harness.engine.dispatch({ + type: "thread.turn.start", + commandId: CommandId.make(`cmd-${messageId}`), + threadId, + message: { + messageId: MessageId.make(messageId), + role: "user", + text, + attachments: [], + }, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "approval-required", + createdAt, + }); + yield* Effect.promise(() => harness.drain()); + } + expect(harness.sendTurn).toHaveBeenCalledTimes(2); + for (const [index, createdAt] of [ + ["1", "2099-01-01T00:00:03.000Z"], + ["2", "2099-01-01T00:00:04.000Z"], + ] as const) { + yield* harness.engine.dispatch({ + type: "thread.activity.append", + commandId: CommandId.make(`cmd-crown-context-${index}`), + threadId, + activity: { + id: EventId.make(`activity-crown-context-${index}`), + tone: "info", + kind: "context-window.updated", + summary: "Context window updated", + payload: { + totalProcessedTokens: Number(index), + lastInputTokens: 1, + lastCachedInputTokens: 0, + lastOutputTokens: 0, + }, + turnId: null, + createdAt, + }, + createdAt, + }); + } + yield* harness.engine.dispatch({ + type: "thread.turn.start", + commandId: CommandId.make("cmd-crown-after-ceiling"), + threadId, + message: { + messageId: MessageId.make("message-crown-after-ceiling"), + role: "user", + text: "Status?", + attachments: [], + }, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "approval-required", + createdAt: "2099-01-01T00:00:05.000Z", + }); + yield* Effect.promise(() => harness.drain()); + const thread = (yield* Effect.promise(() => harness.readModel())).threads.find( + (entry) => entry.id === threadId, + ); + expect(thread?.pullRequests).toEqual( + expect.arrayContaining([expect.objectContaining({ number: 1, source: "created" })]), + ); + const refusal = thread?.activities.find( + (activity) => activity.kind === "provider.turn.start.failed", + ); + expect(refusal).toMatchObject({ + summary: "Turn refused by the session spend guard", + }); + expect(String(refusal?.payload && (refusal.payload as { detail?: string }).detail)).toContain( + "session ceiling", + ); + expect(String(refusal?.payload && (refusal.payload as { detail?: string }).detail)).not.toContain( + "idle limit", + ); + expect(harness.sendTurn).toHaveBeenCalledTimes(2); + NodeFS.rmSync(workspacePath, { recursive: true, force: true }); + }), + ); + + effectIt.effect( + "counts context-window rows older than the thread detail cap toward the session ceiling", + () => + Effect.gen(function* () { + const harness = yield* Effect.promise(() => + createHarness({ + serverSettingsOverrides: { + sessionSpendGuard: { + mode: "enforce", + sessionCeilingUsd: 1, + idleStopUsd: 100, + idleFlagUsd: 50, + }, + usagePriceOverrides: { + "gpt-5-codex": { + inputCostPerMillionTokens: 1_000_000, + outputCostPerMillionTokens: 0, + cacheReadCostPerMillionTokens: 0, + }, + }, + }, + }), + ); + const threadId = ThreadId.make("thread-cap"); + NodeFS.mkdirSync("/tmp/provider-project", { recursive: true }); + const workspacePath = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "crown-cap-")); + yield* Effect.gen(function* () { + yield* harness.engine.dispatch({ + type: "thread.create", + commandId: CommandId.make("cmd-cap-thread"), + threadId, + projectId: asProjectId("project-1"), + title: "Manager work", + modelSelection: { + instanceId: ProviderInstanceId.make("codex"), + model: "gpt-5-codex", + }, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "approval-required", + branch: null, + worktreePath: workspacePath, + executionScope: "coordinator", + createdAt: "2099-01-01T00:00:00.000Z", + }); + yield* Effect.promise(() => + harness.runEffect( + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + for (let sequence = 1; sequence <= 501; sequence += 1) { + const costly = sequence === 1; + yield* sql` + INSERT INTO projection_thread_activities ( + activity_id, thread_id, turn_id, tone, kind, summary, payload_json, sequence, created_at + ) VALUES ( + ${`activity-cap-${sequence}`}, + ${threadId}, + NULL, + 'info', + 'context-window.updated', + 'Context window updated', + ${costly + ? '{"totalProcessedTokens":1,"lastInputTokens":1,"lastOutputTokens":0}' + : '{"totalProcessedTokens":0,"lastInputTokens":0,"lastOutputTokens":0}'}, + ${sequence}, + '2099-01-01T00:00:00.000Z' + ) + `; + } + }), + ), + ); + yield* harness.engine.dispatch({ + type: "thread.turn.start", + commandId: CommandId.make("cmd-cap-turn"), + threadId, + message: { + messageId: MessageId.make("message-cap-turn"), + role: "user", + text: "Status?", + attachments: [], + }, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "approval-required", + createdAt: "2099-01-01T00:01:00.000Z", + }); + yield* Effect.promise(() => harness.drain()); + const thread = (yield* Effect.promise(() => harness.readModel())).threads.find( + (entry) => entry.id === threadId, + ); + const refusal = thread?.activities.find( + (activity) => activity.kind === "provider.turn.start.failed", + ); + expect(refusal).toMatchObject({ + summary: "Turn refused by the session spend guard", + }); + expect( + String(refusal?.payload && (refusal.payload as { detail?: string }).detail), + ).toContain("session ceiling"); + expect(harness.sendTurn).not.toHaveBeenCalled(); + }).pipe( + Effect.ensuring( + Effect.sync(() => NodeFS.rmSync(workspacePath, { recursive: true, force: true })), + ), + ); + }), + ); + it("reacts to thread.turn.start by ensuring session and sending provider turn", async () => { const harness = await createHarness(); const now = "2026-01-01T00:00:00.000Z"; diff --git a/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts b/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts index 7d8ec755f2833..9d0f33c9f29ca 100644 --- a/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts +++ b/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts @@ -66,6 +66,13 @@ import { type ThreadTitleMessage, } from "../../textGeneration/ThreadTitleContext.ts"; import { canReplaceThreadTitle, DEFAULT_THREAD_TITLE } from "../threadTitles.ts"; +import { + checkLaunch, + describeLaunchRefusal, + describeSpendStop, + evaluateThreadSpend, + SPEND_GUARD_ACTIVITY_KINDS, +} from "../SessionSpendGuard.ts"; import { resolveSourceControlWriterModelSelection, ServerSettingsService, @@ -356,17 +363,20 @@ const make = Effect.gen(function* () { resumedTurnStarts.set(commandId, { event, queued, sent }); const { messageId, ...request } = event.payload; yield* orchestrationEngine - .dispatch({ - type: "thread.turn.start", - commandId, - ...request, - message: { - messageId, - role: "user", - text: turnStart.value.message.text, - attachments: turnStart.value.message.attachments ?? [], + .dispatch( + { + type: "thread.turn.start", + commandId, + ...request, + message: { + messageId, + role: "user", + text: turnStart.value.message.text, + attachments: turnStart.value.message.attachments ?? [], + }, }, - }) + event.metadata.origin === undefined ? undefined : { origin: event.metadata.origin }, + ) .pipe( Effect.onError(() => Effect.sync(() => { @@ -1379,6 +1389,37 @@ const make = Effect.gen(function* () { ); } + const guardSettings = yield* serverSettingsService.getSettings; + const launchRefusal = checkLaunch({ + title: thread.title, + titleSeed: event.payload.titleSeed, + executionScope: thread.executionScope, + hasOtherUserMessages, + clientSurface: event.metadata.origin?.surface, + mode: guardSettings.sessionSpendGuard.mode, + }); + if (launchRefusal !== null) { + return yield* appendTurnStartFailure( + "Turn refused by the session spend guard", + describeLaunchRefusal(launchRefusal), + ); + } + if (guardSettings.sessionSpendGuard.mode === "enforce") { + const guardThread = yield* projectionSnapshotQuery.getThreadDetailById(thread.id, { + activityKinds: SPEND_GUARD_ACTIVITY_KINDS, + unboundedActivities: true, + }); + const spend = Option.isSome(guardThread) + ? evaluateThreadSpend(guardThread.value, guardSettings) + : undefined; + if (spend?.level === "stop") { + return yield* appendTurnStartFailure( + "Turn refused by the session spend guard", + describeSpendStop(spend, guardSettings.sessionSpendGuard), + ); + } + } + const handleTurnStartFailure = (cause: Cause.Cause) => { if (Cause.hasInterruptsOnly(cause)) { return Effect.void; diff --git a/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts b/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts index eac3ede9c1eea..b5ef8c22338cd 100644 --- a/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts +++ b/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts @@ -71,6 +71,7 @@ export interface ProjectionThreadDetailQuery { * the activity query. Omit this option to preserve the full detail response. */ readonly activityKinds?: ReadonlyArray; + readonly unboundedActivities?: boolean; } /** diff --git a/apps/server/src/orchestration/SessionSpendGuard.test.ts b/apps/server/src/orchestration/SessionSpendGuard.test.ts new file mode 100644 index 0000000000000..2300fa3455312 --- /dev/null +++ b/apps/server/src/orchestration/SessionSpendGuard.test.ts @@ -0,0 +1,307 @@ +import { describe, expect, it } from "vite-plus/test"; +import { ProviderInstanceId, ServerSettings } from "@t3tools/contracts"; +import * as Schema from "effect/Schema"; + +import { DEFAULT_THREAD_TITLE } from "./threadTitles.ts"; +import { + checkLaunch, + describeLaunchRefusal, + describeSpendStop, + evaluateSpendGuard, + evaluateThreadSpend, + priceSnapshots, + rateForModel, + workSignalAts, + type SpendGuardActivity, + type SpendGuardInput, +} from "./SessionSpendGuard.ts"; + +const decodeSettings = Schema.decodeUnknownSync(ServerSettings); + +const rate = { + inputCostPerToken: 1, + outputCostPerToken: 2, + cacheReadCostPerToken: 0.1, + cacheCreationCostPerToken: 3, +}; + +const settings = { + ...decodeSettings({}).sessionSpendGuard, + idleFlagUsd: 3, + idleStopUsd: 5, + sessionCeilingUsd: 10, +}; + +const activity = ( + kind: string, + createdAt: string, + payload: SpendGuardActivity["payload"] = null, + sequence?: number, +): SpendGuardActivity => ({ kind, createdAt, payload, sequence }); + +const priced = (createdAt: string, usdTokens: number, sequence?: number): SpendGuardActivity => + activity( + "context-window.updated", + createdAt, + { totalProcessedTokens: usdTokens, lastInputTokens: usdTokens }, + sequence, + ); + +const evaluate = (overrides: Partial = {}) => + evaluateSpendGuard({ + activities: [], + workSignalAts: [], + rate, + settings, + ...overrides, + }); + +describe("priceSnapshots", () => { + it("prices the latest uncached input, cache read, and output", () => { + expect( + priceSnapshots( + [ + activity("context-window.updated", "2026-01-01T00:00:00.000Z", { + totalProcessedTokens: 100, + lastInputTokens: 10, + lastCachedInputTokens: 4, + lastOutputTokens: 2, + }), + ], + rate, + ), + ).toEqual([{ createdAt: "2026-01-01T00:00:00.000Z", usd: 10.4 }]); + }); + + it("prices a token delta at the cache-read rate and skips a repeated total", () => { + expect( + priceSnapshots( + [ + activity("context-window.updated", "2026-01-01T00:00:00.000Z", { + totalProcessedTokens: 100, + }), + activity("context-window.updated", "2026-01-01T00:01:00.000Z", { + totalProcessedTokens: 100, + lastInputTokens: 9, + }), + activity("context-window.updated", "2026-01-01T00:02:00.000Z", { + totalProcessedTokens: 150, + }), + activity("context-window.updated", "2026-01-01T00:03:00.000Z", { + totalProcessedTokens: 40, + }), + ], + rate, + ), + ).toEqual([ + { createdAt: "2026-01-01T00:00:00.000Z", usd: 10 }, + { createdAt: "2026-01-01T00:02:00.000Z", usd: 5 }, + { createdAt: "2026-01-01T00:03:00.000Z", usd: 4 }, + ]); + }); + + it("orders snapshots by sequence when both sides have one", () => { + expect( + priceSnapshots( + [ + activity( + "context-window.updated", + "2026-01-01T00:02:00.000Z", + { lastOutputTokens: 1 }, + 2, + ), + activity( + "context-window.updated", + "2026-01-01T00:03:00.000Z", + { lastOutputTokens: 1 }, + 1, + ), + ], + rate, + ).map((entry) => entry.createdAt), + ).toEqual(["2026-01-01T00:03:00.000Z", "2026-01-01T00:02:00.000Z"]); + }); +}); + +describe("evaluateSpendGuard", () => { + it("counts idle spend only after the latest pull request or file-changing checkpoint", () => { + const signals = workSignalAts({ + pullRequests: [ + { linkedAt: "2026-01-01T00:02:00.000Z", source: "manual" }, + { linkedAt: "2026-01-01T00:09:00.000Z", source: "stack-dismissed" }, + ], + checkpoints: [ + { completedAt: "2026-01-01T00:03:00.000Z", files: ["src/app.ts"] }, + { completedAt: "2026-01-01T00:08:00.000Z", files: [] }, + ], + }); + const result = evaluate({ + workSignalAts: signals, + activities: [ + priced("2026-01-01T00:01:00.000Z", 2), + priced("2026-01-01T00:02:30.000Z", 1), + priced("2026-01-01T00:04:00.000Z", 6), + ], + }); + expect(result.idleSince).toBe("2026-01-01T00:03:00.000Z"); + expect(result.sessionUsd).toBe(9); + expect(result.idleUsd).toBe(6); + expect(result.level).toBe("stop"); + expect(result.stopReason).toBe("idle"); + }); + + it("starts a new session after spend-guard.released", () => { + const result = evaluate({ + activities: [ + priced("2026-01-01T00:01:00.000Z", 9), + activity("spend-guard.released", "2026-01-01T00:02:00.000Z"), + priced("2026-01-01T00:03:00.000Z", 2), + ], + }); + expect(result.sessionUsd).toBe(2); + expect(result.idleUsd).toBe(2); + expect(result.level).toBe("ok"); + }); + + it("flags idle spend without stopping when mode is flag", () => { + const result = evaluate({ + settings: { ...settings, mode: "flag" }, + activities: [priced("2026-01-01T00:01:00.000Z", 6)], + }); + expect(result).toMatchObject({ level: "flag", stopReason: "idle", idleUsd: 6 }); + }); + + it("stops for the session ceiling ahead of the idle limit", () => { + const result = evaluate({ + activities: [priced("2026-01-01T00:01:00.000Z", 10)], + }); + expect(result).toMatchObject({ level: "stop", stopReason: "ceiling", sessionUsd: 10 }); + }); + + it("stops at the session ceiling when a recent pull request keeps idle spend under the idle limit", () => { + const result = evaluate({ + workSignalAts: ["2026-01-01T00:02:00.000Z"], + activities: [ + priced("2026-01-01T00:01:00.000Z", 6), + priced("2026-01-01T00:03:00.000Z", 4), + ], + }); + expect(result.idleUsd).toBe(4); + expect(result.sessionUsd).toBe(10); + expect(result.level).toBe("stop"); + expect(result.stopReason).toBe("ceiling"); + }); + + it("does not stop or flag when the guard is off", () => { + const result = evaluate({ + settings: { ...settings, mode: "off" }, + activities: [priced("2026-01-01T00:01:00.000Z", 20)], + }); + expect(result).toMatchObject({ level: "ok", stopReason: null }); + }); + + it("remembers a flag or stop only until the next idle boundary", () => { + const idle = evaluate({ + workSignalAts: ["2026-01-01T00:02:00.000Z"], + activities: [ + activity("spend-guard.flagged", "2026-01-01T00:01:00.000Z"), + activity("spend-guard.stopped", "2026-01-01T00:01:30.000Z", { reason: "idle" }), + activity("spend-guard.stopped", "2026-01-01T00:03:00.000Z", { reason: "ceiling" }), + activity("spend-guard.flagged", "2026-01-01T00:04:00.000Z"), + ], + }); + expect(idle.flaggedSinceIdle).toBe(true); + expect(idle.stoppedSinceIdle).toBe(false); + expect(idle.stoppedForCeiling).toBe(true); + expect(idle.flaggedForCeiling).toBe(false); + }); + + it("keeps a ceiling flag visible when an earlier idle flag is already recorded", () => { + const flagged = evaluate({ + settings: { ...settings, mode: "flag" }, + workSignalAts: ["2026-01-01T00:02:00.000Z"], + activities: [ + activity("spend-guard.flagged", "2026-01-01T00:03:00.000Z", { reason: "idle" }), + priced("2026-01-01T00:01:00.000Z", 6), + priced("2026-01-01T00:04:00.000Z", 4), + ], + }); + expect(flagged).toMatchObject({ + level: "flag", + stopReason: "ceiling", + flaggedSinceIdle: true, + flaggedForCeiling: false, + idleUsd: 4, + sessionUsd: 10, + }); + const recorded = evaluate({ + settings: { ...settings, mode: "flag" }, + activities: [ + priced("2026-01-01T00:01:00.000Z", 10), + activity("spend-guard.flagged", "2026-01-01T00:02:00.000Z", { reason: "ceiling" }), + ], + }); + expect(recorded.flaggedForCeiling).toBe(true); + expect(recorded.stopReason).toBe("ceiling"); + }); +}); + +describe("launch and stop copy", () => { + const base = { + title: DEFAULT_THREAD_TITLE, + titleSeed: undefined, + executionScope: "coding" as const, + hasOtherUserMessages: true, + clientSurface: undefined, + mode: "enforce" as const, + }; + + it("refuses an untitled or ownerless automated follow-up", () => { + expect(checkLaunch(base)).toBe("untitled"); + expect(checkLaunch({ ...base, executionScope: null, hasOtherUserMessages: false })).toBe( + "ownerless", + ); + expect(checkLaunch({ ...base, title: "Named thread" })).toBeNull(); + expect(checkLaunch({ ...base, titleSeed: "seed" })).toBeNull(); + expect(checkLaunch({ ...base, hasOtherUserMessages: false })).toBeNull(); + expect(checkLaunch({ ...base, clientSurface: "desktop" })).toBeNull(); + expect(checkLaunch({ ...base, clientSurface: "cli" })).toBe("untitled"); + expect(checkLaunch({ ...base, mode: "flag" })).toBeNull(); + }); + + it("describes how to continue", () => { + const evaluation = evaluate({ activities: [priced("2026-01-01T00:01:00.000Z", 6)] }); + expect(describeSpendStop(evaluation, settings)).toContain("$6.00"); + expect(describeSpendStop(evaluation, settings)).toContain("$5.00"); + expect(describeLaunchRefusal("untitled")).toContain("default title"); + expect(describeLaunchRefusal("ownerless")).toContain("execution scope"); + }); +}); + +describe("evaluateThreadSpend", () => { + it("uses the model override and otherwise the fallback rate", () => { + const decoded = decodeSettings({ + sessionSpendGuard: { idleStopUsd: 1, sessionCeilingUsd: 100 }, + usagePriceOverrides: { + "custom-model": { + inputCostPerMillionTokens: 1_000_000, + outputCostPerMillionTokens: 0, + }, + }, + }); + const thread = { + activities: [priced("2026-01-01T00:01:00.000Z", 2)], + pullRequests: [], + checkpoints: [], + modelSelection: { + instanceId: ProviderInstanceId.make("codex"), + model: "custom-model", + }, + }; + expect(evaluateThreadSpend(thread, decoded).sessionUsd).toBe(2); + expect( + rateForModel("missing", decoded.usagePriceOverrides, decoded.sessionSpendGuard.fallbackRate) + .inputCostPerToken, + ).toBe(10 / 1_000_000); + }); +}); diff --git a/apps/server/src/orchestration/SessionSpendGuard.ts b/apps/server/src/orchestration/SessionSpendGuard.ts new file mode 100644 index 0000000000000..8c40b62c554ee --- /dev/null +++ b/apps/server/src/orchestration/SessionSpendGuard.ts @@ -0,0 +1,255 @@ +import type { + ClientSurface, + OrchestrationCheckpointSummary, + OrchestrationThread, + OrchestrationThreadActivity, + ServerSettings, + SessionSpendGuardSettings, + ThreadExecutionScope, + ThreadPullRequestLink, + UsageModelPriceOverride, +} from "@t3tools/contracts"; +import * as Predicate from "effect/Predicate"; + +import { createOverrideRateTable, lookupRate, type ModelRate } from "../usage/usagePricing.ts"; +import { DEFAULT_THREAD_TITLE } from "./threadTitles.ts"; + +export const SPEND_GUARD_FLAGGED_KIND = "spend-guard.flagged"; +export const SPEND_GUARD_STOPPED_KIND = "spend-guard.stopped"; +export const SPEND_GUARD_RELEASED_KIND = "spend-guard.released"; +export const CONTEXT_WINDOW_KIND = "context-window.updated"; + +export const SPEND_GUARD_ACTIVITY_KINDS = [ + CONTEXT_WINDOW_KIND, + SPEND_GUARD_FLAGGED_KIND, + SPEND_GUARD_STOPPED_KIND, + SPEND_GUARD_RELEASED_KIND, +] as const; + +export type SpendGuardActivity = Pick< + OrchestrationThreadActivity, + "kind" | "payload" | "createdAt" | "sequence" +>; + +export type SpendGuardLevel = "ok" | "flag" | "stop"; +export type SpendGuardStopReason = "idle" | "ceiling"; + +export interface SpendGuardEvaluation { + readonly level: SpendGuardLevel; + readonly stopReason: SpendGuardStopReason | null; + readonly sessionUsd: number; + readonly idleUsd: number; + readonly idleSince: string | null; + readonly flaggedSinceIdle: boolean; + readonly flaggedForCeiling: boolean; + readonly stoppedSinceIdle: boolean; + readonly stoppedForCeiling: boolean; +} + +export interface SpendGuardInput { + readonly activities: ReadonlyArray; + readonly workSignalAts: ReadonlyArray; + readonly rate: ModelRate; + readonly settings: SessionSpendGuardSettings; +} + +export function rateForModel( + model: string, + overrides: Readonly>, + fallback: UsageModelPriceOverride, +): ModelRate { + return ( + lookupRate(createOverrideRateTable(overrides), model) ?? + createOverrideRateTable({ fallback }).get("fallback")! + ); +} + +export function workSignalAts(thread: { + readonly pullRequests: ReadonlyArray>; + readonly checkpoints: ReadonlyArray< + Pick + >; +}): ReadonlyArray { + return [ + ...thread.pullRequests + .filter((link) => link.source !== "stack-dismissed") + .map((link) => link.linkedAt), + ...thread.checkpoints + .filter((checkpoint) => checkpoint.files.length > 0) + .map((checkpoint) => checkpoint.completedAt), + ]; +} + +function count(record: Record, key: string): number | undefined { + const value = record[key]; + return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : undefined; +} + +function orderActivities( + activities: ReadonlyArray, +): ReadonlyArray { + return activities.toSorted((left, right) => + left.sequence !== undefined && right.sequence !== undefined + ? left.sequence - right.sequence + : left.createdAt.localeCompare(right.createdAt), + ); +} + +export function priceSnapshots( + activities: ReadonlyArray, + rate: ModelRate, +): ReadonlyArray<{ readonly createdAt: string; readonly usd: number }> { + const priced: Array<{ createdAt: string; usd: number }> = []; + let previousTotal: number | undefined; + for (const activity of orderActivities(activities)) { + if (activity.kind !== CONTEXT_WINDOW_KIND || !Predicate.isObject(activity.payload)) continue; + const payload = activity.payload as Record; + const total = count(payload, "totalProcessedTokens"); + if (total !== undefined && total === previousTotal) continue; + const delta = + total === undefined + ? undefined + : previousTotal === undefined || total < previousTotal + ? total + : total - previousTotal; + if (total !== undefined) previousTotal = total; + const lastInput = count(payload, "lastInputTokens"); + const lastOutput = count(payload, "lastOutputTokens"); + let usd: number; + if (lastInput !== undefined || lastOutput !== undefined) { + const input = lastInput ?? 0; + const cached = Math.min(count(payload, "lastCachedInputTokens") ?? 0, input); + usd = + (input - cached) * rate.inputCostPerToken + + cached * rate.cacheReadCostPerToken + + (lastOutput ?? 0) * rate.outputCostPerToken; + } else if (delta !== undefined) { + usd = delta * rate.cacheReadCostPerToken; + } else { + continue; + } + if (usd > 0) priced.push({ createdAt: activity.createdAt, usd }); + } + return priced; +} + +function latest(values: ReadonlyArray): string | null { + return values.reduce( + (current, value) => (current === null || value > current ? value : current), + null, + ); +} + +export function evaluateSpendGuard(input: SpendGuardInput): SpendGuardEvaluation { + const releasedAt = latest( + input.activities + .filter((activity) => activity.kind === SPEND_GUARD_RELEASED_KIND) + .map((activity) => activity.createdAt), + ); + const idleSince = latest([...(releasedAt === null ? [] : [releasedAt]), ...input.workSignalAts]); + const after = (since: string | null) => (createdAt: string) => + since === null || createdAt > since; + const afterRelease = after(releasedAt); + const afterIdle = after(idleSince); + const spend = priceSnapshots(input.activities, input.rate); + const sessionUsd = spend + .filter((entry) => afterRelease(entry.createdAt)) + .reduce((sum, entry) => sum + entry.usd, 0); + const idleUsd = spend + .filter((entry) => afterIdle(entry.createdAt)) + .reduce((sum, entry) => sum + entry.usd, 0); + const guardActivities = (kind: string, since: (createdAt: string) => boolean) => + input.activities.filter((activity) => activity.kind === kind && since(activity.createdAt)); + const stoppedPayloadReason = (activity: SpendGuardActivity) => + Predicate.isObject(activity.payload) + ? (activity.payload as Record).reason + : undefined; + const flaggedSinceIdle = guardActivities(SPEND_GUARD_FLAGGED_KIND, afterIdle).length > 0; + const flaggedForCeiling = guardActivities(SPEND_GUARD_FLAGGED_KIND, afterRelease).some( + (activity) => stoppedPayloadReason(activity) === "ceiling", + ); + const stoppedSinceIdle = guardActivities(SPEND_GUARD_STOPPED_KIND, afterIdle).some( + (activity) => stoppedPayloadReason(activity) === "idle", + ); + const stoppedForCeiling = guardActivities(SPEND_GUARD_STOPPED_KIND, afterRelease).some( + (activity) => stoppedPayloadReason(activity) === "ceiling", + ); + const { mode, idleFlagUsd, idleStopUsd, sessionCeilingUsd } = input.settings; + const base = { + sessionUsd, + idleUsd, + idleSince, + flaggedSinceIdle, + flaggedForCeiling, + stoppedSinceIdle, + stoppedForCeiling, + }; + if (mode === "off") return { ...base, level: "ok", stopReason: null }; + const stopReason: SpendGuardStopReason | null = + sessionUsd >= sessionCeilingUsd ? "ceiling" : idleUsd >= idleStopUsd ? "idle" : null; + if (stopReason !== null && mode === "enforce") { + return { ...base, level: "stop", stopReason }; + } + if (stopReason !== null || idleUsd >= idleFlagUsd) { + return { ...base, level: "flag", stopReason }; + } + return { ...base, level: "ok", stopReason: null }; +} + +export function evaluateThreadSpend( + thread: Pick< + OrchestrationThread, + "activities" | "pullRequests" | "checkpoints" | "modelSelection" + >, + settings: Pick, +): SpendGuardEvaluation { + return evaluateSpendGuard({ + activities: thread.activities, + workSignalAts: workSignalAts(thread), + rate: rateForModel( + thread.modelSelection.model, + settings.usagePriceOverrides, + settings.sessionSpendGuard.fallbackRate, + ), + settings: settings.sessionSpendGuard, + }); +} + +export function describeSpendStop( + evaluation: SpendGuardEvaluation, + settings: SessionSpendGuardSettings, +): string { + const usd = (value: number) => `$${value.toFixed(2)}`; + return evaluation.stopReason === "ceiling" + ? `This thread has spent about ${usd(evaluation.sessionUsd)}, past its ${usd(settings.sessionCeilingUsd)} session ceiling. Raise sessionSpendGuard.sessionCeilingUsd to continue.` + : `This thread has spent about ${usd(evaluation.idleUsd)} with no linked pull request or file-changing turn, past the ${usd(settings.idleStopUsd)} idle limit. Link a pull request or change files to continue.`; +} + +export type LaunchRefusal = "untitled" | "ownerless"; + +export interface LaunchCheckInput { + readonly title: string; + readonly titleSeed: string | undefined; + readonly executionScope: ThreadExecutionScope | null | undefined; + readonly hasOtherUserMessages: boolean; + readonly clientSurface: ClientSurface | undefined; + readonly mode: SessionSpendGuardSettings["mode"]; +} + +const INTERACTIVE_SURFACES: ReadonlySet = new Set(["web", "desktop", "mobile"]); + +export function checkLaunch(input: LaunchCheckInput): LaunchRefusal | null { + const interactive = + input.clientSurface !== undefined && INTERACTIVE_SURFACES.has(input.clientSurface); + if (input.mode !== "enforce" || interactive) return null; + if (input.executionScope === null || input.executionScope === undefined) return "ownerless"; + const untitled = input.title.trim() === "" || input.title.trim() === DEFAULT_THREAD_TITLE; + if (untitled && input.titleSeed === undefined && input.hasOtherUserMessages) return "untitled"; + return null; +} + +export function describeLaunchRefusal(refusal: LaunchRefusal): string { + return refusal === "ownerless" + ? "Automated turns are refused on a thread with no execution scope, such as an imported session or a legacy thread. Continue it from the T3 app, or launch a new thread with an owner." + : "Automated turns are refused on a thread that still has the default title after its first turn. Give the thread a title, then send again."; +} diff --git a/apps/server/src/orchestration/SessionSpendGuardReactor.ts b/apps/server/src/orchestration/SessionSpendGuardReactor.ts new file mode 100644 index 0000000000000..62a264e83cf53 --- /dev/null +++ b/apps/server/src/orchestration/SessionSpendGuardReactor.ts @@ -0,0 +1,181 @@ +import { CommandId, EventId, type OrchestrationEvent, type ThreadId } from "@t3tools/contracts"; +import { makeDrainableWorker } from "@t3tools/shared/DrainableWorker"; +import * as Cause from "effect/Cause"; +import * as Context from "effect/Context"; +import * as Crypto from "effect/Crypto"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import type * as Scope from "effect/Scope"; +import * as Stream from "effect/Stream"; + +import * as ServerSettings from "../serverSettings.ts"; +import { forkParked } from "../serverActivation.ts"; +import * as OrchestrationEngine from "./Services/OrchestrationEngine.ts"; +import * as ProjectionSnapshotQuery from "./Services/ProjectionSnapshotQuery.ts"; +import { + CONTEXT_WINDOW_KIND, + describeSpendStop, + evaluateThreadSpend, + SPEND_GUARD_ACTIVITY_KINDS, + SPEND_GUARD_FLAGGED_KIND, + SPEND_GUARD_STOPPED_KIND, +} from "./SessionSpendGuard.ts"; + +export class SessionSpendGuardReactor extends Context.Service< + SessionSpendGuardReactor, + { + readonly start: () => Effect.Effect; + readonly drain: Effect.Effect; + } +>()("t3/orchestration/SessionSpendGuardReactor") {} + +const usd = (value: number) => Math.round(value * 100) / 100; + +/** @public Service construction is part of the canonical Effect module API. */ +export const make = Effect.gen(function* () { + const engine = yield* OrchestrationEngine.OrchestrationEngineService; + const snapshots = yield* ProjectionSnapshotQuery.ProjectionSnapshotQuery; + const settingsService = yield* ServerSettings.ServerSettingsService; + const crypto = yield* Crypto.Crypto; + const pending = new Set(); + + const appendGuardActivity = Effect.fn("SessionSpendGuardReactor.appendGuardActivity")(function* ( + threadId: ThreadId, + kind: string, + summary: string, + payload: Record, + ) { + const uuid = yield* crypto.randomUUIDv4; + const createdAt = DateTime.formatIso(yield* DateTime.now); + yield* engine.dispatch({ + type: "thread.activity.append", + commandId: CommandId.make(`server:spend-guard:${threadId}:${uuid}`), + threadId, + activity: { + id: EventId.make(`spend-guard:${uuid}`), + tone: "error", + kind, + summary, + payload, + turnId: null, + createdAt, + }, + createdAt, + }); + }); + + const check = Effect.fn("SessionSpendGuardReactor.check")(function* (threadId: ThreadId) { + pending.delete(threadId); + const settings = yield* settingsService.getSettings; + if (settings.sessionSpendGuard.mode === "off") return; + const thread = yield* snapshots.getThreadDetailById(threadId, { + activityKinds: SPEND_GUARD_ACTIVITY_KINDS, + unboundedActivities: true, + }); + if (Option.isNone(thread)) return; + const evaluation = evaluateThreadSpend(thread.value, settings); + const amounts = { + sessionUsd: usd(evaluation.sessionUsd), + idleUsd: usd(evaluation.idleUsd), + idleSince: evaluation.idleSince, + }; + if (evaluation.level === "stop") { + const alreadyStopped = + evaluation.stopReason === "ceiling" + ? evaluation.stoppedForCeiling + : evaluation.stoppedSinceIdle; + if (alreadyStopped) return; + yield* appendGuardActivity( + threadId, + SPEND_GUARD_STOPPED_KIND, + evaluation.stopReason === "ceiling" + ? "Stopped: session spend ceiling reached" + : "Stopped: spending with no pull request or file changes", + { + ...amounts, + reason: evaluation.stopReason, + detail: describeSpendStop(evaluation, settings.sessionSpendGuard), + }, + ); + const session = thread.value.session; + if ( + evaluation.stopReason === "ceiling" && + session !== null && + (session.status === "running" || session.status === "starting") + ) { + const uuid = yield* crypto.randomUUIDv4; + yield* engine.dispatch({ + type: "thread.turn.interrupt", + commandId: CommandId.make(`server:spend-guard-interrupt:${threadId}:${uuid}`), + threadId, + createdAt: DateTime.formatIso(yield* DateTime.now), + }); + } + return; + } + if (evaluation.level === "flag" && evaluation.stopReason === "ceiling") { + if (evaluation.flaggedForCeiling) return; + yield* appendGuardActivity( + threadId, + SPEND_GUARD_FLAGGED_KIND, + "Session spend ceiling reached", + { + ...amounts, + reason: "ceiling", + detail: describeSpendStop(evaluation, settings.sessionSpendGuard), + }, + ); + return; + } + if (evaluation.level === "flag" && !evaluation.flaggedSinceIdle) { + yield* appendGuardActivity( + threadId, + SPEND_GUARD_FLAGGED_KIND, + "Spending with no pull request or file changes", + { + ...amounts, + reason: "idle", + detail: `This thread has spent about $${amounts.idleUsd.toFixed(2)} since its last linked pull request or file-changing turn. It stops at $${settings.sessionSpendGuard.idleStopUsd.toFixed(2)}.`, + }, + ); + } + }); + + const worker = yield* makeDrainableWorker((threadId: ThreadId) => + check(threadId).pipe( + Effect.catchCause((cause) => + Cause.hasInterruptsOnly(cause) + ? Effect.failCause(cause) + : Effect.logWarning("session spend guard check failed", { + threadId, + cause: Cause.pretty(cause), + }), + ), + ), + ); + + const processEvent = (event: OrchestrationEvent) => { + if ( + event.type !== "thread.activity-appended" || + event.payload.activity.kind !== CONTEXT_WINDOW_KIND || + pending.has(event.payload.threadId) + ) { + return Effect.void; + } + pending.add(event.payload.threadId); + return worker.enqueue(event.payload.threadId); + }; + + const start: SessionSpendGuardReactor["Service"]["start"] = Effect.fn( + "SessionSpendGuardReactor.start", + )(function* () { + const events = yield* engine.subscribeDomainEvents; + yield* forkParked(Stream.runForEach(events, processEvent)); + }); + + return { start, drain: worker.drain } satisfies SessionSpendGuardReactor["Service"]; +}); + +export const layer = Layer.effect(SessionSpendGuardReactor, make); diff --git a/apps/server/src/orchestration/decider.settled.test.ts b/apps/server/src/orchestration/decider.settled.test.ts index bcb2408c21e17..752a256da478d 100644 --- a/apps/server/src/orchestration/decider.settled.test.ts +++ b/apps/server/src/orchestration/decider.settled.test.ts @@ -797,6 +797,32 @@ it.layer(NodeServices.layer)("settled thread decider", (it) => { }); const routineEvents = Array.isArray(routineResult) ? routineResult : [routineResult]; expect(routineEvents.map((event) => event.type)).toEqual(["thread.activity-appended"]); + + for (const kind of ["spend-guard.flagged", "spend-guard.stopped"] as const) { + const guardResult = yield* decideOrchestrationCommand({ + command: { + type: "thread.activity.append", + commandId: CommandId.make(`cmd-activity-${kind}`), + threadId: ThreadId.make("thread-1"), + activity: { + id: EventId.make(`activity-${kind}`), + tone: "error", + kind, + summary: kind, + payload: null, + turnId: null, + createdAt: NOW, + }, + createdAt: NOW, + }, + readModel: makeReadModel("settled"), + }); + const guardEvents = Array.isArray(guardResult) ? guardResult : [guardResult]; + expect(guardEvents.map((event) => event.type)).toEqual([ + "thread.unsettled", + "thread.activity-appended", + ]); + } }), ); diff --git a/apps/server/src/orchestration/decider.ts b/apps/server/src/orchestration/decider.ts index db8796de67fe2..7b32a0f1258cf 100644 --- a/apps/server/src/orchestration/decider.ts +++ b/apps/server/src/orchestration/decider.ts @@ -46,6 +46,7 @@ import { } from "./commandInvariants.ts"; import { projectEvent } from "./projector.ts"; import { threadHasQueuedTurnStart } from "./ThreadSettlementPolicy.ts"; +import { SPEND_GUARD_FLAGGED_KIND, SPEND_GUARD_STOPPED_KIND } from "./SessionSpendGuard.ts"; const monogramSegmenter = new Intl.Segmenter(undefined, { granularity: "grapheme" }); @@ -2175,7 +2176,9 @@ export const decideOrchestrationCommand = Effect.fn("decideOrchestrationCommand" // never stay hidden inside a settled slim row. const wakesSettledThread = command.activity.kind === "approval.requested" || - command.activity.kind === "user-input.requested"; + command.activity.kind === "user-input.requested" || + command.activity.kind === SPEND_GUARD_FLAGGED_KIND || + command.activity.kind === SPEND_GUARD_STOPPED_KIND; // Real activity resets ANY override (settled wakes, active unpins). if (thread.settledOverride === null || !wakesSettledThread) { return activityAppendedEvent; diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 7f3165d7c96ca..730855aa83614 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -91,6 +91,7 @@ import { ProviderCommandReactorLive } from "./orchestration/Layers/ProviderComma import { CheckpointReactorLive } from "./orchestration/Layers/CheckpointReactor.ts"; import { ThreadDeletionReactorLive } from "./orchestration/Layers/ThreadDeletionReactor.ts"; import * as ThreadSettlementReactor from "./orchestration/ThreadSettlementReactor.ts"; +import * as SessionSpendGuardReactor from "./orchestration/SessionSpendGuardReactor.ts"; import * as StorageCleanup from "./storageCleanup.ts"; import * as PullRequestSyncReactor from "./orchestration/PullRequestSyncReactor.ts"; import * as ThreadPullRequestReactor from "./orchestration/ThreadPullRequestReactor.ts"; @@ -278,6 +279,7 @@ const ReactorLayerLive = Layer.empty.pipe( Layer.provideMerge(StorageCleanup.layer), Layer.provideMerge(ThreadDeletionReactorLive), Layer.provideMerge(ThreadSettlementReactor.layer), + Layer.provideMerge(SessionSpendGuardReactor.layer), Layer.provideMerge(PullRequestSyncReactor.layer), Layer.provideMerge(ThreadPullRequestReactor.layer), Layer.provideMerge( diff --git a/packages/contracts/src/settings.test.ts b/packages/contracts/src/settings.test.ts index 8ffe207fed73f..5a1c2ea0b27f7 100644 --- a/packages/contracts/src/settings.test.ts +++ b/packages/contracts/src/settings.test.ts @@ -1026,6 +1026,33 @@ describe("ServerSettingsPatch string normalization", () => { }); }); +describe("session spend guard", () => { + it("enforces the idle and session ceilings by default", () => { + expect(decodeServerSettings({}).sessionSpendGuard).toMatchObject({ + mode: "enforce", + idleFlagUsd: 25, + idleStopUsd: 50, + sessionCeilingUsd: 200, + fallbackRate: { + inputCostPerMillionTokens: 10, + outputCostPerMillionTokens: 50, + cacheReadCostPerMillionTokens: 1, + cacheWriteCostPerMillionTokens: 12.5, + }, + }); + }); + + it("patches one field and rejects a non-positive amount", () => { + expect(decodeServerSettingsPatch({ sessionSpendGuard: { mode: "flag" } })).toEqual({ + sessionSpendGuard: { mode: "flag" }, + }); + expect(() => decodeServerSettings({ sessionSpendGuard: { idleStopUsd: 0 } })).toThrow(); + expect(() => + decodeServerSettingsPatch({ sessionSpendGuard: { sessionCeilingUsd: -1 } }), + ).toThrow(); + }); +}); + describe("ServerSettings environment icon", () => { it("defaults to null", () => { expect(decodeServerSettings({}).environmentIcon).toBeNull(); diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index b89064d2db308..d953bb56592b2 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -600,6 +600,29 @@ export const CodexSessionGuardSettings = Schema.Struct({ }); export type CodexSessionGuardSettings = typeof CodexSessionGuardSettings.Type; +export const SessionSpendGuardMode = Schema.Literals(["off", "flag", "enforce"]); +export type SessionSpendGuardMode = typeof SessionSpendGuardMode.Type; + +const SpendGuardUsd = Schema.Number.check(Schema.isFinite(), Schema.isGreaterThan(0)); + +export const SessionSpendGuardSettings = Schema.Struct({ + mode: SessionSpendGuardMode.pipe(Schema.withDecodingDefault(Effect.succeed("enforce" as const))), + idleFlagUsd: SpendGuardUsd.pipe(Schema.withDecodingDefault(Effect.succeed(25))), + idleStopUsd: SpendGuardUsd.pipe(Schema.withDecodingDefault(Effect.succeed(50))), + sessionCeilingUsd: SpendGuardUsd.pipe(Schema.withDecodingDefault(Effect.succeed(200))), + fallbackRate: UsageModelPriceOverride.pipe( + Schema.withDecodingDefault( + Effect.succeed({ + inputCostPerMillionTokens: 10, + outputCostPerMillionTokens: 50, + cacheReadCostPerMillionTokens: 1, + cacheWriteCostPerMillionTokens: 12.5, + }), + ), + ), +}); +export type SessionSpendGuardSettings = typeof SessionSpendGuardSettings.Type; + export const CodexSettings = makeProviderSettingsSchema( { enabled: Schema.Boolean.pipe( @@ -1333,6 +1356,12 @@ export const ServerSettings = Schema.Struct({ * cannot be partly enrolled by leaving another instance unguarded. */ codexSessionGuard: CodexSessionGuardSettings.pipe(Schema.withDecodingDefault(Effect.succeed({}))), + /** + * Closura: idle sessions stop spending by default. `enforce` refuses the + * next automated turn past the idle or session ceiling. `flag` records the + * warning without stopping, and `off` disables the guard. + */ + sessionSpendGuard: SessionSpendGuardSettings.pipe(Schema.withDecodingDefault(Effect.succeed({}))), // Legacy single-instance-per-driver settings. Continues to be the source // of truth until `providerInstances` (below) lands per-driver migration @@ -1628,6 +1657,15 @@ export const ServerSettingsPatch = Schema.Struct({ stateRoot: Schema.optionalKey(TrimmedString), }), ), + sessionSpendGuard: Schema.optionalKey( + Schema.Struct({ + mode: Schema.optionalKey(SessionSpendGuardMode), + idleFlagUsd: Schema.optionalKey(SpendGuardUsd), + idleStopUsd: Schema.optionalKey(SpendGuardUsd), + sessionCeilingUsd: Schema.optionalKey(SpendGuardUsd), + fallbackRate: Schema.optionalKey(UsageModelPriceOverride), + }), + ), observability: Schema.optionalKey( Schema.Struct({ otlpTracesUrl: Schema.optionalKey(TrimmedString),