From e86d9edf05bd77b6ab44c47428b48c7a1f38d50e Mon Sep 17 00:00:00 2001 From: nullStack65 <112618179+nullStack65@users.noreply.github.com> Date: Fri, 20 Nov 2026 02:26:04 +0000 Subject: [PATCH 1/3] feat(server): stop idle sessions that spend without delivery Idle threads could keep spending after the useful work stopped. Price context-window updates and stop or flag a thread once idle spend or the session ceiling is crossed. Enforce is the default. The next automated turn is also refused on an untitled thread or one with no execution scope. A linked pull request, a file-changing checkpoint, or a spend-guard.released activity starts the window over. Settled threads wake when the guard flags or stops them. Model: grok-4.7 Task: feature Agent: grok Harness: t3/codex --- .../OrchestrationEngineHarness.integration.ts | 7 + .../Layers/OrchestrationReactor.test.ts | 11 + .../Layers/OrchestrationReactor.ts | 3 + .../Layers/ProviderCommandReactor.test.ts | 67 ++++- .../Layers/ProviderCommandReactor.ts | 37 +++ .../orchestration/SessionSpendGuard.test.ts | 262 ++++++++++++++++++ .../src/orchestration/SessionSpendGuard.ts | 245 ++++++++++++++++ .../orchestration/SessionSpendGuardReactor.ts | 161 +++++++++++ .../src/orchestration/decider.settled.test.ts | 26 ++ apps/server/src/orchestration/decider.ts | 5 +- apps/server/src/server.ts | 2 + packages/contracts/src/settings.test.ts | 27 ++ packages/contracts/src/settings.ts | 38 +++ 13 files changed, 889 insertions(+), 2 deletions(-) create mode 100644 apps/server/src/orchestration/SessionSpendGuard.test.ts create mode 100644 apps/server/src/orchestration/SessionSpendGuard.ts create mode 100644 apps/server/src/orchestration/SessionSpendGuardReactor.ts diff --git a/apps/server/integration/OrchestrationEngineHarness.integration.ts b/apps/server/integration/OrchestrationEngineHarness.integration.ts index 751c71cabebcc..4c48ec4e7e736 100644 --- a/apps/server/integration/OrchestrationEngineHarness.integration.ts +++ b/apps/server/integration/OrchestrationEngineHarness.integration.ts @@ -68,6 +68,7 @@ import { } from "../src/orchestration/Services/OrchestrationEngine.ts"; import { ThreadDeletionReactor } from "../src/orchestration/Services/ThreadDeletionReactor.ts"; import * as ThreadSettlementReactor from "../src/orchestration/ThreadSettlementReactor.ts"; +import * as SessionSpendGuardReactor from "../src/orchestration/SessionSpendGuardReactor.ts"; import * as PullRequestSyncReactor from "../src/orchestration/PullRequestSyncReactor.ts"; import * as ThreadPullRequestReactor from "../src/orchestration/ThreadPullRequestReactor.ts"; import { OrchestrationReactor } from "../src/orchestration/Services/OrchestrationReactor.ts"; @@ -421,6 +422,12 @@ export const makeOrchestrationIntegrationHarness = ( drain: Effect.void, }), ), + Layer.provideMerge( + Layer.succeed(SessionSpendGuardReactor.SessionSpendGuardReactor, { + start: () => Effect.void, + drain: Effect.void, + }), + ), Layer.provideMerge( Layer.succeed(PullRequestSyncReactor.PullRequestSyncReactor, { start: () => Effect.void, diff --git a/apps/server/src/orchestration/Layers/OrchestrationReactor.test.ts b/apps/server/src/orchestration/Layers/OrchestrationReactor.test.ts index d25c442ac8b23..6b06359e47ad0 100644 --- a/apps/server/src/orchestration/Layers/OrchestrationReactor.test.ts +++ b/apps/server/src/orchestration/Layers/OrchestrationReactor.test.ts @@ -10,6 +10,7 @@ import { ProviderCommandReactor } from "../Services/ProviderCommandReactor.ts"; import { ProviderRuntimeIngestionService } from "../Services/ProviderRuntimeIngestion.ts"; import { ThreadDeletionReactor } from "../Services/ThreadDeletionReactor.ts"; import * as ThreadSettlementReactor from "../ThreadSettlementReactor.ts"; +import * as SessionSpendGuardReactor from "../SessionSpendGuardReactor.ts"; import * as PullRequestSyncReactor from "../PullRequestSyncReactor.ts"; import * as ThreadPullRequestReactor from "../ThreadPullRequestReactor.ts"; import { OrchestrationReactor } from "../Services/OrchestrationReactor.ts"; @@ -95,6 +96,15 @@ describe("OrchestrationReactor", () => { drain: Effect.void, }), ), + Layer.provideMerge( + Layer.succeed(SessionSpendGuardReactor.SessionSpendGuardReactor, { + start: () => { + started.push("session-spend-guard-reactor"); + return Effect.void; + }, + drain: Effect.void, + }), + ), Layer.provideMerge( Layer.succeed(PullRequestSyncReactor.PullRequestSyncReactor, { start: () => { @@ -128,6 +138,7 @@ describe("OrchestrationReactor", () => { "thread-deletion-reactor", "thread-pull-request-reactor", "thread-settlement-reactor", + "session-spend-guard-reactor", "pull-request-sync-reactor", "agent-awareness-relay", "storage-cleanup", diff --git a/apps/server/src/orchestration/Layers/OrchestrationReactor.ts b/apps/server/src/orchestration/Layers/OrchestrationReactor.ts index da5cc337e4208..18845b485b7f3 100644 --- a/apps/server/src/orchestration/Layers/OrchestrationReactor.ts +++ b/apps/server/src/orchestration/Layers/OrchestrationReactor.ts @@ -10,6 +10,7 @@ import { ProviderCommandReactor } from "../Services/ProviderCommandReactor.ts"; import { ProviderRuntimeIngestionService } from "../Services/ProviderRuntimeIngestion.ts"; import { ThreadDeletionReactor } from "../Services/ThreadDeletionReactor.ts"; import * as ThreadSettlementReactor from "../ThreadSettlementReactor.ts"; +import * as SessionSpendGuardReactor from "../SessionSpendGuardReactor.ts"; import * as PullRequestSyncReactor from "../PullRequestSyncReactor.ts"; import * as ThreadPullRequestReactor from "../ThreadPullRequestReactor.ts"; import * as AgentAwarenessRelay from "../../relay/AgentAwarenessRelay.ts"; @@ -21,6 +22,7 @@ export const makeOrchestrationReactor = Effect.gen(function* () { const checkpointReactor = yield* CheckpointReactor; const threadDeletionReactor = yield* ThreadDeletionReactor; const threadSettlementReactor = yield* ThreadSettlementReactor.ThreadSettlementReactor; + const sessionSpendGuardReactor = yield* SessionSpendGuardReactor.SessionSpendGuardReactor; const pullRequestSyncReactor = yield* PullRequestSyncReactor.PullRequestSyncReactor; const threadPullRequestReactor = yield* ThreadPullRequestReactor.ThreadPullRequestReactor; const agentAwarenessRelay = yield* AgentAwarenessRelay.AgentAwarenessRelay; @@ -33,6 +35,7 @@ export const makeOrchestrationReactor = Effect.gen(function* () { yield* threadDeletionReactor.start(); yield* threadPullRequestReactor.start(); yield* threadSettlementReactor.start(); + yield* sessionSpendGuardReactor.start(); yield* pullRequestSyncReactor.start(); yield* agentAwarenessRelay.start(); yield* storageCleanup.start(); diff --git a/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts b/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts index 49b8a4126f68a..4fbc62ace8c18 100644 --- a/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts +++ b/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts @@ -516,7 +516,17 @@ describe("ProviderCommandReactor", () => { generateThreadTitle, }), ), - Layer.provideMerge(ServerSettingsService.layerTest(input?.serverSettingsOverrides ?? {})), + Layer.provideMerge( + ServerSettingsService.layerTest({ + ...input?.serverSettingsOverrides, + // These fixtures are imported threads with no scope, which the guard + // refuses. Leave it off unless a test opts back into enforcement. + sessionSpendGuard: { + mode: "off", + ...input?.serverSettingsOverrides?.sessionSpendGuard, + }, + }), + ), Layer.provideMerge(serverEnvironmentIdentityTestLayer), Layer.provideMerge(SqlitePersistenceMemory), Layer.provideMerge(ServerConfig.layerTest(process.cwd(), baseDir)), @@ -878,6 +888,61 @@ describe("ProviderCommandReactor", () => { }), ); + effectIt.effect("refuses an automated turn on an imported thread with no execution scope", () => + Effect.gen(function* () { + const harness = yield* Effect.promise(() => + createHarness({ + serverSettingsOverrides: { sessionSpendGuard: { mode: "enforce" } }, + }), + ); + const threadId = ThreadId.make("thread-imported"); + const createdAt = "2026-01-01T00:00:00.000Z"; + yield* harness.engine.dispatch({ + type: "thread.create", + commandId: CommandId.make("cmd-imported-thread"), + threadId, + projectId: asProjectId("project-1"), + title: "Imported session", + modelSelection: { + instanceId: ProviderInstanceId.make("codex"), + model: "gpt-5-codex", + }, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "approval-required", + branch: null, + worktreePath: null, + historyImport: true, + createdAt, + }); + yield* harness.engine.dispatch({ + type: "thread.turn.start", + commandId: CommandId.make("cmd-imported-turn"), + threadId, + message: { + messageId: MessageId.make("message-imported-turn"), + role: "user", + text: "continue the imported session", + attachments: [], + }, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "approval-required", + createdAt, + }); + yield* Effect.promise(() => harness.drain()); + const thread = (yield* Effect.promise(() => harness.readModel())).threads.find( + (entry) => entry.id === threadId, + ); + expect(thread?.activities).toContainEqual( + expect.objectContaining({ + kind: "provider.turn.start.failed", + summary: "Turn refused by the session spend guard", + }), + ); + expect(harness.sendTurn).not.toHaveBeenCalled(); + expect(harness.startSession).not.toHaveBeenCalled(); + }), + ); + it("reacts to thread.turn.start by ensuring session and sending provider turn", async () => { const harness = await createHarness(); const now = "2026-01-01T00:00:00.000Z"; diff --git a/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts b/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts index 7d8ec755f2833..07582d7f75773 100644 --- a/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts +++ b/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts @@ -66,6 +66,13 @@ import { type ThreadTitleMessage, } from "../../textGeneration/ThreadTitleContext.ts"; import { canReplaceThreadTitle, DEFAULT_THREAD_TITLE } from "../threadTitles.ts"; +import { + checkLaunch, + describeLaunchRefusal, + describeSpendStop, + evaluateThreadSpend, + SPEND_GUARD_ACTIVITY_KINDS, +} from "../SessionSpendGuard.ts"; import { resolveSourceControlWriterModelSelection, ServerSettingsService, @@ -1379,6 +1386,36 @@ const make = Effect.gen(function* () { ); } + const guardSettings = yield* serverSettingsService.getSettings; + const launchRefusal = checkLaunch({ + title: thread.title, + titleSeed: event.payload.titleSeed, + executionScope: thread.executionScope, + hasOtherUserMessages, + fromClientSurface: event.metadata.origin?.surface !== undefined, + mode: guardSettings.sessionSpendGuard.mode, + }); + if (launchRefusal !== null) { + return yield* appendTurnStartFailure( + "Turn refused by the session spend guard", + describeLaunchRefusal(launchRefusal), + ); + } + if (guardSettings.sessionSpendGuard.mode === "enforce") { + const guardThread = yield* projectionSnapshotQuery.getThreadDetailById(thread.id, { + activityKinds: SPEND_GUARD_ACTIVITY_KINDS, + }); + const spend = Option.isSome(guardThread) + ? evaluateThreadSpend(guardThread.value, guardSettings) + : undefined; + if (spend?.level === "stop") { + return yield* appendTurnStartFailure( + "Turn refused by the session spend guard", + describeSpendStop(spend, guardSettings.sessionSpendGuard), + ); + } + } + const handleTurnStartFailure = (cause: Cause.Cause) => { if (Cause.hasInterruptsOnly(cause)) { return Effect.void; diff --git a/apps/server/src/orchestration/SessionSpendGuard.test.ts b/apps/server/src/orchestration/SessionSpendGuard.test.ts new file mode 100644 index 0000000000000..566c190b431bc --- /dev/null +++ b/apps/server/src/orchestration/SessionSpendGuard.test.ts @@ -0,0 +1,262 @@ +import { describe, expect, it } from "vite-plus/test"; +import { ProviderInstanceId, ServerSettings } from "@t3tools/contracts"; +import * as Schema from "effect/Schema"; + +import { DEFAULT_THREAD_TITLE } from "./threadTitles.ts"; +import { + checkLaunch, + describeLaunchRefusal, + describeSpendStop, + evaluateSpendGuard, + evaluateThreadSpend, + priceSnapshots, + rateForModel, + workSignalAts, + type SpendGuardActivity, + type SpendGuardInput, +} from "./SessionSpendGuard.ts"; + +const decodeSettings = Schema.decodeUnknownSync(ServerSettings); + +const rate = { + inputCostPerToken: 1, + outputCostPerToken: 2, + cacheReadCostPerToken: 0.1, + cacheCreationCostPerToken: 3, +}; + +const settings = { + ...decodeSettings({}).sessionSpendGuard, + idleFlagUsd: 3, + idleStopUsd: 5, + sessionCeilingUsd: 10, +}; + +const activity = ( + kind: string, + createdAt: string, + payload: SpendGuardActivity["payload"] = null, + sequence?: number, +): SpendGuardActivity => ({ kind, createdAt, payload, sequence }); + +const priced = (createdAt: string, usdTokens: number, sequence?: number): SpendGuardActivity => + activity( + "context-window.updated", + createdAt, + { totalProcessedTokens: usdTokens, lastInputTokens: usdTokens }, + sequence, + ); + +const evaluate = (overrides: Partial = {}) => + evaluateSpendGuard({ + activities: [], + workSignalAts: [], + rate, + settings, + ...overrides, + }); + +describe("priceSnapshots", () => { + it("prices the latest uncached input, cache read, and output", () => { + expect( + priceSnapshots( + [ + activity("context-window.updated", "2026-01-01T00:00:00.000Z", { + totalProcessedTokens: 100, + lastInputTokens: 10, + lastCachedInputTokens: 4, + lastOutputTokens: 2, + }), + ], + rate, + ), + ).toEqual([{ createdAt: "2026-01-01T00:00:00.000Z", usd: 10.4 }]); + }); + + it("prices a token delta at the cache-read rate and skips a repeated total", () => { + expect( + priceSnapshots( + [ + activity("context-window.updated", "2026-01-01T00:00:00.000Z", { + totalProcessedTokens: 100, + }), + activity("context-window.updated", "2026-01-01T00:01:00.000Z", { + totalProcessedTokens: 100, + lastInputTokens: 9, + }), + activity("context-window.updated", "2026-01-01T00:02:00.000Z", { + totalProcessedTokens: 150, + }), + activity("context-window.updated", "2026-01-01T00:03:00.000Z", { + totalProcessedTokens: 40, + }), + ], + rate, + ), + ).toEqual([ + { createdAt: "2026-01-01T00:00:00.000Z", usd: 10 }, + { createdAt: "2026-01-01T00:02:00.000Z", usd: 5 }, + { createdAt: "2026-01-01T00:03:00.000Z", usd: 4 }, + ]); + }); + + it("orders snapshots by sequence when both sides have one", () => { + expect( + priceSnapshots( + [ + activity( + "context-window.updated", + "2026-01-01T00:02:00.000Z", + { lastOutputTokens: 1 }, + 2, + ), + activity( + "context-window.updated", + "2026-01-01T00:03:00.000Z", + { lastOutputTokens: 1 }, + 1, + ), + ], + rate, + ).map((entry) => entry.createdAt), + ).toEqual(["2026-01-01T00:03:00.000Z", "2026-01-01T00:02:00.000Z"]); + }); +}); + +describe("evaluateSpendGuard", () => { + it("counts idle spend only after the latest pull request or file-changing checkpoint", () => { + const signals = workSignalAts({ + pullRequests: [ + { linkedAt: "2026-01-01T00:02:00.000Z", source: "manual" }, + { linkedAt: "2026-01-01T00:09:00.000Z", source: "stack-dismissed" }, + ], + checkpoints: [ + { completedAt: "2026-01-01T00:03:00.000Z", files: ["src/app.ts"] }, + { completedAt: "2026-01-01T00:08:00.000Z", files: [] }, + ], + }); + const result = evaluate({ + workSignalAts: signals, + activities: [ + priced("2026-01-01T00:01:00.000Z", 2), + priced("2026-01-01T00:02:30.000Z", 1), + priced("2026-01-01T00:04:00.000Z", 6), + ], + }); + expect(result.idleSince).toBe("2026-01-01T00:03:00.000Z"); + expect(result.sessionUsd).toBe(9); + expect(result.idleUsd).toBe(6); + expect(result.level).toBe("stop"); + expect(result.stopReason).toBe("idle"); + }); + + it("starts a new session after spend-guard.released", () => { + const result = evaluate({ + activities: [ + priced("2026-01-01T00:01:00.000Z", 9), + activity("spend-guard.released", "2026-01-01T00:02:00.000Z"), + priced("2026-01-01T00:03:00.000Z", 2), + ], + }); + expect(result.sessionUsd).toBe(2); + expect(result.idleUsd).toBe(2); + expect(result.level).toBe("ok"); + }); + + it("flags idle spend without stopping when mode is flag", () => { + const result = evaluate({ + settings: { ...settings, mode: "flag" }, + activities: [priced("2026-01-01T00:01:00.000Z", 6)], + }); + expect(result).toMatchObject({ level: "flag", stopReason: null, idleUsd: 6 }); + }); + + it("stops for the session ceiling ahead of the idle limit", () => { + const result = evaluate({ + activities: [priced("2026-01-01T00:01:00.000Z", 10)], + }); + expect(result).toMatchObject({ level: "stop", stopReason: "ceiling", sessionUsd: 10 }); + }); + + it("does not stop or flag when the guard is off", () => { + const result = evaluate({ + settings: { ...settings, mode: "off" }, + activities: [priced("2026-01-01T00:01:00.000Z", 20)], + }); + expect(result).toMatchObject({ level: "ok", stopReason: null }); + }); + + it("remembers a flag or stop only until the next idle boundary", () => { + const idle = evaluate({ + workSignalAts: ["2026-01-01T00:02:00.000Z"], + activities: [ + activity("spend-guard.flagged", "2026-01-01T00:01:00.000Z"), + activity("spend-guard.stopped", "2026-01-01T00:01:30.000Z", { reason: "idle" }), + activity("spend-guard.stopped", "2026-01-01T00:03:00.000Z", { reason: "ceiling" }), + activity("spend-guard.flagged", "2026-01-01T00:04:00.000Z"), + ], + }); + expect(idle.flaggedSinceIdle).toBe(true); + expect(idle.stoppedSinceIdle).toBe(false); + expect(idle.stoppedForCeiling).toBe(true); + }); +}); + +describe("launch and stop copy", () => { + const base = { + title: DEFAULT_THREAD_TITLE, + titleSeed: undefined, + executionScope: "coding" as const, + hasOtherUserMessages: true, + fromClientSurface: false, + mode: "enforce" as const, + }; + + it("refuses an untitled or ownerless automated follow-up", () => { + expect(checkLaunch(base)).toBe("untitled"); + expect(checkLaunch({ ...base, executionScope: null, hasOtherUserMessages: false })).toBe( + "ownerless", + ); + expect(checkLaunch({ ...base, title: "Named thread" })).toBeNull(); + expect(checkLaunch({ ...base, titleSeed: "seed" })).toBeNull(); + expect(checkLaunch({ ...base, hasOtherUserMessages: false })).toBeNull(); + expect(checkLaunch({ ...base, fromClientSurface: true })).toBeNull(); + expect(checkLaunch({ ...base, mode: "flag" })).toBeNull(); + }); + + it("describes how to continue", () => { + const evaluation = evaluate({ activities: [priced("2026-01-01T00:01:00.000Z", 6)] }); + expect(describeSpendStop(evaluation, settings)).toContain("$6.00"); + expect(describeSpendStop(evaluation, settings)).toContain("$5.00"); + expect(describeLaunchRefusal("untitled")).toContain("default title"); + expect(describeLaunchRefusal("ownerless")).toContain("execution scope"); + }); +}); + +describe("evaluateThreadSpend", () => { + it("uses the model override and otherwise the fallback rate", () => { + const decoded = decodeSettings({ + sessionSpendGuard: { idleStopUsd: 1, sessionCeilingUsd: 100 }, + usagePriceOverrides: { + "custom-model": { + inputCostPerMillionTokens: 1_000_000, + outputCostPerMillionTokens: 0, + }, + }, + }); + const thread = { + activities: [priced("2026-01-01T00:01:00.000Z", 2)], + pullRequests: [], + checkpoints: [], + modelSelection: { + instanceId: ProviderInstanceId.make("codex"), + model: "custom-model", + }, + }; + expect(evaluateThreadSpend(thread, decoded).sessionUsd).toBe(2); + expect( + rateForModel("missing", decoded.usagePriceOverrides, decoded.sessionSpendGuard.fallbackRate) + .inputCostPerToken, + ).toBe(10 / 1_000_000); + }); +}); diff --git a/apps/server/src/orchestration/SessionSpendGuard.ts b/apps/server/src/orchestration/SessionSpendGuard.ts new file mode 100644 index 0000000000000..5b3d401ffdddd --- /dev/null +++ b/apps/server/src/orchestration/SessionSpendGuard.ts @@ -0,0 +1,245 @@ +import type { + OrchestrationCheckpointSummary, + OrchestrationThread, + OrchestrationThreadActivity, + ServerSettings, + SessionSpendGuardSettings, + ThreadExecutionScope, + ThreadPullRequestLink, + UsageModelPriceOverride, +} from "@t3tools/contracts"; +import * as Predicate from "effect/Predicate"; + +import { createOverrideRateTable, lookupRate, type ModelRate } from "../usage/usagePricing.ts"; +import { DEFAULT_THREAD_TITLE } from "./threadTitles.ts"; + +export const SPEND_GUARD_FLAGGED_KIND = "spend-guard.flagged"; +export const SPEND_GUARD_STOPPED_KIND = "spend-guard.stopped"; +export const SPEND_GUARD_RELEASED_KIND = "spend-guard.released"; +export const CONTEXT_WINDOW_KIND = "context-window.updated"; + +export const SPEND_GUARD_ACTIVITY_KINDS = [ + CONTEXT_WINDOW_KIND, + SPEND_GUARD_FLAGGED_KIND, + SPEND_GUARD_STOPPED_KIND, + SPEND_GUARD_RELEASED_KIND, +] as const; + +export type SpendGuardActivity = Pick< + OrchestrationThreadActivity, + "kind" | "payload" | "createdAt" | "sequence" +>; + +export type SpendGuardLevel = "ok" | "flag" | "stop"; +export type SpendGuardStopReason = "idle" | "ceiling"; + +export interface SpendGuardEvaluation { + readonly level: SpendGuardLevel; + readonly stopReason: SpendGuardStopReason | null; + readonly sessionUsd: number; + readonly idleUsd: number; + readonly idleSince: string | null; + readonly flaggedSinceIdle: boolean; + readonly stoppedSinceIdle: boolean; + readonly stoppedForCeiling: boolean; +} + +export interface SpendGuardInput { + readonly activities: ReadonlyArray; + readonly workSignalAts: ReadonlyArray; + readonly rate: ModelRate; + readonly settings: SessionSpendGuardSettings; +} + +export function rateForModel( + model: string, + overrides: Readonly>, + fallback: UsageModelPriceOverride, +): ModelRate { + return ( + lookupRate(createOverrideRateTable(overrides), model) ?? + createOverrideRateTable({ fallback }).get("fallback")! + ); +} + +export function workSignalAts(thread: { + readonly pullRequests: ReadonlyArray>; + readonly checkpoints: ReadonlyArray< + Pick + >; +}): ReadonlyArray { + return [ + ...thread.pullRequests + .filter((link) => link.source !== "stack-dismissed") + .map((link) => link.linkedAt), + ...thread.checkpoints + .filter((checkpoint) => checkpoint.files.length > 0) + .map((checkpoint) => checkpoint.completedAt), + ]; +} + +function count(record: Record, key: string): number | undefined { + const value = record[key]; + return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : undefined; +} + +function orderActivities( + activities: ReadonlyArray, +): ReadonlyArray { + return activities.toSorted((left, right) => + left.sequence !== undefined && right.sequence !== undefined + ? left.sequence - right.sequence + : left.createdAt.localeCompare(right.createdAt), + ); +} + +export function priceSnapshots( + activities: ReadonlyArray, + rate: ModelRate, +): ReadonlyArray<{ readonly createdAt: string; readonly usd: number }> { + const priced: Array<{ createdAt: string; usd: number }> = []; + let previousTotal: number | undefined; + for (const activity of orderActivities(activities)) { + if (activity.kind !== CONTEXT_WINDOW_KIND || !Predicate.isObject(activity.payload)) continue; + const payload = activity.payload as Record; + const total = count(payload, "totalProcessedTokens"); + if (total !== undefined && total === previousTotal) continue; + const delta = + total === undefined + ? undefined + : previousTotal === undefined || total < previousTotal + ? total + : total - previousTotal; + if (total !== undefined) previousTotal = total; + const lastInput = count(payload, "lastInputTokens"); + const lastOutput = count(payload, "lastOutputTokens"); + let usd: number; + if (lastInput !== undefined || lastOutput !== undefined) { + const input = lastInput ?? 0; + const cached = Math.min(count(payload, "lastCachedInputTokens") ?? 0, input); + usd = + (input - cached) * rate.inputCostPerToken + + cached * rate.cacheReadCostPerToken + + (lastOutput ?? 0) * rate.outputCostPerToken; + } else if (delta !== undefined) { + usd = delta * rate.cacheReadCostPerToken; + } else { + continue; + } + if (usd > 0) priced.push({ createdAt: activity.createdAt, usd }); + } + return priced; +} + +function latest(values: ReadonlyArray): string | null { + return values.reduce( + (current, value) => (current === null || value > current ? value : current), + null, + ); +} + +export function evaluateSpendGuard(input: SpendGuardInput): SpendGuardEvaluation { + const releasedAt = latest( + input.activities + .filter((activity) => activity.kind === SPEND_GUARD_RELEASED_KIND) + .map((activity) => activity.createdAt), + ); + const idleSince = latest([...(releasedAt === null ? [] : [releasedAt]), ...input.workSignalAts]); + const after = (since: string | null) => (createdAt: string) => + since === null || createdAt > since; + const afterRelease = after(releasedAt); + const afterIdle = after(idleSince); + const spend = priceSnapshots(input.activities, input.rate); + const sessionUsd = spend + .filter((entry) => afterRelease(entry.createdAt)) + .reduce((sum, entry) => sum + entry.usd, 0); + const idleUsd = spend + .filter((entry) => afterIdle(entry.createdAt)) + .reduce((sum, entry) => sum + entry.usd, 0); + const guardActivities = (kind: string, since: (createdAt: string) => boolean) => + input.activities.filter((activity) => activity.kind === kind && since(activity.createdAt)); + const stoppedPayloadReason = (activity: SpendGuardActivity) => + Predicate.isObject(activity.payload) + ? (activity.payload as Record).reason + : undefined; + const flaggedSinceIdle = guardActivities(SPEND_GUARD_FLAGGED_KIND, afterIdle).length > 0; + const stoppedSinceIdle = guardActivities(SPEND_GUARD_STOPPED_KIND, afterIdle).some( + (activity) => stoppedPayloadReason(activity) === "idle", + ); + const stoppedForCeiling = guardActivities(SPEND_GUARD_STOPPED_KIND, afterRelease).some( + (activity) => stoppedPayloadReason(activity) === "ceiling", + ); + const { mode, idleFlagUsd, idleStopUsd, sessionCeilingUsd } = input.settings; + const base = { + sessionUsd, + idleUsd, + idleSince, + flaggedSinceIdle, + stoppedSinceIdle, + stoppedForCeiling, + }; + if (mode === "off") return { ...base, level: "ok", stopReason: null }; + const stopReason: SpendGuardStopReason | null = + sessionUsd >= sessionCeilingUsd ? "ceiling" : idleUsd >= idleStopUsd ? "idle" : null; + if (stopReason !== null && mode === "enforce") { + return { ...base, level: "stop", stopReason }; + } + if (stopReason !== null || idleUsd >= idleFlagUsd) { + return { ...base, level: "flag", stopReason: null }; + } + return { ...base, level: "ok", stopReason: null }; +} + +export function evaluateThreadSpend( + thread: Pick< + OrchestrationThread, + "activities" | "pullRequests" | "checkpoints" | "modelSelection" + >, + settings: Pick, +): SpendGuardEvaluation { + return evaluateSpendGuard({ + activities: thread.activities, + workSignalAts: workSignalAts(thread), + rate: rateForModel( + thread.modelSelection.model, + settings.usagePriceOverrides, + settings.sessionSpendGuard.fallbackRate, + ), + settings: settings.sessionSpendGuard, + }); +} + +export function describeSpendStop( + evaluation: SpendGuardEvaluation, + settings: SessionSpendGuardSettings, +): string { + const usd = (value: number) => `$${value.toFixed(2)}`; + return evaluation.stopReason === "ceiling" + ? `This thread has spent about ${usd(evaluation.sessionUsd)}, past its ${usd(settings.sessionCeilingUsd)} session ceiling. Append a spend-guard.released activity or raise sessionSpendGuard.sessionCeilingUsd to continue.` + : `This thread has spent about ${usd(evaluation.idleUsd)} with no linked pull request or file-changing turn, past the ${usd(settings.idleStopUsd)} idle limit. Link a pull request, change files, or append a spend-guard.released activity to continue.`; +} + +export type LaunchRefusal = "untitled" | "ownerless"; + +export interface LaunchCheckInput { + readonly title: string; + readonly titleSeed: string | undefined; + readonly executionScope: ThreadExecutionScope | null | undefined; + readonly hasOtherUserMessages: boolean; + readonly fromClientSurface: boolean; + readonly mode: SessionSpendGuardSettings["mode"]; +} + +export function checkLaunch(input: LaunchCheckInput): LaunchRefusal | null { + if (input.mode !== "enforce" || input.fromClientSurface) return null; + if (input.executionScope === null || input.executionScope === undefined) return "ownerless"; + const untitled = input.title.trim() === "" || input.title.trim() === DEFAULT_THREAD_TITLE; + if (untitled && input.titleSeed === undefined && input.hasOtherUserMessages) return "untitled"; + return null; +} + +export function describeLaunchRefusal(refusal: LaunchRefusal): string { + return refusal === "ownerless" + ? "Automated turns are refused on a thread with no execution scope, such as an imported session or a legacy thread. Continue it from the T3 app, or launch a new thread with an owner." + : "Automated turns are refused on a thread that still has the default title after its first turn. Give the thread a title, then send again."; +} diff --git a/apps/server/src/orchestration/SessionSpendGuardReactor.ts b/apps/server/src/orchestration/SessionSpendGuardReactor.ts new file mode 100644 index 0000000000000..09a70129edb6d --- /dev/null +++ b/apps/server/src/orchestration/SessionSpendGuardReactor.ts @@ -0,0 +1,161 @@ +import { CommandId, EventId, type OrchestrationEvent, type ThreadId } from "@t3tools/contracts"; +import { makeDrainableWorker } from "@t3tools/shared/DrainableWorker"; +import * as Cause from "effect/Cause"; +import * as Context from "effect/Context"; +import * as Crypto from "effect/Crypto"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import type * as Scope from "effect/Scope"; +import * as Stream from "effect/Stream"; + +import * as ServerSettings from "../serverSettings.ts"; +import { forkParked } from "../serverActivation.ts"; +import * as OrchestrationEngine from "./Services/OrchestrationEngine.ts"; +import * as ProjectionSnapshotQuery from "./Services/ProjectionSnapshotQuery.ts"; +import { + CONTEXT_WINDOW_KIND, + describeSpendStop, + evaluateThreadSpend, + SPEND_GUARD_ACTIVITY_KINDS, + SPEND_GUARD_FLAGGED_KIND, + SPEND_GUARD_STOPPED_KIND, +} from "./SessionSpendGuard.ts"; + +export class SessionSpendGuardReactor extends Context.Service< + SessionSpendGuardReactor, + { + readonly start: () => Effect.Effect; + readonly drain: Effect.Effect; + } +>()("t3/orchestration/SessionSpendGuardReactor") {} + +const usd = (value: number) => Math.round(value * 100) / 100; + +/** @public Service construction is part of the canonical Effect module API. */ +export const make = Effect.gen(function* () { + const engine = yield* OrchestrationEngine.OrchestrationEngineService; + const snapshots = yield* ProjectionSnapshotQuery.ProjectionSnapshotQuery; + const settingsService = yield* ServerSettings.ServerSettingsService; + const crypto = yield* Crypto.Crypto; + const pending = new Set(); + + const appendGuardActivity = Effect.fn("SessionSpendGuardReactor.appendGuardActivity")(function* ( + threadId: ThreadId, + kind: string, + summary: string, + payload: Record, + ) { + const uuid = yield* crypto.randomUUIDv4; + const createdAt = DateTime.formatIso(yield* DateTime.now); + yield* engine.dispatch({ + type: "thread.activity.append", + commandId: CommandId.make(`server:spend-guard:${threadId}:${uuid}`), + threadId, + activity: { + id: EventId.make(`spend-guard:${uuid}`), + tone: "error", + kind, + summary, + payload, + turnId: null, + createdAt, + }, + createdAt, + }); + }); + + const check = Effect.fn("SessionSpendGuardReactor.check")(function* (threadId: ThreadId) { + pending.delete(threadId); + const settings = yield* settingsService.getSettings; + if (settings.sessionSpendGuard.mode === "off") return; + const thread = yield* snapshots.getThreadDetailById(threadId, { + activityKinds: SPEND_GUARD_ACTIVITY_KINDS, + }); + if (Option.isNone(thread)) return; + const evaluation = evaluateThreadSpend(thread.value, settings); + const amounts = { + sessionUsd: usd(evaluation.sessionUsd), + idleUsd: usd(evaluation.idleUsd), + idleSince: evaluation.idleSince, + }; + if (evaluation.level === "stop") { + const alreadyStopped = + evaluation.stopReason === "ceiling" + ? evaluation.stoppedForCeiling + : evaluation.stoppedSinceIdle; + if (alreadyStopped) return; + yield* appendGuardActivity( + threadId, + SPEND_GUARD_STOPPED_KIND, + evaluation.stopReason === "ceiling" + ? "Stopped: session spend ceiling reached" + : "Stopped: spending with no pull request or file changes", + { + ...amounts, + reason: evaluation.stopReason, + detail: describeSpendStop(evaluation, settings.sessionSpendGuard), + }, + ); + const session = thread.value.session; + if (session !== null && (session.status === "running" || session.status === "starting")) { + const uuid = yield* crypto.randomUUIDv4; + yield* engine.dispatch({ + type: "thread.turn.interrupt", + commandId: CommandId.make(`server:spend-guard-interrupt:${threadId}:${uuid}`), + threadId, + createdAt: DateTime.formatIso(yield* DateTime.now), + }); + } + return; + } + if (evaluation.level === "flag" && !evaluation.flaggedSinceIdle) { + yield* appendGuardActivity( + threadId, + SPEND_GUARD_FLAGGED_KIND, + "Spending with no pull request or file changes", + { + ...amounts, + detail: `This thread has spent about $${amounts.idleUsd.toFixed(2)} since its last linked pull request or file-changing turn. It stops at $${settings.sessionSpendGuard.idleStopUsd.toFixed(2)}.`, + }, + ); + } + }); + + const worker = yield* makeDrainableWorker((threadId: ThreadId) => + check(threadId).pipe( + Effect.catchCause((cause) => + Cause.hasInterruptsOnly(cause) + ? Effect.failCause(cause) + : Effect.logWarning("session spend guard check failed", { + threadId, + cause: Cause.pretty(cause), + }), + ), + ), + ); + + const processEvent = (event: OrchestrationEvent) => { + if ( + event.type !== "thread.activity-appended" || + event.payload.activity.kind !== CONTEXT_WINDOW_KIND || + pending.has(event.payload.threadId) + ) { + return Effect.void; + } + pending.add(event.payload.threadId); + return worker.enqueue(event.payload.threadId); + }; + + const start: SessionSpendGuardReactor["Service"]["start"] = Effect.fn( + "SessionSpendGuardReactor.start", + )(function* () { + const events = yield* engine.subscribeDomainEvents; + yield* forkParked(Stream.runForEach(events, processEvent)); + }); + + return { start, drain: worker.drain } satisfies SessionSpendGuardReactor["Service"]; +}); + +export const layer = Layer.effect(SessionSpendGuardReactor, make); diff --git a/apps/server/src/orchestration/decider.settled.test.ts b/apps/server/src/orchestration/decider.settled.test.ts index bcb2408c21e17..752a256da478d 100644 --- a/apps/server/src/orchestration/decider.settled.test.ts +++ b/apps/server/src/orchestration/decider.settled.test.ts @@ -797,6 +797,32 @@ it.layer(NodeServices.layer)("settled thread decider", (it) => { }); const routineEvents = Array.isArray(routineResult) ? routineResult : [routineResult]; expect(routineEvents.map((event) => event.type)).toEqual(["thread.activity-appended"]); + + for (const kind of ["spend-guard.flagged", "spend-guard.stopped"] as const) { + const guardResult = yield* decideOrchestrationCommand({ + command: { + type: "thread.activity.append", + commandId: CommandId.make(`cmd-activity-${kind}`), + threadId: ThreadId.make("thread-1"), + activity: { + id: EventId.make(`activity-${kind}`), + tone: "error", + kind, + summary: kind, + payload: null, + turnId: null, + createdAt: NOW, + }, + createdAt: NOW, + }, + readModel: makeReadModel("settled"), + }); + const guardEvents = Array.isArray(guardResult) ? guardResult : [guardResult]; + expect(guardEvents.map((event) => event.type)).toEqual([ + "thread.unsettled", + "thread.activity-appended", + ]); + } }), ); diff --git a/apps/server/src/orchestration/decider.ts b/apps/server/src/orchestration/decider.ts index db8796de67fe2..7b32a0f1258cf 100644 --- a/apps/server/src/orchestration/decider.ts +++ b/apps/server/src/orchestration/decider.ts @@ -46,6 +46,7 @@ import { } from "./commandInvariants.ts"; import { projectEvent } from "./projector.ts"; import { threadHasQueuedTurnStart } from "./ThreadSettlementPolicy.ts"; +import { SPEND_GUARD_FLAGGED_KIND, SPEND_GUARD_STOPPED_KIND } from "./SessionSpendGuard.ts"; const monogramSegmenter = new Intl.Segmenter(undefined, { granularity: "grapheme" }); @@ -2175,7 +2176,9 @@ export const decideOrchestrationCommand = Effect.fn("decideOrchestrationCommand" // never stay hidden inside a settled slim row. const wakesSettledThread = command.activity.kind === "approval.requested" || - command.activity.kind === "user-input.requested"; + command.activity.kind === "user-input.requested" || + command.activity.kind === SPEND_GUARD_FLAGGED_KIND || + command.activity.kind === SPEND_GUARD_STOPPED_KIND; // Real activity resets ANY override (settled wakes, active unpins). if (thread.settledOverride === null || !wakesSettledThread) { return activityAppendedEvent; diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 7f3165d7c96ca..730855aa83614 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -91,6 +91,7 @@ import { ProviderCommandReactorLive } from "./orchestration/Layers/ProviderComma import { CheckpointReactorLive } from "./orchestration/Layers/CheckpointReactor.ts"; import { ThreadDeletionReactorLive } from "./orchestration/Layers/ThreadDeletionReactor.ts"; import * as ThreadSettlementReactor from "./orchestration/ThreadSettlementReactor.ts"; +import * as SessionSpendGuardReactor from "./orchestration/SessionSpendGuardReactor.ts"; import * as StorageCleanup from "./storageCleanup.ts"; import * as PullRequestSyncReactor from "./orchestration/PullRequestSyncReactor.ts"; import * as ThreadPullRequestReactor from "./orchestration/ThreadPullRequestReactor.ts"; @@ -278,6 +279,7 @@ const ReactorLayerLive = Layer.empty.pipe( Layer.provideMerge(StorageCleanup.layer), Layer.provideMerge(ThreadDeletionReactorLive), Layer.provideMerge(ThreadSettlementReactor.layer), + Layer.provideMerge(SessionSpendGuardReactor.layer), Layer.provideMerge(PullRequestSyncReactor.layer), Layer.provideMerge(ThreadPullRequestReactor.layer), Layer.provideMerge( diff --git a/packages/contracts/src/settings.test.ts b/packages/contracts/src/settings.test.ts index 8ffe207fed73f..5a1c2ea0b27f7 100644 --- a/packages/contracts/src/settings.test.ts +++ b/packages/contracts/src/settings.test.ts @@ -1026,6 +1026,33 @@ describe("ServerSettingsPatch string normalization", () => { }); }); +describe("session spend guard", () => { + it("enforces the idle and session ceilings by default", () => { + expect(decodeServerSettings({}).sessionSpendGuard).toMatchObject({ + mode: "enforce", + idleFlagUsd: 25, + idleStopUsd: 50, + sessionCeilingUsd: 200, + fallbackRate: { + inputCostPerMillionTokens: 10, + outputCostPerMillionTokens: 50, + cacheReadCostPerMillionTokens: 1, + cacheWriteCostPerMillionTokens: 12.5, + }, + }); + }); + + it("patches one field and rejects a non-positive amount", () => { + expect(decodeServerSettingsPatch({ sessionSpendGuard: { mode: "flag" } })).toEqual({ + sessionSpendGuard: { mode: "flag" }, + }); + expect(() => decodeServerSettings({ sessionSpendGuard: { idleStopUsd: 0 } })).toThrow(); + expect(() => + decodeServerSettingsPatch({ sessionSpendGuard: { sessionCeilingUsd: -1 } }), + ).toThrow(); + }); +}); + describe("ServerSettings environment icon", () => { it("defaults to null", () => { expect(decodeServerSettings({}).environmentIcon).toBeNull(); diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index b89064d2db308..d953bb56592b2 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -600,6 +600,29 @@ export const CodexSessionGuardSettings = Schema.Struct({ }); export type CodexSessionGuardSettings = typeof CodexSessionGuardSettings.Type; +export const SessionSpendGuardMode = Schema.Literals(["off", "flag", "enforce"]); +export type SessionSpendGuardMode = typeof SessionSpendGuardMode.Type; + +const SpendGuardUsd = Schema.Number.check(Schema.isFinite(), Schema.isGreaterThan(0)); + +export const SessionSpendGuardSettings = Schema.Struct({ + mode: SessionSpendGuardMode.pipe(Schema.withDecodingDefault(Effect.succeed("enforce" as const))), + idleFlagUsd: SpendGuardUsd.pipe(Schema.withDecodingDefault(Effect.succeed(25))), + idleStopUsd: SpendGuardUsd.pipe(Schema.withDecodingDefault(Effect.succeed(50))), + sessionCeilingUsd: SpendGuardUsd.pipe(Schema.withDecodingDefault(Effect.succeed(200))), + fallbackRate: UsageModelPriceOverride.pipe( + Schema.withDecodingDefault( + Effect.succeed({ + inputCostPerMillionTokens: 10, + outputCostPerMillionTokens: 50, + cacheReadCostPerMillionTokens: 1, + cacheWriteCostPerMillionTokens: 12.5, + }), + ), + ), +}); +export type SessionSpendGuardSettings = typeof SessionSpendGuardSettings.Type; + export const CodexSettings = makeProviderSettingsSchema( { enabled: Schema.Boolean.pipe( @@ -1333,6 +1356,12 @@ export const ServerSettings = Schema.Struct({ * cannot be partly enrolled by leaving another instance unguarded. */ codexSessionGuard: CodexSessionGuardSettings.pipe(Schema.withDecodingDefault(Effect.succeed({}))), + /** + * Closura: idle sessions stop spending by default. `enforce` refuses the + * next automated turn past the idle or session ceiling. `flag` records the + * warning without stopping, and `off` disables the guard. + */ + sessionSpendGuard: SessionSpendGuardSettings.pipe(Schema.withDecodingDefault(Effect.succeed({}))), // Legacy single-instance-per-driver settings. Continues to be the source // of truth until `providerInstances` (below) lands per-driver migration @@ -1628,6 +1657,15 @@ export const ServerSettingsPatch = Schema.Struct({ stateRoot: Schema.optionalKey(TrimmedString), }), ), + sessionSpendGuard: Schema.optionalKey( + Schema.Struct({ + mode: Schema.optionalKey(SessionSpendGuardMode), + idleFlagUsd: Schema.optionalKey(SpendGuardUsd), + idleStopUsd: Schema.optionalKey(SpendGuardUsd), + sessionCeilingUsd: Schema.optionalKey(SpendGuardUsd), + fallbackRate: Schema.optionalKey(UsageModelPriceOverride), + }), + ), observability: Schema.optionalKey( Schema.Struct({ otlpTracesUrl: Schema.optionalKey(TrimmedString), From ae80a452936cd819f2c92c92ae1f95b85250a628 Mon Sep 17 00:00:00 2001 From: nullStack65 <112618179+nullStack65@users.noreply.github.com> Date: Fri, 20 Nov 2026 03:44:56 +0000 Subject: [PATCH 2/3] test(server): refuse a session past its ceiling after a linked pull request A standing manager can keep spending through status nudges and child callbacks after a real pull request, which resets the idle window. Cover that ceiling stop in the evaluator and in the provider reactor. Model: grok-4.7 Task: feature Agent: grok Lane: magpie Harness: t3/codex Model-family: Grok Model-generation: 4.7 --- .../Layers/ProviderCommandReactor.test.ts | 141 ++++++++++++++++++ .../orchestration/SessionSpendGuard.test.ts | 14 ++ 2 files changed, 155 insertions(+) diff --git a/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts b/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts index 4fbc62ace8c18..49021e16f9ed4 100644 --- a/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts +++ b/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts @@ -943,6 +943,147 @@ describe("ProviderCommandReactor", () => { }), ); + effectIt.effect( + "refuses the next user turn at the session ceiling even after a linked pull request", + () => + Effect.gen(function* () { + const harness = yield* Effect.promise(() => + createHarness({ + serverSettingsOverrides: { + sessionSpendGuard: { + mode: "enforce", + sessionCeilingUsd: 1, + idleStopUsd: 50, + idleFlagUsd: 25, + }, + usagePriceOverrides: { + "gpt-5-codex": { + inputCostPerMillionTokens: 1_000_000, + outputCostPerMillionTokens: 0, + cacheReadCostPerMillionTokens: 0, + }, + }, + }, + }), + ); + const threadId = ThreadId.make("thread-crown"); + const modelSelection = { + instanceId: ProviderInstanceId.make("codex"), + model: "gpt-5-codex", + }; + NodeFS.mkdirSync("/tmp/provider-project", { recursive: true }); + const workspacePath = NodeFS.mkdtempSync( + NodePath.join(NodeOS.tmpdir(), "crown-coordinator-"), + ); + yield* harness.engine.dispatch({ + type: "thread.create", + commandId: CommandId.make("cmd-crown-thread"), + threadId, + projectId: asProjectId("project-1"), + title: "Manager work", + modelSelection, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "approval-required", + branch: null, + worktreePath: workspacePath, + executionScope: "coordinator", + createdAt: "2099-01-01T00:00:00.000Z", + }); + yield* harness.engine.dispatch({ + type: "thread.pull-request.link", + commandId: CommandId.make("cmd-crown-pr"), + threadId, + host: "github.com", + repository: "nullStack65/t3code", + number: 1, + url: "https://github.com/nullStack65/t3code/pull/1", + source: "created", + }); + const nudges = [ + ["message-crown-status", "Status? Move all work forward.", "2099-01-01T00:00:01.000Z"], + ["message-crown-callback", "Here is another handoff.", "2099-01-01T00:00:02.000Z"], + ] as const; + for (const [messageId, text, createdAt] of nudges) { + yield* harness.engine.dispatch({ + type: "thread.turn.start", + commandId: CommandId.make(`cmd-${messageId}`), + threadId, + message: { + messageId: MessageId.make(messageId), + role: "user", + text, + attachments: [], + }, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "approval-required", + createdAt, + }); + yield* Effect.promise(() => harness.drain()); + } + expect(harness.sendTurn).toHaveBeenCalledTimes(2); + for (const [index, createdAt] of [ + ["1", "2099-01-01T00:00:03.000Z"], + ["2", "2099-01-01T00:00:04.000Z"], + ] as const) { + yield* harness.engine.dispatch({ + type: "thread.activity.append", + commandId: CommandId.make(`cmd-crown-context-${index}`), + threadId, + activity: { + id: EventId.make(`activity-crown-context-${index}`), + tone: "info", + kind: "context-window.updated", + summary: "Context window updated", + payload: { + totalProcessedTokens: Number(index), + lastInputTokens: 1, + lastCachedInputTokens: 0, + lastOutputTokens: 0, + }, + turnId: null, + createdAt, + }, + createdAt, + }); + } + yield* harness.engine.dispatch({ + type: "thread.turn.start", + commandId: CommandId.make("cmd-crown-after-ceiling"), + threadId, + message: { + messageId: MessageId.make("message-crown-after-ceiling"), + role: "user", + text: "Status?", + attachments: [], + }, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "approval-required", + createdAt: "2099-01-01T00:00:05.000Z", + }); + yield* Effect.promise(() => harness.drain()); + const thread = (yield* Effect.promise(() => harness.readModel())).threads.find( + (entry) => entry.id === threadId, + ); + expect(thread?.pullRequests).toEqual( + expect.arrayContaining([expect.objectContaining({ number: 1, source: "created" })]), + ); + const refusal = thread?.activities.find( + (activity) => activity.kind === "provider.turn.start.failed", + ); + expect(refusal).toMatchObject({ + summary: "Turn refused by the session spend guard", + }); + expect(String(refusal?.payload && (refusal.payload as { detail?: string }).detail)).toContain( + "session ceiling", + ); + expect(String(refusal?.payload && (refusal.payload as { detail?: string }).detail)).not.toContain( + "idle limit", + ); + expect(harness.sendTurn).toHaveBeenCalledTimes(2); + NodeFS.rmSync(workspacePath, { recursive: true, force: true }); + }), + ); + it("reacts to thread.turn.start by ensuring session and sending provider turn", async () => { const harness = await createHarness(); const now = "2026-01-01T00:00:00.000Z"; diff --git a/apps/server/src/orchestration/SessionSpendGuard.test.ts b/apps/server/src/orchestration/SessionSpendGuard.test.ts index 566c190b431bc..1e2f0546d161b 100644 --- a/apps/server/src/orchestration/SessionSpendGuard.test.ts +++ b/apps/server/src/orchestration/SessionSpendGuard.test.ts @@ -178,6 +178,20 @@ describe("evaluateSpendGuard", () => { expect(result).toMatchObject({ level: "stop", stopReason: "ceiling", sessionUsd: 10 }); }); + it("stops at the session ceiling when a recent pull request keeps idle spend under the idle limit", () => { + const result = evaluate({ + workSignalAts: ["2026-01-01T00:02:00.000Z"], + activities: [ + priced("2026-01-01T00:01:00.000Z", 6), + priced("2026-01-01T00:03:00.000Z", 4), + ], + }); + expect(result.idleUsd).toBe(4); + expect(result.sessionUsd).toBe(10); + expect(result.level).toBe("stop"); + expect(result.stopReason).toBe("ceiling"); + }); + it("does not stop or flag when the guard is off", () => { const result = evaluate({ settings: { ...settings, mode: "off" }, From 0d990dcd53cb56fd17768336a3a466b7b2e444b3 Mon Sep 17 00:00:00 2001 From: nullStack65 <112618179+nullStack65@users.noreply.github.com> Date: Fri, 20 Nov 2026 04:01:00 +0000 Subject: [PATCH 3/3] fix(server): count the whole session toward the spend ceiling The thread detail query capped activities at 500 rows, so a long manager thread could stay under the ceiling. Read every spend-guard activity, interrupt a live turn only for the ceiling, and keep untitled or ownerless refusals on cli and missing surfaces. Model: grok-4.7 Task: fix Agent: grok Lane: magpie Harness: t3/codex Model-family: Grok Model-generation: 4.7 --- .../Layers/ProjectionSnapshotQuery.ts | 29 ++++- .../Layers/ProviderCommandReactor.test.ts | 107 ++++++++++++++++++ .../Layers/ProviderCommandReactor.ts | 26 +++-- .../Services/ProjectionSnapshotQuery.ts | 1 + .../orchestration/SessionSpendGuard.test.ts | 37 +++++- .../src/orchestration/SessionSpendGuard.ts | 20 +++- .../orchestration/SessionSpendGuardReactor.ts | 22 +++- 7 files changed, 221 insertions(+), 21 deletions(-) diff --git a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts index 0c80c49b15508..e3a1a1d454560 100644 --- a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts +++ b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts @@ -1469,6 +1469,31 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { `, }); + const listThreadActivityRowsByThreadAndKindsUnbounded = SqlSchema.findAll({ + Request: ThreadActivityKindsLookupInput, + Result: ProjectionThreadActivityDbRowSchema, + execute: ({ threadId, activityKinds }) => + sql` + SELECT + activity_id AS "activityId", + thread_id AS "threadId", + turn_id AS "turnId", + tone, + kind, + summary, + payload_json AS "payload", + sequence, + created_at AS "createdAt" + FROM projection_thread_activities + WHERE thread_id = ${threadId} + AND ${sql.in("kind", activityKinds)} + ORDER BY + sequence ASC, + created_at ASC, + activity_id ASC + `, + }); + const getUserInputActivityRow = SqlSchema.findOneOption({ Request: Schema.Struct({ threadId: ThreadId, requestId: ApprovalRequestId }), Result: ProjectionThreadActivityDbRowSchema, @@ -3442,7 +3467,9 @@ pending_approval_requests AS ( : listThreadActivityRowsByThreadWindow({ threadId, ...bounds }) : activityRead.query.activityKinds.length === 0 ? Effect.succeed([]) - : listThreadActivityRowsByThreadAndKinds({ + : (activityRead.query.unboundedActivities === true + ? listThreadActivityRowsByThreadAndKindsUnbounded + : listThreadActivityRowsByThreadAndKinds)({ threadId, activityKinds: activityRead.query.activityKinds, }) diff --git a/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts b/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts index 49021e16f9ed4..ad193fccedc56 100644 --- a/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts +++ b/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts @@ -1084,6 +1084,113 @@ describe("ProviderCommandReactor", () => { }), ); + effectIt.effect( + "counts context-window rows older than the thread detail cap toward the session ceiling", + () => + Effect.gen(function* () { + const harness = yield* Effect.promise(() => + createHarness({ + serverSettingsOverrides: { + sessionSpendGuard: { + mode: "enforce", + sessionCeilingUsd: 1, + idleStopUsd: 100, + idleFlagUsd: 50, + }, + usagePriceOverrides: { + "gpt-5-codex": { + inputCostPerMillionTokens: 1_000_000, + outputCostPerMillionTokens: 0, + cacheReadCostPerMillionTokens: 0, + }, + }, + }, + }), + ); + const threadId = ThreadId.make("thread-cap"); + NodeFS.mkdirSync("/tmp/provider-project", { recursive: true }); + const workspacePath = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "crown-cap-")); + yield* Effect.gen(function* () { + yield* harness.engine.dispatch({ + type: "thread.create", + commandId: CommandId.make("cmd-cap-thread"), + threadId, + projectId: asProjectId("project-1"), + title: "Manager work", + modelSelection: { + instanceId: ProviderInstanceId.make("codex"), + model: "gpt-5-codex", + }, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "approval-required", + branch: null, + worktreePath: workspacePath, + executionScope: "coordinator", + createdAt: "2099-01-01T00:00:00.000Z", + }); + yield* Effect.promise(() => + harness.runEffect( + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + for (let sequence = 1; sequence <= 501; sequence += 1) { + const costly = sequence === 1; + yield* sql` + INSERT INTO projection_thread_activities ( + activity_id, thread_id, turn_id, tone, kind, summary, payload_json, sequence, created_at + ) VALUES ( + ${`activity-cap-${sequence}`}, + ${threadId}, + NULL, + 'info', + 'context-window.updated', + 'Context window updated', + ${costly + ? '{"totalProcessedTokens":1,"lastInputTokens":1,"lastOutputTokens":0}' + : '{"totalProcessedTokens":0,"lastInputTokens":0,"lastOutputTokens":0}'}, + ${sequence}, + '2099-01-01T00:00:00.000Z' + ) + `; + } + }), + ), + ); + yield* harness.engine.dispatch({ + type: "thread.turn.start", + commandId: CommandId.make("cmd-cap-turn"), + threadId, + message: { + messageId: MessageId.make("message-cap-turn"), + role: "user", + text: "Status?", + attachments: [], + }, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "approval-required", + createdAt: "2099-01-01T00:01:00.000Z", + }); + yield* Effect.promise(() => harness.drain()); + const thread = (yield* Effect.promise(() => harness.readModel())).threads.find( + (entry) => entry.id === threadId, + ); + const refusal = thread?.activities.find( + (activity) => activity.kind === "provider.turn.start.failed", + ); + expect(refusal).toMatchObject({ + summary: "Turn refused by the session spend guard", + }); + expect( + String(refusal?.payload && (refusal.payload as { detail?: string }).detail), + ).toContain("session ceiling"); + expect(harness.sendTurn).not.toHaveBeenCalled(); + }).pipe( + Effect.ensuring( + Effect.sync(() => NodeFS.rmSync(workspacePath, { recursive: true, force: true })), + ), + ); + }), + ); + it("reacts to thread.turn.start by ensuring session and sending provider turn", async () => { const harness = await createHarness(); const now = "2026-01-01T00:00:00.000Z"; diff --git a/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts b/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts index 07582d7f75773..9d0f33c9f29ca 100644 --- a/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts +++ b/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts @@ -363,17 +363,20 @@ const make = Effect.gen(function* () { resumedTurnStarts.set(commandId, { event, queued, sent }); const { messageId, ...request } = event.payload; yield* orchestrationEngine - .dispatch({ - type: "thread.turn.start", - commandId, - ...request, - message: { - messageId, - role: "user", - text: turnStart.value.message.text, - attachments: turnStart.value.message.attachments ?? [], + .dispatch( + { + type: "thread.turn.start", + commandId, + ...request, + message: { + messageId, + role: "user", + text: turnStart.value.message.text, + attachments: turnStart.value.message.attachments ?? [], + }, }, - }) + event.metadata.origin === undefined ? undefined : { origin: event.metadata.origin }, + ) .pipe( Effect.onError(() => Effect.sync(() => { @@ -1392,7 +1395,7 @@ const make = Effect.gen(function* () { titleSeed: event.payload.titleSeed, executionScope: thread.executionScope, hasOtherUserMessages, - fromClientSurface: event.metadata.origin?.surface !== undefined, + clientSurface: event.metadata.origin?.surface, mode: guardSettings.sessionSpendGuard.mode, }); if (launchRefusal !== null) { @@ -1404,6 +1407,7 @@ const make = Effect.gen(function* () { if (guardSettings.sessionSpendGuard.mode === "enforce") { const guardThread = yield* projectionSnapshotQuery.getThreadDetailById(thread.id, { activityKinds: SPEND_GUARD_ACTIVITY_KINDS, + unboundedActivities: true, }); const spend = Option.isSome(guardThread) ? evaluateThreadSpend(guardThread.value, guardSettings) diff --git a/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts b/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts index eac3ede9c1eea..b5ef8c22338cd 100644 --- a/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts +++ b/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts @@ -71,6 +71,7 @@ export interface ProjectionThreadDetailQuery { * the activity query. Omit this option to preserve the full detail response. */ readonly activityKinds?: ReadonlyArray; + readonly unboundedActivities?: boolean; } /** diff --git a/apps/server/src/orchestration/SessionSpendGuard.test.ts b/apps/server/src/orchestration/SessionSpendGuard.test.ts index 1e2f0546d161b..2300fa3455312 100644 --- a/apps/server/src/orchestration/SessionSpendGuard.test.ts +++ b/apps/server/src/orchestration/SessionSpendGuard.test.ts @@ -168,7 +168,7 @@ describe("evaluateSpendGuard", () => { settings: { ...settings, mode: "flag" }, activities: [priced("2026-01-01T00:01:00.000Z", 6)], }); - expect(result).toMatchObject({ level: "flag", stopReason: null, idleUsd: 6 }); + expect(result).toMatchObject({ level: "flag", stopReason: "idle", idleUsd: 6 }); }); it("stops for the session ceiling ahead of the idle limit", () => { @@ -213,6 +213,36 @@ describe("evaluateSpendGuard", () => { expect(idle.flaggedSinceIdle).toBe(true); expect(idle.stoppedSinceIdle).toBe(false); expect(idle.stoppedForCeiling).toBe(true); + expect(idle.flaggedForCeiling).toBe(false); + }); + + it("keeps a ceiling flag visible when an earlier idle flag is already recorded", () => { + const flagged = evaluate({ + settings: { ...settings, mode: "flag" }, + workSignalAts: ["2026-01-01T00:02:00.000Z"], + activities: [ + activity("spend-guard.flagged", "2026-01-01T00:03:00.000Z", { reason: "idle" }), + priced("2026-01-01T00:01:00.000Z", 6), + priced("2026-01-01T00:04:00.000Z", 4), + ], + }); + expect(flagged).toMatchObject({ + level: "flag", + stopReason: "ceiling", + flaggedSinceIdle: true, + flaggedForCeiling: false, + idleUsd: 4, + sessionUsd: 10, + }); + const recorded = evaluate({ + settings: { ...settings, mode: "flag" }, + activities: [ + priced("2026-01-01T00:01:00.000Z", 10), + activity("spend-guard.flagged", "2026-01-01T00:02:00.000Z", { reason: "ceiling" }), + ], + }); + expect(recorded.flaggedForCeiling).toBe(true); + expect(recorded.stopReason).toBe("ceiling"); }); }); @@ -222,7 +252,7 @@ describe("launch and stop copy", () => { titleSeed: undefined, executionScope: "coding" as const, hasOtherUserMessages: true, - fromClientSurface: false, + clientSurface: undefined, mode: "enforce" as const, }; @@ -234,7 +264,8 @@ describe("launch and stop copy", () => { expect(checkLaunch({ ...base, title: "Named thread" })).toBeNull(); expect(checkLaunch({ ...base, titleSeed: "seed" })).toBeNull(); expect(checkLaunch({ ...base, hasOtherUserMessages: false })).toBeNull(); - expect(checkLaunch({ ...base, fromClientSurface: true })).toBeNull(); + expect(checkLaunch({ ...base, clientSurface: "desktop" })).toBeNull(); + expect(checkLaunch({ ...base, clientSurface: "cli" })).toBe("untitled"); expect(checkLaunch({ ...base, mode: "flag" })).toBeNull(); }); diff --git a/apps/server/src/orchestration/SessionSpendGuard.ts b/apps/server/src/orchestration/SessionSpendGuard.ts index 5b3d401ffdddd..8c40b62c554ee 100644 --- a/apps/server/src/orchestration/SessionSpendGuard.ts +++ b/apps/server/src/orchestration/SessionSpendGuard.ts @@ -1,4 +1,5 @@ import type { + ClientSurface, OrchestrationCheckpointSummary, OrchestrationThread, OrchestrationThreadActivity, @@ -40,6 +41,7 @@ export interface SpendGuardEvaluation { readonly idleUsd: number; readonly idleSince: string | null; readonly flaggedSinceIdle: boolean; + readonly flaggedForCeiling: boolean; readonly stoppedSinceIdle: boolean; readonly stoppedForCeiling: boolean; } @@ -163,6 +165,9 @@ export function evaluateSpendGuard(input: SpendGuardInput): SpendGuardEvaluation ? (activity.payload as Record).reason : undefined; const flaggedSinceIdle = guardActivities(SPEND_GUARD_FLAGGED_KIND, afterIdle).length > 0; + const flaggedForCeiling = guardActivities(SPEND_GUARD_FLAGGED_KIND, afterRelease).some( + (activity) => stoppedPayloadReason(activity) === "ceiling", + ); const stoppedSinceIdle = guardActivities(SPEND_GUARD_STOPPED_KIND, afterIdle).some( (activity) => stoppedPayloadReason(activity) === "idle", ); @@ -175,6 +180,7 @@ export function evaluateSpendGuard(input: SpendGuardInput): SpendGuardEvaluation idleUsd, idleSince, flaggedSinceIdle, + flaggedForCeiling, stoppedSinceIdle, stoppedForCeiling, }; @@ -185,7 +191,7 @@ export function evaluateSpendGuard(input: SpendGuardInput): SpendGuardEvaluation return { ...base, level: "stop", stopReason }; } if (stopReason !== null || idleUsd >= idleFlagUsd) { - return { ...base, level: "flag", stopReason: null }; + return { ...base, level: "flag", stopReason }; } return { ...base, level: "ok", stopReason: null }; } @@ -215,8 +221,8 @@ export function describeSpendStop( ): string { const usd = (value: number) => `$${value.toFixed(2)}`; return evaluation.stopReason === "ceiling" - ? `This thread has spent about ${usd(evaluation.sessionUsd)}, past its ${usd(settings.sessionCeilingUsd)} session ceiling. Append a spend-guard.released activity or raise sessionSpendGuard.sessionCeilingUsd to continue.` - : `This thread has spent about ${usd(evaluation.idleUsd)} with no linked pull request or file-changing turn, past the ${usd(settings.idleStopUsd)} idle limit. Link a pull request, change files, or append a spend-guard.released activity to continue.`; + ? `This thread has spent about ${usd(evaluation.sessionUsd)}, past its ${usd(settings.sessionCeilingUsd)} session ceiling. Raise sessionSpendGuard.sessionCeilingUsd to continue.` + : `This thread has spent about ${usd(evaluation.idleUsd)} with no linked pull request or file-changing turn, past the ${usd(settings.idleStopUsd)} idle limit. Link a pull request or change files to continue.`; } export type LaunchRefusal = "untitled" | "ownerless"; @@ -226,12 +232,16 @@ export interface LaunchCheckInput { readonly titleSeed: string | undefined; readonly executionScope: ThreadExecutionScope | null | undefined; readonly hasOtherUserMessages: boolean; - readonly fromClientSurface: boolean; + readonly clientSurface: ClientSurface | undefined; readonly mode: SessionSpendGuardSettings["mode"]; } +const INTERACTIVE_SURFACES: ReadonlySet = new Set(["web", "desktop", "mobile"]); + export function checkLaunch(input: LaunchCheckInput): LaunchRefusal | null { - if (input.mode !== "enforce" || input.fromClientSurface) return null; + const interactive = + input.clientSurface !== undefined && INTERACTIVE_SURFACES.has(input.clientSurface); + if (input.mode !== "enforce" || interactive) return null; if (input.executionScope === null || input.executionScope === undefined) return "ownerless"; const untitled = input.title.trim() === "" || input.title.trim() === DEFAULT_THREAD_TITLE; if (untitled && input.titleSeed === undefined && input.hasOtherUserMessages) return "untitled"; diff --git a/apps/server/src/orchestration/SessionSpendGuardReactor.ts b/apps/server/src/orchestration/SessionSpendGuardReactor.ts index 09a70129edb6d..62a264e83cf53 100644 --- a/apps/server/src/orchestration/SessionSpendGuardReactor.ts +++ b/apps/server/src/orchestration/SessionSpendGuardReactor.ts @@ -72,6 +72,7 @@ export const make = Effect.gen(function* () { if (settings.sessionSpendGuard.mode === "off") return; const thread = yield* snapshots.getThreadDetailById(threadId, { activityKinds: SPEND_GUARD_ACTIVITY_KINDS, + unboundedActivities: true, }); if (Option.isNone(thread)) return; const evaluation = evaluateThreadSpend(thread.value, settings); @@ -99,7 +100,11 @@ export const make = Effect.gen(function* () { }, ); const session = thread.value.session; - if (session !== null && (session.status === "running" || session.status === "starting")) { + if ( + evaluation.stopReason === "ceiling" && + session !== null && + (session.status === "running" || session.status === "starting") + ) { const uuid = yield* crypto.randomUUIDv4; yield* engine.dispatch({ type: "thread.turn.interrupt", @@ -110,6 +115,20 @@ export const make = Effect.gen(function* () { } return; } + if (evaluation.level === "flag" && evaluation.stopReason === "ceiling") { + if (evaluation.flaggedForCeiling) return; + yield* appendGuardActivity( + threadId, + SPEND_GUARD_FLAGGED_KIND, + "Session spend ceiling reached", + { + ...amounts, + reason: "ceiling", + detail: describeSpendStop(evaluation, settings.sessionSpendGuard), + }, + ); + return; + } if (evaluation.level === "flag" && !evaluation.flaggedSinceIdle) { yield* appendGuardActivity( threadId, @@ -117,6 +136,7 @@ export const make = Effect.gen(function* () { "Spending with no pull request or file changes", { ...amounts, + reason: "idle", detail: `This thread has spent about $${amounts.idleUsd.toFixed(2)} since its last linked pull request or file-changing turn. It stops at $${settings.sessionSpendGuard.idleStopUsd.toFixed(2)}.`, }, );