Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -295,7 +295,8 @@ it.each([

function startWithSpendGuard(input: {
readonly title: string;
readonly creationSource: "web" | "mcp";
readonly creationSource: "web" | "mcp" | "server";
readonly restartContinuationOf?: "web" | "mcp";
readonly pullRequestLinkedAt?: string;
readonly providerTurns: ReadonlyArray<{
readonly startedAt: string;
Expand All @@ -308,6 +309,8 @@ function startWithSpendGuard(input: {
const rootNodeId = NodeId.make("node_provider_turn_start_spend_guard");
const providerThreadId = ProviderThreadId.make("provider_thread_provider_turn_start_spend_guard");
const messageId = MessageId.make("message_provider_turn_start_spend_guard");
const sourceRunId = RunId.make("run_provider_turn_start_spend_guard_source");
const sourceMessageId = MessageId.make("message_provider_turn_start_spend_guard_source");
const projection = {
thread: {
id: threadId,
Expand All @@ -323,8 +326,14 @@ function startWithSpendGuard(input: {
},
runs: [
{ id: RunId.make("run_provider_turn_start_spend_guard_first"), ordinal: 1 },
...(input.restartContinuationOf === undefined
? []
: [{ id: sourceRunId, userMessageId: sourceMessageId, ordinal: 1 }]),
{
id: runId,
...(input.restartContinuationOf === undefined
? {}
: { restartContinuationOfRunId: sourceRunId }),
status: "starting",
rootNodeId,
activeAttemptId: attemptId,
Expand All @@ -349,6 +358,17 @@ function startWithSpendGuard(input: {
attachments: [],
creationSource: input.creationSource,
},
...(input.restartContinuationOf === undefined
? []
: [
{
id: sourceMessageId,
role: "user",
text: "continue anyway",
attachments: [],
creationSource: input.restartContinuationOf,
},
]),
],
checkpointScopes: [{ id: CheckpointScopeId.make("scope_spend_guard") }],
checkpoints: [],
Expand Down Expand Up @@ -458,6 +478,33 @@ effectIt.effect("lets a founder's own turn through at the session ceiling", () =
}),
);

effectIt.effect("lets a restart continuation of a founder's own turn through at the ceiling", () =>
Effect.gen(function* () {
const harness = startWithSpendGuard({
title: "Manager work",
creationSource: "server",
restartContinuationOf: "web",
providerTurns: [{ startedAt: "2099-01-01T00:00:01.000Z", inputTokens: 1 }],
});
yield* Effect.exit(harness.start);
expect(harness.refusal()).toBeUndefined();
}),
);

effectIt.effect("still refuses a restart continuation of an automated turn at the ceiling", () =>
Effect.gen(function* () {
const harness = startWithSpendGuard({
title: "Manager work",
creationSource: "server",
restartContinuationOf: "mcp",
providerTurns: [{ startedAt: "2099-01-01T00:00:01.000Z", inputTokens: 1 }],
});
yield* harness.start;
expect(harness.refusal()?.failure.message).toContain("session ceiling");
expect(harness.startRootRun).not.toHaveBeenCalled();
}),
);

effectIt.effect("counts every provider turn in the thread toward the session ceiling", () =>
Effect.gen(function* () {
const harness = startWithSpendGuard({
Expand Down
11 changes: 9 additions & 2 deletions apps/server/src/orchestration-v2/ProviderTurnStartService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ import * as RunExecutionService from "./RunExecutionService.ts";
import * as RuntimePolicy from "./RuntimePolicy.ts";
import {
checkLaunch,
originCreationSource,
refusesAutomatedTurn,
describeSpendStop,
evaluateThreadSpend,
Expand Down Expand Up @@ -563,11 +564,17 @@ export const layer: Layer.Layer<
},
);
const guardSettings = yield* serverSettings.getSettings;
const guardCreationSource =
originCreationSource({
run,
runs: projection.runs,
messages: projection.messages,
}) ?? message.creationSource;
const launchRefusal = checkLaunch({
title: projection.thread.title,
titlePending: projection.thread.titleRegeneration != null,
hasOtherUserMessages: projection.runs.some((candidate) => candidate.ordinal < run.ordinal),
creationSource: message.creationSource,
creationSource: guardCreationSource,
mode: guardSettings.sessionSpendGuard.mode,
});
if (launchRefusal !== null) {
Expand All @@ -579,7 +586,7 @@ export const layer: Layer.Layer<
});
return;
}
if (refusesAutomatedTurn(guardSettings.sessionSpendGuard.mode, message.creationSource)) {
if (refusesAutomatedTurn(guardSettings.sessionSpendGuard.mode, guardCreationSource)) {
const { checkpoints } = yield* projectionStore.getThreadRecords(input.threadId, [
"checkpoints",
]);
Expand Down
139 changes: 119 additions & 20 deletions apps/server/src/orchestration-v2/SessionSpendGuard.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import {
describeSpendStop,
evaluateSpendGuard,
evaluateThreadSpend,
originCreationSource,
priceProviderTurns,
rateForModel,
spendGuardMarkers,
Expand All @@ -30,6 +31,7 @@ const rate = {

const settings = {
...decodeSettings({}).sessionSpendGuard,
mode: "enforce" as const,
idleFlagUsd: 3,
idleStopUsd: 5,
sessionCeilingUsd: 10,
Expand All @@ -56,26 +58,69 @@ const evaluate = (overrides: Partial<SpendGuardInput> = {}) =>

describe("priceProviderTurns", () => {
it("prices uncached input, cache read, output, and other usage", () => {
expect(
priceProviderTurns(
[
{
startedAt: at("2026-01-01T00:00:00.000Z"),
completedAt: at("2026-01-01T00:05:00.000Z"),
turnTokenUsage: {
usageScope: "main_agent",
usageStatus: "complete",
hasSubagents: true,
inputTokens: 10,
cachedInputTokens: 4,
outputTokens: 2,
otherUsage: { inputTokens: 1, outputTokens: 1 },
},
const [entry] = priceProviderTurns(
[
{
startedAt: at("2026-01-01T00:00:00.000Z"),
completedAt: at("2026-01-01T00:05:00.000Z"),
turnTokenUsage: {
usageScope: "main_agent",
usageStatus: "complete",
hasSubagents: true,
inputTokens: 10,
cachedInputTokens: 4,
outputTokens: 2,
otherUsage: { inputTokens: 1, outputTokens: 1 },
},
],
rate,
),
).toEqual([{ createdAt: "2026-01-01T00:00:00.000Z", usd: 13.4 }]);
},
],
rate,
);
expect(entry?.createdAt).toBe("2026-01-01T00:00:00.000Z");
expect(entry?.usd).toBeCloseTo(13.04, 6);
});

it("prices other-agent input at the main agent's cache mix", () => {
const [entry] = priceProviderTurns(
[
{
startedAt: at("2026-01-01T00:00:00.000Z"),
completedAt: at("2026-01-01T00:05:00.000Z"),
turnTokenUsage: {
usageScope: "main_agent",
usageStatus: "complete",
hasSubagents: true,
inputTokens: 1000,
cachedInputTokens: 980,
outputTokens: 0,
otherUsage: { inputTokens: 1_000_000, outputTokens: 0 },
},
},
],
rate,
);
expect(entry?.usd).toBeCloseTo(20 + 98 + 1_000_000 * (0.02 + 0.98 * 0.1), 3);
});

it("prices other-agent input as uncached when the main agent reports no cache reads", () => {
const [entry] = priceProviderTurns(
[
{
startedAt: at("2026-01-01T00:00:00.000Z"),
completedAt: null,
turnTokenUsage: {
usageScope: "main_agent",
usageStatus: "complete",
hasSubagents: true,
inputTokens: 0,
outputTokens: 0,
otherUsage: { inputTokens: 10, outputTokens: 0 },
},
},
],
rate,
);
expect(entry?.usd).toBe(10);
});

it("falls back to the context snapshot and skips a turn with no counts", () => {
Expand Down Expand Up @@ -283,14 +328,25 @@ describe("launch and stop copy", () => {
const evaluation = evaluate({ spend: [priced("2026-01-01T00:01:00.000Z", 6)] });
expect(describeSpendStop(evaluation, settings)).toContain("$6.00");
expect(describeSpendStop(evaluation, settings)).toContain("$5.00");
expect(describeSpendStop(evaluation, settings)).toContain("with no linked pull request");
expect(UNTITLED_LAUNCH_REFUSAL).toContain("default title");
});

it("does not say no pull request is linked when one is", () => {
const evaluation = evaluate({
spend: [priced("2026-01-01T00:02:00.000Z", 6)],
workSignalAts: ["2026-01-01T00:01:00.000Z"],
});
const detail = describeSpendStop(evaluation, settings);
expect(detail).toContain("since its last linked pull request or file-changing turn");
expect(detail).not.toContain("with no linked pull request");
});
});

describe("evaluateThreadSpend", () => {
it("uses the model override and otherwise the fallback rate", () => {
const decoded = decodeSettings({
sessionSpendGuard: { idleStopUsd: 1, sessionCeilingUsd: 100 },
sessionSpendGuard: { mode: "enforce", idleStopUsd: 1, sessionCeilingUsd: 100 },
usagePriceOverrides: {
"custom-model": {
inputCostPerMillionTokens: 1_000_000,
Expand Down Expand Up @@ -323,3 +379,46 @@ describe("evaluateThreadSpend", () => {
).toBe(10 / 1_000_000);
});
});

describe("originCreationSource", () => {
const messages = [
{ id: "m-web", creationSource: "web" as const },
{ id: "m-mcp", creationSource: "mcp" as const },
{ id: "m-server", creationSource: "server" as const },
] as never;
const runs = [
{ id: "r1", userMessageId: "m-web" },
{ id: "r2", userMessageId: "m-server", restartContinuationOfRunId: "r1" },
{ id: "r3", userMessageId: "m-server", restartContinuationOfRunId: "r2" },
{ id: "r4", userMessageId: "m-mcp" },
{ id: "r5", userMessageId: "m-server", restartContinuationOfRunId: "r4" },
{ id: "r6", userMessageId: "m-server", restartContinuationOfRunId: "missing" },
{ id: "r7", userMessageId: "m-server", usageLimitContinuationOfRunId: "r1" },
{ id: "r8", userMessageId: "m-server", restartContinuationOfRunId: "r7" },
] as never;
const origin = (id: string) =>
originCreationSource({
run: (runs as ReadonlyArray<{ id: string }>).find((run) => run.id === id) as never,
runs,
messages,
});

it("follows restart continuations back to the founder's message", () => {
expect(origin("r1")).toBe("web");
expect(origin("r2")).toBe("web");
expect(origin("r3")).toBe("web");
});

it("follows usage limit continuations and mixed chains", () => {
expect(origin("r7")).toBe("web");
expect(origin("r8")).toBe("web");
});

it("keeps an automated origin automated", () => {
expect(origin("r5")).toBe("mcp");
});

it("falls back to the run's own message when the source run is gone", () => {
expect(origin("r6")).toBe("server");
});
});
40 changes: 38 additions & 2 deletions apps/server/src/orchestration-v2/SessionSpendGuard.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import type {
OrchestrationV2Checkpoint,
OrchestrationV2ConversationMessage,
OrchestrationV2ProviderTurn,
OrchestrationV2Run,
OrchestrationV2TurnItem,
ServerSettings,
SessionSpendGuardSettings,
Expand Down Expand Up @@ -97,11 +98,14 @@ export function priceProviderTurns(
const input = usage.inputTokens ?? 0;
const cached = Math.min(usage.cachedInputTokens ?? 0, input);
const other = turn.turnTokenUsage?.otherUsage;
const cachedShare = input > 0 ? cached / input : 0;
const otherInputRate =
(1 - cachedShare) * rate.inputCostPerToken + cachedShare * rate.cacheReadCostPerToken;
const usd =
(input - cached) * rate.inputCostPerToken +
cached * rate.cacheReadCostPerToken +
(usage.outputTokens ?? 0) * rate.outputCostPerToken +
(other?.inputTokens ?? 0) * rate.inputCostPerToken +
(other?.inputTokens ?? 0) * otherInputRate +
(other?.outputTokens ?? 0) * rate.outputCostPerToken;
const at = turn.startedAt ?? turn.completedAt;
const createdAt = at === null ? (turn.tokenUsage?.updatedAt ?? null) : DateTime.formatIso(at);
Expand Down Expand Up @@ -219,7 +223,11 @@ export function describeSpendStop(
const usd = (value: number) => `$${value.toFixed(2)}`;
return evaluation.stopReason === "ceiling"
? `This thread has spent about ${usd(evaluation.sessionUsd)}, past its ${usd(settings.sessionCeilingUsd)} session ceiling. Raise sessionSpendGuard.sessionCeilingUsd to continue.`
: `This thread has spent about ${usd(evaluation.idleUsd)} with no linked pull request or file-changing turn, past the ${usd(settings.idleStopUsd)} idle limit. Link a pull request or change files to continue.`;
: `This thread has spent about ${usd(evaluation.idleUsd)} ${
evaluation.idleSince === null
? "with no linked pull request or file-changing turn"
: "since its last linked pull request or file-changing turn"
}, past the ${usd(settings.idleStopUsd)} idle limit. Link a pull request or change files to continue.`;
}

export interface LaunchCheckInput {
Expand All @@ -240,6 +248,34 @@ export function refusesAutomatedTurn(
return mode === "enforce" && !INTERACTIVE_SOURCES.has(creationSource);
}

const MAX_ORIGIN_CHAIN = 50;

export function originCreationSource(input: {
readonly run: Pick<
OrchestrationV2Run,
"userMessageId" | "restartContinuationOfRunId" | "usageLimitContinuationOfRunId"
>;
readonly runs: ReadonlyArray<
Pick<
OrchestrationV2Run,
"id" | "userMessageId" | "restartContinuationOfRunId" | "usageLimitContinuationOfRunId"
>
>;
readonly messages: ReadonlyArray<
Pick<OrchestrationV2ConversationMessage, "id" | "creationSource">
>;
}): OrchestrationV2ConversationMessage["creationSource"] | undefined {
let current = input.run;
for (let depth = 0; depth < MAX_ORIGIN_CHAIN; depth += 1) {
const sourceRunId = current.restartContinuationOfRunId ?? current.usageLimitContinuationOfRunId;
if (sourceRunId === undefined) break;
const source = input.runs.find((candidate) => candidate.id === sourceRunId);
if (source === undefined) break;
current = source;
}
return input.messages.find((message) => message.id === current.userMessageId)?.creationSource;
}

export function checkLaunch(input: LaunchCheckInput): "untitled" | null {
if (!refusesAutomatedTurn(input.mode, input.creationSource)) return null;
const untitled = input.title.trim() === "" || input.title.trim() === "New thread";
Expand Down
Loading
Loading