From 2f0e16026715366d674c8b774a4ed2f18e0b2e8e Mon Sep 17 00:00:00 2001 From: sabhatinas Date: Fri, 13 Nov 2026 14:32:32 -0800 Subject: [PATCH 1/5] feat(pi): compare LightGBM and Jev token forecasts Signed-off-by: sabhatinas --- examples/pi/FORECAST_COMPARISON.md | 178 +++++++++++++++ examples/pi/README.md | 3 + examples/pi/completion-time.ts | 31 +++ examples/pi/forecast-classifier.ts | 51 +++++ examples/pi/forecast-error.ts | 60 +++++ examples/pi/forecast-worker.ts | 108 +++++++++ examples/pi/forecast/lightgbm_worker.py | 206 ++++++++++++++++++ examples/pi/forecast/test_lightgbm_worker.py | 61 ++++++ examples/pi/metrics.ts | 16 +- examples/pi/prediction-total.ts | 47 ++++ examples/pi/remaining-error.ts | 46 ++++ examples/pi/run-forecast.sh | 33 +++ examples/pi/switchyard-compare.ts | 173 +++++++++++++++ examples/pi/switchyard-forecast.example.json | 7 + examples/pi/test/compare.test.mjs | 118 ++++++++++ examples/pi/test/completion-time.test.mjs | 22 ++ examples/pi/test/fixtures/forecast-worker.mjs | 14 ++ examples/pi/test/forecast-error.test.mjs | 66 ++++++ examples/pi/test/metrics.test.mjs | 27 +++ examples/pi/test/prediction-total.test.mjs | 37 ++++ examples/pi/test/remaining-error.test.mjs | 40 ++++ 21 files changed, 1342 insertions(+), 2 deletions(-) create mode 100644 examples/pi/FORECAST_COMPARISON.md create mode 100644 examples/pi/forecast-classifier.ts create mode 100644 examples/pi/forecast-error.ts create mode 100644 examples/pi/forecast-worker.ts create mode 100644 examples/pi/forecast/lightgbm_worker.py create mode 100644 examples/pi/forecast/test_lightgbm_worker.py create mode 100644 examples/pi/prediction-total.ts create mode 100644 examples/pi/remaining-error.ts create mode 100755 examples/pi/run-forecast.sh create mode 100644 examples/pi/switchyard-compare.ts create mode 100644 examples/pi/switchyard-forecast.example.json create mode 100644 examples/pi/test/compare.test.mjs create mode 100644 examples/pi/test/fixtures/forecast-worker.mjs create mode 100644 examples/pi/test/forecast-error.test.mjs create mode 100644 examples/pi/test/prediction-total.test.mjs create mode 100644 examples/pi/test/remaining-error.test.mjs diff --git a/examples/pi/FORECAST_COMPARISON.md b/examples/pi/FORECAST_COMPARISON.md new file mode 100644 index 0000000000..b74a8c84bd --- /dev/null +++ b/examples/pi/FORECAST_COMPARISON.md @@ -0,0 +1,178 @@ +# Compare LightGBM and Jev forecasts in Pi + +`switchyard/compare` runs both predictors before each acting-model call. One +fixed completion model executes the task, so both forecasts observe the same +pre-call conversation. The display has two rows below the editor: + +```text +LightGBM · ~$0.0200 · ETA ~20s · next ~8.0k tok (avg err +4.0%) | remaining ~30.0k tok (avg err −8.0%) +Jev · ~$0.0400 · ETA ~40s · remaining ~60.0k tok (avg err +12.0%) +``` + +These are illustrative values. Cost and ETA describe remaining work. The two +rows replace the Switchyard footer. `medium` and `max` are passed through to the +completion model, which must support the selected level. + +## Setup + +First follow the Node/Rust build instructions in [README.md](README.md). Pi +1.0.4 is the tested version. Provider credentials are owned by Pi; for the +example configuration, make `OPENROUTER_API_KEY` available in each terminal. +Jev requests are additional API calls and may incur fees. + +The LightGBM worker needs three external inputs, which are not bundled: + +- An `agent-progress-research` checkout with its `v1` scripts and grouped-IR + checkpoints. The tested revision is `d87391d39`. Access to the research + repository is required. Download the actual Git LFS checkpoint files rather + than pointer files. +- [TokenCast](https://github.com/DEFENSE-SEU/TokenCast) checked out at + `d2e2490cb794dc57dd5bb03f732bb33fd2ec5d19`. +- The research project's Python 3.12 environment, installed with + `uv sync --project /path/to/agent-progress-research/v1 --frozen`. + +The worker uses the first published checkpoint, +`v1/checkpoints/grouped-ir/seed-20260719-window-8/pipeline.pkl`, and verifies its +manifest SHA256 before loading. It reuses the fitted research pipeline; there +is no retraining or checkpoint selection in this example. External research +and TokenCast code remain in their own checkouts under their respective licenses. + +Copy and edit the configuration with **absolute paths**: + +```bash +mkdir -p "$HOME/.config/switchyard-pi-forecast" +cp examples/pi/switchyard-forecast.example.json \ + "$HOME/.config/switchyard-pi-forecast/switchyard-forecast.json" +``` + +The first slash in each model reference separates the provider from its model +ID. `decision_model` must be registered as a classifier in Pi. The Python +worker paths can alternatively be set with `SY_FORECAST_PYTHON`, +`SY_FORECAST_RESEARCH_ROOT`, and `SY_FORECAST_TOKENCAST_ROOT`. + +Run from a task directory, using an absolute path to this checkout's launcher: + +```bash +/path/to/Switchyard/examples/pi/run-forecast.sh --thinking max +``` + +In another task directory: + +```bash +/path/to/Switchyard/examples/pi/run-forecast.sh --thinking medium +``` + +Use separate directories and the same prompt to compare reasoning settings +without editing the same task files. Each level has its own Pi settings and +sessions under `~/.config/switchyard-pi-forecast/{max,medium}`. Add `--continue` +from the same task directory to resume. `SY_FORECAST_AGENT_ROOT` overrides the +settings root; `SY_FORECAST_CONFIG` selects a shared configuration file. When +set, `XDG_CONFIG_HOME` replaces `~/.config` for the default locations. + +For direct Pi usage, put `switchyard-forecast.json` in the selected Pi agent +directory, or set `SY_FORECAST_CONFIG`, then load `switchyard-compare.ts` with +`--model switchyard/compare --thinking medium`. + +## Shared prediction target + +Both remaining forecasts count **all provider input plus output**: + +```text +input + cacheRead + cacheWrite + output +``` + +Input counts again on every call, including repeated cached context. Reasoning +is already part of output and is counted once. Predictor API usage is excluded +from these acting-model token targets and recorded separately. + +LightGBM forecasts next-call and remaining-task tokens from its existing +grouped IR and causal features. The worker replays only the observed branch; +historical tools are parsed, never executed. Pi's standard tools have explicit +adapters. Unsupported tools make the forecast unavailable without blocking the +completion model. Requirement outputs remain null. + +Jev sees Greg's bounded task-history representation, full per-call and +cumulative token usage, completion model, and selected reasoning level. It +scores the original 80 token-budget bands through 1.5M tokens plus `impossible`. +Its numeric total estimate is the probability-weighted mean of finite band +**midpoints**, conditional on finite-budget probability. This comparison uses +that point estimate rather than the original display's upper-bound mean. +Remaining is `max(0, estimated total - tokens already used)`. Jev has no +next-call forecast. + +Both predictors run concurrently before each normal user, continuation, or +retry call. Direct helper calls skip forecasting. Cancellation propagates to +Pi. Failed forecasts are recorded independently and allow completion to proceed. +The displayed remaining values consume observed usage until the next forecast. + +## Cost, ETA, and errors + +Both rows use the same calibration from successful, timed acting-model calls +in their session. Let `F` be their observed full-token total and `R` the +remaining forecast: + +```text +projected channel tokens = R × observed channel tokens / F +ETA = R / F × (observed answer duration + tool wall time) +``` + +The channels are uncached input, cache reads, cache writes, and output. Integer +allocation preserves the token budget. One-hour cache writes are a subset of +cache writes. Switchyard prices the projected channels using the selected +model's rates, with pricing tiers selected by mean observed prompt size. +Tool timing retains Greg's non-overlap accounting. Answer timing starts after +forecasting, so predictor latency is excluded from completion timing. + +These estimates exclude future predictor fees and latency. First-call cost +and ETA are unavailable until there is a usable timed response. They assume +the observed channel mix, prompt sizes, and speed continue. + +Next-call error updates after each valid completed call. Remaining errors for +both predictors become available when Pi produces a normal final response and +settles the user turn. Each saved pre-call forecast is scored against actual +full-token usage from its own cutoff to that boundary: + +```text +signed percentage error = 100 × (predicted remaining - actual remaining) + / actual remaining +``` + +`avg err` averages these signed percentages across eligible forecasts in the +session branch. Positive means overprediction; negative means underprediction. +Signs can cancel, so this measures bias rather than MAE or MAPE. Aborted/failed +turns are not scored for remaining usage. Missing usage and unavailable +forecasts are skipped; zero actual usage has no percentage error. A normal Pi +stop defines the measured horizon, not independent task correctness. + +Predictions and scores persist as `switchyard.compare.forecast.v2` and +`switchyard.compare.remaining-errors.v2`. Resume deduplicates scores. Earlier +uncached Jev entries are excluded. Actual metrics keep their existing schema. + +The checkpoint was trained on Opus mini-SWE-agent traces with a 30-call limit. +Pi and other model/reasoning settings are transfer settings. No Pi call cap is +invented. The checkpoint has no explicit reasoning-level or cache-split head; +passing reasoning metadata does not change its fitted features. + +## Validation + +The default Node suite uses a local JSON-lines worker fixture and needs no +checkpoint, Python environment, or API calls: + +```bash +npm --prefix examples/pi run typecheck +npm --prefix examples/pi test +``` + +To exercise the real checkpoint and native-tool adapters as well: + +```bash +export SY_FORECAST_RESEARCH_ROOT=/path/to/agent-progress-research/v1 +export SY_FORECAST_TOKENCAST_ROOT=/path/to/TokenCast +/path/to/agent-progress-research/v1/.venv/bin/python \ + -m unittest discover -s examples/pi/forecast -p 'test_*.py' +``` + +For another compatible JSON-lines forecaster, `worker_command` may contain an +executable and argument array in the configuration. The worker receives one +JSON request per line and returns `{ "id": ..., "result": ... }` or +`{ "id": ..., "error": ... }`. See the Node fixture for the response contract. diff --git a/examples/pi/README.md b/examples/pi/README.md index bcd9b98eac..e20f582e53 100644 --- a/examples/pi/README.md +++ b/examples/pi/README.md @@ -1,5 +1,8 @@ # Pi virtual model using Switchyard composite routing +For a fixed-model demo comparing LightGBM and Jev forecasts, see +[FORECAST_COMPARISON.md](FORECAST_COMPARISON.md). + This extension registers `switchyard/auto`. Pi calls a Decision classifier on user turns. Switchyard's composite algorithm uses that answer and recent tool activity to select a completion model. Thinking stays `medium`. diff --git a/examples/pi/completion-time.ts b/examples/pi/completion-time.ts index 99b5dda5cf..4657dd9743 100644 --- a/examples/pi/completion-time.ts +++ b/examples/pi/completion-time.ts @@ -18,6 +18,11 @@ export class CompletionTiming { #calls = 0; #toolMs = 0; #timedTurns = new Set(); + #fullInput = 0; + #fullOutput = 0; + #fullRead = 0; + #fullWrite = 0; + #fullLongWrite = 0; record(call: CallMetrics) { if (call.call_kind !== "answer" || !call.is_success || call.ttft_ms === undefined @@ -36,6 +41,11 @@ export class CompletionTiming { this.#ttftMs += call.ttft_ms; this.#outputMs += call.duration_ms - call.ttft_ms; this.#calls++; + this.#fullInput += call.usage.input_tokens; + this.#fullOutput += call.usage.output_tokens; + this.#fullRead += call.usage.cached_input_tokens ?? 0; + this.#fullWrite += call.usage.cache_creation_input_tokens ?? 0; + this.#fullLongWrite += call.usage.cache_creation_1h_input_tokens ?? 0; this.#timedTurns.add(turnKey(call.context)); } @@ -47,6 +57,27 @@ export class CompletionTiming { /** Mean prompt size per call, for selecting the current model's pricing tier. */ get meanPromptTokens(): number { return this.#calls ? this.#promptTokens / this.#calls : 0; } + get fullTokens(): number { return this.#fullInput + this.#fullOutput + this.#fullRead + this.#fullWrite; } + + /** Allocate one full-token budget across disjoint channels using observed session totals. */ + remainingFullUsage(remainingTokens: number): TokenUsage | undefined { + if (this.fullTokens === 0 && remainingTokens > 0) return undefined; + const total = Math.round(remainingTokens); + const share = (value: number) => this.fullTokens ? Math.round(total * value / this.fullTokens) : 0; + const output = Math.min(total, share(this.#fullOutput)); + const read = Math.min(total - output, share(this.#fullRead)); + const write = Math.min(total - output - read, share(this.#fullWrite)); + return { input_tokens: total - output - read - write, output_tokens: output, + cached_input_tokens: read, cache_creation_input_tokens: write, + cache_creation_1h_input_tokens: Math.min(write, share(this.#fullLongWrite)) }; + } + + /** Scale observed model-plus-tool wall time by the full-token budget. */ + estimateFullMs(remainingTokens: number): number | undefined { + if (remainingTokens === 0) return 0; + return this.fullTokens ? remainingTokens * (this.#ttftMs + this.#outputMs + this.#toolMs) / this.fullTokens : undefined; + } + /** Splits the uncached budget and projects additional cache reads for monetary pricing. */ remainingUsage(remainingTokens: number): TokenUsage | undefined { if (this.#calls === 0 && remainingTokens > 0) return undefined; diff --git a/examples/pi/forecast-classifier.ts b/examples/pi/forecast-classifier.ts new file mode 100644 index 0000000000..2534510e4e --- /dev/null +++ b/examples/pi/forecast-classifier.ts @@ -0,0 +1,51 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { DecisionRequest, DecisionResponse } from "@switchyard/libsy"; +import type { ClassifierContext, ClassifierResult } from "@earendil-works/pi-ai"; +import { tokenUsage } from "./metrics.ts"; + +export function modelIdentity(reference: unknown) { + if (typeof reference !== "string") throw new Error("Expected a model in provider/model form"); + // Model IDs may themselves contain slashes, as on OpenRouter. + const slash = reference.indexOf("/"); + if (slash <= 0 || slash === reference.length - 1) throw new Error("Expected a model in provider/model form"); + return { reference, provider: reference.slice(0, slash), id: reference.slice(slash + 1) }; +} + +function text(value: unknown): string { + return typeof value === "string" ? value : JSON.stringify(value ?? null); +} + +export function classifierContext(request: DecisionRequest): ClassifierContext { + const questions: ClassifierContext["questions"] = {}; + for (const [id, question] of Object.entries(request.questions)) { + if (question.kind.type !== "choice") throw new Error(`Expected a choice question: ${id}`); + questions[id] = { + type: "choice", + instructions: text(question.instructions), + criteria: Object.fromEntries(question.kind.data.options.map((option) => [option.id, text(option.description)])), + }; + } + return { state: request.context as ClassifierContext["state"], questions }; +} + +export function decisionResponse(request: DecisionRequest, result: ClassifierResult): DecisionResponse { + if (result.stopReason === "aborted") { + throw Object.assign(new Error(result.errorMessage ?? "Classification cancelled"), { name: "AbortError", code: "ABORT_ERR" }); + } + if (result.stopReason === "error") throw new Error(result.errorMessage ?? "Classification failed"); + const answers = Object.fromEntries(Object.entries(request.questions).map(([id, question]) => { + const answer = result.answers[id]; + if (question.kind.type !== "choice" || answer?.type !== "choice" + || !question.kind.data.options.some((option) => option.id === answer.choice)) { + throw new Error(`Expected a declared choice for question: ${id}`); + } + return [id, { + value: { type: "choice" as const, data: { selected: answer.choice, probabilities: answer.probabilities } }, + provider_confidence: answer.confidence, + }]; + })); + return { model: `${result.provider}/${result.model}`, answers, + ...(result.usage ? { usage: tokenUsage(result.usage) } : {}) }; +} diff --git a/examples/pi/forecast-error.ts b/examples/pi/forecast-error.ts new file mode 100644 index 0000000000..26536a6c39 --- /dev/null +++ b/examples/pi/forecast-error.ts @@ -0,0 +1,60 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export const ERROR_ENTRY = "switchyard.lightgbm.error.v1"; + +export interface ErrorSample { + sample_id: string; + session_id: string; + predicted_tokens: number; + actual_tokens: number; + absolute_error: number; + signed_error?: number; // Predicted minus actual; optional on older saved records. +} + +export class NextCallMAE { + private samples = new Map(); + private pending?: { sampleId: string; predicted: number }; + private sessionId: string; + constructor(sessionId: string, saved: ErrorSample[] = []) { + this.sessionId = sessionId; + for (const sample of saved) { + if (sample.session_id === sessionId && Number.isFinite(sample.absolute_error)) { + this.samples.set(sample.sample_id, sample); + } + } + } + begin(sampleId: string, predicted: number | undefined) { + this.pending = predicted !== undefined && Number.isFinite(predicted) ? { sampleId, predicted } : undefined; + } + finish(usage: { input: number; output: number; cacheRead: number; cacheWrite: number } | undefined, success: boolean) { + const pending = this.pending; + this.pending = undefined; + if (!pending || !success || !usage || this.samples.has(pending.sampleId)) return; + const channels = [usage.input, usage.output, usage.cacheRead, usage.cacheWrite]; + if (!channels.every((value) => Number.isFinite(value) && value >= 0)) return; + // Same target as the checkpoint: all provider input + output, reasoning once. + const actual = channels.reduce((sum, value) => sum + value, 0); + const sample: ErrorSample = { sample_id: pending.sampleId, session_id: this.sessionId, + predicted_tokens: pending.predicted, actual_tokens: actual, signed_error: pending.predicted - actual, absolute_error: Math.abs(pending.predicted - actual) }; + this.samples.set(sample.sample_id, sample); + return sample; + } + summary() { + const samples = [...this.samples.values()]; + const total = samples.reduce((sum, s) => sum + s.absolute_error, 0); + // Zero-token responses have no defined percentage error. + const percentages = samples.filter(s => s.actual_tokens > 0) + .map(s => 100 * (s.predicted_tokens - s.actual_tokens) / s.actual_tokens); + const meanPercentageError = percentages.length + ? percentages.reduce((sum, value) => sum + value, 0) / percentages.length : null; + return { percentageCount: percentages.length, meanPercentageError, count: samples.length, mae: samples.length ? total / samples.length : null, + last: samples.at(-1)?.absolute_error ?? null, + lastSigned: samples.length ? samples.at(-1)!.predicted_tokens - samples.at(-1)!.actual_tokens : null }; + } +} + +export function errorStatus(stats: ReturnType) { + return stats.meanPercentageError === null ? " | avg err — (n=0)" + : ` | avg err ${stats.meanPercentageError.toLocaleString("en-US", { signDisplay: "always", minimumFractionDigits: 1, maximumFractionDigits: 1 })}% (n=${stats.percentageCount})`; +} diff --git a/examples/pi/forecast-worker.ts b/examples/pi/forecast-worker.ts new file mode 100644 index 0000000000..d8a5091401 --- /dev/null +++ b/examples/pi/forecast-worker.ts @@ -0,0 +1,108 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; +import { createInterface } from "node:readline"; +import { fileURLToPath } from "node:url"; + +interface ForecastWorkerOptions { + python?: string; + researchRoot?: string; + tokenCastRoot?: string; + command?: string[]; +} + +export interface HorizonForecast { + next_call: { tokens: number; requirements: unknown | null }; + remaining_task: { tokens: number; requirements: unknown | null }; + checkpoint: string; + completed_calls: number; + inference_ms: number; +} + +/** One model load per Pi process; each line contains only source-visible state. */ +export class ForecastWorker { + private options: ForecastWorkerOptions; + private child?: ChildProcessWithoutNullStreams; + private sequence = 0; + private pending = new Map void; reject: (e: Error) => void }>(); + + constructor(options: ForecastWorkerOptions = {}) { this.options = options; } + + close() { + this.child?.kill(); + this.child = undefined; + for (const entry of this.pending.values()) entry.reject(new Error("Forecast worker stopped")); + this.pending.clear(); + } + + private start() { + if (this.child) return this.child; + const [command, ...args] = this.options.command ?? [this.options.python ?? process.env.SY_FORECAST_PYTHON ?? "python3", + "-u", fileURLToPath(new URL("./forecast/lightgbm_worker.py", import.meta.url))]; + if (!command) throw new Error("Forecast worker command cannot be empty"); + const child = spawn(command, args, { stdio: ["pipe", "pipe", "pipe"], env: { + ...process.env, OMP_NUM_THREADS: "2", OPENBLAS_NUM_THREADS: "1", + ...(this.options.researchRoot ? { SY_FORECAST_RESEARCH_ROOT: this.options.researchRoot } : {}), + ...(this.options.tokenCastRoot ? { SY_FORECAST_TOKENCAST_ROOT: this.options.tokenCastRoot } : {}), + } }); + this.child = child; + let stderr = ""; + child.stderr.on("data", (data) => { stderr = (stderr + data).slice(-2000); }); + createInterface({ input: child.stdout }).on("line", (line) => { + try { + const value = JSON.parse(line); + const pending = this.pending.get(value.id); + if (!pending) return; + this.pending.delete(value.id); + if (value.error) pending.reject(new Error(value.error)); + else if (![value.result?.next_call?.tokens, value.result?.remaining_task?.tokens] + .every(tokens => typeof tokens === "number" && Number.isFinite(tokens) && tokens >= 0)) { + pending.reject(new Error("Forecast worker returned invalid token predictions")); + } + else pending.resolve(value.result); + } catch { /* Non-protocol output cannot become a forecast. */ } + }); + const fail = (error: Error) => { + if (this.child !== child) return; + this.child = undefined; + for (const entry of this.pending.values()) entry.reject(error); + this.pending.clear(); + }; + child.on("error", fail); + child.on("exit", () => fail(new Error(`Forecast worker exited: ${stderr}`))); + child.stdin.on("error", fail); + return child; + } + + predict(input: object, signal?: AbortSignal): Promise { + if (signal?.aborted) return Promise.reject(new Error("Forecast cancelled")); + const child = this.start(); + const id = ++this.sequence; + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { this.close(); }, 15000); + const cleanup = () => { clearTimeout(timer); signal?.removeEventListener("abort", abort); }; + const abort = () => { cleanup(); this.pending.delete(id); reject(new Error("Forecast cancelled")); }; + signal?.addEventListener("abort", abort, { once: true }); + this.pending.set(id, { + resolve: (value) => { cleanup(); resolve(value); }, + reject: (error) => { cleanup(); reject(error); }, + }); + child.stdin.write(JSON.stringify({ ...input, id }) + "\n"); + }); + } +} + +/** Total observed provider input + output; unknown usage stays unavailable. */ +export function observedTokens(messages: readonly { role: string; usage?: { input: number; output: number; cacheRead: number; cacheWrite: number } }[]): number | undefined { + let total = 0; + for (const message of messages) { + if (message.role !== "assistant") continue; + const usage = message.usage; + if (!usage) return undefined; + const channels = [usage.input, usage.output, usage.cacheRead, usage.cacheWrite]; + if (!channels.every((value) => Number.isFinite(value) && value >= 0)) return undefined; + total += channels.reduce((sum, value) => sum + value, 0); + } + return total; +} diff --git a/examples/pi/forecast/lightgbm_worker.py b/examples/pi/forecast/lightgbm_worker.py new file mode 100644 index 0000000000..2479aec5d9 --- /dev/null +++ b/examples/pi/forecast/lightgbm_worker.py @@ -0,0 +1,206 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Local JSON-lines inference worker; recorded Pi tools are parsed, never executed.""" +import hashlib +import json +import math +import os +from pathlib import Path +import pickle +import shlex +import sys +import time + +def required_root(variable): + value = os.environ.get(variable) + if not value: + raise RuntimeError(f'Set {variable} or configure the equivalent forecast checkout path') + root = Path(value).expanduser().resolve() + if not root.is_dir(): + raise RuntimeError(f'{variable} does not name a directory: {root}') + return root + + +RESEARCH = required_root('SY_FORECAST_RESEARCH_ROOT') +TOKENCAST = required_root('SY_FORECAST_TOKENCAST_ROOT') +sys.path[:0] = [str(RESEARCH / 'scripts'), str(TOKENCAST / 'src')] +from token_prediction.contracts import CanonicalEvent, EventType +from token_prediction.features.reducer import FeatureState +from ir_grouped import GroupedObserver, grouped_project + + +def load_model(): + root = RESEARCH / 'checkpoints/grouped-ir' + manifest = json.loads((root / 'manifest.json').read_text()) + # First published seed, not a winner selected on validation. + entry = manifest['checkpoints'][0] + path = root / entry['path'] + if hashlib.sha256(path.read_bytes()).hexdigest() != entry['sha256']: + raise ValueError('Checkpoint hash mismatch') + with path.open('rb') as f: + model = pickle.load(f) + return model, entry + + +def text(message): + content = message.get('content', '') + if isinstance(content, str): + return content + return '\n'.join(b.get('text', '') for b in content if b.get('type') == 'text') + + +def shell_surface(name, args): + """Explicit native-tool adaptation for the checkpoint's shell-only IR parser.""" + q = shlex.quote + path = args.get('path', '.') + if name == 'bash': + return args['command'] + if name == 'read': + return 'cat -- ' + q(path) + if name == 'write': + marker = 'PI_CONTENT_' + hashlib.sha256(args.get('content', '').encode()).hexdigest()[:12] + return f'cat > {q(path)} <<\'{marker}\'\n{args.get("content", "")}\n{marker}' + if name == 'edit': + # Preserve requested old/new text without claiming an observed filesystem image. + old = args.get('oldText', '').splitlines() + new = args.get('newText', '').splitlines() + patch = '\n'.join(['*** Begin Patch', f'*** Update File: {path}', '@@', + *('-' + l for l in old), *('+' + l for l in new), '*** End Patch']) + return "apply_patch <<'PI_PATCH'\n" + patch + '\nPI_PATCH' + if name == 'grep': + return 'rg -- ' + q(args.get('pattern', '')) + ' ' + q(path) + if name == 'find': + return 'find ' + q(path) + ' -name ' + q(args.get('pattern', '*')) + if name == 'ls': + return 'ls -- ' + q(path) + raise ValueError(f'No IR adapter for tool {name!r}') + + +def build_rows(request, config): + messages = request['messages'] + task = next((text(m) for m in messages if m['role'] == 'user'), '') + tid = request['session_id'] + state = FeatureState(trajectory_id=tid) + observer = GroupedObserver({**config, 'working_directory': request['cwd']}) + seq = -1 + latest = None + completed = 0 + pending = {} + adapted = set() + + def emit(kind, payload, cid=None, *, ir_payload=None): + nonlocal seq, state, latest + seq += 1 + event = CanonicalEvent(1, f'{tid}:{seq}', tid, seq, EventType(kind), + '1970-01-01T00:00:00+00:00', json.dumps(payload), + logical_call_id=cid, + attempt_id=f'{cid}:observed' if kind in ('api_attempt_started', 'api_completed', 'api_failed') else None) + state = state.apply(event) + observer.apply({'event_type': kind, 'event_seq': seq, 'trajectory_id': tid, + 'logical_call_id': cid, 'payload': ir_payload or payload}) + latest = event + + emit('task_started', {'task_id': tid, 'task_text': task, + 'model_id': request['model'], 'agent_id': 'pi'}) + for i, message in enumerate(messages): + if message['role'] == 'assistant': + cid = f'call:{completed}' + prefix = messages[:i] + emit('request_built', {'model': message.get('model', request['model']), + 'message_count': len(prefix)}, cid) + emit('api_attempt_started', {}, cid) + usage = message.get('usage') or {} + # Pi splits cache tokens from input; training uses total provider input. + inputs = usage.get('input') + if inputs is not None: + inputs += usage.get('cacheRead', 0) + usage.get('cacheWrite', 0) + emit('api_completed', {'usage': { + 'input_tokens': inputs, 'output_tokens': usage.get('output'), + 'cached_input_tokens': usage.get('cacheRead'), + 'cache_write_input_tokens': usage.get('cacheWrite'), + 'reasoning_output_tokens': usage.get('reasoning') }}, cid) + content = message.get('content', []) + for block in content if isinstance(content, list) else []: + if block.get('type') != 'toolCall': + continue + name, args, call_id = block['name'], block['arguments'], block['id'] + command = shell_surface(name, args) + payload = {'tool_call_id': call_id, 'tool_name': name, 'arguments': args} + ir_payload = {**payload, 'tool_name': 'bash', 'arguments': {'command': command}} + emit('tool_started', payload, cid, ir_payload=ir_payload) + pending[call_id] = cid + if name != 'bash': + adapted.add(name) + completed += 1 + elif message['role'] == 'toolResult': + call_id = message['toolCallId'] + if call_id not in pending: + raise ValueError(f'Missing source action for tool result {call_id}') + result = text(message) + # Native Pi success is not an observed shell exit code. + code = (message.get('details') or {}).get('exitCode') + payload = {'tool_call_id': call_id, 'exit_code': code, + 'result_text': result, 'result_chars': len(result), + 'result_utf8_bytes': len(result.encode()), 'result_lines': len(result.splitlines()), + 'result_fields': {'is_error': message.get('isError', False)}} + emit('tool_failed' if message.get('isError') else 'tool_completed', payload, pending.pop(call_id)) + + def row(position, observed): + f = dict(observed.snapshot(latest).values) + f.update(prediction_position=position, configured_call_limit=None, + completed_call_count=completed, remaining_call_allowance=None, + current_request_input_known=0.0, + request_logged_utf8_bytes=len(json.dumps(request['request_messages'], ensure_ascii=False).encode()) + if position == 'call_start' else None) + if position != 'call_start': + for key in ['current_request_tokens_local', 'request_message_count', 'request_tool_count', + 'request_content_chars', 'request_system_utf8_bytes', 'request_delta_tokens']: + f[key] = None + r = {'task_id': tid, 'point_id': f'{tid}:{seq}:{position}', 'position': position, + 'completed_calls': completed, 'task_text': task, 'features': f} + r['ir_features'] = grouped_project(observer.snapshot(r), observer.config) + return r + + remaining = row('task_start' if completed == 0 else 'task_update', + state._start_logical_call('snapshot-only') if completed else state) + emit('request_built', {'model': request['model'], 'message_count': len(request['request_messages'])}, + f'call:{completed}') + upcoming = row('call_start', state) + return [upcoming, remaining], completed, sorted(adapted) + + +def forecast(request, pipeline, entry): + started = time.perf_counter() + rows, completed, adapted = build_rows(request, pipeline.config['ir_features']) + pred = pipeline.predict(rows) + def horizon(i): + value = float(pred['final'][i]) + if not math.isfinite(value): + raise ValueError('Non-finite token forecast') + return {'tokens': round(max(0, value)), + 'token_interval': [round(float(pred['lower'][i])), round(float(pred['upper'][i]))], + 'requirements': None, + 'requirements_unavailable_reason': 'checkpoint_has_no_requirement_head'} + return {'next_call': horizon(0), 'remaining_task': horizon(1), + 'completed_calls': completed, 'inference_ms': round((time.perf_counter()-started)*1000, 2), + 'checkpoint': entry['path'], 'checkpoint_sha256': entry['sha256'], + 'token_definition': 'provider_input_including_cache_plus_output; reasoning_in_output', + 'warnings': ['Cross-harness/model transfer: trained on Opus mini-SWE-agent traces.', + 'No Pi call cap is configured; cap features remain missing.'], + 'adapted_native_tools': adapted} + + +if __name__ == '__main__': + pipeline, entry = load_model() + for line in sys.stdin: + request = {} + try: + parsed = json.loads(line) + if not isinstance(parsed, dict): + raise ValueError('Expected a JSON request object') + request = parsed + response = {'id': request['id'], 'result': forecast(request, pipeline, entry)} + except Exception as exc: + response = {'id': request.get('id'), 'error': f'{type(exc).__name__}: {exc}'} + print(json.dumps(response, allow_nan=False), flush=True) diff --git a/examples/pi/forecast/test_lightgbm_worker.py b/examples/pi/forecast/test_lightgbm_worker.py new file mode 100644 index 0000000000..742abc2162 --- /dev/null +++ b/examples/pi/forecast/test_lightgbm_worker.py @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import sys +import unittest +from pathlib import Path +sys.path.insert(0, str(Path(__file__).parent)) +from lightgbm_worker import build_rows, load_model, forecast + +class AdapterTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.model, cls.entry = load_model() + + def request(self): + messages = [ + {'role': 'user', 'content': 'Fix parser and test it'}, + {'role': 'assistant', 'model': 'openai/gpt-6.1-sol', + 'usage': {'input': 100, 'cacheRead': 20, 'cacheWrite': 5, 'output': 10, 'reasoning': 3}, + 'content': [{'type': 'toolCall', 'name': 'bash', 'id': 'a', 'arguments': {'command': 'pytest tests/test_parser.py'}}]}, + {'role': 'toolResult', 'toolCallId': 'a', 'toolName': 'bash', 'isError': True, + 'details': {'exitCode': 1}, 'content': [{'type': 'text', 'text': 'FAILED tests/test_parser.py - AssertionError: wrong value'}]}, + ] + return {'session_id': 'fixture', 'cwd': '/workspace', 'model': 'openrouter/openai/gpt-6.1-sol', + 'messages': messages, 'request_messages': messages} + + def test_usage_and_horizons_are_source_only(self): + request = self.request() + rows, completed, adapted = build_rows(request, self.model.config['ir_features']) + self.assertEqual(completed, 1) + self.assertEqual([r['position'] for r in rows], ['call_start', 'task_update']) + self.assertEqual(rows[0]['features']['last_call_input_tokens'], 125) + self.assertEqual(rows[0]['features']['last_call_output_tokens'], 10) + self.assertIsNone(rows[0]['features']['current_request_tokens_local']) + self.assertIsNone(rows[0]['features']['configured_call_limit']) + self.assertIn('AssertionError', rows[0]['ir_features']['execution_text']) + self.assertNotIn('labels', rows[0]) + result = forecast(request, self.model, self.entry) + self.assertGreaterEqual(result['next_call']['tokens'], 0) + self.assertIsNone(result['next_call']['requirements']) + self.assertIsNone(result['remaining_task']['requirements']) + + def test_native_read_and_write_have_explicit_ir_adapters(self): + request = self.request() + request['messages'][1]['content'][0].update(name='write', arguments={'path': 'result.py', 'content': 'print(1)\n'}) + request['messages'][2].update(isError=False, details={}, content=[{'type': 'text', 'text': 'Successfully wrote result.py'}]) + rows, _, adapted = build_rows(request, self.model.config['ir_features']) + self.assertEqual(adapted, ['write']) + self.assertIn('result.py', rows[0]['ir_features']['artifact_text']) + request['messages'][1]['content'][0].update(name='read', arguments={'path': 'result.py'}) + rows, _, adapted = build_rows(request, self.model.config['ir_features']) + self.assertEqual(adapted, ['read']) + + def test_unknown_tool_does_not_produce_fake_ir(self): + request = self.request() + request['messages'][1]['content'][0]['name'] = 'unknown_custom_tool' + with self.assertRaisesRegex(ValueError, 'No IR adapter'): + build_rows(request, self.model.config['ir_features']) + +if __name__ == '__main__': + unittest.main() diff --git a/examples/pi/metrics.ts b/examples/pi/metrics.ts index 25689462a1..12cc3aca27 100644 --- a/examples/pi/metrics.ts +++ b/examples/pi/metrics.ts @@ -96,7 +96,7 @@ interface ToolWindow { } /** Feed Pi-owned calls into Switchyard and read status data from its metric snapshots. */ -export function registerMetrics(pi: ExtensionAPI) { +export function registerMetrics(pi: ExtensionAPI, options: { status?: boolean } = {}) { let collector: Metrics | undefined; const metrics = () => collector ??= new Metrics(); const sessions = new Map(); @@ -142,7 +142,7 @@ export function registerMetrics(pi: ExtensionAPI) { } function render(ctx: ExtensionContext) { - if (!ctx.hasUI) return; + if (!ctx.hasUI || options.status === false) return; const state = session(ctx); const filter = { session_id: ctx.sessionManager.getSessionId(), agent_id: "root" }; const all = metrics().stats(filter); @@ -234,6 +234,16 @@ export function registerMetrics(pi: ExtensionAPI) { }); return { + /** Shared Greg-style pricing and timing for forecasts in either token basis. */ + projectRemaining(ctx: ExtensionContext, remainingTokens: number, basis: "uncached" | "full", modelCost?: ModelCost) { + const state = session(ctx); + const usage = basis === "full" ? state.completion.remainingFullUsage(remainingTokens) : state.completion.remainingUsage(remainingTokens); + const rates = tokenRates(modelCost ?? state.modelCost, { input: state.completion.meanPromptTokens, cacheRead: 0, cacheWrite: 0 }); + return { + usd: usage && rates ? estimateCost(usage, rates)?.usd : undefined, + eta_ms: basis === "full" ? state.completion.estimateFullMs(remainingTokens) : state.completion.estimateMs(remainingTokens), + }; + }, begin(reason: "user" | "continuation" | "retry" | "direct", ctx: ExtensionContext) { // Direct helper requests do not emit the agent message events used to finish measurements. if (reason === "direct") return undefined; @@ -249,6 +259,8 @@ export function registerMetrics(pi: ExtensionAPI) { active = current; return { context: current.context, + /** Forecasting is measured separately; start answer latency when routing finishes. */ + answerStarted() { current.started = performance.now(); current.firstToken = undefined; }, modelCost(cost: ModelCost | undefined) { current.cost = cost; session(ctx).modelCost = cost; }, prediction(value: TaskPrediction | null, tokensUsed?: number) { if (ctx.sessionManager.getSessionId() !== current.context.session_id || active !== current) return; diff --git a/examples/pi/prediction-total.ts b/examples/pi/prediction-total.ts new file mode 100644 index 0000000000..f3109c0263 --- /dev/null +++ b/examples/pi/prediction-total.ts @@ -0,0 +1,47 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { Message, ModelThinkingLevel } from "@earendil-works/pi-ai"; +import { predictionRequest } from "./prediction.ts"; +import { observedTokens } from "./forecast-worker.ts"; + +/** Reuse Greg's history buffer and budget bands with the checkpoint's full-token target. */ +export function totalTokenPredictionRequest(judge: string, messages: readonly Message[], + completionModel: string, thinkingLevel: ModelThinkingLevel) { + const used = observedTokens(messages); + if (used === undefined) throw new Error("Full token usage is missing from the observed history"); + const request = predictionRequest(judge, messages); + const context = request.context as Record; + const fullTokens = (text: string) => text + .replaceAll("uncached input plus output", "all input (including cache reads and writes) plus output") + .replaceAll("uncached-input-plus-output", "full-input-plus-output"); + request.context = { + ...context, + instructions: fullTokens(String(context.instructions)) + + " Count input again on every model call, including repeated and cached context. Cache discounts affect dollars, not token counts." + + " Include reasoning once as part of output. The predicted total includes all tokens already used." + + " Forecast through the final response to the latest user request, not future user requests." + + " Exclude this forecasting model's own usage.", + completion_model: completionModel, + thinking_level: thinkingLevel, + token_usage: { + definition: "All provider input plus output: input + cacheRead + cacheWrite + output. Reasoning is included in output.", + tokens_used_so_far: used, + per_step: messages.flatMap(message => message.role === "assistant" && message.usage ? [{ + step: 0, input_tokens: message.usage.input + message.usage.cacheRead + message.usage.cacheWrite, + cache_read_tokens: message.usage.cacheRead, cache_write_tokens: message.usage.cacheWrite, + output_tokens: message.usage.output, reasoning_output_tokens: message.usage.reasoning ?? 0, + tokens_used: message.usage.input + message.usage.cacheRead + message.usage.cacheWrite + message.usage.output, + }] : []).map((usage, index) => ({ ...usage, step: index + 1 })), + }, + }; + for (const question of Object.values(request.questions)) { + const instructions = question.instructions as Record; + question.instructions = { ...instructions, task: fullTokens(String(instructions.task)), + use_state_fields: ["instructions", "task_history", "token_usage", "completion_model", "thinking_level"] }; + if (question.kind.type === "choice") for (const option of question.kind.data.options) { + if (typeof option.description === "string") option.description = fullTokens(option.description); + } + } + return request; +} diff --git a/examples/pi/remaining-error.ts b/examples/pi/remaining-error.ts new file mode 100644 index 0000000000..a8fe5e0e20 --- /dev/null +++ b/examples/pi/remaining-error.ts @@ -0,0 +1,46 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { ErrorSample } from "./forecast-error.ts"; + +export const REMAINING_ERROR_ENTRY = "switchyard.compare.remaining-errors.v2"; +export interface RemainingForecast { + session_id: string; + agent_turn_idx: number; + lightgbm: { remaining_task: { tokens: number } } | null; + greg: { predicted_tokens?: number | null } | null; + full_used?: number; +} +export interface RemainingError extends ErrorSample { + predictor: "lightgbm" | "greg"; +} +export interface RemainingSettlement { + session_id: string; + completed: boolean; + samples: RemainingError[]; +} + +/** Score original pre-call forecasts against usage from their cutoff to the final response. */ +export function scoreRemainingForecasts(forecasts: RemainingForecast[], fullTotal: number | undefined): RemainingError[] { + const samples = new Map(); + const add = (forecast: RemainingForecast, predictor: RemainingError["predictor"], + predicted: number | null | undefined, cutoff: number | undefined, total: number | undefined) => { + if (predicted == null || cutoff === undefined || total === undefined + || ![predicted, cutoff, total].every(n => Number.isFinite(n) && n >= 0) || total < cutoff) return; + const actual = total - cutoff; + const sample: RemainingError = { + predictor, session_id: forecast.session_id, sample_id: `${forecast.session_id}:${forecast.agent_turn_idx}:${predictor}`, + predicted_tokens: predicted, actual_tokens: actual, + signed_error: predicted - actual, absolute_error: Math.abs(predicted - actual), + }; + samples.set(sample.sample_id, sample); + }; + for (const forecast of forecasts) { + add(forecast, "lightgbm", forecast.lightgbm?.remaining_task.tokens, forecast.full_used, fullTotal); + const total = forecast.greg?.predicted_tokens; + const predicted = total == null || forecast.full_used === undefined ? undefined + : Math.max(0, total - forecast.full_used); + add(forecast, "greg", predicted, forecast.full_used, fullTotal); + } + return [...samples.values()]; +} diff --git a/examples/pi/run-forecast.sh b/examples/pi/run-forecast.sh new file mode 100755 index 0000000000..234529c460 --- /dev/null +++ b/examples/pi/run-forecast.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +thinking_level=medium +pi_args=(--model switchyard/compare) +while (( $# )); do + case "$1" in + --) pi_args+=("$@"); break ;; + --thinking) + if (( $# < 2 )); then + printf '%s\n' 'Use --thinking medium or --thinking max.' >&2 + exit 2 + fi + thinking_level="$2"; shift 2 ;; + --thinking=*) thinking_level="${1#*=}"; shift ;; + *) pi_args+=("$1"); shift ;; + esac +done +case "$thinking_level" in + medium|max) ;; + *) printf '%s\n' 'Use --thinking medium or --thinking max.' >&2; exit 2 ;; +esac + +agent_root="${SY_FORECAST_AGENT_ROOT:-${XDG_CONFIG_HOME:-$HOME/.config}/switchyard-pi-forecast}" +export SY_FORECAST_CONFIG="${SY_FORECAST_CONFIG:-$agent_root/switchyard-forecast.json}" +export PI_CODING_AGENT_DIR="$agent_root/$thinking_level" +export PI_TELEMETRY="${PI_TELEMETRY:-0}" +mkdir -p "$PI_CODING_AGENT_DIR" +exec "$script_dir/node_modules/.bin/pi" -e "$script_dir/switchyard-compare.ts" \ + --thinking "$thinking_level" "${pi_args[@]}" diff --git a/examples/pi/switchyard-compare.ts b/examples/pi/switchyard-compare.ts new file mode 100644 index 0000000000..656a1e09e7 --- /dev/null +++ b/examples/pi/switchyard-compare.ts @@ -0,0 +1,173 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { taskPrediction, type TaskPrediction } from "@switchyard/libsy"; +import type { Message, ModelCost, ModelThinkingLevel } from "@earendil-works/pi-ai"; +import { getAgentDir, type ExtensionAPI, type ExtensionContext } from "@earendil-works/pi-coding-agent"; +import { registerMetrics } from "./metrics.ts"; +import { completionDuration } from "./completion-time.ts"; +import { totalTokenPredictionRequest } from "./prediction-total.ts"; +import { classifierContext, decisionResponse, modelIdentity } from "./forecast-classifier.ts"; +import { ForecastWorker, observedTokens, type HorizonForecast } from "./forecast-worker.ts"; +import { NextCallMAE, ERROR_ENTRY, errorStatus, type ErrorSample } from "./forecast-error.ts"; +import { scoreRemainingForecasts, REMAINING_ERROR_ENTRY, + type RemainingSettlement } from "./remaining-error.ts"; + +const ENTRY = "switchyard.compare.forecast.v2"; +interface Pair { + remaining_error_version?: 1; + session_id: string; + agent_turn_idx: number; + lightgbm: HorizonForecast | null; + greg: TaskPrediction | null; + full_used?: number; + token_basis: "full_input_plus_output"; + thinking_level: ModelThinkingLevel; + model_cost?: ModelCost; + errors: { lightgbm?: string; greg?: string }; +} +const count = (n: number) => n >= 1000 ? `${(n / 1000).toFixed(1)}k` : String(Math.round(n)); + +export default function (pi: ExtensionAPI) { + const metrics = registerMetrics(pi, { status: false }); + let worker: ForecastWorker | undefined; + const latest = new Map(); + const errors = new Map(); + const messages = (ctx: ExtensionContext): Message[] => ctx.sessionManager.getBranch().flatMap(entry => + entry.type === "message" && ["assistant", "user", "toolResult"].includes(entry.message.role) ? [entry.message as Message] : []); + function tracker(ctx: ExtensionContext) { + const id = ctx.sessionManager.getSessionId(); + if (!errors.has(id)) errors.set(id, new NextCallMAE(id, ctx.sessionManager.getEntries().flatMap(entry => + entry.type === "custom" && entry.customType === ERROR_ENTRY ? [entry.data as ErrorSample] : []))); + return errors.get(id)!; + } + function render(ctx: ExtensionContext) { + if (!ctx.hasUI) return; + // The two widget rows are the entire comparison display. + ctx.ui.setStatus("switchyard", undefined); + const pair = latest.get(ctx.sessionManager.getSessionId()); + if (!pair) { + ctx.ui.setWidget("switchyard-forecast-compare", ["LightGBM · waiting for forecasts…", "Jev · waiting for forecast…"], { placement: "belowEditor" }); + return; + } + const history = messages(ctx); + const fullNow = observedTokens(history); + const lgbmRemaining = pair.lightgbm && fullNow !== undefined && pair.full_used !== undefined + ? Math.max(0, pair.lightgbm.remaining_task.tokens - (fullNow - pair.full_used)) : undefined; + // Use the distribution's point estimate (band midpoints), not its upper-bound buffer. + const gregRemaining = pair.greg?.predicted_tokens == null || fullNow === undefined ? undefined + : Math.max(0, pair.greg.predicted_tokens - fullNow); + const savedErrors = ctx.sessionManager.getBranch().flatMap(entry => entry.type === "custom" + && entry.customType === REMAINING_ERROR_ENTRY ? [(entry.data as RemainingSettlement)] : []) + .filter(record => record.session_id === pair.session_id).flatMap(record => record.samples); + const formatError = (stats: ReturnType) => errorStatus(stats) + .replace(/^ \| /, "").replace(/ \(n=\d+\)/, ""); + const row = (label: string, predictor: "lightgbm" | "greg", remaining: number | undefined, next?: string) => { + const stats = new NextCallMAE(pair.session_id, savedErrors.filter(sample => sample.predictor === predictor)).summary(); + const err = formatError(stats); + const estimate = remaining === undefined ? undefined : metrics.projectRemaining(ctx, remaining, "full", pair.model_cost); + const cost = estimate?.usd === undefined ? "$—" : `~$${estimate.usd.toFixed(4)}`; + const prefix = `${label} · ${cost} · ETA ${completionDuration(estimate?.eta_ms)} · ${next ? `${next} | ` : ""}remaining`; + if (remaining === undefined) return `${prefix} unavailable (${err})`; + return `${prefix} ~${count(remaining)} tok (${err})`; + }; + const next = pair.lightgbm ? `next ~${count(pair.lightgbm.next_call.tokens)} tok (${formatError(tracker(ctx).summary())})` : "next unavailable"; + const lines = [row("LightGBM", "lightgbm", lgbmRemaining, next), row("Jev", "greg", gregRemaining)]; + ctx.ui.setWidget("switchyard-forecast-compare", lines, { placement: "belowEditor" }); + } + pi.on("session_start", (_, ctx) => { + const saved = ctx.sessionManager.getBranch().filter(entry => entry.type === "custom" && entry.customType === ENTRY).at(-1); + if (saved?.type === "custom") latest.set(ctx.sessionManager.getSessionId(), saved.data as Pair); + render(ctx); + }); + pi.on("message_end", (event, ctx) => { + if (event.message.role !== "assistant") return; + const sample = tracker(ctx).finish(event.message.usage, !["error", "aborted"].includes(event.message.stopReason)); + if (sample) pi.appendEntry(ERROR_ENTRY, sample); + render(ctx); + }); + pi.on("tool_execution_end", (_, ctx) => render(ctx)); + pi.on("agent_settled", (_, ctx) => { + const id = ctx.sessionManager.getSessionId(); + const branch = ctx.sessionManager.getBranch(); + const boundary = branch.findLastIndex(entry => entry.type === "custom" && entry.customType === REMAINING_ERROR_ENTRY + && (entry.data as RemainingSettlement).session_id === id); + const pending = branch.slice(boundary + 1).flatMap(entry => entry.type === "custom" && entry.customType === ENTRY + && (entry.data as Pair).session_id === id && (entry.data as Pair).remaining_error_version === 1 ? [entry.data as Pair] : []); + if (pending.length) { + const history = messages(ctx); + const lastForecast = branch.findLastIndex(entry => entry.type === "custom" && entry.customType === ENTRY + && (entry.data as Pair).session_id === id); + const finalEntry = branch.slice(lastForecast + 1).findLast(entry => entry.type === "message" && entry.message.role === "assistant"); + // A normal final response defines this measured horizon. Aborts/errors close it without scores. + const completed = finalEntry?.type === "message" && finalEntry.message.role === "assistant" && finalEntry.message.stopReason === "stop"; + const record: RemainingSettlement = { session_id: id, completed, + samples: completed ? scoreRemainingForecasts(pending, observedTokens(history)) : [] }; + pi.appendEntry(REMAINING_ERROR_ENTRY, record); + } + render(ctx); + }); + pi.on("session_shutdown", () => worker?.close()); + pi.registerVirtualModel({ + provider: "switchyard", id: "compare", name: "LightGBM + Greg forecasts (passthrough)", + thinkingLevels: ["medium", "max"], + async route(request, ctx) { + const measurement = metrics.begin(request.reason, ctx); + try { + const thinkingLevel = request.thinkingLevel ?? "medium"; + if (thinkingLevel !== "medium" && thinkingLevel !== "max") throw new Error("Select --thinking medium or --thinking max"); + const config = JSON.parse(readFileSync(process.env.SY_FORECAST_CONFIG ?? join(getAgentDir(), "switchyard-forecast.json"), "utf8")); + const identity = modelIdentity(config.completion_model); + const model = ctx.modelRegistry.find(identity.provider, identity.id); + if (!model) throw new Error(`Unknown completion model: ${identity.reference}`); + measurement?.modelCost(model.cost); + if (measurement) { + const id = ctx.sessionManager.getSessionId(); + tracker(ctx).begin("", undefined); + const branch = messages(ctx); + const history = branch.length ? branch : request.messages; + if (ctx.hasUI) { + ctx.ui.setStatus("switchyard", undefined); + ctx.ui.setWidget("switchyard-forecast-compare", ["LightGBM · predicting…", "Jev · predicting…"], { placement: "belowEditor" }); + } + // Both forecasts use this prefix and the same full-input-plus-output token target. + const results = await Promise.allSettled([ + (async () => { + worker ??= new ForecastWorker({ python: config.python, command: config.worker_command, researchRoot: config.research_root, + tokenCastRoot: config.tokencast_root }); + return worker.predict({ session_id: id, cwd: ctx.cwd, model: identity.reference, thinking_level: thinkingLevel, + messages: history, request_messages: request.messages }, request.signal); + })(), + (async () => { + const classifier = modelIdentity(config.decision_model); + const judge = ctx.modelRegistry.findOfType("classifier", classifier.provider, classifier.id); + if (!judge) throw new Error(`Unknown classifier: ${classifier.reference}`); + const prediction = totalTokenPredictionRequest(classifier.reference, history, identity.reference, thinkingLevel); + const result = await measurement.classify(classifier.reference, + () => ctx.modelRegistry.classify(judge, classifierContext(prediction), { signal: request.signal }), judge.cost); + return taskPrediction(decisionResponse(prediction, result)); + })(), + ]); + if (request.signal?.aborted) throw Object.assign(new Error("Forecast cancelled"), { name: "AbortError" }); + if (ctx.sessionManager.getSessionId() === id) { + const [lgbm, greg] = results; + const pair: Pair = { remaining_error_version: 1, session_id: id, agent_turn_idx: measurement.context.agent_turn_idx, + lightgbm: lgbm.status === "fulfilled" ? lgbm.value : null, + greg: greg.status === "fulfilled" ? greg.value : null, + full_used: observedTokens(history), token_basis: "full_input_plus_output", thinking_level: thinkingLevel, model_cost: model.cost, + errors: { lightgbm: lgbm.status === "rejected" ? String(lgbm.reason?.message ?? lgbm.reason) : undefined, + greg: greg.status === "rejected" ? String(greg.reason?.message ?? greg.reason) : undefined } }; + latest.set(id, pair); + pi.appendEntry(ENTRY, pair); + tracker(ctx).begin(`${id}:${pair.agent_turn_idx}`, pair.lightgbm?.next_call.tokens); + render(ctx); + } + } + measurement?.answerStarted(); + return { model, thinkingLevel }; + } catch (error) { measurement?.fail(); throw error; } + }, + }); +} diff --git a/examples/pi/switchyard-forecast.example.json b/examples/pi/switchyard-forecast.example.json new file mode 100644 index 0000000000..6a893118c4 --- /dev/null +++ b/examples/pi/switchyard-forecast.example.json @@ -0,0 +1,7 @@ +{ + "completion_model": "openrouter/openai/gpt-6.1-sol", + "decision_model": "openrouter/~typesafe/jev-latest", + "python": "/absolute/path/to/agent-progress-research/v1/.venv/bin/python", + "research_root": "/absolute/path/to/agent-progress-research/v1", + "tokencast_root": "/absolute/path/to/TokenCast" +} diff --git a/examples/pi/test/compare.test.mjs b/examples/pi/test/compare.test.mjs new file mode 100644 index 0000000000..93a6004bbe --- /dev/null +++ b/examples/pi/test/compare.test.mjs @@ -0,0 +1,118 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { mkdtempSync, writeFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { fileURLToPath } from "node:url"; +import extension from "../switchyard-compare.ts"; + +test("both forecasts share a cutoff and completion; rendering survives classifier failure and resume", async (t) => { + const directory = mkdtempSync(join(tmpdir(), "pi-compare-")); + const previous = process.env.PI_CODING_AGENT_DIR; + process.env.PI_CODING_AGENT_DIR = directory; + t.after(() => { + if (previous === undefined) delete process.env.PI_CODING_AGENT_DIR; + else process.env.PI_CODING_AGENT_DIR = previous; + rmSync(directory, { recursive: true, force: true }); + }); + writeFileSync(join(directory, "switchyard-forecast.json"), JSON.stringify({ + decision_model: "openrouter/~typesafe/jev-latest", completion_model: "openrouter/openai/gpt-6.1-sol", + worker_command: [process.execPath, fileURLToPath(new URL("./fixtures/forecast-worker.mjs", import.meta.url))], + })); + const handlers = new Map(), entries = [], statuses = [], widgets = [], contexts = []; + // Earlier uncached forecasts and scores must not appear in the full-token comparison. + entries.push({ type: "custom", customType: "switchyard.compare.forecast.v1", + data: { session_id: "comparison", greg: { expected_upper_tokens: 999999 } } }); + entries.push({ type: "custom", customType: "switchyard.compare.remaining-errors.v1", + data: { session_id: "comparison", samples: [{ predictor: "greg", sample_id: "old", session_id: "comparison", + predicted_tokens: 999999, actual_tokens: 1, absolute_error: 999998 }] } }); + let definition, fail = false; + const model = { provider: "openrouter", id: "openai/gpt-6.1-sol", + cost: { input: 1, output: 4, cacheRead: 0.1, cacheWrite: 1 } }; + const ctx = { + hasUI: true, cwd: directory, + ui: { setStatus: (_key, value) => statuses.push(value), setWidget: (_key, lines) => widgets.push(lines) }, + sessionManager: { getSessionId: () => "comparison", getEntries: () => entries, getBranch: () => entries }, + modelRegistry: { + find: () => model, findOfType: () => ({ provider: "openrouter", id: "~typesafe/jev-latest" }), + classify: async (_judge, context) => { + contexts.push(context); + if (fail) throw new Error("offline"); + return { provider: "openrouter", model: "~typesafe/jev-latest", stopReason: "stop", answers: { + minimum_likely_budget: { type: "choice", choice: "b010", probabilities: + Object.fromEntries(Object.keys(context.questions.minimum_likely_budget.criteria).map(id => [id, id === "b010" ? 1 : 0])) }, + } }; + }, + }, + }; + const emit = (name, event = {}) => { for (const handler of handlers.get(name) ?? []) handler(event, ctx); }; + extension({ registerVirtualModel: value => { definition = value; }, + on: (name, handler) => handlers.set(name, [...handlers.get(name) ?? [], handler]), + appendEntry: (customType, data) => entries.push({ type: "custom", customType, data }) }); + t.after(() => emit("session_shutdown")); + const user = { role: "user", content: "Read the parser and explain it" }; + entries.push({ type: "message", message: user }); + const request = { reason: "user", thinkingLevel: "max", messages: [user] }; + assert.deepEqual(definition.thinkingLevels, ["medium", "max"]); + emit("session_start"); + assert.match(widgets.at(-1)[1], /^Jev · waiting/); + const firstRoute = await definition.route(request, ctx); + assert.equal(firstRoute.model, model); + assert.equal(firstRoute.thinkingLevel, "max"); + const pair = entries.find(e => e.customType === "switchyard.compare.forecast.v2").data; + assert.ok(pair.lightgbm.remaining_task.tokens > 0); + assert.equal(pair.greg.predicted_tokens, 46000); + assert.equal(pair.token_basis, "full_input_plus_output"); + assert.equal(pair.thinking_level, "max"); + assert.equal(pair.full_used, 0); + assert.equal(contexts[0].state.task_history[0].message, user.content); + assert.equal(contexts[0].state.thinking_level, "max"); + assert.equal(widgets.at(-1).length, 2); + assert.match(widgets.at(-1)[0], /^LightGBM · \$— · ETA — · next .* \| remaining .*\(avg err/); + assert.match(widgets.at(-1)[1], /^Jev · \$— · ETA — · remaining .*\(avg err/); + assert.ok(statuses.every(value => value === undefined)); + emit("message_update", { message: { role: "assistant" }, assistantMessageEvent: { type: "text_delta", delta: "a" } }); + await new Promise(resolve => setTimeout(resolve, 2)); + const response = { role: "assistant", provider: model.provider, model: model.id, content: [{ type: "text", text: "Done" }], + stopReason: "stop", usage: { input: 100, output: 10, cacheRead: 1000, cacheWrite: 0, totalTokens: 1110 } }; + entries.push({ type: "message", message: response }); + emit("message_end", { message: response }); + assert.match(widgets.at(-1)[0], /^LightGBM · ~\$[0-9.]+ · ETA ~[^·]+ · next/); + assert.match(widgets.at(-1)[1], /^Jev · ~\$[0-9.]+ · ETA ~[^·]+ · remaining/); + assert.match(widgets.at(-1)[0], /next .*\(avg err [+-][\d,]+\.\d%\) \| remaining/); + assert.match(widgets.at(-1)[1], /avg err —\)/); + emit("agent_settled"); + const settlements = () => entries.filter(e => e.customType === "switchyard.compare.remaining-errors.v2"); + assert.equal(settlements().length, 1); + assert.equal(settlements()[0].data.samples.length, 2); + assert.equal(settlements()[0].data.samples.find(s => s.predictor === "greg").actual_tokens, 1110); + assert.equal(settlements()[0].data.samples.find(s => s.predictor === "lightgbm").actual_tokens, 1110); + assert.match(widgets.at(-1)[0], /remaining .*avg err \+[\d,]+\.\d%\)/); + assert.match(widgets.at(-1)[1], /remaining .*avg err \+[\d,]+\.\d%\)/); + emit("agent_settled"); + assert.equal(settlements().length, 1); + const before = widgets.at(-1); + emit("session_start"); + assert.deepEqual(widgets.at(-1), before); + fail = true; + const secondRoute = await definition.route({ ...request, thinkingLevel: "medium", reason: "continuation" }, ctx); + assert.equal(secondRoute.model, model); + assert.equal(secondRoute.thinkingLevel, "medium"); + assert.equal(contexts.at(-1).state.thinking_level, "medium"); + assert.equal(contexts.at(-1).state.token_usage.tokens_used_so_far, 1110); + assert.match(widgets.at(-1)[0], /^LightGBM · .* · ETA .* · next .* \| remaining/); + assert.match(widgets.at(-1)[1], /^Jev · \$— · ETA — · remaining unavailable/); + const number = contexts.length; + assert.equal((await definition.route({ ...request, reason: "direct" }, ctx)).model, model); + assert.equal(contexts.length, number); + // Cancelling before a new response must not reuse the previous turn's final response. + emit("agent_settled"); + assert.equal(settlements().length, 2); + assert.equal(settlements()[1].data.completed, false); + assert.deepEqual(settlements()[1].data.samples, []); + assert.match(widgets.at(-1)[1], /avg err \+[\d,]+\.\d%\)/); + assert.ok(statuses.every(value => value === undefined)); +}); diff --git a/examples/pi/test/completion-time.test.mjs b/examples/pi/test/completion-time.test.mjs index b30f0b9540..f4762a80b9 100644 --- a/examples/pi/test/completion-time.test.mjs +++ b/examples/pi/test/completion-time.test.mjs @@ -12,6 +12,28 @@ const call = (changes = {}) => ({ output_tokens: 200, reasoning_tokens: 50 }, ...changes, }); +test("full-token projections preserve the budget, including cache, and scale observed wall time", () => { + const timing = new CompletionTiming(); + assert.equal(timing.remainingFullUsage(1000), undefined); + assert.equal(timing.estimateFullMs(1000), undefined); + assert.equal(timing.estimateFullMs(0), 0); + const response = call({ usage: { input_tokens: 700, output_tokens: 200, cached_input_tokens: 10000, + cache_creation_input_tokens: 100, cache_creation_1h_input_tokens: 20, reasoning_tokens: 50 } }); + timing.record(response); + timing.recordTools(response.context, 8000); + assert.deepEqual(timing.remainingFullUsage(22000), { + input_tokens: 1400, output_tokens: 400, cached_input_tokens: 20000, + cache_creation_input_tokens: 200, cache_creation_1h_input_tokens: 40, + }); + assert.equal(timing.estimateFullMs(22000), 28000); + for (let budget = 0; budget < 30; budget++) { + const usage = timing.remainingFullUsage(budget); + assert.equal(usage.input_tokens + usage.output_tokens + usage.cached_input_tokens + usage.cache_creation_input_tokens, budget); + assert.ok(Object.values(usage).every(value => value >= 0)); + assert.ok(usage.cache_creation_1h_input_tokens <= usage.cache_creation_input_tokens); + } +}); + test("splits the combined budget before applying input latency and output throughput", () => { const timing = new CompletionTiming(); timing.record(call()); diff --git a/examples/pi/test/fixtures/forecast-worker.mjs b/examples/pi/test/fixtures/forecast-worker.mjs new file mode 100644 index 0000000000..313353787d --- /dev/null +++ b/examples/pi/test/fixtures/forecast-worker.mjs @@ -0,0 +1,14 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createInterface } from "node:readline"; + +for await (const line of createInterface({ input: process.stdin })) { + const request = JSON.parse(line); + process.stdout.write(JSON.stringify({ id: request.id, result: { + next_call: { tokens: 2000, requirements: null }, + remaining_task: { tokens: 6000, requirements: null }, + checkpoint: "fixture", inference_ms: 0, + completed_calls: request.messages.filter(message => message.role === "assistant").length, + } }) + "\n"); +} diff --git a/examples/pi/test/forecast-error.test.mjs b/examples/pi/test/forecast-error.test.mjs new file mode 100644 index 0000000000..dc56801e57 --- /dev/null +++ b/examples/pi/test/forecast-error.test.mjs @@ -0,0 +1,66 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { NextCallMAE, errorStatus } from "../forecast-error.ts"; + +test("MAE pairs each old forecast with actual input/cache/output once", () => { + const tracker = new NextCallMAE("s"); + tracker.begin("s:0", 100); + const first = tracker.finish({ input: 20, cacheRead: 30, cacheWrite: 10, output: 20, reasoning: 5 }, true); + assert.equal(first.actual_tokens, 80); + assert.equal(first.absolute_error, 20); + assert.equal(tracker.finish({ input: 20, cacheRead: 30, cacheWrite: 10, output: 20 }, true), undefined); + tracker.begin("s:1", 200); + tracker.finish({ input: 100, cacheRead: 0, cacheWrite: 0, output: 50 }, true); + assert.equal(tracker.summary().mae, 35); + assert.ok(Math.abs(tracker.summary().meanPercentageError - (25 + 100/3) / 2) < 1e-10); + assert.match(errorStatus(tracker.summary()), /avg err \+29.2% \(n=2\)/); +}); + +test("missing forecasts, invalid usage and failed calls never count as zero error", () => { + const tracker = new NextCallMAE("s"); + const usage = { input: 10, cacheRead: 0, cacheWrite: 0, output: 2 }; + tracker.begin("s:0", undefined); + tracker.finish(usage, true); + tracker.begin("s:1", 100); + tracker.finish(undefined, true); + tracker.begin("s:2", 100); + tracker.finish(usage, false); + tracker.begin("s:3", 100); + tracker.finish({ ...usage, input: NaN }, true); + assert.equal(tracker.summary().count, 0); +}); + +test("resume deduplicates saved records and a new session starts fresh", () => { + const sample = { sample_id: "s:0", session_id: "s", predicted_tokens: 100, actual_tokens: 80, absolute_error: 20 }; + const tracker = new NextCallMAE("s", [sample, sample]); + assert.equal(tracker.summary().count, 1); + tracker.begin("s:0", 100); + assert.equal(tracker.finish({ input: 70, output: 10, cacheRead: 0, cacheWrite: 0 }, true), undefined); + tracker.begin("s:1", 200); + tracker.finish({ input: 100, output: 50, cacheRead: 0, cacheWrite: 0 }, true); + assert.equal(tracker.summary().mae, 35); + assert.equal(new NextCallMAE("fork", [sample]).summary().count, 0); +}); + +test("signed error distinguishes underprediction and restores old records", () => { + const tracker = new NextCallMAE("s"); + tracker.begin("s:0", 50); + const sample = tracker.finish({ input: 100, output: 10, cacheRead: 0, cacheWrite: 0 }, true); + assert.equal(sample.signed_error, -60); + assert.match(errorStatus(tracker.summary()), /avg err -54.5%/); + const old = { ...sample }; delete old.signed_error; + assert.match(errorStatus(new NextCallMAE("s", [old]).summary()), /avg err -54.5%/); +}); + +test("percentage error excludes zero actual tokens and survives resume", () => { + const tracker = new NextCallMAE("s"); + tracker.begin("s:0", 50); + tracker.finish({ input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, true); + assert.equal(tracker.summary().percentageCount, 0); + assert.match(errorStatus(tracker.summary()), /avg err — \(n=0\)/); + const saved = { sample_id: "s:1", session_id: "s", predicted_tokens: 150, actual_tokens: 100, absolute_error: 50 }; + assert.match(errorStatus(new NextCallMAE("s", [saved]).summary()), /avg err \+50.0% \(n=1\)/); +}); diff --git a/examples/pi/test/metrics.test.mjs b/examples/pi/test/metrics.test.mjs index 997772b46a..c55bc76767 100644 --- a/examples/pi/test/metrics.test.mjs +++ b/examples/pi/test/metrics.test.mjs @@ -30,6 +30,7 @@ function setup(t) { return { ctx, emit, sessions, begin: (reason = "user") => measurements.begin(reason, ctx), + project: (remaining, cost) => measurements.projectRemaining(ctx, remaining, "full", cost), time: (value) => { now = value; }, status: () => ctx.ui.setStatus.mock.calls.at(-1).arguments[1], entries: () => sessions.get(sessionId), @@ -46,6 +47,32 @@ function setup(t) { const assistant = (reportedUsage, stopReason = "stop") => ({ role: "assistant", provider: "provider", model: "answer", usage: reportedUsage, stopReason, }); + +test("shared full-token cost and ETA exclude forecast latency, count cache once, and restore", (t) => { + const fixture = setup(t); + const cost = { input: 2, output: 10, cacheRead: 0.1, cacheWrite: 2.5 }; + const request = fixture.begin(); + request.modelCost(cost); + assert.equal(fixture.project(2440).usd, undefined); + assert.equal(fixture.project(2440).eta_ms, undefined); + fixture.time(5000); + request.answerStarted(); + fixture.time(7000); + update(fixture, "text_delta", { delta: "answer" }); + fixture.time(11000); + fixture.emit("message_end", { message: assistant(usage(100, 20, 1000, 100, 5), "toolUse") }); + fixture.emit("tool_execution_start", { toolCallId: "tool" }); + fixture.time(13000); + fixture.emit("tool_execution_end", { toolCallId: "tool" }); + const estimate = fixture.project(2440); + assert.ok(Math.abs(estimate.usd - 0.0015) < 1e-12); + assert.equal(estimate.eta_ms, 16000); + assert.equal(fixture.entries().find(entry => entry.data.call_kind === "answer").data.duration_ms, 6000); + fixture.reload(); + assert.deepEqual(fixture.project(2440, cost), estimate); + assert.equal(fixture.project(0, cost).usd, 0); + assert.equal(fixture.project(0, cost).eta_ms, 0); +}); const update = (fixture, type, data = {}) => fixture.emit("message_update", { message: { role: "assistant" }, assistantMessageEvent: { type, ...data }, }); diff --git a/examples/pi/test/prediction-total.test.mjs b/examples/pi/test/prediction-total.test.mjs new file mode 100644 index 0000000000..44d58b2007 --- /dev/null +++ b/examples/pi/test/prediction-total.test.mjs @@ -0,0 +1,37 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { totalTokenPredictionRequest } from "../prediction-total.ts"; + +test("Jev's full-token prompt includes cache usage, model and selected reasoning without rewriting task text", () => { + const messages = [ + { role: "user", content: "Explain uncached input plus output" }, + { role: "assistant", content: [{ type: "text", text: "Inspecting" }], + usage: { input: 800, cacheRead: 5000, cacheWrite: 200, output: 100, reasoning: 40 } }, + ]; + for (const level of ["medium", "max"]) { + const request = totalTokenPredictionRequest("provider/judge", messages, "provider/worker", level); + assert.equal(request.context.token_usage.tokens_used_so_far, 6100); + assert.equal(request.context.token_usage.per_step[0].input_tokens, 6000); + assert.equal(request.context.token_usage.per_step[0].output_tokens, 100); + assert.equal(request.context.token_usage.per_step[0].tokens_used, 6100); + assert.equal(request.context.completion_model, "provider/worker"); + assert.equal(request.context.thinking_level, level); + assert.match(request.context.instructions, /including cache reads and writes/); + assert.match(request.context.instructions, /latest user request/); + assert.equal(request.context.task_history[0].message, messages[0].content); + assert.doesNotMatch(request.context.instructions, /uncached input plus output/); + assert.doesNotMatch(JSON.stringify(request.questions), /uncached/); + assert.equal(request.questions.minimum_likely_budget.kind.data.options.length, 81); + } +}); + +test("missing or invalid cache usage cannot silently change Jev's target", () => { + for (const usage of [undefined, { input: 100, output: 10, cacheWrite: 0 }, + { input: 100, output: 10, cacheRead: -1, cacheWrite: 0 }]) { + assert.throws(() => totalTokenPredictionRequest("provider/judge", [{ role: "assistant", content: [], usage }], + "provider/worker", "max"), /usage is missing/); + } +}); diff --git a/examples/pi/test/remaining-error.test.mjs b/examples/pi/test/remaining-error.test.mjs new file mode 100644 index 0000000000..2a915c26e0 --- /dev/null +++ b/examples/pi/test/remaining-error.test.mjs @@ -0,0 +1,40 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { scoreRemainingForecasts } from "../remaining-error.ts"; +import { NextCallMAE } from "../forecast-error.ts"; + +const forecast = (overrides = {}) => ({ session_id: "s", agent_turn_idx: 1, + lightgbm: { remaining_task: { tokens: 1200 } }, greg: { predicted_tokens: 1000 }, + full_used: 400, ...overrides }); + +test("scores remaining tokens from each forecast cutoff using the same full-token definition for both predictors", () => { + const samples = scoreRemainingForecasts([forecast()], 1200); + assert.equal(samples.length, 2); + assert.deepEqual(samples.map(s => [s.predictor, s.predicted_tokens, s.actual_tokens, s.signed_error]), + [["lightgbm", 1200, 800, 400], ["greg", 600, 800, -200]]); + const greg = new NextCallMAE("s", samples.filter(s => s.predictor === "greg")).summary(); + assert.equal(greg.meanPercentageError, -25); + assert.equal(greg.mae, 200); +}); + +test("averages independent forecast cutoffs, deduplicates records, and excludes undefined percentages", () => { + const first = forecast(); + const second = forecast({ agent_turn_idx: 2, full_used: 800, + lightgbm: { remaining_task: { tokens: 200 } }, greg: { predicted_tokens: 1400 } }); + const samples = scoreRemainingForecasts([first, first, second], 1200); + assert.equal(samples.length, 4); + assert.equal(new NextCallMAE("s", samples.filter(s => s.predictor === "lightgbm")).summary().meanPercentageError, 0); + assert.equal(new NextCallMAE("s", samples.filter(s => s.predictor === "greg")).summary().meanPercentageError, 12.5); + const zero = scoreRemainingForecasts([forecast()], 400); + assert.equal(new NextCallMAE("s", zero).summary().percentageCount, 0); +}); + +test("missing usage, impossible forecasts and invalid cutoffs stay unavailable", () => { + assert.equal(scoreRemainingForecasts([forecast()], undefined).length, 0); + assert.equal(scoreRemainingForecasts([forecast()], 399).length, 0); + assert.equal(scoreRemainingForecasts([forecast({ lightgbm: null, greg: { predicted_tokens: null } })], 1200).length, 0); + assert.equal(scoreRemainingForecasts([forecast({ full_used: undefined })], 1200).length, 0); +}); From 0e26a8953c2c33c6987fccffd2a38b7fb9a5e51b Mon Sep 17 00:00:00 2001 From: sabhatinas Date: Fri, 13 Nov 2026 16:28:14 -0800 Subject: [PATCH 2/5] chore(pi): default forecast demo to Opus 4.8 Signed-off-by: sabhatinas --- examples/pi/FORECAST_COMPARISON.md | 4 +++- examples/pi/switchyard-forecast.example.json | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/pi/FORECAST_COMPARISON.md b/examples/pi/FORECAST_COMPARISON.md index b74a8c84bd..767bc6fd27 100644 --- a/examples/pi/FORECAST_COMPARISON.md +++ b/examples/pi/FORECAST_COMPARISON.md @@ -2,7 +2,9 @@ `switchyard/compare` runs both predictors before each acting-model call. One fixed completion model executes the task, so both forecasts observe the same -pre-call conversation. The display has two rows below the editor: +pre-call conversation. The example configuration uses Opus 4.8 through +OpenRouter (`openrouter/anthropic/claude-opus-4.8`). The display has two rows +below the editor: ```text LightGBM · ~$0.0200 · ETA ~20s · next ~8.0k tok (avg err +4.0%) | remaining ~30.0k tok (avg err −8.0%) diff --git a/examples/pi/switchyard-forecast.example.json b/examples/pi/switchyard-forecast.example.json index 6a893118c4..a2387c8c91 100644 --- a/examples/pi/switchyard-forecast.example.json +++ b/examples/pi/switchyard-forecast.example.json @@ -1,5 +1,5 @@ { - "completion_model": "openrouter/openai/gpt-6.1-sol", + "completion_model": "openrouter/anthropic/claude-opus-4.8", "decision_model": "openrouter/~typesafe/jev-latest", "python": "/absolute/path/to/agent-progress-research/v1/.venv/bin/python", "research_root": "/absolute/path/to/agent-progress-research/v1", From f74a18f1f35a268dc67dad3773ca35cc15cfccf0 Mon Sep 17 00:00:00 2001 From: sabhatinas Date: Fri, 13 Nov 2026 16:38:37 -0800 Subject: [PATCH 3/5] fix(pi): clear remaining forecasts when a turn completes Signed-off-by: sabhatinas --- examples/pi/FORECAST_COMPARISON.md | 4 ++++ examples/pi/switchyard-compare.ts | 23 ++++++++++++++++------- examples/pi/test/compare.test.mjs | 9 +++++++++ 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/examples/pi/FORECAST_COMPARISON.md b/examples/pi/FORECAST_COMPARISON.md index 767bc6fd27..ecf11a445c 100644 --- a/examples/pi/FORECAST_COMPARISON.md +++ b/examples/pi/FORECAST_COMPARISON.md @@ -106,6 +106,10 @@ Both predictors run concurrently before each normal user, continuation, or retry call. Direct helper calls skip forecasting. Cancellation propagates to Pi. Failed forecasts are recorded independently and allow completion to proceed. The displayed remaining values consume observed usage until the next forecast. +When Pi finishes a turn normally, both rows show remaining `0 tok`, `$0.0000`, +and `ETA 0s`; LightGBM shows `next —`. The original forecasts remain unchanged +for error scoring. This completed display restores on resume and resets at +the next forecast. Aborted or failed turns retain their forecast estimates. ## Cost, ETA, and errors diff --git a/examples/pi/switchyard-compare.ts b/examples/pi/switchyard-compare.ts index 656a1e09e7..80028745b6 100644 --- a/examples/pi/switchyard-compare.ts +++ b/examples/pi/switchyard-compare.ts @@ -53,13 +53,20 @@ export default function (pi: ExtensionAPI) { return; } const history = messages(ctx); + const branch = ctx.sessionManager.getBranch(); + const lastState = branch.findLast(entry => entry.type === "custom" + && (entry.customType === ENTRY || entry.customType === REMAINING_ERROR_ENTRY) + && (entry.data as { session_id: string }).session_id === pair.session_id); + // A completed turn has no future usage. Keep its original forecasts for error scoring. + const completed = lastState?.type === "custom" && lastState.customType === REMAINING_ERROR_ENTRY + && (lastState.data as RemainingSettlement).completed; const fullNow = observedTokens(history); - const lgbmRemaining = pair.lightgbm && fullNow !== undefined && pair.full_used !== undefined + const lgbmRemaining = completed ? 0 : pair.lightgbm && fullNow !== undefined && pair.full_used !== undefined ? Math.max(0, pair.lightgbm.remaining_task.tokens - (fullNow - pair.full_used)) : undefined; // Use the distribution's point estimate (band midpoints), not its upper-bound buffer. - const gregRemaining = pair.greg?.predicted_tokens == null || fullNow === undefined ? undefined + const gregRemaining = completed ? 0 : pair.greg?.predicted_tokens == null || fullNow === undefined ? undefined : Math.max(0, pair.greg.predicted_tokens - fullNow); - const savedErrors = ctx.sessionManager.getBranch().flatMap(entry => entry.type === "custom" + const savedErrors = branch.flatMap(entry => entry.type === "custom" && entry.customType === REMAINING_ERROR_ENTRY ? [(entry.data as RemainingSettlement)] : []) .filter(record => record.session_id === pair.session_id).flatMap(record => record.samples); const formatError = (stats: ReturnType) => errorStatus(stats) @@ -68,12 +75,14 @@ export default function (pi: ExtensionAPI) { const stats = new NextCallMAE(pair.session_id, savedErrors.filter(sample => sample.predictor === predictor)).summary(); const err = formatError(stats); const estimate = remaining === undefined ? undefined : metrics.projectRemaining(ctx, remaining, "full", pair.model_cost); - const cost = estimate?.usd === undefined ? "$—" : `~$${estimate.usd.toFixed(4)}`; - const prefix = `${label} · ${cost} · ETA ${completionDuration(estimate?.eta_ms)} · ${next ? `${next} | ` : ""}remaining`; + const cost = completed ? "$0.0000" : estimate?.usd === undefined ? "$—" : `~$${estimate.usd.toFixed(4)}`; + const eta = completed ? "0s" : completionDuration(estimate?.eta_ms); + const prefix = `${label} · ${cost} · ETA ${eta} · ${next ? `${next} | ` : ""}remaining`; if (remaining === undefined) return `${prefix} unavailable (${err})`; - return `${prefix} ~${count(remaining)} tok (${err})`; + return `${prefix} ${completed ? "" : "~"}${count(remaining)} tok (${err})`; }; - const next = pair.lightgbm ? `next ~${count(pair.lightgbm.next_call.tokens)} tok (${formatError(tracker(ctx).summary())})` : "next unavailable"; + const next = completed ? `next — (${formatError(tracker(ctx).summary())})` + : pair.lightgbm ? `next ~${count(pair.lightgbm.next_call.tokens)} tok (${formatError(tracker(ctx).summary())})` : "next unavailable"; const lines = [row("LightGBM", "lightgbm", lgbmRemaining, next), row("Jev", "greg", gregRemaining)]; ctx.ui.setWidget("switchyard-forecast-compare", lines, { placement: "belowEditor" }); } diff --git a/examples/pi/test/compare.test.mjs b/examples/pi/test/compare.test.mjs index 93a6004bbe..e01b55b5dd 100644 --- a/examples/pi/test/compare.test.mjs +++ b/examples/pi/test/compare.test.mjs @@ -90,6 +90,13 @@ test("both forecasts share a cutoff and completion; rendering survives classifie assert.equal(settlements()[0].data.samples.length, 2); assert.equal(settlements()[0].data.samples.find(s => s.predictor === "greg").actual_tokens, 1110); assert.equal(settlements()[0].data.samples.find(s => s.predictor === "lightgbm").actual_tokens, 1110); + for (const line of widgets.at(-1)) { + assert.match(line, / · \$0\.0000 · ETA 0s · /); + assert.match(line, /remaining 0 tok \(avg err /); + } + assert.match(widgets.at(-1)[0], /next — \(avg err .*\) \| remaining 0 tok/); + assert.equal(pair.lightgbm.remaining_task.tokens, 6000); + assert.equal(pair.greg.predicted_tokens, 46000); assert.match(widgets.at(-1)[0], /remaining .*avg err \+[\d,]+\.\d%\)/); assert.match(widgets.at(-1)[1], /remaining .*avg err \+[\d,]+\.\d%\)/); emit("agent_settled"); @@ -103,6 +110,7 @@ test("both forecasts share a cutoff and completion; rendering survives classifie assert.equal(secondRoute.thinkingLevel, "medium"); assert.equal(contexts.at(-1).state.thinking_level, "medium"); assert.equal(contexts.at(-1).state.token_usage.tokens_used_so_far, 1110); + assert.match(widgets.at(-1)[0], /ETA ~.* \| remaining ~/); assert.match(widgets.at(-1)[0], /^LightGBM · .* · ETA .* · next .* \| remaining/); assert.match(widgets.at(-1)[1], /^Jev · \$— · ETA — · remaining unavailable/); const number = contexts.length; @@ -113,6 +121,7 @@ test("both forecasts share a cutoff and completion; rendering survives classifie assert.equal(settlements().length, 2); assert.equal(settlements()[1].data.completed, false); assert.deepEqual(settlements()[1].data.samples, []); + assert.doesNotMatch(widgets.at(-1)[0], /ETA 0s|remaining 0 tok/); assert.match(widgets.at(-1)[1], /avg err \+[\d,]+\.\d%\)/); assert.ok(statuses.every(value => value === undefined)); }); From b3710ae6a0ac9cc21114b800fd7396e87e149614 Mon Sep 17 00:00:00 2001 From: sabhatinas Date: Fri, 13 Nov 2026 16:53:52 -0800 Subject: [PATCH 4/5] feat(pi): add high reasoning comparison mode Signed-off-by: sabhatinas --- examples/pi/FORECAST_COMPARISON.md | 10 ++++++++-- examples/pi/run-forecast.sh | 6 +++--- examples/pi/switchyard-compare.ts | 4 ++-- examples/pi/test/compare.test.mjs | 2 +- examples/pi/test/prediction-total.test.mjs | 2 +- 5 files changed, 15 insertions(+), 9 deletions(-) diff --git a/examples/pi/FORECAST_COMPARISON.md b/examples/pi/FORECAST_COMPARISON.md index ecf11a445c..73eed920a7 100644 --- a/examples/pi/FORECAST_COMPARISON.md +++ b/examples/pi/FORECAST_COMPARISON.md @@ -12,7 +12,7 @@ Jev · ~$0.0400 · ETA ~40s · remaining ~60.0k tok (avg err +12.0%) ``` These are illustrative values. Cost and ETA describe remaining work. The two -rows replace the Switchyard footer. `medium` and `max` are passed through to the +rows replace the Switchyard footer. `medium`, `high`, and `max` are passed through to the completion model, which must support the selected level. ## Setup @@ -64,9 +64,15 @@ In another task directory: /path/to/Switchyard/examples/pi/run-forecast.sh --thinking medium ``` +For a third comparison, run from another task directory: + +```bash +/path/to/Switchyard/examples/pi/run-forecast.sh --thinking high +``` + Use separate directories and the same prompt to compare reasoning settings without editing the same task files. Each level has its own Pi settings and -sessions under `~/.config/switchyard-pi-forecast/{max,medium}`. Add `--continue` +sessions under `~/.config/switchyard-pi-forecast/{max,high,medium}`. Add `--continue` from the same task directory to resume. `SY_FORECAST_AGENT_ROOT` overrides the settings root; `SY_FORECAST_CONFIG` selects a shared configuration file. When set, `XDG_CONFIG_HOME` replaces `~/.config` for the default locations. diff --git a/examples/pi/run-forecast.sh b/examples/pi/run-forecast.sh index 234529c460..e9ef72c403 100755 --- a/examples/pi/run-forecast.sh +++ b/examples/pi/run-forecast.sh @@ -11,7 +11,7 @@ while (( $# )); do --) pi_args+=("$@"); break ;; --thinking) if (( $# < 2 )); then - printf '%s\n' 'Use --thinking medium or --thinking max.' >&2 + printf '%s\n' 'Use --thinking medium, --thinking high, or --thinking max.' >&2 exit 2 fi thinking_level="$2"; shift 2 ;; @@ -20,8 +20,8 @@ while (( $# )); do esac done case "$thinking_level" in - medium|max) ;; - *) printf '%s\n' 'Use --thinking medium or --thinking max.' >&2; exit 2 ;; + medium|high|max) ;; + *) printf '%s\n' 'Use --thinking medium, --thinking high, or --thinking max.' >&2; exit 2 ;; esac agent_root="${SY_FORECAST_AGENT_ROOT:-${XDG_CONFIG_HOME:-$HOME/.config}/switchyard-pi-forecast}" diff --git a/examples/pi/switchyard-compare.ts b/examples/pi/switchyard-compare.ts index 80028745b6..e2292a7376 100644 --- a/examples/pi/switchyard-compare.ts +++ b/examples/pi/switchyard-compare.ts @@ -121,12 +121,12 @@ export default function (pi: ExtensionAPI) { pi.on("session_shutdown", () => worker?.close()); pi.registerVirtualModel({ provider: "switchyard", id: "compare", name: "LightGBM + Greg forecasts (passthrough)", - thinkingLevels: ["medium", "max"], + thinkingLevels: ["medium", "high", "max"], async route(request, ctx) { const measurement = metrics.begin(request.reason, ctx); try { const thinkingLevel = request.thinkingLevel ?? "medium"; - if (thinkingLevel !== "medium" && thinkingLevel !== "max") throw new Error("Select --thinking medium or --thinking max"); + if (thinkingLevel !== "medium" && thinkingLevel !== "high" && thinkingLevel !== "max") throw new Error("Select --thinking medium, --thinking high, or --thinking max"); const config = JSON.parse(readFileSync(process.env.SY_FORECAST_CONFIG ?? join(getAgentDir(), "switchyard-forecast.json"), "utf8")); const identity = modelIdentity(config.completion_model); const model = ctx.modelRegistry.find(identity.provider, identity.id); diff --git a/examples/pi/test/compare.test.mjs b/examples/pi/test/compare.test.mjs index e01b55b5dd..c28459fbd9 100644 --- a/examples/pi/test/compare.test.mjs +++ b/examples/pi/test/compare.test.mjs @@ -56,7 +56,7 @@ test("both forecasts share a cutoff and completion; rendering survives classifie const user = { role: "user", content: "Read the parser and explain it" }; entries.push({ type: "message", message: user }); const request = { reason: "user", thinkingLevel: "max", messages: [user] }; - assert.deepEqual(definition.thinkingLevels, ["medium", "max"]); + assert.deepEqual(definition.thinkingLevels, ["medium", "high", "max"]); emit("session_start"); assert.match(widgets.at(-1)[1], /^Jev · waiting/); const firstRoute = await definition.route(request, ctx); diff --git a/examples/pi/test/prediction-total.test.mjs b/examples/pi/test/prediction-total.test.mjs index 44d58b2007..2ba5306357 100644 --- a/examples/pi/test/prediction-total.test.mjs +++ b/examples/pi/test/prediction-total.test.mjs @@ -11,7 +11,7 @@ test("Jev's full-token prompt includes cache usage, model and selected reasoning { role: "assistant", content: [{ type: "text", text: "Inspecting" }], usage: { input: 800, cacheRead: 5000, cacheWrite: 200, output: 100, reasoning: 40 } }, ]; - for (const level of ["medium", "max"]) { + for (const level of ["medium", "high", "max"]) { const request = totalTokenPredictionRequest("provider/judge", messages, "provider/worker", level); assert.equal(request.context.token_usage.tokens_used_so_far, 6100); assert.equal(request.context.token_usage.per_step[0].input_tokens, 6000); From 2c77d654eae4bac3ca26d77aa0d4dcc07c53aea5 Mon Sep 17 00:00:00 2001 From: sabhatinas Date: Fri, 13 Nov 2026 16:55:33 -0800 Subject: [PATCH 5/5] feat(pi): track cumulative forecast costs Signed-off-by: sabhatinas --- examples/pi/FORECAST_COMPARISON.md | 46 ++++++++- examples/pi/cost-forecast.ts | 123 ++++++++++++++++++++++++ examples/pi/remaining-error.ts | 2 + examples/pi/switchyard-compare.ts | 38 +++++++- examples/pi/test/compare.test.mjs | 46 +++++++-- examples/pi/test/cost-forecast.test.mjs | 81 ++++++++++++++++ 6 files changed, 320 insertions(+), 16 deletions(-) create mode 100644 examples/pi/cost-forecast.ts create mode 100644 examples/pi/test/cost-forecast.test.mjs diff --git a/examples/pi/FORECAST_COMPARISON.md b/examples/pi/FORECAST_COMPARISON.md index 73eed920a7..c1bdb7489a 100644 --- a/examples/pi/FORECAST_COMPARISON.md +++ b/examples/pi/FORECAST_COMPARISON.md @@ -7,11 +7,12 @@ OpenRouter (`openrouter/anthropic/claude-opus-4.8`). The display has two rows below the editor: ```text -LightGBM · ~$0.0200 · ETA ~20s · next ~8.0k tok (avg err +4.0%) | remaining ~30.0k tok (avg err −8.0%) -Jev · ~$0.0400 · ETA ~40s · remaining ~60.0k tok (avg err +12.0%) +LightGBM · rem ~$0.0200 · total ~$0.0500 · ETA ~20s · next ~8.0k tok (avg err +4.0%) | remaining ~30.0k tok (avg err −8.0%) +Jev · rem ~$0.0400 · total ~$0.0700 · ETA ~40s · remaining ~60.0k tok (avg err +12.0%) ``` -These are illustrative values. Cost and ETA describe remaining work. The two +These are illustrative values. `rem` and ETA describe remaining work; `total` +adds observed acting-model spending for the current task/turn. The two rows replace the Switchyard footer. `medium`, `high`, and `max` are passed through to the completion model, which must support the selected level. @@ -119,6 +120,45 @@ the next forecast. Aborted or failed turns retain their forecast estimates. ## Cost, ETA, and errors +For a task/turn, let `C[t]` be the remaining-cost prediction before call `t`, +`c[t]` its observed cost, and `S[t]` the sum of observed acting-model costs so +far. The cost-to-go identity is `C[t] = c[t] + C[t+1]` for a consistent forecast; +the projected total is `S[t] + C[t]`. Remaining forecasts are not summed across +calls. Predictions may revise this total, so their differences are not treated +as observed charges. + +Every saved forecast has a typed `cost` frame with USD fields for each +predictor: `remaining_usd`, `next_call_usd` (LightGBM), `total_usd`, +`session_total_usd`, `initial_total_usd`, the first-priced baseline, and forecast +revisions. The frame records observed task and session spend, plus forecasting +API spend and the current forecasting API call's cost separately. See +[`cost-forecast.ts`](cost-forecast.ts) for the API types. Charges are computed +from actual recorded usage and its saved per-call catalog rates; missing prices +or usage remain unknown, not zero. Retries and partially charged failed calls +are included when their usage and prices are available. + +Each forecast's dollar value is frozen at its pre-call boundary. Observed +dollars consume that budget until a new prediction arrives; updated token mix +does not retroactively reprice an old forecast. `initial_total_usd` freezes the +first total-cost prediction (`C[0]` at zero task spend). At normal completion, +the settlement's `cost_outcome` reports observed acting-model cost minus that +original prediction, and the widget prints `C0 err` in dollars. Positive means +actual cost exceeded the predicted budget; this sign is the reverse of the +token error convention below. Remaining cost and ETA still become zero. + +A completely fresh session initially has no observed token/cache mix, so its +first dollar forecast and `C0` are unknown. A later estimate is saved as +`first_priced_total_usd` and is scored/displayed as `first err`, not relabeled +as `C0`. Existing timed observations can price the first forecast of a new +turn. Baselines persist on resume and reset after each settled turn, since +future user requests are outside the current forecast horizon. + +`revision_usd = total[t] - total[t-1]` records each forecast update. The API also +retains `C[t-1] - C[t]` as an implied preceding-call cost and the actual observed +cost separately. Including the final revision to actual spend makes revisions +telescope to `actual - C[0]` when all values are known. Predictor API charges +remain outside the acting-model forecast/error target. + Both rows use the same calibration from successful, timed acting-model calls in their session. Let `F` be their observed full-token total and `R` the remaining forecast: diff --git a/examples/pi/cost-forecast.ts b/examples/pi/cost-forecast.ts new file mode 100644 index 0000000000..3112f2e128 --- /dev/null +++ b/examples/pi/cost-forecast.ts @@ -0,0 +1,123 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { estimateCost, type CallMetrics } from "@switchyard/libsy"; + +export type Predictor = "lightgbm" | "greg"; +type Dollars = number | null; +type Entries = readonly { type: string; customType?: string; data?: unknown }[]; + +export interface ObservedSpend { + completion_usd: Dollars; + prediction_api_usd: Dollars; +} +export interface CostPrediction { + remaining_usd: Dollars; + next_call_usd: Dollars; + total_usd: Dollars; // Observed task spend plus the current remaining forecast. + session_total_usd: Dollars; + initial_total_usd: Dollars; // C0, frozen only at the first forecast of this task. + first_priced_total_usd: Dollars; + first_priced_agent_turn_idx: number | null; + revision_usd: Dollars; // Change to the projected total since the previous forecast. + implied_previous_call_usd: Dollars; // Previous remaining minus current remaining, not an observed charge. + actual_since_previous_forecast_usd: Dollars; +} +export interface CostFrame { + version: 1; + currency: "USD"; + basis: "catalog_priced_acting_model_calls"; + task_start_agent_turn_idx: number; + agent_turn_idx: number; + spent_usd: Dollars; + session_spent_usd: Dollars; + prediction_api_spent_usd: Dollars; + prediction_api_call_usd: Dollars; + predictors: Record; +} + +const add = (a: Dollars, b: Dollars): Dollars => a === null || b === null ? null : a + b; +const subtract = (a: Dollars, b: Dollars): Dollars => a === null || b === null ? null : a - b; + +/** Price recorded acting calls and forecasting API calls separately; unknown charges stay unknown. */ +export function observedSpend(entries: Entries, sessionId: string, fromAgentTurn = 0): ObservedSpend { + let completion: Dollars = 0, prediction: Dollars = 0; + const seen = new Set(); + for (const entry of entries) { + if (entry.type !== "custom" || entry.customType !== "switchyard.metrics.v1") continue; + const call = entry.data as CallMetrics; + if (call.context.session_id !== sessionId || call.context.agent_turn_idx < fromAgentTurn || seen.has(call.id)) continue; + seen.add(call.id); + if (call.call_kind !== "answer" && call.call_kind !== "decision") continue; + const charge = call.usage && call.rates ? estimateCost(call.usage, call.rates)?.usd ?? null : null; + if (call.call_kind === "answer") completion = add(completion, charge); + else prediction = add(prediction, charge); + } + return { completion_usd: completion, prediction_api_usd: prediction }; +} + +/** Freeze each cost forecast at its pre-call boundary, including the original C0. */ +export function costFrame(input: { + taskStart: number; agentTurn: number; spent: ObservedSpend; sessionSpent: Dollars; + predictionApiCall: Dollars; remaining: Record; nextCall: Dollars; previous: CostFrame[]; +}): CostFrame { + const first = input.previous[0], previous = input.previous.at(-1); + const prediction = (name: Predictor): CostPrediction => { + const total = add(input.spent.completion_usd, input.remaining[name]); + const prior = previous?.predictors[name]; + const initial = first ? first.predictors[name].initial_total_usd + : input.agentTurn === input.taskStart ? total : null; + return { + remaining_usd: input.remaining[name], next_call_usd: name === "lightgbm" ? input.nextCall : null, + total_usd: total, session_total_usd: add(input.sessionSpent, input.remaining[name]), initial_total_usd: initial, + first_priced_total_usd: prior?.first_priced_total_usd ?? total, + first_priced_agent_turn_idx: prior?.first_priced_agent_turn_idx ?? (total === null ? null : input.agentTurn), + revision_usd: prior ? subtract(total, prior.total_usd) : null, + implied_previous_call_usd: prior ? subtract(prior.remaining_usd, input.remaining[name]) : null, + actual_since_previous_forecast_usd: previous ? subtract(input.spent.completion_usd, previous.spent_usd) : null, + }; + }; + return { + version: 1, currency: "USD", basis: "catalog_priced_acting_model_calls", + task_start_agent_turn_idx: input.taskStart, agent_turn_idx: input.agentTurn, + spent_usd: input.spent.completion_usd, session_spent_usd: input.sessionSpent, + prediction_api_spent_usd: input.spent.prediction_api_usd, prediction_api_call_usd: input.predictionApiCall, + predictors: { lightgbm: prediction("lightgbm"), greg: prediction("greg") }, + }; +} + +/** Consume observed dollars, not future forecasts; terminal errors compare against the frozen baseline. */ +export function costView(frame: CostFrame, name: Predictor, spentNow: Dollars, completed: boolean) { + const estimate = frame.predictors[name]; + const used = subtract(spentNow, frame.spent_usd); + const remaining = completed ? 0 : used === null || estimate.remaining_usd === null || used < 0 + ? null : Math.max(0, estimate.remaining_usd - used); + return { + spent_usd: spentNow, remaining_usd: remaining, total_usd: add(spentNow, remaining), + session_total_usd: add(add(frame.session_spent_usd, used), remaining), + spent_since_forecast_usd: used, + terminal_revision_usd: completed ? subtract(spentNow, estimate.total_usd) : null, + initial_total_usd: estimate.initial_total_usd, + actual_minus_initial_usd: completed ? subtract(spentNow, estimate.initial_total_usd) : null, + first_priced_total_usd: estimate.first_priced_total_usd, + first_priced_agent_turn_idx: estimate.first_priced_agent_turn_idx, + actual_minus_first_priced_usd: completed ? subtract(spentNow, estimate.first_priced_total_usd) : null, + }; +} + +export function finalCost(frame: CostFrame, spend: ObservedSpend) { + return { actual_usd: spend.completion_usd, prediction_api_spent_usd: spend.prediction_api_usd, + lightgbm: costView(frame, "lightgbm", spend.completion_usd, true), + greg: costView(frame, "greg", spend.completion_usd, true) }; +} +export type CostOutcome = ReturnType; + +export function formatDollars(value: Dollars, estimated = false) { + return value === null ? "$—" : `${estimated ? "~" : ""}$${value.toFixed(4)}`; +} +export function formatCostError(view: ReturnType) { + const initial = view.actual_minus_initial_usd; + const error = initial ?? view.actual_minus_first_priced_usd; + if (error === null) return "C0 err —"; + return `${initial === null ? "first err" : "C0 err"} ${error < 0 ? "−" : "+"}$${Math.abs(error).toFixed(4)}`; +} diff --git a/examples/pi/remaining-error.ts b/examples/pi/remaining-error.ts index a8fe5e0e20..89493be95a 100644 --- a/examples/pi/remaining-error.ts +++ b/examples/pi/remaining-error.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import type { ErrorSample } from "./forecast-error.ts"; +import type { CostOutcome } from "./cost-forecast.ts"; export const REMAINING_ERROR_ENTRY = "switchyard.compare.remaining-errors.v2"; export interface RemainingForecast { @@ -18,6 +19,7 @@ export interface RemainingSettlement { session_id: string; completed: boolean; samples: RemainingError[]; + cost_outcome?: CostOutcome; } /** Score original pre-call forecasts against usage from their cutoff to the final response. */ diff --git a/examples/pi/switchyard-compare.ts b/examples/pi/switchyard-compare.ts index e2292a7376..bbbff1c668 100644 --- a/examples/pi/switchyard-compare.ts +++ b/examples/pi/switchyard-compare.ts @@ -14,6 +14,8 @@ import { ForecastWorker, observedTokens, type HorizonForecast } from "./forecast import { NextCallMAE, ERROR_ENTRY, errorStatus, type ErrorSample } from "./forecast-error.ts"; import { scoreRemainingForecasts, REMAINING_ERROR_ENTRY, type RemainingSettlement } from "./remaining-error.ts"; +import { observedSpend, costFrame, costView, finalCost, formatDollars, formatCostError, + type CostFrame, type Predictor } from "./cost-forecast.ts"; const ENTRY = "switchyard.compare.forecast.v2"; interface Pair { @@ -26,6 +28,7 @@ interface Pair { token_basis: "full_input_plus_output"; thinking_level: ModelThinkingLevel; model_cost?: ModelCost; + cost?: CostFrame; errors: { lightgbm?: string; greg?: string }; } const count = (n: number) => n >= 1000 ? `${(n / 1000).toFixed(1)}k` : String(Math.round(n)); @@ -43,6 +46,14 @@ export default function (pi: ExtensionAPI) { entry.type === "custom" && entry.customType === ERROR_ENTRY ? [entry.data as ErrorSample] : []))); return errors.get(id)!; } + function pendingPairs(ctx: ExtensionContext): Pair[] { + const id = ctx.sessionManager.getSessionId(); + const branch = ctx.sessionManager.getBranch(); + const boundary = branch.findLastIndex(entry => entry.type === "custom" && entry.customType === REMAINING_ERROR_ENTRY + && (entry.data as RemainingSettlement).session_id === id); + return branch.slice(boundary + 1).flatMap(entry => entry.type === "custom" && entry.customType === ENTRY + && (entry.data as Pair).session_id === id ? [entry.data as Pair] : []); + } function render(ctx: ExtensionContext) { if (!ctx.hasUI) return; // The two widget rows are the entire comparison display. @@ -69,15 +80,20 @@ export default function (pi: ExtensionAPI) { const savedErrors = branch.flatMap(entry => entry.type === "custom" && entry.customType === REMAINING_ERROR_ENTRY ? [(entry.data as RemainingSettlement)] : []) .filter(record => record.session_id === pair.session_id).flatMap(record => record.samples); + const spend = pair.cost ? observedSpend(branch, pair.session_id, pair.cost.task_start_agent_turn_idx) : undefined; const formatError = (stats: ReturnType) => errorStatus(stats) .replace(/^ \| /, "").replace(/ \(n=\d+\)/, ""); - const row = (label: string, predictor: "lightgbm" | "greg", remaining: number | undefined, next?: string) => { + const row = (label: string, predictor: Predictor, remaining: number | undefined, next?: string) => { const stats = new NextCallMAE(pair.session_id, savedErrors.filter(sample => sample.predictor === predictor)).summary(); const err = formatError(stats); const estimate = remaining === undefined ? undefined : metrics.projectRemaining(ctx, remaining, "full", pair.model_cost); - const cost = completed ? "$0.0000" : estimate?.usd === undefined ? "$—" : `~$${estimate.usd.toFixed(4)}`; + const view = pair.cost && spend ? costView(pair.cost, predictor, spend.completion_usd, Boolean(completed)) : undefined; + const remainingCost = view ? view.remaining_usd : completed ? 0 : estimate?.usd ?? null; + const cost = formatDollars(remainingCost, !completed); + const total = formatDollars(view?.total_usd ?? null, !completed); + const finalError = completed && view ? ` · ${formatCostError(view)}` : ""; const eta = completed ? "0s" : completionDuration(estimate?.eta_ms); - const prefix = `${label} · ${cost} · ETA ${eta} · ${next ? `${next} | ` : ""}remaining`; + const prefix = `${label} · rem ${cost} · total ${total}${finalError} · ETA ${eta} · ${next ? `${next} | ` : ""}remaining`; if (remaining === undefined) return `${prefix} unavailable (${err})`; return `${prefix} ${completed ? "" : "~"}${count(remaining)} tok (${err})`; }; @@ -114,6 +130,9 @@ export default function (pi: ExtensionAPI) { const completed = finalEntry?.type === "message" && finalEntry.message.role === "assistant" && finalEntry.message.stopReason === "stop"; const record: RemainingSettlement = { session_id: id, completed, samples: completed ? scoreRemainingForecasts(pending, observedTokens(history)) : [] }; + const costs = pending.findLast(pair => pair.cost)?.cost; + if (completed && costs) record.cost_outcome = finalCost(costs, + observedSpend(branch, id, costs.task_start_agent_turn_idx)); pi.appendEntry(REMAINING_ERROR_ENTRY, record); } render(ctx); @@ -168,6 +187,19 @@ export default function (pi: ExtensionAPI) { full_used: observedTokens(history), token_basis: "full_input_plus_output", thinking_level: thinkingLevel, model_cost: model.cost, errors: { lightgbm: lgbm.status === "rejected" ? String(lgbm.reason?.message ?? lgbm.reason) : undefined, greg: greg.status === "rejected" ? String(greg.reason?.message ?? greg.reason) : undefined } }; + const previous = pendingPairs(ctx); + const taskStart = previous[0]?.agent_turn_idx ?? pair.agent_turn_idx; + const branch = ctx.sessionManager.getBranch(); + const projection = (tokens: number | undefined) => tokens === undefined ? null + : metrics.projectRemaining(ctx, tokens, "full", model.cost).usd ?? null; + pair.cost = costFrame({ taskStart, agentTurn: pair.agent_turn_idx, + spent: observedSpend(branch, id, taskStart), sessionSpent: observedSpend(branch, id).completion_usd, + predictionApiCall: observedSpend(branch, id, pair.agent_turn_idx).prediction_api_usd, + remaining: { lightgbm: projection(pair.lightgbm?.remaining_task.tokens), + greg: projection(pair.greg?.predicted_tokens == null || pair.full_used === undefined ? undefined + : Math.max(0, pair.greg.predicted_tokens - pair.full_used)) }, + nextCall: projection(pair.lightgbm?.next_call.tokens), previous: previous.flatMap(p => p.cost ? [p.cost] : []), + }); latest.set(id, pair); pi.appendEntry(ENTRY, pair); tracker(ctx).begin(`${id}:${pair.agent_turn_idx}`, pair.lightgbm?.next_call.tokens); diff --git a/examples/pi/test/compare.test.mjs b/examples/pi/test/compare.test.mjs index c28459fbd9..f4df6726d9 100644 --- a/examples/pi/test/compare.test.mjs +++ b/examples/pi/test/compare.test.mjs @@ -68,11 +68,14 @@ test("both forecasts share a cutoff and completion; rendering survives classifie assert.equal(pair.token_basis, "full_input_plus_output"); assert.equal(pair.thinking_level, "max"); assert.equal(pair.full_used, 0); + assert.equal(pair.cost.spent_usd, 0); + assert.equal(pair.cost.predictors.lightgbm.initial_total_usd, null); + assert.equal(pair.cost.predictors.greg.initial_total_usd, null); assert.equal(contexts[0].state.task_history[0].message, user.content); assert.equal(contexts[0].state.thinking_level, "max"); assert.equal(widgets.at(-1).length, 2); - assert.match(widgets.at(-1)[0], /^LightGBM · \$— · ETA — · next .* \| remaining .*\(avg err/); - assert.match(widgets.at(-1)[1], /^Jev · \$— · ETA — · remaining .*\(avg err/); + assert.match(widgets.at(-1)[0], /^LightGBM · rem \$— · total \$— · ETA — · next .* \| remaining .*\(avg err/); + assert.match(widgets.at(-1)[1], /^Jev · rem \$— · total \$— · ETA — · remaining .*\(avg err/); assert.ok(statuses.every(value => value === undefined)); emit("message_update", { message: { role: "assistant" }, assistantMessageEvent: { type: "text_delta", delta: "a" } }); await new Promise(resolve => setTimeout(resolve, 2)); @@ -80,8 +83,8 @@ test("both forecasts share a cutoff and completion; rendering survives classifie stopReason: "stop", usage: { input: 100, output: 10, cacheRead: 1000, cacheWrite: 0, totalTokens: 1110 } }; entries.push({ type: "message", message: response }); emit("message_end", { message: response }); - assert.match(widgets.at(-1)[0], /^LightGBM · ~\$[0-9.]+ · ETA ~[^·]+ · next/); - assert.match(widgets.at(-1)[1], /^Jev · ~\$[0-9.]+ · ETA ~[^·]+ · remaining/); + assert.match(widgets.at(-1)[0], /^LightGBM · rem \$— · total \$— · ETA ~[^·]+ · next/); + assert.match(widgets.at(-1)[1], /^Jev · rem \$— · total \$— · ETA ~[^·]+ · remaining/); assert.match(widgets.at(-1)[0], /next .*\(avg err [+-][\d,]+\.\d%\) \| remaining/); assert.match(widgets.at(-1)[1], /avg err —\)/); emit("agent_settled"); @@ -90,8 +93,10 @@ test("both forecasts share a cutoff and completion; rendering survives classifie assert.equal(settlements()[0].data.samples.length, 2); assert.equal(settlements()[0].data.samples.find(s => s.predictor === "greg").actual_tokens, 1110); assert.equal(settlements()[0].data.samples.find(s => s.predictor === "lightgbm").actual_tokens, 1110); + assert.ok(Math.abs(settlements()[0].data.cost_outcome.actual_usd - 0.00024) < 1e-12); + assert.equal(settlements()[0].data.cost_outcome.lightgbm.actual_minus_initial_usd, null); for (const line of widgets.at(-1)) { - assert.match(line, / · \$0\.0000 · ETA 0s · /); + assert.match(line, / · rem \$0\.0000 · total \$0\.0002 · C0 err — · ETA 0s · /); assert.match(line, /remaining 0 tok \(avg err /); } assert.match(widgets.at(-1)[0], /next — \(avg err .*\) \| remaining 0 tok/); @@ -105,23 +110,44 @@ test("both forecasts share a cutoff and completion; rendering survives classifie emit("session_start"); assert.deepEqual(widgets.at(-1), before); fail = true; - const secondRoute = await definition.route({ ...request, thinkingLevel: "medium", reason: "continuation" }, ctx); + const secondRoute = await definition.route({ ...request, thinkingLevel: "high", reason: "continuation" }, ctx); assert.equal(secondRoute.model, model); - assert.equal(secondRoute.thinkingLevel, "medium"); - assert.equal(contexts.at(-1).state.thinking_level, "medium"); + assert.equal(secondRoute.thinkingLevel, "high"); + assert.equal(contexts.at(-1).state.thinking_level, "high"); assert.equal(contexts.at(-1).state.token_usage.tokens_used_so_far, 1110); assert.match(widgets.at(-1)[0], /ETA ~.* \| remaining ~/); assert.match(widgets.at(-1)[0], /^LightGBM · .* · ETA .* · next .* \| remaining/); - assert.match(widgets.at(-1)[1], /^Jev · \$— · ETA — · remaining unavailable/); + assert.match(widgets.at(-1)[1], /^Jev · rem \$— · total \$— · ETA — · remaining unavailable/); const number = contexts.length; - assert.equal((await definition.route({ ...request, reason: "direct" }, ctx)).model, model); + const direct = await definition.route({ ...request, thinkingLevel: "medium", reason: "direct" }, ctx); + assert.equal(direct.model, model); + assert.equal(direct.thinkingLevel, "medium"); assert.equal(contexts.length, number); // Cancelling before a new response must not reuse the previous turn's final response. emit("agent_settled"); assert.equal(settlements().length, 2); assert.equal(settlements()[1].data.completed, false); assert.deepEqual(settlements()[1].data.samples, []); + assert.equal(settlements()[1].data.cost_outcome, undefined); assert.doesNotMatch(widgets.at(-1)[0], /ETA 0s|remaining 0 tok/); assert.match(widgets.at(-1)[1], /avg err \+[\d,]+\.\d%\)/); + fail = false; + entries.push({ type: "message", message: user }); + await definition.route({ ...request, thinkingLevel: "high" }, ctx); + const priced = entries.filter(e => e.customType === "switchyard.compare.forecast.v2").at(-1).data; + assert.equal(priced.cost.spent_usd, 0); + assert.ok(priced.cost.session_spent_usd > 0); + assert.ok(priced.cost.predictors.lightgbm.initial_total_usd > 0); + emit("message_update", { message: { role: "assistant" }, assistantMessageEvent: { type: "text_delta", delta: "a" } }); + await new Promise(resolve => setTimeout(resolve, 2)); + entries.push({ type: "message", message: response }); + emit("message_end", { message: response }); + emit("agent_settled"); + const outcome = settlements().at(-1).data.cost_outcome; + assert.ok(Math.abs(outcome.actual_usd - 0.00024) < 1e-12); + assert.equal(outcome.lightgbm.initial_total_usd, priced.cost.predictors.lightgbm.initial_total_usd); + assert.ok(Math.abs(outcome.lightgbm.actual_minus_initial_usd + - (0.00024 - priced.cost.predictors.lightgbm.initial_total_usd)) < 1e-12); + assert.match(widgets.at(-1)[0], /C0 err [−+]\$/); assert.ok(statuses.every(value => value === undefined)); }); diff --git a/examples/pi/test/cost-forecast.test.mjs b/examples/pi/test/cost-forecast.test.mjs new file mode 100644 index 0000000000..0905d9d7eb --- /dev/null +++ b/examples/pi/test/cost-forecast.test.mjs @@ -0,0 +1,81 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { observedSpend, costFrame, costView, formatCostError } from "../cost-forecast.ts"; + +const frame = (turn, spent, remaining, previous = [], taskStart = 0) => costFrame({ + taskStart, agentTurn: turn, spent: { completion_usd: spent, prediction_api_usd: 0.01 }, + sessionSpent: spent, predictionApiCall: 0.001, + remaining: { lightgbm: remaining, greg: remaining === null ? null : remaining * 2 }, nextCall: 0.1, previous, +}); + +test("projected totals freeze C0 and final errors use actual minus the original forecast", () => { + const first = frame(0, 0, 1); + const second = frame(1, 0.2, 0.7, [first]); + const third = frame(2, 0.45, 0.4, [first, second]); + assert.ok(Math.abs(second.predictors.lightgbm.total_usd - 0.9) < 1e-12); + assert.equal(third.predictors.lightgbm.initial_total_usd, 1); + assert.equal(third.predictors.greg.initial_total_usd, 2); + assert.ok(Math.abs(second.predictors.lightgbm.revision_usd + 0.1) < 1e-12); + const final = costView(third, "lightgbm", 0.95, true); + assert.equal(final.remaining_usd, 0); + assert.equal(final.total_usd, 0.95); + assert.ok(Math.abs(final.actual_minus_initial_usd + 0.05) < 1e-12); + const terminalRevision = 0.95 - third.predictors.lightgbm.total_usd; + const revisions = second.predictors.lightgbm.revision_usd + third.predictors.lightgbm.revision_usd + terminalRevision; + assert.ok(Math.abs(revisions - final.actual_minus_initial_usd) < 1e-12); + assert.equal(formatCostError(final), "C0 err −$0.0500"); + assert.equal(third.predictors.lightgbm.remaining_usd, 0.4); +}); + +test("cold-start and mid-task upgrades do not invent C0; first-priced estimates stay frozen", () => { + const first = frame(0, 0, null); + const second = frame(1, 0.2, 0.7, [first]); + const third = frame(2, 0.45, 0.1, [first, second]); + assert.equal(third.predictors.lightgbm.initial_total_usd, null); + assert.equal(third.predictors.lightgbm.first_priced_agent_turn_idx, 1); + assert.equal(third.predictors.lightgbm.first_priced_total_usd, second.predictors.lightgbm.total_usd); + const final = costView(third, "lightgbm", 1, true); + assert.equal(final.actual_minus_initial_usd, null); + assert.equal(formatCostError(final), "first err +$0.1000"); + assert.equal(frame(7, 0.2, 1, [], 5).predictors.lightgbm.initial_total_usd, null); +}); + +test("spent dollars consume one frozen budget without summing remaining forecasts", () => { + const estimate = frame(0, 0, 1); + assert.equal(costView(estimate, "lightgbm", 0.25, false).remaining_usd, 0.75); + assert.equal(costView(estimate, "lightgbm", 0.25, false).total_usd, 1); + assert.equal(costView(estimate, "lightgbm", 1.2, false).total_usd, 1.2); + assert.equal(costView(estimate, "lightgbm", null, false).remaining_usd, null); + assert.equal(costView(estimate, "lightgbm", 0.25, false).actual_minus_initial_usd, null); +}); + +test("actual acting cost uses saved per-call prices, excludes predictor fees, and deduplicates branches", () => { + const call = (id, turn, kind, usage, rates) => ({ type: "custom", customType: "switchyard.metrics.v1", data: { + id, context: { session_id: "s", agent_turn_idx: turn }, call_kind: kind, usage, rates, + } }); + const rates = { input: 2, output: 10, cache_read: 0.1, cache_write: 2.5, cache_write_1h: 4 }; + const answer = call("a", 0, "answer", { input_tokens: 100, output_tokens: 20, + cached_input_tokens: 1000, cache_creation_input_tokens: 100, reasoning_tokens: 5 }, rates); + const decision = call("d", 0, "decision", { input_tokens: 100, output_tokens: 10 }, rates); + const retry = call("b", 1, "answer", { input_tokens: 100, output_tokens: 10 }, rates); + const foreign = structuredClone(answer); foreign.data.context.session_id = "parent"; + const entries = [answer, answer, decision, retry, foreign]; + assert.ok(Math.abs(observedSpend(entries, "s").completion_usd - 0.00105) < 1e-12); + assert.ok(Math.abs(observedSpend(entries, "s").prediction_api_usd - 0.0003) < 1e-12); + assert.ok(Math.abs(observedSpend(entries, "s", 1).completion_usd - 0.0003) < 1e-12); + assert.deepEqual(observedSpend(entries, "new"), { completion_usd: 0, prediction_api_usd: 0 }); + entries.push(call("unknown", 2, "answer", undefined, rates)); + assert.equal(observedSpend(entries, "s").completion_usd, null); + assert.equal(observedSpend(entries, "s", 3).completion_usd, 0); +}); + +test("reloading cost frames preserves baseline errors and a new task starts a new C0", () => { + const first = frame(3, 0, 0.5, [], 3); + const restored = JSON.parse(JSON.stringify(first)); + assert.equal(costView(restored, "lightgbm", 0.75, true).actual_minus_initial_usd, 0.25); + const next = frame(9, 0, 2, [], 9); + assert.equal(next.predictors.lightgbm.initial_total_usd, 2); +});