diff --git a/scripts/usage-summary.test.mjs b/scripts/usage-summary.test.mjs new file mode 100644 index 0000000..5ca003c --- /dev/null +++ b/scripts/usage-summary.test.mjs @@ -0,0 +1,307 @@ +// Contract tests for usage-summary.mjs — turns fact-proxy's per-call metering +// (CR_USAGE_FILE, one JSON object per line) into a token + cost picture. +// +// What these pin, in order of how much a silent break would cost: +// 1. The billing math. ratio 1 == $2.00/1M input tokens. Output is +// ratio * completion_ratio; cached input is ratio * cache_ratio, and a +// price entry with NO cache_ratio bills cached tokens at the FULL input +// rate (that is what the gateway does) and says so in the output. +// 2. Price-list resolution. A metering record carries the name the UPSTREAM +// provider echoed; the price list is keyed by the gateway's own +// vendor-prefixed alias. Exact-match-only lookup would silently price +// nothing, so there are three fallbacks — and an AMBIGUOUS basename must +// stay unpriced rather than be guessed. +// 3. Best-effort I/O. Metering is observability, never a gate: a malformed +// line or an unreadable price list degrades the report but still exits 0. +// Only a missing file argument is a usage error (exit 2). + +import { spawnSync } from "node:child_process"; +import { mkdtempSync, writeFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; +import { after, before, describe, test } from "node:test"; +import assert from "node:assert/strict"; + +const SCRIPTS = dirname(fileURLToPath(import.meta.url)); +const USAGE_SUMMARY = join(SCRIPTS, "usage-summary.mjs"); + +let dir; +let seq = 0; + +before(() => { + dir = mkdtempSync(join(tmpdir(), "usage-summary-test-")); +}); +after(() => { + rmSync(dir, { recursive: true, force: true }); +}); + +function writeTmp(name, body) { + const file = join(dir, `${(seq += 1)}-${name}`); + writeFileSync(file, body); + return file; +} + +// Rows may be objects (serialized here) or raw strings, so a test can inject a +// malformed line deliberately. +const writeUsage = (rows) => + writeTmp( + "usage.jsonl", + `${rows.map((r) => (typeof r === "string" ? r : JSON.stringify(r))).join("\n")}\n`, + ); + +const writePricing = (entries) => writeTmp("pricing.json", JSON.stringify({ data: entries })); + +// spawnSync is fine here (unlike report.test.mjs): this script talks to no +// network, so the test's event loop does not need to stay live for the child. +function run(args) { + const r = spawnSync("node", [USAGE_SUMMARY, ...args], { encoding: "utf8" }); + return { status: r.status, stdout: r.stdout ?? "", stderr: r.stderr ?? "" }; +} + +// The flag order matters: the file is the first argv entry not starting with +// "--", so the usage file must precede --pricing. That is the documented order. +function runJson(args) { + const r = run([...args, "--json"]); + assert.equal(r.status, 0, r.stderr); + return JSON.parse(r.stdout); +} + +describe("aggregation", () => { + test("sums tokens and groups by model", () => { + const usage = writeUsage([ + { model: "m1", prompt: 100, completion: 10, cached: 40 }, + { model: "m1", prompt: 200, completion: 20, cached: 60 }, + { model: "m2", prompt: 500, completion: 50, cached: 0 }, + ]); + const out = runJson([usage]); + + assert.equal(out.calls, 3); + assert.equal(out.prompt_tokens, 800); + assert.equal(out.completion_tokens, 80); + assert.equal(out.cached_tokens, 100); + assert.equal(out.cache_hit_rate, 100 / 800); + + assert.equal(out.by_model.length, 2); + const m1 = out.by_model.find((m) => m.model === "m1"); + assert.deepEqual( + { calls: m1.calls, prompt: m1.prompt, completion: m1.completion, cached: m1.cached }, + { calls: 2, prompt: 300, completion: 30, cached: 100 }, + ); + }); + + test("missing numeric fields count as 0, missing model is (unknown)", () => { + const out = runJson([writeUsage([{}, { prompt: 5 }])]); + assert.equal(out.calls, 2); + assert.equal(out.prompt_tokens, 5); + assert.equal(out.completion_tokens, 0); + assert.equal(out.cached_tokens, 0); + assert.equal(out.by_model.length, 1); + assert.equal(out.by_model[0].model, "(unknown)"); + }); + + test("only status >= 400 counts as blocked; retries are summed", () => { + const out = runJson([ + writeUsage([ + { model: "m", status: 200, retries: 1 }, + { model: "m", status: 399 }, + { model: "m", status: 400 }, + { model: "m", status: 500, retries: 2 }, + { model: "m" }, // no status at all + ]), + ]); + assert.equal(out.blocked_calls, 2); + assert.equal(out.retries, 3); + }); + + test("cache_hit_rate is null when there are no prompt tokens", () => { + const out = runJson([writeUsage([{ model: "m" }])]); + assert.equal(out.cache_hit_rate, null); + }); +}); + +describe("best-effort input handling", () => { + test("blank lines are skipped and are NOT counted as malformed", () => { + const usage = writeTmp("usage.jsonl", '\n{"model":"m","prompt":10}\n\n \n'); + const out = runJson([usage]); + assert.equal(out.calls, 1); + assert.equal(out.malformed_lines, 0); + }); + + test("a malformed line is counted and skipped, and the run still exits 0", () => { + const usage = writeUsage([{ model: "m", prompt: 10 }, "{not json", { model: "m", prompt: 5 }]); + const out = runJson([usage]); + assert.equal(out.calls, 2); + assert.equal(out.prompt_tokens, 15); + assert.equal(out.malformed_lines, 1); + }); + + test("text mode notes skipped lines", () => { + const r = run([writeUsage([{ model: "m", prompt: 10 }, "nope"])]); + assert.equal(r.status, 0); + assert.match(r.stdout, /1 malformed line\(s\) skipped/); + }); + + test("a missing file argument is a usage error (exit 2)", () => { + const r = run([]); + assert.equal(r.status, 2); + assert.match(r.stderr, /usage: node usage-summary\.mjs/); + }); + + test("an unreadable price list degrades to tokens-only, still exit 0", () => { + const usage = writeUsage([{ model: "m", prompt: 1000 }]); + const r = run([usage, "--pricing", join(dir, "does-not-exist.json"), "--json"]); + assert.equal(r.status, 0); + assert.match(r.stderr, /could not read pricing/); + const out = JSON.parse(r.stdout); + assert.equal(out.prompt_tokens, 1000, "tokens are still reported"); + assert.equal(out.usd, null, "but nothing is priced"); + }); +}); + +describe("price-list resolution", () => { + const MILLION = { model: "TARGET", prompt: 1_000_000 }; + // ratio 1 => $2.00/1M input, so a priced 1M-prompt run is exactly $2.0000. + const priceOf = (entries, model = "TARGET") => { + const usage = writeUsage([{ ...MILLION, model }]); + return runJson([usage, "--pricing", writePricing(entries)]).usd; + }; + + test("exact key", () => { + assert.equal(priceOf([{ model_name: "TARGET", model_ratio: 1 }]), 2); + }); + + test("vendor prefix is dropped", () => { + assert.equal(priceOf([{ model_name: "openai/TARGET", model_ratio: 1 }]), 2); + }); + + test("basename match is case-insensitive", () => { + assert.equal(priceOf([{ model_name: "openai/target", model_ratio: 1 }]), 2); + }); + + test("a trailing -YYYY-MM-DD snapshot suffix is stripped", () => { + const usage = writeUsage([{ model: "TARGET-2024-07-18", prompt: 1_000_000 }]); + const pricing = writePricing([{ model_name: "openai/TARGET", model_ratio: 1 }]); + assert.equal(runJson([usage, "--pricing", pricing]).usd, 2); + }); + + test("a release-channel suffix is stripped", () => { + for (const suffix of ["preview", "latest", "stable"]) { + const usage = writeUsage([{ model: `TARGET-${suffix}`, prompt: 1_000_000 }]); + const pricing = writePricing([{ model_name: "openai/TARGET", model_ratio: 1 }]); + assert.equal(runJson([usage, "--pricing", pricing]).usd, 2, suffix); + } + }); + + test("an AMBIGUOUS basename is left unpriced rather than guessed", () => { + const usd = priceOf([ + { model_name: "openai/TARGET", model_ratio: 1 }, + { model_name: "azure/TARGET", model_ratio: 99 }, + ]); + assert.equal(usd, null, "two vendors share the basename, so neither ratio may be assumed"); + }); + + test("a model absent from the price list is reported as such", () => { + const usage = writeUsage([{ model: "TARGET", prompt: 1000 }]); + const r = run([usage, "--pricing", writePricing([{ model_name: "other", model_ratio: 1 }])]); + assert.equal(r.status, 0); + assert.match(r.stdout, /not in price list/); + assert.doesNotMatch(r.stdout, /TOTAL/, "an unpriced model suppresses the run total"); + }); + + test("a non-numeric model_ratio is treated as unpriced", () => { + assert.equal(priceOf([{ model_name: "TARGET", model_ratio: "1" }]), null); + }); +}); + +describe("cost math", () => { + const usdFor = (row, priceEntry) => { + const usage = writeUsage([{ model: "TARGET", ...row }]); + const pricing = writePricing([{ model_name: "TARGET", ...priceEntry }]); + return runJson([usage, "--pricing", pricing]).usd; + }; + + test("ratio 1 bills input at $2.00/1M", () => { + assert.equal(usdFor({ prompt: 1_000_000 }, { model_ratio: 1 }), 2); + assert.equal(usdFor({ prompt: 500_000 }, { model_ratio: 1 }), 1); + assert.equal(usdFor({ prompt: 1_000_000 }, { model_ratio: 3 }), 6); + }); + + test("completion_ratio multiplies output only", () => { + assert.equal(usdFor({ completion: 1_000_000 }, { model_ratio: 1, completion_ratio: 4 }), 8); + }); + + test("output defaults to the input rate when completion_ratio is absent", () => { + assert.equal(usdFor({ completion: 1_000_000 }, { model_ratio: 1 }), 2); + }); + + test("cached tokens get the cache_ratio discount", () => { + const usd = usdFor( + { prompt: 1_000_000, cached: 1_000_000 }, + { model_ratio: 1, cache_ratio: 0.5 }, + ); + assert.equal(usd, 1, "fully cached at half rate"); + }); + + test("NO cache_ratio means cached tokens are billed at the FULL input rate", () => { + const usd = usdFor({ prompt: 1_000_000, cached: 1_000_000 }, { model_ratio: 1 }); + assert.equal(usd, 2, "same tokens cost double vs. a gateway entry that caches"); + }); + + test("the missing-cache-discount case is flagged in the text report", () => { + const usage = writeUsage([{ model: "TARGET", prompt: 1_000_000, cached: 1_000_000 }]); + const pricing = writePricing([{ model_name: "TARGET", model_ratio: 1 }]); + const r = run([usage, "--pricing", pricing]); + assert.equal(r.status, 0); + assert.match(r.stdout, /no cache discount at this gateway/); + }); + + test("only the non-cached remainder is billed at the fresh rate", () => { + // 600k fresh @ $2/1M + 400k cached @ $0.50/1M = 1.20 + 0.20 + const usd = usdFor( + { prompt: 1_000_000, cached: 400_000 }, + { model_ratio: 1, cache_ratio: 0.25 }, + ); + assert.equal(usd, 1.4); + }); + + test("cached exceeding prompt never bills negative fresh tokens", () => { + const usd = usdFor( + { prompt: 100, cached: 5_000_000 }, + { model_ratio: 1, cache_ratio: 0 }, + ); + assert.equal(usd, 0, "fresh clamps at 0 and a zero cache_ratio is free"); + }); + + test("per-model costs are summed into the run total", () => { + const usage = writeUsage([ + { model: "a", prompt: 1_000_000 }, + { model: "b", prompt: 1_000_000 }, + ]); + const pricing = writePricing([ + { model_name: "a", model_ratio: 1 }, + { model_name: "b", model_ratio: 2 }, + ]); + const out = runJson([usage, "--pricing", pricing]); + assert.equal(out.usd, 6); + assert.deepEqual( + out.by_model.map((m) => [m.model, m.usd]), + [ + ["a", 2], + ["b", 4], + ], + ); + }); + + test("text mode prints the total and the per-call amplification signal", () => { + const usage = writeUsage([ + { model: "TARGET", prompt: 400 }, + { model: "TARGET", prompt: 600 }, + ]); + const pricing = writePricing([{ model_name: "TARGET", model_ratio: 1 }]); + const r = run([usage, "--pricing", pricing]); + assert.equal(r.status, 0); + assert.match(r.stdout, /TOTAL \$0\.0020 for this run/); + assert.match(r.stdout, /500 input tokens per call/); + }); +});