Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 18 additions & 3 deletions scripts/usage-summary.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,28 @@ import fs from "node:fs";

const USD_PER_1M_AT_RATIO_1 = 2.0;

// One pass, because a flag's VALUE is not a positional argument. --pricing
// takes a path, and a path does not start with "--", so scanning separately
// for "the first argv entry that isn't a flag" picks up the price list as the
// usage file whenever --pricing comes first. That failure is silent rather
// than loud: a price list is itself valid JSON, so it parses as a single
// metering row with no token fields and reports a run that cost nothing.
// Advancing the index past a flag's value keeps both argument orders working.
const argv = process.argv.slice(2);
const file = argv.find((a) => !a.startsWith("--"));
let file = null;
let pricingPath = null;
let asJson = false;
for (let i = 0; i < argv.length; i += 1) {
if (argv[i] === "--pricing") pricingPath = argv[i + 1];
else if (argv[i] === "--json") asJson = true;
if (argv[i] === "--pricing") {
// Only consume the next entry when it is actually a value. A trailing
// --pricing, or one followed by another flag, must not swallow that flag.
const next = argv[i + 1];
if (next !== undefined && !next.startsWith("--")) {
pricingPath = next;
i += 1;
}
} else if (argv[i] === "--json") asJson = true;
else if (!argv[i].startsWith("--") && file === null) file = argv[i];
}
if (!file) {
console.error("usage: node usage-summary.mjs <usage.jsonl> [--pricing pricing.json] [--json]");
Expand Down
57 changes: 55 additions & 2 deletions scripts/usage-summary.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,8 @@ function run(args) {
return { status: r.status, stdout: r.stdout ?? "", stderr: r.stderr ?? "" };
}

// The flag order matters: the file is the first argv entry not starting with
// "--", so the usage file must precede --pricing. That is the documented order.
// Most cases below use the documented order (usage file first); the
// "argument order" block pins that the reverse order works too.
function runJson(args) {
const r = run([...args, "--json"]);
assert.equal(r.status, 0, r.stderr);
Expand Down Expand Up @@ -159,6 +159,59 @@ describe("best-effort input handling", () => {
});
});

describe("argument order", () => {
// A flag's value is not a positional argument. --pricing takes a path, and a
// path does not start with "--", so a "first non-flag argv entry" scan used
// to pick the price list as the usage file when the flag came first. It
// failed silently: a price list is valid JSON, so it parsed as one metering
// row with no token fields and reported a run that cost nothing.
test("--pricing before the usage file does not steal it", () => {
const usage = writeUsage([{ model: "m", prompt: 1_000_000 }]);
const pricing = writePricing([{ model_name: "m", model_ratio: 1 }]);
const r = run(["--pricing", pricing, usage, "--json"]);
assert.equal(r.status, 0, r.stderr);
const out = JSON.parse(r.stdout);
assert.equal(out.prompt_tokens, 1_000_000, "the usage file must be read, not the price list");
assert.equal(out.usd, 2, "and the price list must still be applied");
});

test("both orders produce the same result", () => {
const usage = writeUsage([{ model: "m", prompt: 500_000, completion: 1000 }]);
const pricing = writePricing([{ model_name: "m", model_ratio: 1 }]);
const flagFirst = JSON.parse(run(["--pricing", pricing, usage, "--json"]).stdout);
const fileFirst = JSON.parse(run([usage, "--pricing", pricing, "--json"]).stdout);
assert.deepEqual(flagFirst, fileFirst);
});

test("the first positional wins; a stray extra one does not override it", () => {
const usage = writeUsage([{ model: "m", prompt: 10 }]);
const other = writeUsage([{ model: "m", prompt: 999 }]);
const out = JSON.parse(run([usage, other, "--json"]).stdout);
assert.equal(out.prompt_tokens, 10);
});

test("--json is recognized wherever it appears", () => {
const usage = writeUsage([{ model: "m", prompt: 7 }]);
const out = JSON.parse(run(["--json", usage]).stdout);
assert.equal(out.prompt_tokens, 7);
});

test("a valueless --pricing does not swallow the flag after it", () => {
const usage = writeUsage([{ model: "m", prompt: 7 }]);
const r = run([usage, "--pricing", "--json"]);
assert.equal(r.status, 0, r.stderr);
const out = JSON.parse(r.stdout);
assert.equal(out.prompt_tokens, 7, "--json must still be honored");
assert.equal(out.usd, null, "no price list was supplied");
});

test("a trailing --pricing with nothing after it is harmless", () => {
const usage = writeUsage([{ model: "m", prompt: 7 }]);
const r = run([usage, "--pricing"]);
assert.equal(r.status, 0, r.stderr);
});
});

describe("price-list resolution", () => {
const MILLION = { model: "TARGET", prompt: 1_000_000 };
// ratio 1 => $2.00/1M input, so a priced 1M-prompt run is exactly $2.0000.
Expand Down
Loading