diff --git a/.changeset/forty-wasps-jump.md b/.changeset/forty-wasps-jump.md deleted file mode 100644 index 1d127dac..00000000 --- a/.changeset/forty-wasps-jump.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -'@primer/agent-eval': patch ---- - -Fix output-token totals for Copilot CLI and SDK trials by collecting usage -statistics instead of relying on message events that no longer include token -counts. Preserve compatibility with older session output. diff --git a/.changeset/itchy-weeks-arrive.md b/.changeset/itchy-weeks-arrive.md deleted file mode 100644 index c4690555..00000000 --- a/.changeset/itchy-weeks-arrive.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -'@primer/agent-eval': patch ---- - -Validate manifest consistency through schemas before loading configurations or -trial artifacts. Reject metadata IDs that differ from their manifest keys, -duplicate benchmark capability scenario IDs, and conflicting metadata across -benchmark result shards. Duplicate trial IDs and other merge conflicts now report -schema errors with field paths. diff --git a/.changeset/silver-toys-travel.md b/.changeset/silver-toys-travel.md deleted file mode 100644 index f94e0317..00000000 --- a/.changeset/silver-toys-travel.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -'@primer/agent-eval': minor ---- - -Require `runner` on parsed experiment and benchmark trial outputs. Legacy output -files without a runner default to `copilot-cli` when parsed. Consumers constructing -trial output objects directly must include `runner: 'copilot-cli'` or -`runner: 'copilot-sdk'`. diff --git a/packages/agent-eval/CHANGELOG.md b/packages/agent-eval/CHANGELOG.md index e474e565..c2b43259 100644 --- a/packages/agent-eval/CHANGELOG.md +++ b/packages/agent-eval/CHANGELOG.md @@ -1,5 +1,25 @@ # @primer/agent-eval +## 0.13.0 + +### Minor Changes + +- 72742af: Require `runner` on parsed experiment and benchmark trial outputs. Legacy output + files without a runner default to `copilot-cli` when parsed. Consumers constructing + trial output objects directly must include `runner: 'copilot-cli'` or + `runner: 'copilot-sdk'`. + +### Patch Changes + +- 922afff: Fix output-token totals for Copilot CLI and SDK trials by collecting usage + statistics instead of relying on message events that no longer include token + counts. Preserve compatibility with older session output. +- 7fd247f: Validate manifest consistency through schemas before loading configurations or + trial artifacts. Reject metadata IDs that differ from their manifest keys, + duplicate benchmark capability scenario IDs, and conflicting metadata across + benchmark result shards. Duplicate trial IDs and other merge conflicts now report + schema errors with field paths. + ## 0.12.0 ### Minor Changes diff --git a/packages/agent-eval/package.json b/packages/agent-eval/package.json index e2d9db9a..01bba068 100644 --- a/packages/agent-eval/package.json +++ b/packages/agent-eval/package.json @@ -1,6 +1,6 @@ { "name": "@primer/agent-eval", - "version": "0.12.0", + "version": "0.13.0", "type": "module", "bin": { "agent-eval": "bin/agent-eval"