From 7dc1bcfe4269ad4058d1feabc254d46fcbece186 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 02:54:40 +0000 Subject: [PATCH 01/14] Define the score as answers right in ten notes The notes gave the headline score as answers within $1, but the board scores eligibility flags by exact match (row_hit_scores). Each note now names dollar amounts within $1 of PolicyEngine's reference and eligibility answers that match it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LwQq6GAupb55h8SCa6AP87 --- app/src/notes/2026-09-01-claude-fable-5-1-added.json | 2 +- app/src/notes/2026-09-05-gpt-6-astra-debuts-second.json | 2 +- app/src/notes/2026-09-22-gpt-6-sol-debuts-first.json | 2 +- app/src/notes/2026-09-29-claude-sonnet-5-5-debuts-fifth.json | 2 +- .../2026-10-06-policybench-stops-scoring-eight-tax-outputs.json | 2 +- app/src/notes/2026-10-10-claude-haiku-5-5-joins-the-board.json | 2 +- app/src/notes/2026-10-13-muse-wonder-joins-the-board.json | 2 +- app/src/notes/2026-10-14-gemini-4-argon-joins-the-board.json | 2 +- ...0-given-a-policyengine-tool-three-models-score-above-99.json | 2 +- app/src/notes/2026-10-31-gpt-6-1-terra-joins-the-board.json | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/app/src/notes/2026-09-01-claude-fable-5-1-added.json b/app/src/notes/2026-09-01-claude-fable-5-1-added.json index fabea8fd..09c6c13e 100644 --- a/app/src/notes/2026-09-01-claude-fable-5-1-added.json +++ b/app/src/notes/2026-09-01-claude-fable-5-1-added.json @@ -3,7 +3,7 @@ "date": "2026-09-01", "title": "Claude Fable 5.1 added", "paragraphs": [ - "Claude Fable 5.1 was released on 2026-09-01 and joined the board the same day. Board row: {exactRate}% of answers within $1, #{rank} of {nModels}. {parsed} of {answers} answers parsed.", + "Claude Fable 5.1 was released on 2026-09-01 and joined the board the same day. It gets {exactRate}% of answers right, weighted by household impact, #{rank} of {nModels}. The score counts dollar amounts within $1 of PolicyEngine's reference and eligibility answers that match it. {parsed} of {answers} answers parsed.", "The model's API rejects forced tool calls, which is how the board collects most models' answers, so its row answers as a JSON object and reasons at the provider default. The serving-configuration table in the paper records the transport for every row.", "Sensitivity: with the answer tool declared under tool_choice auto, Fable 5.1 scores {autoRate}% (would rank #{autoRank}). Under the same request Claude Fable 5 scores {fable5AutoRate}%. Fable 5's board row ({fable5BoardRate}%) ran chunked under a forced tool call, which switches off its thinking, so the {boardGap}-point gap between the two board rows is mostly request shape.", "The next board version moves every model to tool_choice auto so each provider's default reasoning engages under the recorded request shape (policybench#139)." diff --git a/app/src/notes/2026-09-05-gpt-6-astra-debuts-second.json b/app/src/notes/2026-09-05-gpt-6-astra-debuts-second.json index af93d23d..36690eae 100644 --- a/app/src/notes/2026-09-05-gpt-6-astra-debuts-second.json +++ b/app/src/notes/2026-09-05-gpt-6-astra-debuts-second.json @@ -3,7 +3,7 @@ "date": "2026-09-05", "title": "GPT-6 Astra debuts second: the two rules it invented", "paragraphs": [ - "GPT-6 Astra reached the API on 2026-09-04 and joined the board on 2026-09-05 at {astraExact}% of answers within $1, #{astraRank} of {nModels}, behind GPT-5.6 Sol ({solExact}%) and ahead of Claude Fable 5.1 ({fableExact}%, #{fableRank}). Every row on this board is scored on {scoredOutputs} of its {totalOutputs} requested outputs: {excludedOutputs} outputs whose reference depends on an engine input the household facts never list are excluded for every model.", + "GPT-6 Astra reached the API on 2026-09-04 and joined the board on 2026-09-05 at {astraExact}% of answers right, weighted by household impact, #{astraRank} of {nModels}, behind GPT-5.6 Sol ({solExact}%) and ahead of Claude Fable 5.1 ({fableExact}%, #{fableRank}). The score counts dollar amounts within $1 of PolicyEngine's reference and eligibility answers that match it. Every row on this board is scored on {scoredOutputs} of its {totalOutputs} requested outputs: {excludedOutputs} outputs whose reference depends on an engine input the household facts never list are excluded for every model.", "Row by row, the two OpenAI models agree far more than they differ. Both are right on {bothRight} scored outputs and both wrong on {bothWrong}. The gap is {astraOnlyMisses} outputs Astra misses and Sol gets right, against {solOnlyMisses} the other way.", "Astra's solo misses cluster in two invented rules. In {esiRows} rows it treats a listed employer-sponsored insurance premium as a pre-tax salary reduction and subtracts it from wages before computing payroll tax, federal income tax or state income tax; the household facts state gross wages, and the reference keeps them. In {medicareRows} rows it marks a household member under 65 who is flagged disabled as Medicare-eligible; the reference grants Medicare before 65 only with 24 months of Social Security disability receipt recorded, and none of these households carry that input. The remaining {otherRows} solo misses are one-off rule errors with no shared mechanism. One further Medicare row of the same kind, on scenario_074, sits on an excluded output and is outside every count here.", "The judge's row notes name an employer-sponsored insurance premium on {astraEsiMentions} of Astra's annotated rows and on {solEsiMentions} of Sol's. Sol netted the same premium once, on scenario_045, which cost it {solEsiRows} of its {solOnlyMisses} solo misses; the rest of Sol's solo misses share no mechanism.", diff --git a/app/src/notes/2026-09-22-gpt-6-sol-debuts-first.json b/app/src/notes/2026-09-22-gpt-6-sol-debuts-first.json index ca8fb480..1cd9a201 100644 --- a/app/src/notes/2026-09-22-gpt-6-sol-debuts-first.json +++ b/app/src/notes/2026-09-22-gpt-6-sol-debuts-first.json @@ -3,7 +3,7 @@ "date": "2026-09-22", "title": "GPT-6 Sol debuts first, Claude Opus 5.5 second", "paragraphs": [ - "GPT-6 Sol, GPT-6 Luna and Claude Opus 5.5 joined the board on 2026-09-22. GPT-6 Sol leads at {solExact}% of answers within $1, weighted by household impact, #{solRank} of {nModels} and {solLead} points above Claude Opus 5.5 ({opusExact}%, #{opusRank}). GPT-5.6 Sol, the previous leader, is #{sol56Rank} at {sol56Exact}%.", + "GPT-6 Sol, GPT-6 Luna and Claude Opus 5.5 joined the board on 2026-09-22. GPT-6 Sol leads at {solExact}% of answers right, weighted by household impact, #{solRank} of {nModels} and {solLead} points above Claude Opus 5.5 ({opusExact}%, #{opusRank}). The score counts dollar amounts within $1 of PolicyEngine's reference and eligibility answers that match it. GPT-5.6 Sol, the previous leader, is #{sol56Rank} at {sol56Exact}%.", "GPT-6 Luna scores {lunaExact}% (#{lunaRank}) at ${lunaCost} a household, against ${solCost} for GPT-6 Sol and ${opusCost} for Claude Opus 5.5.", "Claude Opus 5.5's API rejects forced tool calls, as Claude Fable 5.1's does, so its row answers as a JSON object and reasons at the provider default. Claude Opus 5's board row runs the forced tool call, which switches that model's thinking off, and scores {opus5BoardExact}%; its tool_choice auto re-run, where it reasons, scores {opus5AutoExact}%. The two Opus rows differ in serving shape as well as in model, so the {opusGap}-point gap between them is not a measure of the model change alone.", "Every row is scored on {scoredOutputs} of its {totalOutputs} requested outputs. This release regenerated {regenerated} references and excludes {excluded} outputs from scoring for every model; the reference audit note explains both.", diff --git a/app/src/notes/2026-09-29-claude-sonnet-5-5-debuts-fifth.json b/app/src/notes/2026-09-29-claude-sonnet-5-5-debuts-fifth.json index 58c61e09..ceb0ef49 100644 --- a/app/src/notes/2026-09-29-claude-sonnet-5-5-debuts-fifth.json +++ b/app/src/notes/2026-09-29-claude-sonnet-5-5-debuts-fifth.json @@ -3,7 +3,7 @@ "date": "2026-09-29", "title": "Claude Sonnet 5.5 debuts fifth as the references move to policyengine-us 2.15.17", "paragraphs": [ - "Claude Sonnet 5.5, Grok 4.7 and DeepSeek V4.1 Flash joined the board on 2026-09-29, bringing it to {nModels} models. Claude Sonnet 5.5 scores {sonnetExact}% of answers within $1, weighted by household impact, #{sonnetRank} of {nModels}. GPT-6 Luna, #{lunaRank}, also rounds to {lunaExact}% and sits about level with it, less than {lunaSonnetGapBelow} points ahead. Grok 4.7 scores {grokExact}% (#{grokRank}) and DeepSeek V4.1 Flash {flashExact}% (#{flashRank}). GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%).", + "Claude Sonnet 5.5, Grok 4.7 and DeepSeek V4.1 Flash joined the board on 2026-09-29, bringing it to {nModels} models. Claude Sonnet 5.5 scores {sonnetExact}% of answers right, weighted by household impact, #{sonnetRank} of {nModels}. The score counts dollar amounts within $1 of PolicyEngine's reference and eligibility answers that match it. GPT-6 Luna, #{lunaRank}, also rounds to {lunaExact}% and sits about level with it, less than {lunaSonnetGapBelow} points ahead. Grok 4.7 scores {grokExact}% (#{grokRank}) and DeepSeek V4.1 Flash {flashExact}% (#{flashRank}). GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%).", "Claude Sonnet 5.5 costs ${sonnetCost} a household, against ${lunaCost} for GPT-6 Luna. DeepSeek V4.1 Flash costs ${flashCost} at DeepSeek's peak list price, and Grok 4.7 costs ${grokCost}.", "Claude Sonnet 5.5's API rejects forced tool calls, as Claude Opus 5.5's and Claude Fable 5.1's do, so its row answers as a JSON object and reasons with adaptive thinking, the provider default. Grok 4.7 answers through the forced tool call with a {grokTimeoutSeconds}-second request timeout, because its whole-household onboarding probe ran past a {grokOnboardingTimeoutSeconds}-second timeout. DeepSeek serves V4.1 Flash under the alias deepseek-flash, and all {flashAnswers} of the row's answers report that alias and one system fingerprint; none reports a version. PolicyBench labels the row from DeepSeek's September 10 release note, which put V4.1 Flash on that alias.", "The release also moves PolicyBench's scored references from policyengine-us {previousEngine} to {engineVersion}, the newest release when PolicyBench began sweeping the references on 2026-09-29 (uploaded at {engineUploadedUtc} UTC). The newer version includes the {upstreamFixed:words} upstream fixes that PolicyBench applied as sandbox fixes for the September 22 references, and it encodes law the older version lacked. PolicyBench still builds each scored reference from the stated facts and law published before it froze the references on 2026-07-03, so it ported its {conventions:words} publication conventions to the new version. With those conventions and an adapter that keeps Maryland county tax out of state income tax, policyengine-us {checkEngine}, the newest release when PolicyBench checked PyPI on 2026-09-29 at {checkPypiReadUtc} UTC, gives the same value as {engineVersion} for all {totalOutputs} outputs.", diff --git a/app/src/notes/2026-10-06-policybench-stops-scoring-eight-tax-outputs.json b/app/src/notes/2026-10-06-policybench-stops-scoring-eight-tax-outputs.json index c2e85c32..01dbf396 100644 --- a/app/src/notes/2026-10-06-policybench-stops-scoring-eight-tax-outputs.json +++ b/app/src/notes/2026-10-06-policybench-stops-scoring-eight-tax-outputs.json @@ -8,7 +8,7 @@ "The Virginia household's head is {va114Age}, and the prompt lists no Medicare enrollment or premium. policyengine-us treats everyone eligible for Medicare as enrolled and counts the standard Part B premium, ${partBPremium} for 2026, as a medical expense. The household itemizes, and Virginia's itemized deductions start from the federal ones, so the premium lowers its Virginia income tax: without it, that output is ${va114Alt} rather than the reference's ${va114Ref}. The premium lowers the household's federal income tax too, and that output's exclusion record names both unstated facts.", "The {payrollOutputs:words} payroll outputs count an employee share of a state paid-leave or disability premium: {optionalPrograms}. In each of these states the premium is the employer's, and the employer may deduct up to a set share of it from wages but need not, so the law requires no amount of the employee. The prompts ask for mandatory employee state payroll taxes, and the references count the largest share each state's published 2026 rates let the employer deduct, without the prompts saying whether it does. Without that share, the outputs are ${mnAlt} in Minnesota, ${coAlt} in Colorado, ${maAlt} in Massachusetts, and ${nyAlt} in New York, against references of ${mnRef}, ${coRef}, ${maRef}, and ${nyRef}. PolicyBench still scores the payroll outputs of {mandatoryScored:words} households in {mandatoryStates}, where the law puts the contribution on the worker and requires the employer to withhold it.", "No model's answer is within $1 of the reference on any of the {incomeTaxOutputs:words} income tax outputs. On the {payrollOutputs:words} payroll outputs, {payrollHits} answers match the reference within $1: {mnHits:words} for Minnesota, {coHits} for Colorado, {maHits:words} for Massachusetts, and {nyHits:words} for New York. Another {payrollAltHits} match the amount without the state share.", - "Every model's exact rate, its share of answers within $1 weighted by household impact, rises by {riseMin} to {riseMax} points. GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%). Claude Sonnet 5.5 ({sonnetExact}%) passes GPT-6 Luna ({lunaExact}%) for fourth. Further down, {riserTwo} moves above {riserTwoPassed}, {riserThree} above {riserThreePassed}, and {riserFour} above {riserFourPassed}. Every other pair keeps its order.", + "Every model's exact rate, its share of answers right weighted by household impact, rises by {riseMin} to {riseMax} points. The rate counts dollar amounts within $1 of PolicyEngine's reference and eligibility answers that match it. GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%). Claude Sonnet 5.5 ({sonnetExact}%) passes GPT-6 Luna ({lunaExact}%) for fourth. Further down, {riserTwo} moves above {riserTwoPassed}, {riserThree} above {riserThreePassed}, and {riserFour} above {riserFourPassed}. Every other pair keeps its order.", "PolicyBench's next prompt contract remains an unactivated draft (PolicyEngine/policybench#173). It states the state income tax paid for each tax unit or a named convention for estimating withholding; each person's Medicare Part B enrollment and annual net premium; and whether each person's employer withholds the employee share of state paid-leave or disability premiums. Its payroll output definition counts optional employee shares only when the employer withholds them. A later run can score these outputs after its reference builder and a fresh input sweep validate the stated conventions.", "Two corrections in this release move no score. PolicyBench now reports that it collected model responses from {responseStart} to {responseEnd}, 2026, ending on the UTC date of the last answer; release dashboard-data-20260930 gave {previousResponseEnd}, the date of that release. The audit text for a California household head's Medicaid eligibility now explains the head's income test as the engine applies it: California's ${medicaidDisregard} monthly income disregard, which takes the place of SSI's ${ssiExclusion} general income exclusion, brings countable income under the limit. The earlier text said the engine subtracts a Medicare Part B premium from income, which it does not." ], diff --git a/app/src/notes/2026-10-10-claude-haiku-5-5-joins-the-board.json b/app/src/notes/2026-10-10-claude-haiku-5-5-joins-the-board.json index 00d6c8af..162e4f3c 100644 --- a/app/src/notes/2026-10-10-claude-haiku-5-5-joins-the-board.json +++ b/app/src/notes/2026-10-10-claude-haiku-5-5-joins-the-board.json @@ -3,7 +3,7 @@ "date": "2026-10-10", "title": "Claude Haiku 5.5 joins the board as the references move to policyengine-us 2.38.6", "paragraphs": [ - "Claude Haiku 5.5 joined the board on {releaseDate}, bringing it to {nModels} models. It gets {haikuExact}% of answers within $1, weighted by household impact, #{haikuRank} of {nModels} and {haikuGain} points above Claude Haiku 4.5 ({haiku45Exact}%, #{haiku45Rank}). GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%).", + "Claude Haiku 5.5 joined the board on {releaseDate}, bringing it to {nModels} models. It gets {haikuExact}% of answers right, weighted by household impact, #{haikuRank} of {nModels} and {haikuGain} points above Claude Haiku 4.5 ({haiku45Exact}%, #{haiku45Rank}). The score counts dollar amounts within $1 of PolicyEngine's reference and eligibility answers that match it. GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%).", "Claude Haiku 5.5 cost ${haikuCost} a household, the least of any row with a cost above zero; Claude Haiku 4.5 cost ${haiku45Cost}. Unlike Claude Opus 5.5 and Claude Sonnet 5.5, its API accepts a forced tool call, so its row takes the board's forced answer tool. A probe with that request returned no thinking block, where a request without the forced tool did, so the row answers without extended thinking, as Claude Opus 5's does. Re-run with tool_choice: auto, it gets {haikuAutoExact}%, which would rank #{haikuAutoRank}.", "The release also moves PolicyBench's references from policyengine-us {previousEngine} to {engineVersion}, a release that fixes the engine defects behind {restored:words} outputs. The previous release excluded {newlyScored:words} of them, and they return to scoring at the new version's values. The other {restoredRuled:words} are the October 6 defects below, which stay scored at the new version's values. Each of the {restored:words} lands within $1 of the corrected value PolicyBench's audits computed for it. Between them they take the upstream fixes for {restoredFixes}. PolicyBench still builds each reference from the stated facts and from law published before it froze the references on 2026-07-03, under the same publication conventions.", "Two rulings of October 6 apply too. PolicyBench's reference adversary flags the scored outputs where many models, or several of the strongest, agree on an answer the reference does not give, and a judge that cannot yet see the engine's derivation works each one from primary law. Its findings led to rulings to exclude {adversaryOutputs:words} outputs. Four were engine defects, in Arizona, Ohio, Colorado and New York; the new version fixes them, so PolicyBench keeps scoring those four instead. The other four are the federal and state income tax of a Pennsylvania and a Missouri household whose dependent earns $45,000 of wages and must file their own return. The output definitions do not say whether the household's income tax includes that return, so PolicyBench stops scoring them. It also stops scoring {louisianaOutputs:words} Louisiana households' state income tax, whose reference rests on a 2026 standard deduction computed from published price indexes; Louisiana published the amount itself on September 28, after the freeze.", diff --git a/app/src/notes/2026-10-13-muse-wonder-joins-the-board.json b/app/src/notes/2026-10-13-muse-wonder-joins-the-board.json index 952721eb..07944726 100644 --- a/app/src/notes/2026-10-13-muse-wonder-joins-the-board.json +++ b/app/src/notes/2026-10-13-muse-wonder-joins-the-board.json @@ -3,7 +3,7 @@ "date": "2026-10-13", "title": "Meta's Muse Wonder joins the board as the references move to policyengine-us 2.41.0", "paragraphs": [ - "Meta's Muse Wonder joined the board on {releaseDate}, the day after Meta released it, bringing the board to {nModels} models. It is the first Meta model on the board. It gets {museExact}% of answers within $1, weighted by household impact, #{museRank} of {nModels}. On the bounded score, which gives partial credit for amounts close to the reference, it ranks #{museBoundedRank}, behind only GPT-6 Sol and Claude Opus 5.5. GPT-6 Sol still leads on answers within $1 at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%).", + "Meta's Muse Wonder joined the board on {releaseDate}, the day after Meta released it, bringing the board to {nModels} models. It is the first Meta model on the board. It gets {museExact}% of answers right, weighted by household impact, #{museRank} of {nModels}. The score counts dollar amounts within $1 of PolicyEngine's reference and eligibility answers that match it. On the bounded score, which gives partial credit for amounts close to the reference, it ranks #{museBoundedRank}, behind only GPT-6 Sol and Claude Opus 5.5. GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%).", "Meta released Muse Wonder Full the same day, but only to Muse Code subscribers, with no API. PolicyBench runs every model through its provider's API under one protocol, so Muse Wonder Full is not on the board. PolicyBench reaches Muse Wonder through OpenRouter, whose one endpoint for it is Meta's own API. That API accepts only an automatic tool choice and rejects a forced tool call, so the row answers as a JSON object, as Claude Opus 5.5's and Claude Sonnet 5.5's do, and reasons at Meta's default setting.", "The release also moves PolicyBench's references from policyengine-us {previousEngine} to {engineVersion}, the first release with every fix the previous release was waiting on. Of the outputs the previous release excluded, {restored:words} return to scoring at the new version's values, each within $1 of the corrected value PolicyBench's audits computed for it. Between them they take the upstream fixes for {restoredFixes}. The new version also fixes the defect behind a SNAP output, the heat-and-eat utility allowance, but that output stays excluded because the stated facts do not decide it. No reference PolicyBench already scored changes, and nothing is newly excluded. Every model is now scored on {scoredOutputs} of its {totalOutputs} requested outputs; PolicyBench excludes the other {excluded}.", "Against the previous release, the exact rate of every one of the {incumbents} earlier models falls, by {fallMin} to {fallMax} points. The cause is the {restored:words} restored outputs, which few models get right: half of them are answered within $1 by at most {restoredHalfAtMost:words} of the {nModels} models, and {restoredUnmatched:words} by none. Muse Wonder answers {museRestored:words} of them within $1. Without them, every earlier model's rate would be the previous release's." diff --git a/app/src/notes/2026-10-14-gemini-4-argon-joins-the-board.json b/app/src/notes/2026-10-14-gemini-4-argon-joins-the-board.json index 1a22c33b..125f7c63 100644 --- a/app/src/notes/2026-10-14-gemini-4-argon-joins-the-board.json +++ b/app/src/notes/2026-10-14-gemini-4-argon-joins-the-board.json @@ -3,7 +3,7 @@ "date": "2026-10-14", "title": "Google's Gemini 4 Argon and Meta's Muse Spark 1.3 join the board", "paragraphs": [ - "Google's Gemini 4 Argon joined the board on {releaseDate}, the day after Google opened it to the public Gemini API, and Meta's Muse Spark 1.3 joined with it, bringing the board to {nModels} models. Gemini 4 Argon gets {argonExact}% of answers within $1, weighted by household impact, #{argonRank} of {nModels} and the highest of any Gemini model; Gemini 3.8 Flash, the previous best, gets {flashExact}% (#{flashRank}). Muse Spark 1.3 gets {sparkExact}% (#{sparkRank}), behind Meta's Muse Wonder ({museExact}%, #{museRank}), but on the bounded score, which gives partial credit for amounts close to the reference, it ranks #{sparkBoundedRank}. GPT-6 Sol still leads on answers within $1 at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%).", + "Google's Gemini 4 Argon joined the board on {releaseDate}, the day after Google opened it to the public Gemini API, and Meta's Muse Spark 1.3 joined with it, bringing the board to {nModels} models. Gemini 4 Argon gets {argonExact}% of answers right, weighted by household impact, #{argonRank} of {nModels} and the highest of any Gemini model; Gemini 3.8 Flash, the previous best, gets {flashExact}% (#{flashRank}). The score counts dollar amounts within $1 of PolicyEngine's reference and eligibility answers that match it. Muse Spark 1.3 gets {sparkExact}% (#{sparkRank}), behind Meta's Muse Wonder ({museExact}%, #{museRank}), but on the bounded score, which gives partial credit for amounts close to the reference, it ranks #{sparkBoundedRank}. GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%).", "Google announced Gemini 4 Argon on September 30 for a group of partners and released it on the Gemini API on October 13. PolicyBench runs it there, as it does every Gemini model. The API accepts a forced answer tool, so the row answers through the board's tool and reasons at Google's default setting. A forced tool call suppresses some models' reasoning, as it does Claude Opus 5's, so PolicyBench fixed a test before looking at the results and ran all {households} households both ways. With the forced tool Gemini 4 Argon returned reasoning tokens on {forcedReasoning} of them, and on {boardReasoning} in the board's own run; with an automatic tool choice, on {autoReasoning}. The test found no significant reduction, so the board keeps the forced tool and publishes no second row. At Google's introductory price of $2 per million input tokens and $10 per million output tokens, which doubles on December 1, the {households} households cost ${argonCost}.", "Muse Spark 1.3, which Meta released on September 2, is Meta's second model on the board. PolicyBench reaches it through OpenRouter, whose one endpoint for it is Meta's own API, as it does Muse Wonder. That API rejects a forced tool call, so the row answers as a JSON object and reasons at Meta's default setting.", "Nothing else on the board moves. The newest policyengine-us release, {checkedEngine}, gives the same value as {engineVersion} for every output, so the references stay where they were, and every model is still scored on {scoredOutputs} of its {totalOutputs} requested outputs. Each of the {incumbents} earlier models keeps its score." diff --git a/app/src/notes/2026-10-20-given-a-policyengine-tool-three-models-score-above-99.json b/app/src/notes/2026-10-20-given-a-policyengine-tool-three-models-score-above-99.json index 699595d8..3a0f328e 100644 --- a/app/src/notes/2026-10-20-given-a-policyengine-tool-three-models-score-above-99.json +++ b/app/src/notes/2026-10-20-given-a-policyengine-tool-three-models-score-above-99.json @@ -3,7 +3,7 @@ "date": "2026-10-20", "title": "Given a tool that runs PolicyEngine, three models score above 99%", "paragraphs": [ - "PolicyBench gave three models a tool that runs PolicyEngine on a household the model writes from the prompt's facts, and ran its {households} US households again on October 14 and 15. GPT-6 Sol, which leads the board at {solBoardExact}% of answers within $1, weighted by household impact, gets {solToolExact}% with the tool. Claude Opus 5.5 goes from {opusBoardExact}% to {opusToolExact}%, and Gemini 4 Argon from {argonBoardExact}% to {argonToolExact}%. The runs are a pilot, not board rows, since the board scores models answering alone, and the result is circular: the tool runs the engine that computes PolicyBench's references, so the scores measure agreement with that engine, not whether it is right.", + "PolicyBench gave three models a tool that runs PolicyEngine on a household the model writes from the prompt's facts, and ran its {households} US households again on October 14 and 15. GPT-6 Sol, which leads the board at {solBoardExact}% of answers right, weighted by household impact, gets {solToolExact}% with the tool. Each score counts dollar amounts within $1 of PolicyEngine's reference and eligibility answers that match it. Claude Opus 5.5 goes from {opusBoardExact}% to {opusToolExact}%, and Gemini 4 Argon from {argonBoardExact}% to {argonToolExact}%. The runs are a pilot, not board rows, since the board scores models answering alone, and the result is circular: the tool runs the engine that computes PolicyBench's references, so the scores measure agreement with that engine, not whether it is right.", "Most of the answers the tool fixes are dollar amounts that are not zero. Of the {nonzeroAmounts} nonzero amounts, GPT-6 Sol gets {solNonzeroRate}% within $1 on its own and {solNonzeroToolRate}% with the tool; Claude Opus 5.5, {opusNonzeroRate}% and {opusNonzeroToolRate}%; and Gemini 4 Argon, {argonNonzeroRate}% and {argonNonzeroToolRate}%.", "With the tool, {toolMisses} answers are wrong across the three runs: {solMisses:words} of GPT-6 Sol's, {opusMisses:words} of Claude Opus 5.5's and {argonMisses:words} of Gemini 4 Argon's. Each traces to the household the model described to the engine, or to the output it read back, rather than to the calculation: in {enteredElsewhere:words} the model had entered a stated fact under another variable, in {leftOut:words} it had left one out, and in {wrongOutput:words} it read back the wrong output. Prompt labels share the blame for {labelMisses:words}: \"employer sponsored insurance premiums\" never says the employer paid them, and \"disability benefits\" never says they are not Social Security. In {solCapMisses:words} of GPT-6 Sol's, it had used all {turnCap:words} calculation turns the pilot allows. GPT-6 Sol is also the only model that turned right answers wrong, {solFlips:words} of them.", "The tool costs more. For the {households} households, GPT-6 Sol costs ${solToolCost} with it and ${solBoardCost} without, Claude Opus 5.5 ${opusToolCost} and ${opusBoardCost}, and Gemini 4 Argon ${argonToolCost} and ${argonBoardCost}.", diff --git a/app/src/notes/2026-10-31-gpt-6-1-terra-joins-the-board.json b/app/src/notes/2026-10-31-gpt-6-1-terra-joins-the-board.json index 72f9037a..1ec047f7 100644 --- a/app/src/notes/2026-10-31-gpt-6-1-terra-joins-the-board.json +++ b/app/src/notes/2026-10-31-gpt-6-1-terra-joins-the-board.json @@ -3,7 +3,7 @@ "date": "2026-10-31", "title": "OpenAI's GPT-6.1 Terra joins the board at #3 as the references move to policyengine-us 2.60.0", "paragraphs": [ - "PolicyBench added OpenAI's GPT-6.1 Terra to its {snapshotDate} snapshot, the day after OpenAI released it, bringing the board to {nModels} models. It gets {terraExact}% of answers within $1, weighted by household impact, #{terraRank} of {nModels}. Only GPT-6 Sol ({solExact}%) and Claude Opus 5.5 ({opusExact}%) score higher, and GPT-5.6 Sol, which also rounds to {sol56Exact}%, is less than a tenth of a point behind. On the bounded score, which gives partial credit for amounts close to the reference, GPT-6.1 Terra ranks #{terraBoundedRank}. GPT-6.1 Sol, which OpenAI released on September 29, gets {sol61Exact}% (#{sol61Rank}).", + "PolicyBench added OpenAI's GPT-6.1 Terra to its {snapshotDate} snapshot, the day after OpenAI released it, bringing the board to {nModels} models. It gets {terraExact}% of answers right, weighted by household impact, #{terraRank} of {nModels}. The score counts dollar amounts within $1 of PolicyEngine's reference and eligibility answers that match it. Only GPT-6 Sol ({solExact}%) and Claude Opus 5.5 ({opusExact}%) score higher, and GPT-5.6 Sol, which also rounds to {sol56Exact}%, is less than a tenth of a point behind. On the bounded score, which gives partial credit for amounts close to the reference, GPT-6.1 Terra ranks #{terraBoundedRank}. GPT-6.1 Sol, which OpenAI released on September 29, gets {sol61Exact}% (#{sol61Rank}).", "PolicyBench runs GPT-6.1 Terra as it runs GPT-6.1 Sol, on OpenAI's Responses API with the board's forced answer tool at OpenAI's default reasoning effort, and it returned reasoning tokens on all {households} households. At OpenAI's price of $1 per million input tokens and $5 per million output tokens, the {households} households cost ${terraCost}, against ${sol61Cost} for GPT-6.1 Sol and ${solCost} for GPT-6 Sol. GPT-5.6 Terra, the Terra model before it, gets {terra56Exact}% (#{terra56Rank}) and cost ${terra56Cost}.", "The release also moves PolicyBench's references from policyengine-us {previousEngine} to {engineVersion}, the newest release when PolicyBench began recomputing them on October 23. The move changes {changed:words} scored references, each for law in force before the references were frozen. An elderly Californian who lives alone, whose SNAP was $0, now gets the ${snapMonthly} monthly minimum, ${snap031} a year: California's categorical eligibility for CalFresh applies no net income test, even to a household with an elderly member, and the previous version applied one (policyengine-us #10036). Three households in Pennsylvania and Arkansas have too much income for free or reduced-price school lunch, but those states give students at covered schools a free breakfast, so the output asking whether PolicyEngine returns free school meal support is now yes for each (policyengine-us #10238). The new version models both breakfast laws for a student at a covered school and leaves out their conditions: Pennsylvania reimburses schools for them to the extent funds are available, and Arkansas serves them on request.", "Nothing is newly excluded, and no excluded output returns to scoring, so every model is still scored on {scoredOutputs} of its {totalOutputs} requested outputs. When PolicyBench checked PyPI on {checkedOn}, the newest policyengine-us release was {checkedEngine}, and it gives the same value as {engineVersion} for all {scoredOutputs}; it moves {excludedMoved:words} excluded output, which keeps the value it was decided on.", From 2d8beb120b7e6572464bae9f601b10ad52d0bf90 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 02:55:50 +0000 Subject: [PATCH 02/14] Carry the score definition in paragraph 1 where paragraph 0 runs long Paragraph 0 doubles as the page description and list blurb; in the October 14, 20 and 31 notes the definition would have taken it to 645-737 characters, so it opens paragraph 1 instead. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LwQq6GAupb55h8SCa6AP87 --- app/src/notes/2026-10-14-gemini-4-argon-joins-the-board.json | 4 ++-- ...given-a-policyengine-tool-three-models-score-above-99.json | 4 ++-- app/src/notes/2026-10-31-gpt-6-1-terra-joins-the-board.json | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/src/notes/2026-10-14-gemini-4-argon-joins-the-board.json b/app/src/notes/2026-10-14-gemini-4-argon-joins-the-board.json index 125f7c63..cf237b1f 100644 --- a/app/src/notes/2026-10-14-gemini-4-argon-joins-the-board.json +++ b/app/src/notes/2026-10-14-gemini-4-argon-joins-the-board.json @@ -3,8 +3,8 @@ "date": "2026-10-14", "title": "Google's Gemini 4 Argon and Meta's Muse Spark 1.3 join the board", "paragraphs": [ - "Google's Gemini 4 Argon joined the board on {releaseDate}, the day after Google opened it to the public Gemini API, and Meta's Muse Spark 1.3 joined with it, bringing the board to {nModels} models. Gemini 4 Argon gets {argonExact}% of answers right, weighted by household impact, #{argonRank} of {nModels} and the highest of any Gemini model; Gemini 3.8 Flash, the previous best, gets {flashExact}% (#{flashRank}). The score counts dollar amounts within $1 of PolicyEngine's reference and eligibility answers that match it. Muse Spark 1.3 gets {sparkExact}% (#{sparkRank}), behind Meta's Muse Wonder ({museExact}%, #{museRank}), but on the bounded score, which gives partial credit for amounts close to the reference, it ranks #{sparkBoundedRank}. GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%).", - "Google announced Gemini 4 Argon on September 30 for a group of partners and released it on the Gemini API on October 13. PolicyBench runs it there, as it does every Gemini model. The API accepts a forced answer tool, so the row answers through the board's tool and reasons at Google's default setting. A forced tool call suppresses some models' reasoning, as it does Claude Opus 5's, so PolicyBench fixed a test before looking at the results and ran all {households} households both ways. With the forced tool Gemini 4 Argon returned reasoning tokens on {forcedReasoning} of them, and on {boardReasoning} in the board's own run; with an automatic tool choice, on {autoReasoning}. The test found no significant reduction, so the board keeps the forced tool and publishes no second row. At Google's introductory price of $2 per million input tokens and $10 per million output tokens, which doubles on December 1, the {households} households cost ${argonCost}.", + "Google's Gemini 4 Argon joined the board on {releaseDate}, the day after Google opened it to the public Gemini API, and Meta's Muse Spark 1.3 joined with it, bringing the board to {nModels} models. Gemini 4 Argon gets {argonExact}% of answers right, weighted by household impact, #{argonRank} of {nModels} and the highest of any Gemini model; Gemini 3.8 Flash, the previous best, gets {flashExact}% (#{flashRank}). Muse Spark 1.3 gets {sparkExact}% (#{sparkRank}), behind Meta's Muse Wonder ({museExact}%, #{museRank}), but on the bounded score, which gives partial credit for amounts close to the reference, it ranks #{sparkBoundedRank}. GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%).", + "The board's score counts dollar amounts within $1 of PolicyEngine's reference and eligibility answers that match it. Google announced Gemini 4 Argon on September 30 for a group of partners and released it on the Gemini API on October 13. PolicyBench runs it there, as it does every Gemini model. The API accepts a forced answer tool, so the row answers through the board's tool and reasons at Google's default setting. A forced tool call suppresses some models' reasoning, as it does Claude Opus 5's, so PolicyBench fixed a test before looking at the results and ran all {households} households both ways. With the forced tool Gemini 4 Argon returned reasoning tokens on {forcedReasoning} of them, and on {boardReasoning} in the board's own run; with an automatic tool choice, on {autoReasoning}. The test found no significant reduction, so the board keeps the forced tool and publishes no second row. At Google's introductory price of $2 per million input tokens and $10 per million output tokens, which doubles on December 1, the {households} households cost ${argonCost}.", "Muse Spark 1.3, which Meta released on September 2, is Meta's second model on the board. PolicyBench reaches it through OpenRouter, whose one endpoint for it is Meta's own API, as it does Muse Wonder. That API rejects a forced tool call, so the row answers as a JSON object and reasons at Meta's default setting.", "Nothing else on the board moves. The newest policyengine-us release, {checkedEngine}, gives the same value as {engineVersion} for every output, so the references stay where they were, and every model is still scored on {scoredOutputs} of its {totalOutputs} requested outputs. Each of the {incumbents} earlier models keeps its score." ], diff --git a/app/src/notes/2026-10-20-given-a-policyengine-tool-three-models-score-above-99.json b/app/src/notes/2026-10-20-given-a-policyengine-tool-three-models-score-above-99.json index 3a0f328e..c5deead1 100644 --- a/app/src/notes/2026-10-20-given-a-policyengine-tool-three-models-score-above-99.json +++ b/app/src/notes/2026-10-20-given-a-policyengine-tool-three-models-score-above-99.json @@ -3,8 +3,8 @@ "date": "2026-10-20", "title": "Given a tool that runs PolicyEngine, three models score above 99%", "paragraphs": [ - "PolicyBench gave three models a tool that runs PolicyEngine on a household the model writes from the prompt's facts, and ran its {households} US households again on October 14 and 15. GPT-6 Sol, which leads the board at {solBoardExact}% of answers right, weighted by household impact, gets {solToolExact}% with the tool. Each score counts dollar amounts within $1 of PolicyEngine's reference and eligibility answers that match it. Claude Opus 5.5 goes from {opusBoardExact}% to {opusToolExact}%, and Gemini 4 Argon from {argonBoardExact}% to {argonToolExact}%. The runs are a pilot, not board rows, since the board scores models answering alone, and the result is circular: the tool runs the engine that computes PolicyBench's references, so the scores measure agreement with that engine, not whether it is right.", - "Most of the answers the tool fixes are dollar amounts that are not zero. Of the {nonzeroAmounts} nonzero amounts, GPT-6 Sol gets {solNonzeroRate}% within $1 on its own and {solNonzeroToolRate}% with the tool; Claude Opus 5.5, {opusNonzeroRate}% and {opusNonzeroToolRate}%; and Gemini 4 Argon, {argonNonzeroRate}% and {argonNonzeroToolRate}%.", + "PolicyBench gave three models a tool that runs PolicyEngine on a household the model writes from the prompt's facts, and ran its {households} US households again on October 14 and 15. GPT-6 Sol, which leads the board at {solBoardExact}% of answers right, weighted by household impact, gets {solToolExact}% with the tool. Claude Opus 5.5 goes from {opusBoardExact}% to {opusToolExact}%, and Gemini 4 Argon from {argonBoardExact}% to {argonToolExact}%. The runs are a pilot, not board rows, since the board scores models answering alone, and the result is circular: the tool runs the engine that computes PolicyBench's references, so the scores measure agreement with that engine, not whether it is right.", + "Each score counts dollar amounts within $1 of PolicyEngine's reference and eligibility answers that match it. Most of the answers the tool fixes are dollar amounts that are not zero. Of the {nonzeroAmounts} nonzero amounts, GPT-6 Sol gets {solNonzeroRate}% within $1 on its own and {solNonzeroToolRate}% with the tool; Claude Opus 5.5, {opusNonzeroRate}% and {opusNonzeroToolRate}%; and Gemini 4 Argon, {argonNonzeroRate}% and {argonNonzeroToolRate}%.", "With the tool, {toolMisses} answers are wrong across the three runs: {solMisses:words} of GPT-6 Sol's, {opusMisses:words} of Claude Opus 5.5's and {argonMisses:words} of Gemini 4 Argon's. Each traces to the household the model described to the engine, or to the output it read back, rather than to the calculation: in {enteredElsewhere:words} the model had entered a stated fact under another variable, in {leftOut:words} it had left one out, and in {wrongOutput:words} it read back the wrong output. Prompt labels share the blame for {labelMisses:words}: \"employer sponsored insurance premiums\" never says the employer paid them, and \"disability benefits\" never says they are not Social Security. In {solCapMisses:words} of GPT-6 Sol's, it had used all {turnCap:words} calculation turns the pilot allows. GPT-6 Sol is also the only model that turned right answers wrong, {solFlips:words} of them.", "The tool costs more. For the {households} households, GPT-6 Sol costs ${solToolCost} with it and ${solBoardCost} without, Claude Opus 5.5 ${opusToolCost} and ${opusBoardCost}, and Gemini 4 Argon ${argonToolCost} and ${argonBoardCost}.", "The tool applies the conventions PolicyBench uses for its references, so a model that describes a household exactly as the reference does gets the reference's answer. Describing a household is also easier here than it would be from a filer's documents: {labelMatches} of the {statedFacts} facts the prompts state, not counting ages and wages, carry a label that is the engine's own variable name with spaces.", diff --git a/app/src/notes/2026-10-31-gpt-6-1-terra-joins-the-board.json b/app/src/notes/2026-10-31-gpt-6-1-terra-joins-the-board.json index 1ec047f7..9e49bdb1 100644 --- a/app/src/notes/2026-10-31-gpt-6-1-terra-joins-the-board.json +++ b/app/src/notes/2026-10-31-gpt-6-1-terra-joins-the-board.json @@ -3,8 +3,8 @@ "date": "2026-10-31", "title": "OpenAI's GPT-6.1 Terra joins the board at #3 as the references move to policyengine-us 2.60.0", "paragraphs": [ - "PolicyBench added OpenAI's GPT-6.1 Terra to its {snapshotDate} snapshot, the day after OpenAI released it, bringing the board to {nModels} models. It gets {terraExact}% of answers right, weighted by household impact, #{terraRank} of {nModels}. The score counts dollar amounts within $1 of PolicyEngine's reference and eligibility answers that match it. Only GPT-6 Sol ({solExact}%) and Claude Opus 5.5 ({opusExact}%) score higher, and GPT-5.6 Sol, which also rounds to {sol56Exact}%, is less than a tenth of a point behind. On the bounded score, which gives partial credit for amounts close to the reference, GPT-6.1 Terra ranks #{terraBoundedRank}. GPT-6.1 Sol, which OpenAI released on September 29, gets {sol61Exact}% (#{sol61Rank}).", - "PolicyBench runs GPT-6.1 Terra as it runs GPT-6.1 Sol, on OpenAI's Responses API with the board's forced answer tool at OpenAI's default reasoning effort, and it returned reasoning tokens on all {households} households. At OpenAI's price of $1 per million input tokens and $5 per million output tokens, the {households} households cost ${terraCost}, against ${sol61Cost} for GPT-6.1 Sol and ${solCost} for GPT-6 Sol. GPT-5.6 Terra, the Terra model before it, gets {terra56Exact}% (#{terra56Rank}) and cost ${terra56Cost}.", + "PolicyBench added OpenAI's GPT-6.1 Terra to its {snapshotDate} snapshot, the day after OpenAI released it, bringing the board to {nModels} models. It gets {terraExact}% of answers right, weighted by household impact, #{terraRank} of {nModels}. Only GPT-6 Sol ({solExact}%) and Claude Opus 5.5 ({opusExact}%) score higher, and GPT-5.6 Sol, which also rounds to {sol56Exact}%, is less than a tenth of a point behind. On the bounded score, which gives partial credit for amounts close to the reference, GPT-6.1 Terra ranks #{terraBoundedRank}. GPT-6.1 Sol, which OpenAI released on September 29, gets {sol61Exact}% (#{sol61Rank}).", + "The board's score counts dollar amounts within $1 of PolicyEngine's reference and eligibility answers that match it. PolicyBench runs GPT-6.1 Terra as it runs GPT-6.1 Sol, on OpenAI's Responses API with the board's forced answer tool at OpenAI's default reasoning effort, and it returned reasoning tokens on all {households} households. At OpenAI's price of $1 per million input tokens and $5 per million output tokens, the {households} households cost ${terraCost}, against ${sol61Cost} for GPT-6.1 Sol and ${solCost} for GPT-6 Sol. GPT-5.6 Terra, the Terra model before it, gets {terra56Exact}% (#{terra56Rank}) and cost ${terra56Cost}.", "The release also moves PolicyBench's references from policyengine-us {previousEngine} to {engineVersion}, the newest release when PolicyBench began recomputing them on October 23. The move changes {changed:words} scored references, each for law in force before the references were frozen. An elderly Californian who lives alone, whose SNAP was $0, now gets the ${snapMonthly} monthly minimum, ${snap031} a year: California's categorical eligibility for CalFresh applies no net income test, even to a household with an elderly member, and the previous version applied one (policyengine-us #10036). Three households in Pennsylvania and Arkansas have too much income for free or reduced-price school lunch, but those states give students at covered schools a free breakfast, so the output asking whether PolicyEngine returns free school meal support is now yes for each (policyengine-us #10238). The new version models both breakfast laws for a student at a covered school and leaves out their conditions: Pennsylvania reimburses schools for them to the extent funds are available, and Arkansas serves them on request.", "Nothing is newly excluded, and no excluded output returns to scoring, so every model is still scored on {scoredOutputs} of its {totalOutputs} requested outputs. When PolicyBench checked PyPI on {checkedOn}, the newest policyengine-us release was {checkedEngine}, and it gives the same value as {engineVersion} for all {scoredOutputs}; it moves {excludedMoved:words} excluded output, which keeps the value it was decided on.", "Against the previous release, the exact rates of {fallers:words} of the {incumbents} earlier models fall, by {fallMin} to {fallMax} points, and {risers:words} rise by {rise}. No model answers yes for any of the three households' school meals, and only {snapMatch:words} give the Californian's new SNAP: GPT-5.5, GPT-6 Astra and GPT-6.1 Sol, the three that rise. Claude Sonnet 5.5 falls from #{sonnetThen} to #{sonnetNow}, behind GPT-6.1 Terra and GPT-6 Astra." From ad7ac93dea11b921621f1be50f91a0e84b8da2d6 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 02:59:18 +0000 Subject: [PATCH 03/14] Pin the corrected score sentences and guard against the shorthand tests/test_notes.py pins the September 29 and October 6 definitions beside row_hit_scores, and a new test fails on any note that gives the score as answers within $1. app/tests/notes.test.ts pins the rendered sentences. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LwQq6GAupb55h8SCa6AP87 --- app/tests/notes.test.ts | 12 +++++------ tests/test_notes.py | 47 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 51 insertions(+), 8 deletions(-) diff --git a/app/tests/notes.test.ts b/app/tests/notes.test.ts index 9c3b8642..a134de1f 100644 --- a/app/tests/notes.test.ts +++ b/app/tests/notes.test.ts @@ -64,7 +64,7 @@ describe("notes", () => { expect(markup).toContain("GPT-6 Astra debuts second: the two rules it invented"); expect(markup).toContain("release dashboard-data-20260905c"); // Whole-number board rates keep one decimal. - expect(markup).toContain("at 88.0% of answers within $1"); + expect(markup).toContain("at 88.0% of answers right"); expect(markup).not.toContain("at 88% of answers"); expect(markup).toContain("release dashboard-data-20260901c"); expect(markup).toContain("Six SNAP households the top three models deny"); @@ -141,7 +141,7 @@ describe("notes", () => { ); // Board rates keep one decimal. expect(markup).toContain( - "It gets 93.1% of answers within $1, weighted by household impact, #3 of 51.", + "It gets 93.1% of answers right, weighted by household impact, #3 of 51.", ); expect(markup).toContain("GPT-6.1 Terra ranks #5."); expect(markup).toContain("GPT-6.1 Sol, which OpenAI released on September 29, gets 90.2% (#10)."); @@ -177,7 +177,7 @@ describe("notes", () => { createElement(NoteArticle, { note, titleLevel: "h1" }), ); expect(markup).toContain( - "GPT-6 Sol, which leads the board at 95.2% of answers within $1, weighted by household impact, gets 99.2% with the tool.", + "GPT-6 Sol, which leads the board at 95.2% of answers right, weighted by household impact, gets 99.2% with the tool.", ); expect(markup).toContain( "Claude Opus 5.5 goes from 94.1% to 99.8%, and Gemini 4 Argon from 88.7% to 99.9%.", @@ -248,7 +248,7 @@ describe("notes", () => { "Google's Gemini 4 Argon joined the board on 2026-10-14, the day after Google opened it to the public Gemini API, and Meta's Muse Spark 1.3 joined with it, bringing the board to 50 models.", ); // Board rates keep one decimal. - expect(markup).toContain("Gemini 4 Argon gets 88.7% of answers within $1"); + expect(markup).toContain("Gemini 4 Argon gets 88.7% of answers right"); expect(markup).toContain("#12 of 50 and the highest of any Gemini model"); expect(markup).toContain("Gemini 3.8 Flash, the previous best, gets 85.6% (#21)"); expect(markup).toContain( @@ -282,11 +282,11 @@ describe("notes", () => { "Meta's Muse Wonder joined the board on 2026-10-13, the day after Meta released it, bringing the board to 48 models.", ); // Board rates keep one decimal. - expect(markup).toContain("It gets 90.1% of answers within $1"); + expect(markup).toContain("It gets 90.1% of answers right"); expect(markup).toContain("#10 of 48"); expect(markup).toContain("it ranks #3, behind only GPT-6 Sol and Claude Opus 5.5"); expect(markup).toContain( - "GPT-6 Sol still leads on answers within $1 at 95.2%, ahead of Claude Opus 5.5 (94.1%) and GPT-5.6 Sol (93.1%).", + "GPT-6 Sol still leads at 95.2%, ahead of Claude Opus 5.5 (94.1%) and GPT-5.6 Sol (93.1%).", ); expect(markup).toContain("so Muse Wonder Full is not on the board"); expect(markup).toContain( diff --git a/tests/test_notes.py b/tests/test_notes.py index 4baf5507..1fca40c0 100644 --- a/tests/test_notes.py +++ b/tests/test_notes.py @@ -3408,8 +3408,12 @@ def pin(*sentences: str) -> None: pin( "Claude Sonnet 5.5, Grok 4.7 and DeepSeek V4.1 Flash joined the board on " "2026-09-29, bringing it to {nModels} models.", - "Claude Sonnet 5.5 scores {sonnetExact}% of answers within $1, weighted by " + "Claude Sonnet 5.5 scores {sonnetExact}% of answers right, weighted by " "household impact, #{sonnetRank} of {nModels}.", + ) + _assert_score_definition() + pin( + SCORE_DEFINITION, "GPT-6 Luna, #{lunaRank}, also rounds to {lunaExact}% and sits about level " "with it, less than {lunaSonnetGapBelow} points ahead.", "Grok 4.7 scores {grokExact}% (#{grokRank}) and DeepSeek V4.1 Flash " @@ -4446,8 +4450,13 @@ def rank_then(model: str) -> int: assert rank_now(passed[0]) == rank_then(riser) pin( # The headline the board ranks by, as the September 29 note states it. - "Every model's exact rate, its share of answers within $1 weighted by " + "Every model's exact rate, its share of answers right weighted by " "household impact, rises by {riseMin} to {riseMax} points.", + ) + _assert_score_definition() + pin( + "The rate counts dollar amounts within $1 of PolicyEngine's reference and " + "eligibility answers that match it.", "GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 " "({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%).", "Claude Sonnet 5.5 ({sonnetExact}%) passes GPT-6 Luna ({lunaExact}%) for " @@ -6160,3 +6169,37 @@ def test_release_20261027_note_claims() -> None: ): if href.startswith(prefix): assert (ROOT / href.removeprefix(prefix)).exists(), href + + +SCORE_DEFINITION = ( + "The score counts dollar amounts within $1 of PolicyEngine's reference and " + "eligibility answers that match it." +) + + +def _assert_score_definition() -> None: + """The headline exact score, as ``row_hit_scores`` computes it: a dollar + amount hits within $1 of the reference, and an eligibility flag hits only + when it matches.""" + from policybench.analysis import BINARY_PROGRAMS, row_hit_scores + + assert row_hit_scores("snap", 100.0, 101.0)["exact"] == 1.0 + assert row_hit_scores("snap", 100.0, 101.5)["exact"] == 0.0 + assert "person_medicaid_eligible" in BINARY_PROGRAMS + assert row_hit_scores("person_medicaid_eligible", 1.0, 1.0)["exact"] == 1.0 + assert row_hit_scores("person_medicaid_eligible", 1.0, 0.0)["exact"] == 0.0 + + +def test_no_note_gives_the_score_as_answers_within_one_dollar(): + """The board scores eligibility flags by exact match, not within $1, so no + note gives an all-outputs score as answers within $1.""" + _assert_score_definition() + # PR #254 rewrites this note's opening; drop it from here once that merges. + pending = {"2026-10-19-america-gov-scores-level-with-gpt-5-5"} + for path in sorted(NOTES_DIR.glob("*.json")): + note = _load_json(path) + if note["slug"] in pending: + continue + text = " ".join([note["title"], *note["paragraphs"]]) + for phrase in ("answers within $1", "of answers within"): + assert phrase not in text, (path.name, phrase) From 266998feabb4a447ff4bcfdc3d93aa49ef6b88aa Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 2 Nov 2026 23:04:00 -0500 Subject: [PATCH 04/14] Name PolicyBench in the score definition and give each rank a verb Each note now says "PolicyBench counts a dollar amount as right when it falls within $1 of PolicyEngine's reference, and an eligibility answer as right when it matches." Score sentences end "and ranks #N of M" instead of trailing the rank. The October 14 and October 31 notes define the score beside it in paragraph 0. The September 5 note splits its opening sentence, the September 29 note names Claude Sonnet 5.5 where "it" now sits a sentence away, and the October 6 note drops "exact rate". The guard test now also requires the definition in every note that gives a score as answers right. Co-Authored-By: Claude Opus 5.5 --- .../2026-09-01-claude-fable-5-1-added.json | 2 +- .../2026-09-05-gpt-6-astra-debuts-second.json | 2 +- .../2026-09-22-gpt-6-sol-debuts-first.json | 2 +- ...-09-29-claude-sonnet-5-5-debuts-fifth.json | 2 +- ...bench-stops-scoring-eight-tax-outputs.json | 2 +- ...0-10-claude-haiku-5-5-joins-the-board.json | 2 +- ...026-10-13-muse-wonder-joins-the-board.json | 2 +- ...-10-14-gemini-4-argon-joins-the-board.json | 4 +-- ...gine-tool-three-models-score-above-99.json | 2 +- ...6-10-31-gpt-6-1-terra-joins-the-board.json | 4 +-- app/tests/notes.test.ts | 13 ++++---- tests/test_notes.py | 30 ++++++++++++------- 12 files changed, 40 insertions(+), 27 deletions(-) diff --git a/app/src/notes/2026-09-01-claude-fable-5-1-added.json b/app/src/notes/2026-09-01-claude-fable-5-1-added.json index 09c6c13e..67a1188f 100644 --- a/app/src/notes/2026-09-01-claude-fable-5-1-added.json +++ b/app/src/notes/2026-09-01-claude-fable-5-1-added.json @@ -3,7 +3,7 @@ "date": "2026-09-01", "title": "Claude Fable 5.1 added", "paragraphs": [ - "Claude Fable 5.1 was released on 2026-09-01 and joined the board the same day. It gets {exactRate}% of answers right, weighted by household impact, #{rank} of {nModels}. The score counts dollar amounts within $1 of PolicyEngine's reference and eligibility answers that match it. {parsed} of {answers} answers parsed.", + "Claude Fable 5.1 was released on 2026-09-01 and joined the board the same day. It gets {exactRate}% of answers right, weighted by household impact, and ranks #{rank} of {nModels}. PolicyBench counts a dollar amount as right when it falls within $1 of PolicyEngine's reference, and an eligibility answer as right when it matches. {parsed} of {answers} answers parsed.", "The model's API rejects forced tool calls, which is how the board collects most models' answers, so its row answers as a JSON object and reasons at the provider default. The serving-configuration table in the paper records the transport for every row.", "Sensitivity: with the answer tool declared under tool_choice auto, Fable 5.1 scores {autoRate}% (would rank #{autoRank}). Under the same request Claude Fable 5 scores {fable5AutoRate}%. Fable 5's board row ({fable5BoardRate}%) ran chunked under a forced tool call, which switches off its thinking, so the {boardGap}-point gap between the two board rows is mostly request shape.", "The next board version moves every model to tool_choice auto so each provider's default reasoning engages under the recorded request shape (policybench#139)." diff --git a/app/src/notes/2026-09-05-gpt-6-astra-debuts-second.json b/app/src/notes/2026-09-05-gpt-6-astra-debuts-second.json index 36690eae..18b200cf 100644 --- a/app/src/notes/2026-09-05-gpt-6-astra-debuts-second.json +++ b/app/src/notes/2026-09-05-gpt-6-astra-debuts-second.json @@ -3,7 +3,7 @@ "date": "2026-09-05", "title": "GPT-6 Astra debuts second: the two rules it invented", "paragraphs": [ - "GPT-6 Astra reached the API on 2026-09-04 and joined the board on 2026-09-05 at {astraExact}% of answers right, weighted by household impact, #{astraRank} of {nModels}, behind GPT-5.6 Sol ({solExact}%) and ahead of Claude Fable 5.1 ({fableExact}%, #{fableRank}). The score counts dollar amounts within $1 of PolicyEngine's reference and eligibility answers that match it. Every row on this board is scored on {scoredOutputs} of its {totalOutputs} requested outputs: {excludedOutputs} outputs whose reference depends on an engine input the household facts never list are excluded for every model.", + "GPT-6 Astra reached the API on 2026-09-04 and joined the board on 2026-09-05. It gets {astraExact}% of answers right, weighted by household impact, and ranks #{astraRank} of {nModels}, behind GPT-5.6 Sol ({solExact}%) and ahead of Claude Fable 5.1 ({fableExact}%, #{fableRank}). PolicyBench counts a dollar amount as right when it falls within $1 of PolicyEngine's reference, and an eligibility answer as right when it matches. Every row on this board is scored on {scoredOutputs} of its {totalOutputs} requested outputs: {excludedOutputs} outputs whose reference depends on an engine input the household facts never list are excluded for every model.", "Row by row, the two OpenAI models agree far more than they differ. Both are right on {bothRight} scored outputs and both wrong on {bothWrong}. The gap is {astraOnlyMisses} outputs Astra misses and Sol gets right, against {solOnlyMisses} the other way.", "Astra's solo misses cluster in two invented rules. In {esiRows} rows it treats a listed employer-sponsored insurance premium as a pre-tax salary reduction and subtracts it from wages before computing payroll tax, federal income tax or state income tax; the household facts state gross wages, and the reference keeps them. In {medicareRows} rows it marks a household member under 65 who is flagged disabled as Medicare-eligible; the reference grants Medicare before 65 only with 24 months of Social Security disability receipt recorded, and none of these households carry that input. The remaining {otherRows} solo misses are one-off rule errors with no shared mechanism. One further Medicare row of the same kind, on scenario_074, sits on an excluded output and is outside every count here.", "The judge's row notes name an employer-sponsored insurance premium on {astraEsiMentions} of Astra's annotated rows and on {solEsiMentions} of Sol's. Sol netted the same premium once, on scenario_045, which cost it {solEsiRows} of its {solOnlyMisses} solo misses; the rest of Sol's solo misses share no mechanism.", diff --git a/app/src/notes/2026-09-22-gpt-6-sol-debuts-first.json b/app/src/notes/2026-09-22-gpt-6-sol-debuts-first.json index 1cd9a201..b8ac1d19 100644 --- a/app/src/notes/2026-09-22-gpt-6-sol-debuts-first.json +++ b/app/src/notes/2026-09-22-gpt-6-sol-debuts-first.json @@ -3,7 +3,7 @@ "date": "2026-09-22", "title": "GPT-6 Sol debuts first, Claude Opus 5.5 second", "paragraphs": [ - "GPT-6 Sol, GPT-6 Luna and Claude Opus 5.5 joined the board on 2026-09-22. GPT-6 Sol leads at {solExact}% of answers right, weighted by household impact, #{solRank} of {nModels} and {solLead} points above Claude Opus 5.5 ({opusExact}%, #{opusRank}). The score counts dollar amounts within $1 of PolicyEngine's reference and eligibility answers that match it. GPT-5.6 Sol, the previous leader, is #{sol56Rank} at {sol56Exact}%.", + "GPT-6 Sol, GPT-6 Luna and Claude Opus 5.5 joined the board on 2026-09-22. GPT-6 Sol ranks #{solRank} of {nModels} at {solExact}% of answers right, weighted by household impact, {solLead} points above Claude Opus 5.5 ({opusExact}%, #{opusRank}). PolicyBench counts a dollar amount as right when it falls within $1 of PolicyEngine's reference, and an eligibility answer as right when it matches. GPT-5.6 Sol, the previous leader, is #{sol56Rank} at {sol56Exact}%.", "GPT-6 Luna scores {lunaExact}% (#{lunaRank}) at ${lunaCost} a household, against ${solCost} for GPT-6 Sol and ${opusCost} for Claude Opus 5.5.", "Claude Opus 5.5's API rejects forced tool calls, as Claude Fable 5.1's does, so its row answers as a JSON object and reasons at the provider default. Claude Opus 5's board row runs the forced tool call, which switches that model's thinking off, and scores {opus5BoardExact}%; its tool_choice auto re-run, where it reasons, scores {opus5AutoExact}%. The two Opus rows differ in serving shape as well as in model, so the {opusGap}-point gap between them is not a measure of the model change alone.", "Every row is scored on {scoredOutputs} of its {totalOutputs} requested outputs. This release regenerated {regenerated} references and excludes {excluded} outputs from scoring for every model; the reference audit note explains both.", diff --git a/app/src/notes/2026-09-29-claude-sonnet-5-5-debuts-fifth.json b/app/src/notes/2026-09-29-claude-sonnet-5-5-debuts-fifth.json index ceb0ef49..3399213e 100644 --- a/app/src/notes/2026-09-29-claude-sonnet-5-5-debuts-fifth.json +++ b/app/src/notes/2026-09-29-claude-sonnet-5-5-debuts-fifth.json @@ -3,7 +3,7 @@ "date": "2026-09-29", "title": "Claude Sonnet 5.5 debuts fifth as the references move to policyengine-us 2.15.17", "paragraphs": [ - "Claude Sonnet 5.5, Grok 4.7 and DeepSeek V4.1 Flash joined the board on 2026-09-29, bringing it to {nModels} models. Claude Sonnet 5.5 scores {sonnetExact}% of answers right, weighted by household impact, #{sonnetRank} of {nModels}. The score counts dollar amounts within $1 of PolicyEngine's reference and eligibility answers that match it. GPT-6 Luna, #{lunaRank}, also rounds to {lunaExact}% and sits about level with it, less than {lunaSonnetGapBelow} points ahead. Grok 4.7 scores {grokExact}% (#{grokRank}) and DeepSeek V4.1 Flash {flashExact}% (#{flashRank}). GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%).", + "Claude Sonnet 5.5, Grok 4.7 and DeepSeek V4.1 Flash joined the board on 2026-09-29, bringing it to {nModels} models. Claude Sonnet 5.5 gets {sonnetExact}% of answers right, weighted by household impact, and ranks #{sonnetRank} of {nModels}. PolicyBench counts a dollar amount as right when it falls within $1 of PolicyEngine's reference, and an eligibility answer as right when it matches. GPT-6 Luna, #{lunaRank}, also rounds to {lunaExact}% and sits about level with Claude Sonnet 5.5, less than {lunaSonnetGapBelow} points ahead. Grok 4.7 scores {grokExact}% (#{grokRank}) and DeepSeek V4.1 Flash {flashExact}% (#{flashRank}). GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%).", "Claude Sonnet 5.5 costs ${sonnetCost} a household, against ${lunaCost} for GPT-6 Luna. DeepSeek V4.1 Flash costs ${flashCost} at DeepSeek's peak list price, and Grok 4.7 costs ${grokCost}.", "Claude Sonnet 5.5's API rejects forced tool calls, as Claude Opus 5.5's and Claude Fable 5.1's do, so its row answers as a JSON object and reasons with adaptive thinking, the provider default. Grok 4.7 answers through the forced tool call with a {grokTimeoutSeconds}-second request timeout, because its whole-household onboarding probe ran past a {grokOnboardingTimeoutSeconds}-second timeout. DeepSeek serves V4.1 Flash under the alias deepseek-flash, and all {flashAnswers} of the row's answers report that alias and one system fingerprint; none reports a version. PolicyBench labels the row from DeepSeek's September 10 release note, which put V4.1 Flash on that alias.", "The release also moves PolicyBench's scored references from policyengine-us {previousEngine} to {engineVersion}, the newest release when PolicyBench began sweeping the references on 2026-09-29 (uploaded at {engineUploadedUtc} UTC). The newer version includes the {upstreamFixed:words} upstream fixes that PolicyBench applied as sandbox fixes for the September 22 references, and it encodes law the older version lacked. PolicyBench still builds each scored reference from the stated facts and law published before it froze the references on 2026-07-03, so it ported its {conventions:words} publication conventions to the new version. With those conventions and an adapter that keeps Maryland county tax out of state income tax, policyengine-us {checkEngine}, the newest release when PolicyBench checked PyPI on 2026-09-29 at {checkPypiReadUtc} UTC, gives the same value as {engineVersion} for all {totalOutputs} outputs.", diff --git a/app/src/notes/2026-10-06-policybench-stops-scoring-eight-tax-outputs.json b/app/src/notes/2026-10-06-policybench-stops-scoring-eight-tax-outputs.json index 01dbf396..6d05b1e1 100644 --- a/app/src/notes/2026-10-06-policybench-stops-scoring-eight-tax-outputs.json +++ b/app/src/notes/2026-10-06-policybench-stops-scoring-eight-tax-outputs.json @@ -8,7 +8,7 @@ "The Virginia household's head is {va114Age}, and the prompt lists no Medicare enrollment or premium. policyengine-us treats everyone eligible for Medicare as enrolled and counts the standard Part B premium, ${partBPremium} for 2026, as a medical expense. The household itemizes, and Virginia's itemized deductions start from the federal ones, so the premium lowers its Virginia income tax: without it, that output is ${va114Alt} rather than the reference's ${va114Ref}. The premium lowers the household's federal income tax too, and that output's exclusion record names both unstated facts.", "The {payrollOutputs:words} payroll outputs count an employee share of a state paid-leave or disability premium: {optionalPrograms}. In each of these states the premium is the employer's, and the employer may deduct up to a set share of it from wages but need not, so the law requires no amount of the employee. The prompts ask for mandatory employee state payroll taxes, and the references count the largest share each state's published 2026 rates let the employer deduct, without the prompts saying whether it does. Without that share, the outputs are ${mnAlt} in Minnesota, ${coAlt} in Colorado, ${maAlt} in Massachusetts, and ${nyAlt} in New York, against references of ${mnRef}, ${coRef}, ${maRef}, and ${nyRef}. PolicyBench still scores the payroll outputs of {mandatoryScored:words} households in {mandatoryStates}, where the law puts the contribution on the worker and requires the employer to withhold it.", "No model's answer is within $1 of the reference on any of the {incomeTaxOutputs:words} income tax outputs. On the {payrollOutputs:words} payroll outputs, {payrollHits} answers match the reference within $1: {mnHits:words} for Minnesota, {coHits} for Colorado, {maHits:words} for Massachusetts, and {nyHits:words} for New York. Another {payrollAltHits} match the amount without the state share.", - "Every model's exact rate, its share of answers right weighted by household impact, rises by {riseMin} to {riseMax} points. The rate counts dollar amounts within $1 of PolicyEngine's reference and eligibility answers that match it. GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%). Claude Sonnet 5.5 ({sonnetExact}%) passes GPT-6 Luna ({lunaExact}%) for fourth. Further down, {riserTwo} moves above {riserTwoPassed}, {riserThree} above {riserThreePassed}, and {riserFour} above {riserFourPassed}. Every other pair keeps its order.", + "Every model's share of answers right, weighted by household impact, rises by {riseMin} to {riseMax} points. PolicyBench counts a dollar amount as right when it falls within $1 of PolicyEngine's reference, and an eligibility answer as right when it matches. GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%). Claude Sonnet 5.5 ({sonnetExact}%) passes GPT-6 Luna ({lunaExact}%) for fourth. Further down, {riserTwo} moves above {riserTwoPassed}, {riserThree} above {riserThreePassed}, and {riserFour} above {riserFourPassed}. Every other pair keeps its order.", "PolicyBench's next prompt contract remains an unactivated draft (PolicyEngine/policybench#173). It states the state income tax paid for each tax unit or a named convention for estimating withholding; each person's Medicare Part B enrollment and annual net premium; and whether each person's employer withholds the employee share of state paid-leave or disability premiums. Its payroll output definition counts optional employee shares only when the employer withholds them. A later run can score these outputs after its reference builder and a fresh input sweep validate the stated conventions.", "Two corrections in this release move no score. PolicyBench now reports that it collected model responses from {responseStart} to {responseEnd}, 2026, ending on the UTC date of the last answer; release dashboard-data-20260930 gave {previousResponseEnd}, the date of that release. The audit text for a California household head's Medicaid eligibility now explains the head's income test as the engine applies it: California's ${medicaidDisregard} monthly income disregard, which takes the place of SSI's ${ssiExclusion} general income exclusion, brings countable income under the limit. The earlier text said the engine subtracts a Medicare Part B premium from income, which it does not." ], diff --git a/app/src/notes/2026-10-10-claude-haiku-5-5-joins-the-board.json b/app/src/notes/2026-10-10-claude-haiku-5-5-joins-the-board.json index 162e4f3c..62a84f3b 100644 --- a/app/src/notes/2026-10-10-claude-haiku-5-5-joins-the-board.json +++ b/app/src/notes/2026-10-10-claude-haiku-5-5-joins-the-board.json @@ -3,7 +3,7 @@ "date": "2026-10-10", "title": "Claude Haiku 5.5 joins the board as the references move to policyengine-us 2.38.6", "paragraphs": [ - "Claude Haiku 5.5 joined the board on {releaseDate}, bringing it to {nModels} models. It gets {haikuExact}% of answers right, weighted by household impact, #{haikuRank} of {nModels} and {haikuGain} points above Claude Haiku 4.5 ({haiku45Exact}%, #{haiku45Rank}). The score counts dollar amounts within $1 of PolicyEngine's reference and eligibility answers that match it. GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%).", + "Claude Haiku 5.5 joined the board on {releaseDate}, bringing it to {nModels} models. It gets {haikuExact}% of answers right, weighted by household impact, {haikuGain} points above Claude Haiku 4.5 ({haiku45Exact}%, #{haiku45Rank}), and ranks #{haikuRank} of {nModels}. PolicyBench counts a dollar amount as right when it falls within $1 of PolicyEngine's reference, and an eligibility answer as right when it matches. GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%).", "Claude Haiku 5.5 cost ${haikuCost} a household, the least of any row with a cost above zero; Claude Haiku 4.5 cost ${haiku45Cost}. Unlike Claude Opus 5.5 and Claude Sonnet 5.5, its API accepts a forced tool call, so its row takes the board's forced answer tool. A probe with that request returned no thinking block, where a request without the forced tool did, so the row answers without extended thinking, as Claude Opus 5's does. Re-run with tool_choice: auto, it gets {haikuAutoExact}%, which would rank #{haikuAutoRank}.", "The release also moves PolicyBench's references from policyengine-us {previousEngine} to {engineVersion}, a release that fixes the engine defects behind {restored:words} outputs. The previous release excluded {newlyScored:words} of them, and they return to scoring at the new version's values. The other {restoredRuled:words} are the October 6 defects below, which stay scored at the new version's values. Each of the {restored:words} lands within $1 of the corrected value PolicyBench's audits computed for it. Between them they take the upstream fixes for {restoredFixes}. PolicyBench still builds each reference from the stated facts and from law published before it froze the references on 2026-07-03, under the same publication conventions.", "Two rulings of October 6 apply too. PolicyBench's reference adversary flags the scored outputs where many models, or several of the strongest, agree on an answer the reference does not give, and a judge that cannot yet see the engine's derivation works each one from primary law. Its findings led to rulings to exclude {adversaryOutputs:words} outputs. Four were engine defects, in Arizona, Ohio, Colorado and New York; the new version fixes them, so PolicyBench keeps scoring those four instead. The other four are the federal and state income tax of a Pennsylvania and a Missouri household whose dependent earns $45,000 of wages and must file their own return. The output definitions do not say whether the household's income tax includes that return, so PolicyBench stops scoring them. It also stops scoring {louisianaOutputs:words} Louisiana households' state income tax, whose reference rests on a 2026 standard deduction computed from published price indexes; Louisiana published the amount itself on September 28, after the freeze.", diff --git a/app/src/notes/2026-10-13-muse-wonder-joins-the-board.json b/app/src/notes/2026-10-13-muse-wonder-joins-the-board.json index 07944726..8dcadc1c 100644 --- a/app/src/notes/2026-10-13-muse-wonder-joins-the-board.json +++ b/app/src/notes/2026-10-13-muse-wonder-joins-the-board.json @@ -3,7 +3,7 @@ "date": "2026-10-13", "title": "Meta's Muse Wonder joins the board as the references move to policyengine-us 2.41.0", "paragraphs": [ - "Meta's Muse Wonder joined the board on {releaseDate}, the day after Meta released it, bringing the board to {nModels} models. It is the first Meta model on the board. It gets {museExact}% of answers right, weighted by household impact, #{museRank} of {nModels}. The score counts dollar amounts within $1 of PolicyEngine's reference and eligibility answers that match it. On the bounded score, which gives partial credit for amounts close to the reference, it ranks #{museBoundedRank}, behind only GPT-6 Sol and Claude Opus 5.5. GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%).", + "Meta's Muse Wonder joined the board on {releaseDate}, the day after Meta released it, bringing the board to {nModels} models. It is the first Meta model on the board. It gets {museExact}% of answers right, weighted by household impact, and ranks #{museRank} of {nModels}. PolicyBench counts a dollar amount as right when it falls within $1 of PolicyEngine's reference, and an eligibility answer as right when it matches. On the bounded score, which gives partial credit for amounts close to the reference, it ranks #{museBoundedRank}, behind only GPT-6 Sol and Claude Opus 5.5. GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%).", "Meta released Muse Wonder Full the same day, but only to Muse Code subscribers, with no API. PolicyBench runs every model through its provider's API under one protocol, so Muse Wonder Full is not on the board. PolicyBench reaches Muse Wonder through OpenRouter, whose one endpoint for it is Meta's own API. That API accepts only an automatic tool choice and rejects a forced tool call, so the row answers as a JSON object, as Claude Opus 5.5's and Claude Sonnet 5.5's do, and reasons at Meta's default setting.", "The release also moves PolicyBench's references from policyengine-us {previousEngine} to {engineVersion}, the first release with every fix the previous release was waiting on. Of the outputs the previous release excluded, {restored:words} return to scoring at the new version's values, each within $1 of the corrected value PolicyBench's audits computed for it. Between them they take the upstream fixes for {restoredFixes}. The new version also fixes the defect behind a SNAP output, the heat-and-eat utility allowance, but that output stays excluded because the stated facts do not decide it. No reference PolicyBench already scored changes, and nothing is newly excluded. Every model is now scored on {scoredOutputs} of its {totalOutputs} requested outputs; PolicyBench excludes the other {excluded}.", "Against the previous release, the exact rate of every one of the {incumbents} earlier models falls, by {fallMin} to {fallMax} points. The cause is the {restored:words} restored outputs, which few models get right: half of them are answered within $1 by at most {restoredHalfAtMost:words} of the {nModels} models, and {restoredUnmatched:words} by none. Muse Wonder answers {museRestored:words} of them within $1. Without them, every earlier model's rate would be the previous release's." diff --git a/app/src/notes/2026-10-14-gemini-4-argon-joins-the-board.json b/app/src/notes/2026-10-14-gemini-4-argon-joins-the-board.json index cf237b1f..a67c0b24 100644 --- a/app/src/notes/2026-10-14-gemini-4-argon-joins-the-board.json +++ b/app/src/notes/2026-10-14-gemini-4-argon-joins-the-board.json @@ -3,8 +3,8 @@ "date": "2026-10-14", "title": "Google's Gemini 4 Argon and Meta's Muse Spark 1.3 join the board", "paragraphs": [ - "Google's Gemini 4 Argon joined the board on {releaseDate}, the day after Google opened it to the public Gemini API, and Meta's Muse Spark 1.3 joined with it, bringing the board to {nModels} models. Gemini 4 Argon gets {argonExact}% of answers right, weighted by household impact, #{argonRank} of {nModels} and the highest of any Gemini model; Gemini 3.8 Flash, the previous best, gets {flashExact}% (#{flashRank}). Muse Spark 1.3 gets {sparkExact}% (#{sparkRank}), behind Meta's Muse Wonder ({museExact}%, #{museRank}), but on the bounded score, which gives partial credit for amounts close to the reference, it ranks #{sparkBoundedRank}. GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%).", - "The board's score counts dollar amounts within $1 of PolicyEngine's reference and eligibility answers that match it. Google announced Gemini 4 Argon on September 30 for a group of partners and released it on the Gemini API on October 13. PolicyBench runs it there, as it does every Gemini model. The API accepts a forced answer tool, so the row answers through the board's tool and reasons at Google's default setting. A forced tool call suppresses some models' reasoning, as it does Claude Opus 5's, so PolicyBench fixed a test before looking at the results and ran all {households} households both ways. With the forced tool Gemini 4 Argon returned reasoning tokens on {forcedReasoning} of them, and on {boardReasoning} in the board's own run; with an automatic tool choice, on {autoReasoning}. The test found no significant reduction, so the board keeps the forced tool and publishes no second row. At Google's introductory price of $2 per million input tokens and $10 per million output tokens, which doubles on December 1, the {households} households cost ${argonCost}.", + "Google's Gemini 4 Argon joined the board on {releaseDate}, the day after Google opened it to the public Gemini API, and Meta's Muse Spark 1.3 joined with it, bringing the board to {nModels} models. Gemini 4 Argon gets {argonExact}% of answers right, weighted by household impact, and ranks #{argonRank} of {nModels}, the highest of any Gemini model; Gemini 3.8 Flash, the previous best, gets {flashExact}% (#{flashRank}). PolicyBench counts a dollar amount as right when it falls within $1 of PolicyEngine's reference, and an eligibility answer as right when it matches. Muse Spark 1.3 gets {sparkExact}% (#{sparkRank}), behind Meta's Muse Wonder ({museExact}%, #{museRank}), but on the bounded score, which gives partial credit for amounts close to the reference, it ranks #{sparkBoundedRank}. GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%).", + "Google announced Gemini 4 Argon on September 30 for a group of partners and released it on the Gemini API on October 13. PolicyBench runs it there, as it does every Gemini model. The API accepts a forced answer tool, so the row answers through the board's tool and reasons at Google's default setting. A forced tool call suppresses some models' reasoning, as it does Claude Opus 5's, so PolicyBench fixed a test before looking at the results and ran all {households} households both ways. With the forced tool Gemini 4 Argon returned reasoning tokens on {forcedReasoning} of them, and on {boardReasoning} in the board's own run; with an automatic tool choice, on {autoReasoning}. The test found no significant reduction, so the board keeps the forced tool and publishes no second row. At Google's introductory price of $2 per million input tokens and $10 per million output tokens, which doubles on December 1, the {households} households cost ${argonCost}.", "Muse Spark 1.3, which Meta released on September 2, is Meta's second model on the board. PolicyBench reaches it through OpenRouter, whose one endpoint for it is Meta's own API, as it does Muse Wonder. That API rejects a forced tool call, so the row answers as a JSON object and reasons at Meta's default setting.", "Nothing else on the board moves. The newest policyengine-us release, {checkedEngine}, gives the same value as {engineVersion} for every output, so the references stay where they were, and every model is still scored on {scoredOutputs} of its {totalOutputs} requested outputs. Each of the {incumbents} earlier models keeps its score." ], diff --git a/app/src/notes/2026-10-20-given-a-policyengine-tool-three-models-score-above-99.json b/app/src/notes/2026-10-20-given-a-policyengine-tool-three-models-score-above-99.json index c5deead1..edde8996 100644 --- a/app/src/notes/2026-10-20-given-a-policyengine-tool-three-models-score-above-99.json +++ b/app/src/notes/2026-10-20-given-a-policyengine-tool-three-models-score-above-99.json @@ -4,7 +4,7 @@ "title": "Given a tool that runs PolicyEngine, three models score above 99%", "paragraphs": [ "PolicyBench gave three models a tool that runs PolicyEngine on a household the model writes from the prompt's facts, and ran its {households} US households again on October 14 and 15. GPT-6 Sol, which leads the board at {solBoardExact}% of answers right, weighted by household impact, gets {solToolExact}% with the tool. Claude Opus 5.5 goes from {opusBoardExact}% to {opusToolExact}%, and Gemini 4 Argon from {argonBoardExact}% to {argonToolExact}%. The runs are a pilot, not board rows, since the board scores models answering alone, and the result is circular: the tool runs the engine that computes PolicyBench's references, so the scores measure agreement with that engine, not whether it is right.", - "Each score counts dollar amounts within $1 of PolicyEngine's reference and eligibility answers that match it. Most of the answers the tool fixes are dollar amounts that are not zero. Of the {nonzeroAmounts} nonzero amounts, GPT-6 Sol gets {solNonzeroRate}% within $1 on its own and {solNonzeroToolRate}% with the tool; Claude Opus 5.5, {opusNonzeroRate}% and {opusNonzeroToolRate}%; and Gemini 4 Argon, {argonNonzeroRate}% and {argonNonzeroToolRate}%.", + "PolicyBench counts a dollar amount as right when it falls within $1 of PolicyEngine's reference, and an eligibility answer as right when it matches, with or without the tool. Most of the answers the tool fixes are dollar amounts that are not zero. Of the {nonzeroAmounts} nonzero amounts, GPT-6 Sol gets {solNonzeroRate}% within $1 on its own and {solNonzeroToolRate}% with the tool; Claude Opus 5.5, {opusNonzeroRate}% and {opusNonzeroToolRate}%; and Gemini 4 Argon, {argonNonzeroRate}% and {argonNonzeroToolRate}%.", "With the tool, {toolMisses} answers are wrong across the three runs: {solMisses:words} of GPT-6 Sol's, {opusMisses:words} of Claude Opus 5.5's and {argonMisses:words} of Gemini 4 Argon's. Each traces to the household the model described to the engine, or to the output it read back, rather than to the calculation: in {enteredElsewhere:words} the model had entered a stated fact under another variable, in {leftOut:words} it had left one out, and in {wrongOutput:words} it read back the wrong output. Prompt labels share the blame for {labelMisses:words}: \"employer sponsored insurance premiums\" never says the employer paid them, and \"disability benefits\" never says they are not Social Security. In {solCapMisses:words} of GPT-6 Sol's, it had used all {turnCap:words} calculation turns the pilot allows. GPT-6 Sol is also the only model that turned right answers wrong, {solFlips:words} of them.", "The tool costs more. For the {households} households, GPT-6 Sol costs ${solToolCost} with it and ${solBoardCost} without, Claude Opus 5.5 ${opusToolCost} and ${opusBoardCost}, and Gemini 4 Argon ${argonToolCost} and ${argonBoardCost}.", "The tool applies the conventions PolicyBench uses for its references, so a model that describes a household exactly as the reference does gets the reference's answer. Describing a household is also easier here than it would be from a filer's documents: {labelMatches} of the {statedFacts} facts the prompts state, not counting ages and wages, carry a label that is the engine's own variable name with spaces.", diff --git a/app/src/notes/2026-10-31-gpt-6-1-terra-joins-the-board.json b/app/src/notes/2026-10-31-gpt-6-1-terra-joins-the-board.json index 9e49bdb1..9b753d7a 100644 --- a/app/src/notes/2026-10-31-gpt-6-1-terra-joins-the-board.json +++ b/app/src/notes/2026-10-31-gpt-6-1-terra-joins-the-board.json @@ -3,8 +3,8 @@ "date": "2026-10-31", "title": "OpenAI's GPT-6.1 Terra joins the board at #3 as the references move to policyengine-us 2.60.0", "paragraphs": [ - "PolicyBench added OpenAI's GPT-6.1 Terra to its {snapshotDate} snapshot, the day after OpenAI released it, bringing the board to {nModels} models. It gets {terraExact}% of answers right, weighted by household impact, #{terraRank} of {nModels}. Only GPT-6 Sol ({solExact}%) and Claude Opus 5.5 ({opusExact}%) score higher, and GPT-5.6 Sol, which also rounds to {sol56Exact}%, is less than a tenth of a point behind. On the bounded score, which gives partial credit for amounts close to the reference, GPT-6.1 Terra ranks #{terraBoundedRank}. GPT-6.1 Sol, which OpenAI released on September 29, gets {sol61Exact}% (#{sol61Rank}).", - "The board's score counts dollar amounts within $1 of PolicyEngine's reference and eligibility answers that match it. PolicyBench runs GPT-6.1 Terra as it runs GPT-6.1 Sol, on OpenAI's Responses API with the board's forced answer tool at OpenAI's default reasoning effort, and it returned reasoning tokens on all {households} households. At OpenAI's price of $1 per million input tokens and $5 per million output tokens, the {households} households cost ${terraCost}, against ${sol61Cost} for GPT-6.1 Sol and ${solCost} for GPT-6 Sol. GPT-5.6 Terra, the Terra model before it, gets {terra56Exact}% (#{terra56Rank}) and cost ${terra56Cost}.", + "PolicyBench added OpenAI's GPT-6.1 Terra to its {snapshotDate} snapshot, the day after OpenAI released it, bringing the board to {nModels} models. It gets {terraExact}% of answers right, weighted by household impact, and ranks #{terraRank} of {nModels}. PolicyBench counts a dollar amount as right when it falls within $1 of PolicyEngine's reference, and an eligibility answer as right when it matches. Only GPT-6 Sol ({solExact}%) and Claude Opus 5.5 ({opusExact}%) score higher, and GPT-5.6 Sol, which also rounds to {sol56Exact}%, is less than a tenth of a point behind. On the bounded score, which gives partial credit for amounts close to the reference, GPT-6.1 Terra ranks #{terraBoundedRank}. GPT-6.1 Sol, which OpenAI released on September 29, gets {sol61Exact}% (#{sol61Rank}).", + "PolicyBench runs GPT-6.1 Terra as it runs GPT-6.1 Sol, on OpenAI's Responses API with the board's forced answer tool at OpenAI's default reasoning effort, and it returned reasoning tokens on all {households} households. At OpenAI's price of $1 per million input tokens and $5 per million output tokens, the {households} households cost ${terraCost}, against ${sol61Cost} for GPT-6.1 Sol and ${solCost} for GPT-6 Sol. GPT-5.6 Terra, the Terra model before it, gets {terra56Exact}% (#{terra56Rank}) and cost ${terra56Cost}.", "The release also moves PolicyBench's references from policyengine-us {previousEngine} to {engineVersion}, the newest release when PolicyBench began recomputing them on October 23. The move changes {changed:words} scored references, each for law in force before the references were frozen. An elderly Californian who lives alone, whose SNAP was $0, now gets the ${snapMonthly} monthly minimum, ${snap031} a year: California's categorical eligibility for CalFresh applies no net income test, even to a household with an elderly member, and the previous version applied one (policyengine-us #10036). Three households in Pennsylvania and Arkansas have too much income for free or reduced-price school lunch, but those states give students at covered schools a free breakfast, so the output asking whether PolicyEngine returns free school meal support is now yes for each (policyengine-us #10238). The new version models both breakfast laws for a student at a covered school and leaves out their conditions: Pennsylvania reimburses schools for them to the extent funds are available, and Arkansas serves them on request.", "Nothing is newly excluded, and no excluded output returns to scoring, so every model is still scored on {scoredOutputs} of its {totalOutputs} requested outputs. When PolicyBench checked PyPI on {checkedOn}, the newest policyengine-us release was {checkedEngine}, and it gives the same value as {engineVersion} for all {scoredOutputs}; it moves {excludedMoved:words} excluded output, which keeps the value it was decided on.", "Against the previous release, the exact rates of {fallers:words} of the {incumbents} earlier models fall, by {fallMin} to {fallMax} points, and {risers:words} rise by {rise}. No model answers yes for any of the three households' school meals, and only {snapMatch:words} give the Californian's new SNAP: GPT-5.5, GPT-6 Astra and GPT-6.1 Sol, the three that rise. Claude Sonnet 5.5 falls from #{sonnetThen} to #{sonnetNow}, behind GPT-6.1 Terra and GPT-6 Astra." diff --git a/app/tests/notes.test.ts b/app/tests/notes.test.ts index a134de1f..cd456547 100644 --- a/app/tests/notes.test.ts +++ b/app/tests/notes.test.ts @@ -64,8 +64,8 @@ describe("notes", () => { expect(markup).toContain("GPT-6 Astra debuts second: the two rules it invented"); expect(markup).toContain("release dashboard-data-20260905c"); // Whole-number board rates keep one decimal. - expect(markup).toContain("at 88.0% of answers right"); - expect(markup).not.toContain("at 88% of answers"); + expect(markup).toContain("It gets 88.0% of answers right"); + expect(markup).not.toContain("gets 88% of answers"); expect(markup).toContain("release dashboard-data-20260901c"); expect(markup).toContain("Six SNAP households the top three models deny"); expect(markup).toContain( @@ -141,7 +141,10 @@ describe("notes", () => { ); // Board rates keep one decimal. expect(markup).toContain( - "It gets 93.1% of answers right, weighted by household impact, #3 of 51.", + "It gets 93.1% of answers right, weighted by household impact, and ranks #3 of 51.", + ); + expect(markup).toContain( + "PolicyBench counts a dollar amount as right when it falls within $1 of PolicyEngine's reference, and an eligibility answer as right when it matches.", ); expect(markup).toContain("GPT-6.1 Terra ranks #5."); expect(markup).toContain("GPT-6.1 Sol, which OpenAI released on September 29, gets 90.2% (#10)."); @@ -249,7 +252,7 @@ describe("notes", () => { ); // Board rates keep one decimal. expect(markup).toContain("Gemini 4 Argon gets 88.7% of answers right"); - expect(markup).toContain("#12 of 50 and the highest of any Gemini model"); + expect(markup).toContain("ranks #12 of 50, the highest of any Gemini model"); expect(markup).toContain("Gemini 3.8 Flash, the previous best, gets 85.6% (#21)"); expect(markup).toContain( "Muse Spark 1.3 gets 87.2% (#18), behind Meta's Muse Wonder (90.1%, #10)", @@ -283,7 +286,7 @@ describe("notes", () => { ); // Board rates keep one decimal. expect(markup).toContain("It gets 90.1% of answers right"); - expect(markup).toContain("#10 of 48"); + expect(markup).toContain("and ranks #10 of 48."); expect(markup).toContain("it ranks #3, behind only GPT-6 Sol and Claude Opus 5.5"); expect(markup).toContain( "GPT-6 Sol still leads at 95.2%, ahead of Claude Opus 5.5 (94.1%) and GPT-5.6 Sol (93.1%).", diff --git a/tests/test_notes.py b/tests/test_notes.py index 1fca40c0..bcdba753 100644 --- a/tests/test_notes.py +++ b/tests/test_notes.py @@ -3408,14 +3408,14 @@ def pin(*sentences: str) -> None: pin( "Claude Sonnet 5.5, Grok 4.7 and DeepSeek V4.1 Flash joined the board on " "2026-09-29, bringing it to {nModels} models.", - "Claude Sonnet 5.5 scores {sonnetExact}% of answers right, weighted by " - "household impact, #{sonnetRank} of {nModels}.", + "Claude Sonnet 5.5 gets {sonnetExact}% of answers right, weighted by " + "household impact, and ranks #{sonnetRank} of {nModels}.", ) _assert_score_definition() pin( SCORE_DEFINITION, "GPT-6 Luna, #{lunaRank}, also rounds to {lunaExact}% and sits about level " - "with it, less than {lunaSonnetGapBelow} points ahead.", + "with Claude Sonnet 5.5, less than {lunaSonnetGapBelow} points ahead.", "Grok 4.7 scores {grokExact}% (#{grokRank}) and DeepSeek V4.1 Flash " "{flashExact}% (#{flashRank}).", # The previous release's leader, recomputed on its references. @@ -4450,13 +4450,12 @@ def rank_then(model: str) -> int: assert rank_now(passed[0]) == rank_then(riser) pin( # The headline the board ranks by, as the September 29 note states it. - "Every model's exact rate, its share of answers right weighted by " - "household impact, rises by {riseMin} to {riseMax} points.", + "Every model's share of answers right, weighted by household impact, " + "rises by {riseMin} to {riseMax} points.", ) _assert_score_definition() pin( - "The rate counts dollar amounts within $1 of PolicyEngine's reference and " - "eligibility answers that match it.", + SCORE_DEFINITION, "GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 " "({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%).", "Claude Sonnet 5.5 ({sonnetExact}%) passes GPT-6 Luna ({lunaExact}%) for " @@ -6172,8 +6171,9 @@ def test_release_20261027_note_claims() -> None: SCORE_DEFINITION = ( - "The score counts dollar amounts within $1 of PolicyEngine's reference and " - "eligibility answers that match it." + "PolicyBench counts a dollar amount as right when it falls within $1 of " + "PolicyEngine's reference, and an eligibility answer as right when it " + "matches." ) @@ -6192,7 +6192,8 @@ def _assert_score_definition() -> None: def test_no_note_gives_the_score_as_answers_within_one_dollar(): """The board scores eligibility flags by exact match, not within $1, so no - note gives an all-outputs score as answers within $1.""" + note gives an all-outputs score as answers within $1, and a note that + gives a score as answers right defines it.""" _assert_score_definition() # PR #254 rewrites this note's opening; drop it from here once that merges. pending = {"2026-10-19-america-gov-scores-level-with-gpt-5-5"} @@ -6203,3 +6204,12 @@ def test_no_note_gives_the_score_as_answers_within_one_dollar(): text = " ".join([note["title"], *note["paragraphs"]]) for phrase in ("answers within $1", "of answers within"): assert phrase not in text, (path.name, phrase) + # A note that gives a score as answers right also says what + # PolicyBench counts as right. + if "of answers right" in text: + for clause in ( + "PolicyBench counts a dollar amount as right when it falls within " + "$1 of PolicyEngine's reference", + "an eligibility answer as right when it matches", + ): + assert clause in text, (path.name, clause) From 96a7fae6a32aa123ce4ffbb0c60ad21457e98990 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 2 Nov 2026 23:45:47 -0500 Subject: [PATCH 05/14] Close each opening with the score definition and name who computes the reference Review r1 (REQUEST_CHANGES at 266998fe) found that the definition cut a pronoun off from its antecedent in the October 13 note and cut the comparisons off from Terra's score in the October 31 note. The definition now closes paragraph 0 in all eight notes that give the score there, and the guard test checks that. - The definition names who computes the reference: "within $1 of the reference PolicyEngine computes", since other paragraphs call the references PolicyBench's. - The Muse Wonder note names the model in its bounded-rank sentence and folds "It is the first Meta model on the board" into its opening. - Ranks and comparisons take verbs: GPT-5.6 Sol "ranks #3" and "trails by less than a tenth of a point". - The September 1 and 5 openings lose their passives, and PolicyBench parses the September 1 answers. - The October 20 definition opens with "With or without the tool". - Four notes drop "the exact rate", a second name for the score. - The guard requires the shared definition word for word, and its America.gov skip fails once #254 lands. - docs/terra/design.md stops giving the score as answers within $1. Co-Authored-By: Claude Opus 5.5 --- .../2026-09-01-claude-fable-5-1-added.json | 2 +- .../2026-09-05-gpt-6-astra-debuts-second.json | 2 +- .../2026-09-22-gpt-6-sol-debuts-first.json | 2 +- ...-09-29-claude-sonnet-5-5-debuts-fifth.json | 4 +- ...bench-stops-scoring-eight-tax-outputs.json | 2 +- ...0-10-claude-haiku-5-5-joins-the-board.json | 4 +- ...026-10-13-muse-wonder-joins-the-board.json | 4 +- ...-10-14-gemini-4-argon-joins-the-board.json | 2 +- ...gine-tool-three-models-score-above-99.json | 2 +- ...6-10-31-gpt-6-1-terra-joins-the-board.json | 4 +- app/tests/notes.test.ts | 45 +++++++++++++++++-- docs/terra/design.md | 2 +- tests/test_notes.py | 33 +++++++------- 13 files changed, 74 insertions(+), 34 deletions(-) diff --git a/app/src/notes/2026-09-01-claude-fable-5-1-added.json b/app/src/notes/2026-09-01-claude-fable-5-1-added.json index 67a1188f..5d2220d2 100644 --- a/app/src/notes/2026-09-01-claude-fable-5-1-added.json +++ b/app/src/notes/2026-09-01-claude-fable-5-1-added.json @@ -3,7 +3,7 @@ "date": "2026-09-01", "title": "Claude Fable 5.1 added", "paragraphs": [ - "Claude Fable 5.1 was released on 2026-09-01 and joined the board the same day. It gets {exactRate}% of answers right, weighted by household impact, and ranks #{rank} of {nModels}. PolicyBench counts a dollar amount as right when it falls within $1 of PolicyEngine's reference, and an eligibility answer as right when it matches. {parsed} of {answers} answers parsed.", + "Anthropic released Claude Fable 5.1 on 2026-09-01, and it joined the board the same day. It gets {exactRate}% of answers right, weighted by household impact, and ranks #{rank} of {nModels}. PolicyBench parsed {parsed} of the model's {answers} answers. PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches.", "The model's API rejects forced tool calls, which is how the board collects most models' answers, so its row answers as a JSON object and reasons at the provider default. The serving-configuration table in the paper records the transport for every row.", "Sensitivity: with the answer tool declared under tool_choice auto, Fable 5.1 scores {autoRate}% (would rank #{autoRank}). Under the same request Claude Fable 5 scores {fable5AutoRate}%. Fable 5's board row ({fable5BoardRate}%) ran chunked under a forced tool call, which switches off its thinking, so the {boardGap}-point gap between the two board rows is mostly request shape.", "The next board version moves every model to tool_choice auto so each provider's default reasoning engages under the recorded request shape (policybench#139)." diff --git a/app/src/notes/2026-09-05-gpt-6-astra-debuts-second.json b/app/src/notes/2026-09-05-gpt-6-astra-debuts-second.json index 18b200cf..b028bd42 100644 --- a/app/src/notes/2026-09-05-gpt-6-astra-debuts-second.json +++ b/app/src/notes/2026-09-05-gpt-6-astra-debuts-second.json @@ -3,7 +3,7 @@ "date": "2026-09-05", "title": "GPT-6 Astra debuts second: the two rules it invented", "paragraphs": [ - "GPT-6 Astra reached the API on 2026-09-04 and joined the board on 2026-09-05. It gets {astraExact}% of answers right, weighted by household impact, and ranks #{astraRank} of {nModels}, behind GPT-5.6 Sol ({solExact}%) and ahead of Claude Fable 5.1 ({fableExact}%, #{fableRank}). PolicyBench counts a dollar amount as right when it falls within $1 of PolicyEngine's reference, and an eligibility answer as right when it matches. Every row on this board is scored on {scoredOutputs} of its {totalOutputs} requested outputs: {excludedOutputs} outputs whose reference depends on an engine input the household facts never list are excluded for every model.", + "GPT-6 Astra reached the API on 2026-09-04 and joined the board on 2026-09-05. It gets {astraExact}% of answers right, weighted by household impact, and ranks #{astraRank} of {nModels}, behind GPT-5.6 Sol ({solExact}%) and ahead of Claude Fable 5.1 ({fableExact}%, #{fableRank}). PolicyBench scores every row on {scoredOutputs} of its {totalOutputs} requested outputs. It excludes the other {excludedOutputs} for every model, since their references depend on an engine input the household facts never list. PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches.", "Row by row, the two OpenAI models agree far more than they differ. Both are right on {bothRight} scored outputs and both wrong on {bothWrong}. The gap is {astraOnlyMisses} outputs Astra misses and Sol gets right, against {solOnlyMisses} the other way.", "Astra's solo misses cluster in two invented rules. In {esiRows} rows it treats a listed employer-sponsored insurance premium as a pre-tax salary reduction and subtracts it from wages before computing payroll tax, federal income tax or state income tax; the household facts state gross wages, and the reference keeps them. In {medicareRows} rows it marks a household member under 65 who is flagged disabled as Medicare-eligible; the reference grants Medicare before 65 only with 24 months of Social Security disability receipt recorded, and none of these households carry that input. The remaining {otherRows} solo misses are one-off rule errors with no shared mechanism. One further Medicare row of the same kind, on scenario_074, sits on an excluded output and is outside every count here.", "The judge's row notes name an employer-sponsored insurance premium on {astraEsiMentions} of Astra's annotated rows and on {solEsiMentions} of Sol's. Sol netted the same premium once, on scenario_045, which cost it {solEsiRows} of its {solOnlyMisses} solo misses; the rest of Sol's solo misses share no mechanism.", diff --git a/app/src/notes/2026-09-22-gpt-6-sol-debuts-first.json b/app/src/notes/2026-09-22-gpt-6-sol-debuts-first.json index b8ac1d19..eed63db2 100644 --- a/app/src/notes/2026-09-22-gpt-6-sol-debuts-first.json +++ b/app/src/notes/2026-09-22-gpt-6-sol-debuts-first.json @@ -3,7 +3,7 @@ "date": "2026-09-22", "title": "GPT-6 Sol debuts first, Claude Opus 5.5 second", "paragraphs": [ - "GPT-6 Sol, GPT-6 Luna and Claude Opus 5.5 joined the board on 2026-09-22. GPT-6 Sol ranks #{solRank} of {nModels} at {solExact}% of answers right, weighted by household impact, {solLead} points above Claude Opus 5.5 ({opusExact}%, #{opusRank}). PolicyBench counts a dollar amount as right when it falls within $1 of PolicyEngine's reference, and an eligibility answer as right when it matches. GPT-5.6 Sol, the previous leader, is #{sol56Rank} at {sol56Exact}%.", + "GPT-6 Sol, GPT-6 Luna and Claude Opus 5.5 joined the board on 2026-09-22. GPT-6 Sol gets {solExact}% of answers right, weighted by household impact, {solLead} points above Claude Opus 5.5 ({opusExact}%, #{opusRank}), and ranks #{solRank} of {nModels}. GPT-5.6 Sol, the previous leader, ranks #{sol56Rank} at {sol56Exact}%. PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches.", "GPT-6 Luna scores {lunaExact}% (#{lunaRank}) at ${lunaCost} a household, against ${solCost} for GPT-6 Sol and ${opusCost} for Claude Opus 5.5.", "Claude Opus 5.5's API rejects forced tool calls, as Claude Fable 5.1's does, so its row answers as a JSON object and reasons at the provider default. Claude Opus 5's board row runs the forced tool call, which switches that model's thinking off, and scores {opus5BoardExact}%; its tool_choice auto re-run, where it reasons, scores {opus5AutoExact}%. The two Opus rows differ in serving shape as well as in model, so the {opusGap}-point gap between them is not a measure of the model change alone.", "Every row is scored on {scoredOutputs} of its {totalOutputs} requested outputs. This release regenerated {regenerated} references and excludes {excluded} outputs from scoring for every model; the reference audit note explains both.", diff --git a/app/src/notes/2026-09-29-claude-sonnet-5-5-debuts-fifth.json b/app/src/notes/2026-09-29-claude-sonnet-5-5-debuts-fifth.json index 3399213e..9f3a715b 100644 --- a/app/src/notes/2026-09-29-claude-sonnet-5-5-debuts-fifth.json +++ b/app/src/notes/2026-09-29-claude-sonnet-5-5-debuts-fifth.json @@ -3,13 +3,13 @@ "date": "2026-09-29", "title": "Claude Sonnet 5.5 debuts fifth as the references move to policyengine-us 2.15.17", "paragraphs": [ - "Claude Sonnet 5.5, Grok 4.7 and DeepSeek V4.1 Flash joined the board on 2026-09-29, bringing it to {nModels} models. Claude Sonnet 5.5 gets {sonnetExact}% of answers right, weighted by household impact, and ranks #{sonnetRank} of {nModels}. PolicyBench counts a dollar amount as right when it falls within $1 of PolicyEngine's reference, and an eligibility answer as right when it matches. GPT-6 Luna, #{lunaRank}, also rounds to {lunaExact}% and sits about level with Claude Sonnet 5.5, less than {lunaSonnetGapBelow} points ahead. Grok 4.7 scores {grokExact}% (#{grokRank}) and DeepSeek V4.1 Flash {flashExact}% (#{flashRank}). GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%).", + "Claude Sonnet 5.5, Grok 4.7 and DeepSeek V4.1 Flash joined the board on 2026-09-29, bringing it to {nModels} models. Claude Sonnet 5.5 gets {sonnetExact}% of answers right, weighted by household impact, and ranks #{sonnetRank} of {nModels}. GPT-6 Luna, #{lunaRank}, also rounds to {lunaExact}% and sits about level with Claude Sonnet 5.5, less than {lunaSonnetGapBelow} points ahead. Grok 4.7 scores {grokExact}% (#{grokRank}) and DeepSeek V4.1 Flash {flashExact}% (#{flashRank}). GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%). PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches.", "Claude Sonnet 5.5 costs ${sonnetCost} a household, against ${lunaCost} for GPT-6 Luna. DeepSeek V4.1 Flash costs ${flashCost} at DeepSeek's peak list price, and Grok 4.7 costs ${grokCost}.", "Claude Sonnet 5.5's API rejects forced tool calls, as Claude Opus 5.5's and Claude Fable 5.1's do, so its row answers as a JSON object and reasons with adaptive thinking, the provider default. Grok 4.7 answers through the forced tool call with a {grokTimeoutSeconds}-second request timeout, because its whole-household onboarding probe ran past a {grokOnboardingTimeoutSeconds}-second timeout. DeepSeek serves V4.1 Flash under the alias deepseek-flash, and all {flashAnswers} of the row's answers report that alias and one system fingerprint; none reports a version. PolicyBench labels the row from DeepSeek's September 10 release note, which put V4.1 Flash on that alias.", "The release also moves PolicyBench's scored references from policyengine-us {previousEngine} to {engineVersion}, the newest release when PolicyBench began sweeping the references on 2026-09-29 (uploaded at {engineUploadedUtc} UTC). The newer version includes the {upstreamFixed:words} upstream fixes that PolicyBench applied as sandbox fixes for the September 22 references, and it encodes law the older version lacked. PolicyBench still builds each scored reference from the stated facts and law published before it froze the references on 2026-07-03, so it ported its {conventions:words} publication conventions to the new version. With those conventions and an adapter that keeps Maryland county tax out of state income tax, policyengine-us {checkEngine}, the newest release when PolicyBench checked PyPI on 2026-09-29 at {checkPypiReadUtc} UTC, gives the same value as {engineVersion} for all {totalOutputs} outputs.", "The move changes {scoredChanges:words} scored references. New Jersey's child tax credit schedule for 2026 to 2028, which the state approved on June 30, raises one household's state refundable credits from ${njBefore} to ${njAfter}. Arizona raised the income limit for its broad-based categorical eligibility from {azLimitBefore}% to {azLimitAfter}% of the poverty guideline effective March, and the new limit gives an Arizona household ${azAfter} of SNAP where the reference was ${azBefore}. policyengine-us now counts child support received as school-meal income, which ends a Pennsylvania household's eligibility for reduced-price meals. A fix to the rounding in New York's Empire State child credit phase-out (policyengine-us #9425) raises a New York household's state refundable credits from ${nyBefore} to ${nyAfter}.", "PolicyBench stops scoring {newExclusions:words} federal income tax outputs. policyengine-us {engineVersion} counts the whole of a listed state and local tax refund as income. Federal law counts the refund only to the extent the refunded tax lowered the household's federal tax in the year the household paid it, and the prompts do not say whether it did. PolicyBench also stops scoring one California household head's Medicaid eligibility, which its re-review of the household's excluded SNAP output flagged. On disability, the prompt says only that the head is disabled. The head's income, {medicaidIncomePercent}% of the poverty guideline, is above the {expansionLimitPercent}% limit for the adult expansion group, so only a disability pathway leads to Medi-Cal, California's Medicaid program. Medi-Cal's Working Disabled Program requires SSI's definition of disability, which the prompt does not state, and policyengine-us {engineVersion} tests the general disability flag instead. The same unstated fact already keeps the household's SNAP out of scoring. PolicyBench now scores every model on {scoredOutputs} of its {totalOutputs} requested outputs and excludes {excluded}. Another {withinTolerance:words} references move by less than $1, and PolicyBench re-reviewed the {rechecked} excluded outputs whose values moved; all stay excluded.", - "Together, the new references and the Medicaid exclusion raise the exact rate of every one of the {incumbents} earlier models, by {driftMin} to {driftMax} points. None of the {incumbents} matched any of the {newExclusions:words} federal outputs now excluded, and all {incumbents} had matched the Arizona household's old ${azBefore} SNAP reference, which none matches now. On the Medicaid output, {medicaidMatched} of the {incumbents} had matched the reference and {medicaidMissed} had not. Among the {incumbents}, {riserCount:words} models move up in the order: {riserOne} moves above {riserOnePassed}, {riserTwo} above {riserTwoPassed}, {riserThree} above {riserThreePassed}, {riserFour} above {riserFourPassed}, and {riserFive} above {riserFivePassed}. Every other pair keeps its order.", + "With the new references and the Medicaid exclusion together, every one of the {incumbents} earlier models rises, by {driftMin} to {driftMax} points. None of the {incumbents} matched any of the {newExclusions:words} federal outputs now excluded, and all {incumbents} had matched the Arizona household's old ${azBefore} SNAP reference, which none matches now. On the Medicaid output, {medicaidMatched} of the {incumbents} had matched the reference and {medicaidMissed} had not. Among the {incumbents}, {riserCount:words} models move up in the order: {riserOne} moves above {riserOnePassed}, {riserTwo} above {riserTwoPassed}, {riserThree} above {riserThreePassed}, {riserFour} above {riserFourPassed}, and {riserFive} above {riserFivePassed}. Every other pair keeps its order.", "The Arizona household's income keeps it from qualifying for SNAP under the program's ordinary tests, so it qualifies only through broad-based categorical eligibility, and all {nModels} models answer $0 for it. PolicyBench's October 5 note on such households counts it among five held back by income, with {bbceIncomeHeldCount:words} in {bbceIncomeHeldStates}. That note's data list every model's answer for those five households and for the {bbceAssetHeldCount:words} held back by savings, including the three models this release adds." ], "facts": { diff --git a/app/src/notes/2026-10-06-policybench-stops-scoring-eight-tax-outputs.json b/app/src/notes/2026-10-06-policybench-stops-scoring-eight-tax-outputs.json index 6d05b1e1..23240da1 100644 --- a/app/src/notes/2026-10-06-policybench-stops-scoring-eight-tax-outputs.json +++ b/app/src/notes/2026-10-06-policybench-stops-scoring-eight-tax-outputs.json @@ -8,7 +8,7 @@ "The Virginia household's head is {va114Age}, and the prompt lists no Medicare enrollment or premium. policyengine-us treats everyone eligible for Medicare as enrolled and counts the standard Part B premium, ${partBPremium} for 2026, as a medical expense. The household itemizes, and Virginia's itemized deductions start from the federal ones, so the premium lowers its Virginia income tax: without it, that output is ${va114Alt} rather than the reference's ${va114Ref}. The premium lowers the household's federal income tax too, and that output's exclusion record names both unstated facts.", "The {payrollOutputs:words} payroll outputs count an employee share of a state paid-leave or disability premium: {optionalPrograms}. In each of these states the premium is the employer's, and the employer may deduct up to a set share of it from wages but need not, so the law requires no amount of the employee. The prompts ask for mandatory employee state payroll taxes, and the references count the largest share each state's published 2026 rates let the employer deduct, without the prompts saying whether it does. Without that share, the outputs are ${mnAlt} in Minnesota, ${coAlt} in Colorado, ${maAlt} in Massachusetts, and ${nyAlt} in New York, against references of ${mnRef}, ${coRef}, ${maRef}, and ${nyRef}. PolicyBench still scores the payroll outputs of {mandatoryScored:words} households in {mandatoryStates}, where the law puts the contribution on the worker and requires the employer to withhold it.", "No model's answer is within $1 of the reference on any of the {incomeTaxOutputs:words} income tax outputs. On the {payrollOutputs:words} payroll outputs, {payrollHits} answers match the reference within $1: {mnHits:words} for Minnesota, {coHits} for Colorado, {maHits:words} for Massachusetts, and {nyHits:words} for New York. Another {payrollAltHits} match the amount without the state share.", - "Every model's share of answers right, weighted by household impact, rises by {riseMin} to {riseMax} points. PolicyBench counts a dollar amount as right when it falls within $1 of PolicyEngine's reference, and an eligibility answer as right when it matches. GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%). Claude Sonnet 5.5 ({sonnetExact}%) passes GPT-6 Luna ({lunaExact}%) for fourth. Further down, {riserTwo} moves above {riserTwoPassed}, {riserThree} above {riserThreePassed}, and {riserFour} above {riserFourPassed}. Every other pair keeps its order.", + "Every model's share of answers right, weighted by household impact, rises by {riseMin} to {riseMax} points. PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches. GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%). Claude Sonnet 5.5 ({sonnetExact}%) passes GPT-6 Luna ({lunaExact}%) for fourth. Further down, {riserTwo} moves above {riserTwoPassed}, {riserThree} above {riserThreePassed}, and {riserFour} above {riserFourPassed}. Every other pair keeps its order.", "PolicyBench's next prompt contract remains an unactivated draft (PolicyEngine/policybench#173). It states the state income tax paid for each tax unit or a named convention for estimating withholding; each person's Medicare Part B enrollment and annual net premium; and whether each person's employer withholds the employee share of state paid-leave or disability premiums. Its payroll output definition counts optional employee shares only when the employer withholds them. A later run can score these outputs after its reference builder and a fresh input sweep validate the stated conventions.", "Two corrections in this release move no score. PolicyBench now reports that it collected model responses from {responseStart} to {responseEnd}, 2026, ending on the UTC date of the last answer; release dashboard-data-20260930 gave {previousResponseEnd}, the date of that release. The audit text for a California household head's Medicaid eligibility now explains the head's income test as the engine applies it: California's ${medicaidDisregard} monthly income disregard, which takes the place of SSI's ${ssiExclusion} general income exclusion, brings countable income under the limit. The earlier text said the engine subtracts a Medicare Part B premium from income, which it does not." ], diff --git a/app/src/notes/2026-10-10-claude-haiku-5-5-joins-the-board.json b/app/src/notes/2026-10-10-claude-haiku-5-5-joins-the-board.json index 62a84f3b..c7dd8206 100644 --- a/app/src/notes/2026-10-10-claude-haiku-5-5-joins-the-board.json +++ b/app/src/notes/2026-10-10-claude-haiku-5-5-joins-the-board.json @@ -3,12 +3,12 @@ "date": "2026-10-10", "title": "Claude Haiku 5.5 joins the board as the references move to policyengine-us 2.38.6", "paragraphs": [ - "Claude Haiku 5.5 joined the board on {releaseDate}, bringing it to {nModels} models. It gets {haikuExact}% of answers right, weighted by household impact, {haikuGain} points above Claude Haiku 4.5 ({haiku45Exact}%, #{haiku45Rank}), and ranks #{haikuRank} of {nModels}. PolicyBench counts a dollar amount as right when it falls within $1 of PolicyEngine's reference, and an eligibility answer as right when it matches. GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%).", + "Claude Haiku 5.5 joined the board on {releaseDate}, bringing it to {nModels} models. It gets {haikuExact}% of answers right, weighted by household impact, {haikuGain} points above Claude Haiku 4.5 ({haiku45Exact}%, #{haiku45Rank}), and ranks #{haikuRank} of {nModels}. GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%). PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches.", "Claude Haiku 5.5 cost ${haikuCost} a household, the least of any row with a cost above zero; Claude Haiku 4.5 cost ${haiku45Cost}. Unlike Claude Opus 5.5 and Claude Sonnet 5.5, its API accepts a forced tool call, so its row takes the board's forced answer tool. A probe with that request returned no thinking block, where a request without the forced tool did, so the row answers without extended thinking, as Claude Opus 5's does. Re-run with tool_choice: auto, it gets {haikuAutoExact}%, which would rank #{haikuAutoRank}.", "The release also moves PolicyBench's references from policyengine-us {previousEngine} to {engineVersion}, a release that fixes the engine defects behind {restored:words} outputs. The previous release excluded {newlyScored:words} of them, and they return to scoring at the new version's values. The other {restoredRuled:words} are the October 6 defects below, which stay scored at the new version's values. Each of the {restored:words} lands within $1 of the corrected value PolicyBench's audits computed for it. Between them they take the upstream fixes for {restoredFixes}. PolicyBench still builds each reference from the stated facts and from law published before it froze the references on 2026-07-03, under the same publication conventions.", "Two rulings of October 6 apply too. PolicyBench's reference adversary flags the scored outputs where many models, or several of the strongest, agree on an answer the reference does not give, and a judge that cannot yet see the engine's derivation works each one from primary law. Its findings led to rulings to exclude {adversaryOutputs:words} outputs. Four were engine defects, in Arizona, Ohio, Colorado and New York; the new version fixes them, so PolicyBench keeps scoring those four instead. The other four are the federal and state income tax of a Pennsylvania and a Missouri household whose dependent earns $45,000 of wages and must file their own return. The output definitions do not say whether the household's income tax includes that return, so PolicyBench stops scoring them. It also stops scoring {louisianaOutputs:words} Louisiana households' state income tax, whose reference rests on a 2026 standard deduction computed from published price indexes; Louisiana published the amount itself on September 28, after the freeze.", "The new version counts Indiana county income tax in local income tax, and the prompt names no county, so PolicyBench stops scoring the local income tax of {indianaOutputs:words} Indiana households. It also counts Idaho's $10 permanent building fund tax in state income tax, which moves one household's reference from ${idahoBefore} to ${idahoAfter}. Every model is now scored on {scoredOutputs} of its {totalOutputs} requested outputs; PolicyBench excludes the other {excluded}.", - "Against the previous release, the exact rate of every one of the {incumbents} earlier models falls, by {fallMin} to {fallMax} points. The cause is the {newlyScored} restored outputs that the previous release did not score, which few models get right: half of them are answered within $1 by at most {newlyScoredHalfAtMost:words} of the {nModels} models, and {newlyScoredUnmatched:words} by none. Had they stayed excluded, every earlier model's rate would have risen, by {riseWithoutMin} to {riseWithoutMax} points." + "Against the previous release, every one of the {incumbents} earlier models falls, by {fallMin} to {fallMax} points. The cause is the {newlyScored} restored outputs that the previous release did not score, which few models get right: half of them are answered within $1 by at most {newlyScoredHalfAtMost:words} of the {nModels} models, and {newlyScoredUnmatched:words} by none. Had they stayed excluded, every earlier model's rate would have risen, by {riseWithoutMin} to {riseWithoutMax} points." ], "facts": { "releaseDate": "2026-10-10", diff --git a/app/src/notes/2026-10-13-muse-wonder-joins-the-board.json b/app/src/notes/2026-10-13-muse-wonder-joins-the-board.json index 8dcadc1c..24db92f0 100644 --- a/app/src/notes/2026-10-13-muse-wonder-joins-the-board.json +++ b/app/src/notes/2026-10-13-muse-wonder-joins-the-board.json @@ -3,10 +3,10 @@ "date": "2026-10-13", "title": "Meta's Muse Wonder joins the board as the references move to policyengine-us 2.41.0", "paragraphs": [ - "Meta's Muse Wonder joined the board on {releaseDate}, the day after Meta released it, bringing the board to {nModels} models. It is the first Meta model on the board. It gets {museExact}% of answers right, weighted by household impact, and ranks #{museRank} of {nModels}. PolicyBench counts a dollar amount as right when it falls within $1 of PolicyEngine's reference, and an eligibility answer as right when it matches. On the bounded score, which gives partial credit for amounts close to the reference, it ranks #{museBoundedRank}, behind only GPT-6 Sol and Claude Opus 5.5. GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%).", + "Muse Wonder joined the board on {releaseDate}, the day after Meta released it, bringing the board to {nModels} models, the first of them from Meta. It gets {museExact}% of answers right, weighted by household impact, and ranks #{museRank} of {nModels}. On the bounded score, which gives partial credit for amounts close to the reference, Muse Wonder ranks #{museBoundedRank}, behind only GPT-6 Sol and Claude Opus 5.5. GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%). PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches.", "Meta released Muse Wonder Full the same day, but only to Muse Code subscribers, with no API. PolicyBench runs every model through its provider's API under one protocol, so Muse Wonder Full is not on the board. PolicyBench reaches Muse Wonder through OpenRouter, whose one endpoint for it is Meta's own API. That API accepts only an automatic tool choice and rejects a forced tool call, so the row answers as a JSON object, as Claude Opus 5.5's and Claude Sonnet 5.5's do, and reasons at Meta's default setting.", "The release also moves PolicyBench's references from policyengine-us {previousEngine} to {engineVersion}, the first release with every fix the previous release was waiting on. Of the outputs the previous release excluded, {restored:words} return to scoring at the new version's values, each within $1 of the corrected value PolicyBench's audits computed for it. Between them they take the upstream fixes for {restoredFixes}. The new version also fixes the defect behind a SNAP output, the heat-and-eat utility allowance, but that output stays excluded because the stated facts do not decide it. No reference PolicyBench already scored changes, and nothing is newly excluded. Every model is now scored on {scoredOutputs} of its {totalOutputs} requested outputs; PolicyBench excludes the other {excluded}.", - "Against the previous release, the exact rate of every one of the {incumbents} earlier models falls, by {fallMin} to {fallMax} points. The cause is the {restored:words} restored outputs, which few models get right: half of them are answered within $1 by at most {restoredHalfAtMost:words} of the {nModels} models, and {restoredUnmatched:words} by none. Muse Wonder answers {museRestored:words} of them within $1. Without them, every earlier model's rate would be the previous release's." + "Against the previous release, every one of the {incumbents} earlier models falls, by {fallMin} to {fallMax} points. The cause is the {restored:words} restored outputs, which few models get right: half of them are answered within $1 by at most {restoredHalfAtMost:words} of the {nModels} models, and {restoredUnmatched:words} by none. Muse Wonder answers {museRestored:words} of them within $1. Without them, every earlier model's rate would be the previous release's." ], "facts": { "releaseDate": "2026-10-13", diff --git a/app/src/notes/2026-10-14-gemini-4-argon-joins-the-board.json b/app/src/notes/2026-10-14-gemini-4-argon-joins-the-board.json index a67c0b24..0c7ae06f 100644 --- a/app/src/notes/2026-10-14-gemini-4-argon-joins-the-board.json +++ b/app/src/notes/2026-10-14-gemini-4-argon-joins-the-board.json @@ -3,7 +3,7 @@ "date": "2026-10-14", "title": "Google's Gemini 4 Argon and Meta's Muse Spark 1.3 join the board", "paragraphs": [ - "Google's Gemini 4 Argon joined the board on {releaseDate}, the day after Google opened it to the public Gemini API, and Meta's Muse Spark 1.3 joined with it, bringing the board to {nModels} models. Gemini 4 Argon gets {argonExact}% of answers right, weighted by household impact, and ranks #{argonRank} of {nModels}, the highest of any Gemini model; Gemini 3.8 Flash, the previous best, gets {flashExact}% (#{flashRank}). PolicyBench counts a dollar amount as right when it falls within $1 of PolicyEngine's reference, and an eligibility answer as right when it matches. Muse Spark 1.3 gets {sparkExact}% (#{sparkRank}), behind Meta's Muse Wonder ({museExact}%, #{museRank}), but on the bounded score, which gives partial credit for amounts close to the reference, it ranks #{sparkBoundedRank}. GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%).", + "Google's Gemini 4 Argon joined the board on {releaseDate}, the day after Google opened it to the public Gemini API, and Meta's Muse Spark 1.3 joined with it, bringing the board to {nModels} models. Gemini 4 Argon gets {argonExact}% of answers right, weighted by household impact, and ranks #{argonRank} of {nModels}, the highest of any Gemini model; Gemini 3.8 Flash, the previous best, gets {flashExact}% (#{flashRank}). Muse Spark 1.3 gets {sparkExact}% (#{sparkRank}), behind Meta's Muse Wonder ({museExact}%, #{museRank}), but on the bounded score, which gives partial credit for amounts close to the reference, it ranks #{sparkBoundedRank}. GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%). PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches.", "Google announced Gemini 4 Argon on September 30 for a group of partners and released it on the Gemini API on October 13. PolicyBench runs it there, as it does every Gemini model. The API accepts a forced answer tool, so the row answers through the board's tool and reasons at Google's default setting. A forced tool call suppresses some models' reasoning, as it does Claude Opus 5's, so PolicyBench fixed a test before looking at the results and ran all {households} households both ways. With the forced tool Gemini 4 Argon returned reasoning tokens on {forcedReasoning} of them, and on {boardReasoning} in the board's own run; with an automatic tool choice, on {autoReasoning}. The test found no significant reduction, so the board keeps the forced tool and publishes no second row. At Google's introductory price of $2 per million input tokens and $10 per million output tokens, which doubles on December 1, the {households} households cost ${argonCost}.", "Muse Spark 1.3, which Meta released on September 2, is Meta's second model on the board. PolicyBench reaches it through OpenRouter, whose one endpoint for it is Meta's own API, as it does Muse Wonder. That API rejects a forced tool call, so the row answers as a JSON object and reasons at Meta's default setting.", "Nothing else on the board moves. The newest policyengine-us release, {checkedEngine}, gives the same value as {engineVersion} for every output, so the references stay where they were, and every model is still scored on {scoredOutputs} of its {totalOutputs} requested outputs. Each of the {incumbents} earlier models keeps its score." diff --git a/app/src/notes/2026-10-20-given-a-policyengine-tool-three-models-score-above-99.json b/app/src/notes/2026-10-20-given-a-policyengine-tool-three-models-score-above-99.json index edde8996..3e999f6a 100644 --- a/app/src/notes/2026-10-20-given-a-policyengine-tool-three-models-score-above-99.json +++ b/app/src/notes/2026-10-20-given-a-policyengine-tool-three-models-score-above-99.json @@ -4,7 +4,7 @@ "title": "Given a tool that runs PolicyEngine, three models score above 99%", "paragraphs": [ "PolicyBench gave three models a tool that runs PolicyEngine on a household the model writes from the prompt's facts, and ran its {households} US households again on October 14 and 15. GPT-6 Sol, which leads the board at {solBoardExact}% of answers right, weighted by household impact, gets {solToolExact}% with the tool. Claude Opus 5.5 goes from {opusBoardExact}% to {opusToolExact}%, and Gemini 4 Argon from {argonBoardExact}% to {argonToolExact}%. The runs are a pilot, not board rows, since the board scores models answering alone, and the result is circular: the tool runs the engine that computes PolicyBench's references, so the scores measure agreement with that engine, not whether it is right.", - "PolicyBench counts a dollar amount as right when it falls within $1 of PolicyEngine's reference, and an eligibility answer as right when it matches, with or without the tool. Most of the answers the tool fixes are dollar amounts that are not zero. Of the {nonzeroAmounts} nonzero amounts, GPT-6 Sol gets {solNonzeroRate}% within $1 on its own and {solNonzeroToolRate}% with the tool; Claude Opus 5.5, {opusNonzeroRate}% and {opusNonzeroToolRate}%; and Gemini 4 Argon, {argonNonzeroRate}% and {argonNonzeroToolRate}%.", + "With or without the tool, PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches. Most of the answers the tool fixes are dollar amounts that are not zero. Of the {nonzeroAmounts} nonzero amounts, GPT-6 Sol gets {solNonzeroRate}% within $1 on its own and {solNonzeroToolRate}% with the tool; Claude Opus 5.5, {opusNonzeroRate}% and {opusNonzeroToolRate}%; and Gemini 4 Argon, {argonNonzeroRate}% and {argonNonzeroToolRate}%.", "With the tool, {toolMisses} answers are wrong across the three runs: {solMisses:words} of GPT-6 Sol's, {opusMisses:words} of Claude Opus 5.5's and {argonMisses:words} of Gemini 4 Argon's. Each traces to the household the model described to the engine, or to the output it read back, rather than to the calculation: in {enteredElsewhere:words} the model had entered a stated fact under another variable, in {leftOut:words} it had left one out, and in {wrongOutput:words} it read back the wrong output. Prompt labels share the blame for {labelMisses:words}: \"employer sponsored insurance premiums\" never says the employer paid them, and \"disability benefits\" never says they are not Social Security. In {solCapMisses:words} of GPT-6 Sol's, it had used all {turnCap:words} calculation turns the pilot allows. GPT-6 Sol is also the only model that turned right answers wrong, {solFlips:words} of them.", "The tool costs more. For the {households} households, GPT-6 Sol costs ${solToolCost} with it and ${solBoardCost} without, Claude Opus 5.5 ${opusToolCost} and ${opusBoardCost}, and Gemini 4 Argon ${argonToolCost} and ${argonBoardCost}.", "The tool applies the conventions PolicyBench uses for its references, so a model that describes a household exactly as the reference does gets the reference's answer. Describing a household is also easier here than it would be from a filer's documents: {labelMatches} of the {statedFacts} facts the prompts state, not counting ages and wages, carry a label that is the engine's own variable name with spaces.", diff --git a/app/src/notes/2026-10-31-gpt-6-1-terra-joins-the-board.json b/app/src/notes/2026-10-31-gpt-6-1-terra-joins-the-board.json index 9b753d7a..fa54ecfd 100644 --- a/app/src/notes/2026-10-31-gpt-6-1-terra-joins-the-board.json +++ b/app/src/notes/2026-10-31-gpt-6-1-terra-joins-the-board.json @@ -3,11 +3,11 @@ "date": "2026-10-31", "title": "OpenAI's GPT-6.1 Terra joins the board at #3 as the references move to policyengine-us 2.60.0", "paragraphs": [ - "PolicyBench added OpenAI's GPT-6.1 Terra to its {snapshotDate} snapshot, the day after OpenAI released it, bringing the board to {nModels} models. It gets {terraExact}% of answers right, weighted by household impact, and ranks #{terraRank} of {nModels}. PolicyBench counts a dollar amount as right when it falls within $1 of PolicyEngine's reference, and an eligibility answer as right when it matches. Only GPT-6 Sol ({solExact}%) and Claude Opus 5.5 ({opusExact}%) score higher, and GPT-5.6 Sol, which also rounds to {sol56Exact}%, is less than a tenth of a point behind. On the bounded score, which gives partial credit for amounts close to the reference, GPT-6.1 Terra ranks #{terraBoundedRank}. GPT-6.1 Sol, which OpenAI released on September 29, gets {sol61Exact}% (#{sol61Rank}).", + "PolicyBench added OpenAI's GPT-6.1 Terra to its {snapshotDate} snapshot, the day after OpenAI released it, bringing the board to {nModels} models. It gets {terraExact}% of answers right, weighted by household impact, and ranks #{terraRank} of {nModels}. Only GPT-6 Sol ({solExact}%) and Claude Opus 5.5 ({opusExact}%) score higher, and GPT-5.6 Sol, which also rounds to {sol56Exact}%, trails by less than a tenth of a point. On the bounded score, which gives partial credit for amounts close to the reference, GPT-6.1 Terra ranks #{terraBoundedRank}. GPT-6.1 Sol, which OpenAI released on September 29, gets {sol61Exact}% (#{sol61Rank}). PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches.", "PolicyBench runs GPT-6.1 Terra as it runs GPT-6.1 Sol, on OpenAI's Responses API with the board's forced answer tool at OpenAI's default reasoning effort, and it returned reasoning tokens on all {households} households. At OpenAI's price of $1 per million input tokens and $5 per million output tokens, the {households} households cost ${terraCost}, against ${sol61Cost} for GPT-6.1 Sol and ${solCost} for GPT-6 Sol. GPT-5.6 Terra, the Terra model before it, gets {terra56Exact}% (#{terra56Rank}) and cost ${terra56Cost}.", "The release also moves PolicyBench's references from policyengine-us {previousEngine} to {engineVersion}, the newest release when PolicyBench began recomputing them on October 23. The move changes {changed:words} scored references, each for law in force before the references were frozen. An elderly Californian who lives alone, whose SNAP was $0, now gets the ${snapMonthly} monthly minimum, ${snap031} a year: California's categorical eligibility for CalFresh applies no net income test, even to a household with an elderly member, and the previous version applied one (policyengine-us #10036). Three households in Pennsylvania and Arkansas have too much income for free or reduced-price school lunch, but those states give students at covered schools a free breakfast, so the output asking whether PolicyEngine returns free school meal support is now yes for each (policyengine-us #10238). The new version models both breakfast laws for a student at a covered school and leaves out their conditions: Pennsylvania reimburses schools for them to the extent funds are available, and Arkansas serves them on request.", "Nothing is newly excluded, and no excluded output returns to scoring, so every model is still scored on {scoredOutputs} of its {totalOutputs} requested outputs. When PolicyBench checked PyPI on {checkedOn}, the newest policyengine-us release was {checkedEngine}, and it gives the same value as {engineVersion} for all {scoredOutputs}; it moves {excludedMoved:words} excluded output, which keeps the value it was decided on.", - "Against the previous release, the exact rates of {fallers:words} of the {incumbents} earlier models fall, by {fallMin} to {fallMax} points, and {risers:words} rise by {rise}. No model answers yes for any of the three households' school meals, and only {snapMatch:words} give the Californian's new SNAP: GPT-5.5, GPT-6 Astra and GPT-6.1 Sol, the three that rise. Claude Sonnet 5.5 falls from #{sonnetThen} to #{sonnetNow}, behind GPT-6.1 Terra and GPT-6 Astra." + "Against the previous release, {fallers:words} of the {incumbents} earlier models fall, by {fallMin} to {fallMax} points, and {risers:words} rise by {rise}. No model answers yes for any of the three households' school meals, and only {snapMatch:words} give the Californian's new SNAP: GPT-5.5, GPT-6 Astra and GPT-6.1 Sol, the three that rise. Claude Sonnet 5.5 falls from #{sonnetThen} to #{sonnetNow}, behind GPT-6.1 Terra and GPT-6 Astra." ], "facts": { "snapshotDate": "2026-10-27", diff --git a/app/tests/notes.test.ts b/app/tests/notes.test.ts index cd456547..926f2fc6 100644 --- a/app/tests/notes.test.ts +++ b/app/tests/notes.test.ts @@ -144,7 +144,7 @@ describe("notes", () => { "It gets 93.1% of answers right, weighted by household impact, and ranks #3 of 51.", ); expect(markup).toContain( - "PolicyBench counts a dollar amount as right when it falls within $1 of PolicyEngine's reference, and an eligibility answer as right when it matches.", + "PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches.", ); expect(markup).toContain("GPT-6.1 Terra ranks #5."); expect(markup).toContain("GPT-6.1 Sol, which OpenAI released on September 29, gets 90.2% (#10)."); @@ -161,7 +161,7 @@ describe("notes", () => { "gives the same value as 2.60.0 for all 1,935; it moves one excluded output", ); expect(markup).toContain( - "the exact rates of 47 of the 50 earlier models fall, by 0.03 to 0.07 points, and three rise by 0.02.", + "Against the previous release, 47 of the 50 earlier models fall, by 0.03 to 0.07 points, and three rise by 0.02.", ); expect(markup).toContain("only three give the Californian's new SNAP"); expect(markup).toContain('href="/model/gpt-6.1-terra"'); @@ -282,12 +282,12 @@ describe("notes", () => { createElement(NoteArticle, { note, titleLevel: "h1" }), ); expect(markup).toContain( - "Meta's Muse Wonder joined the board on 2026-10-13, the day after Meta released it, bringing the board to 48 models.", + "Muse Wonder joined the board on 2026-10-13, the day after Meta released it, bringing the board to 48 models, the first of them from Meta.", ); // Board rates keep one decimal. expect(markup).toContain("It gets 90.1% of answers right"); expect(markup).toContain("and ranks #10 of 48."); - expect(markup).toContain("it ranks #3, behind only GPT-6 Sol and Claude Opus 5.5"); + expect(markup).toContain("Muse Wonder ranks #3, behind only GPT-6 Sol and Claude Opus 5.5"); expect(markup).toContain( "GPT-6 Sol still leads at 95.2%, ahead of Claude Opus 5.5 (94.1%) and GPT-5.6 Sol (93.1%).", ); @@ -315,6 +315,43 @@ describe("notes", () => { expect(markup).not.toMatch(/\{[A-Za-z][A-Za-z0-9]*(?::words)?\}/); }); + test("the rewritten score sentences render", () => { + const expected: Record = { + "2026-09-01-claude-fable-5-1-added": [ + "Anthropic released Claude Fable 5.1 on 2026-09-01, and it joined the board the same day.", + "PolicyBench parsed 1,984 of the model's 1,984 answers.", + ], + "2026-09-05-gpt-6-astra-debuts-second": [ + "PolicyBench scores every row on 1,973 of its 1,984 requested outputs. It excludes the other 11 for every model, since their references depend on an engine input the household facts never list.", + ], + "2026-09-22-gpt-6-sol-debuts-first": [ + "GPT-6 Sol gets 94.2% of answers right, weighted by household impact, 1.3 points above Claude Opus 5.5 (92.9%, #2), and ranks #1 of 42. GPT-5.6 Sol, the previous leader, ranks #3 at 92.7%.", + ], + "2026-09-29-claude-sonnet-5-5-debuts-fifth": [ + "With the new references and the Medicaid exclusion together, every one of the 42 earlier models rises, by 0.14 to 0.83 points.", + ], + "2026-10-10-claude-haiku-5-5-joins-the-board": [ + "Against the previous release, every one of the 46 earlier models falls, by 0.24 to 1.37 points.", + ], + "2026-10-13-muse-wonder-joins-the-board": [ + "Against the previous release, every one of the 47 earlier models falls, by 0.18 to 0.42 points.", + ], + "2026-10-20-given-a-policyengine-tool-three-models-score-above-99": [ + "With or without the tool, PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches.", + ], + "2026-10-31-gpt-6-1-terra-joins-the-board": [ + "and GPT-5.6 Sol, which also rounds to 93.1%, trails by less than a tenth of a point.", + ], + }; + for (const [slug, sentences] of Object.entries(expected)) { + const note = notes.find((n) => n.slug === slug)!; + const markup = renderToStaticMarkup( + createElement(NoteArticle, { note, titleLevel: "h1" }), + ); + for (const sentence of sentences) expect(markup).toContain(sentence); + } + }); + test("a words placeholder spells out whole numbers under ten", () => { // Any note serves as the template; its facts and paragraphs are replaced. const note = { diff --git a/docs/terra/design.md b/docs/terra/design.md index cd0be468..4eb4de91 100644 --- a/docs/terra/design.md +++ b/docs/terra/design.md @@ -141,7 +141,7 @@ Export writes `data-board51.json` and a hash-bound `release-ready.json`. The res ## Lead completion record -- **Result.** GPT-6.1 Terra gets 93.14% of answers within $1, weighted by household impact, #3 of 51 on the 1,935 scored outputs, with every output parsed (bounded score 97.19). Its 100 households cost $1.2532 ($0.0125 each) and took a median 22.4 seconds each, all live calls. It answered on 2026-10-26 between 22:11 and 22:28 UTC (board run 2, input pins b0cfa0db). The board's response window is 2026-06-12 to 2026-10-26 and the snapshot is dated 2026-10-27. +- **Result.** GPT-6.1 Terra gets 93.14% of answers right (dollar amounts within $1 of the reference, eligibility answers matching it), weighted by household impact, #3 of 51 on the 1,935 scored outputs, with every output parsed (bounded score 97.19). Its 100 households cost $1.2532 ($0.0125 each) and took a median 22.4 seconds each, all live calls. It answered on 2026-10-26 between 22:11 and 22:28 UTC (board run 2, input pins b0cfa0db). The board's response window is 2026-06-12 to 2026-10-26 and the snapshot is dated 2026-10-27. - **Judge.** The audit covers 681 cases: 551 verdicts carried over, and 127 cases were re-opened, among them scenario_031's `snap`, one of the four changed references, whose earlier verdict install-references set aside because its prompt re-renders on policyengine-us 2.60.0. The three new cases are the other three changed references, `free_school_meals_eligible` for scenarios 028, 117 and 123. Claude Opus 5.5 judged the 130 on 2026-10-27 under the isolated `setup-token 2` login (`docs/terra/judge_provenance.json`). Scenario 008's `payroll_tax` verdict left out one model in nine judgings in a row; it passed at 14:47 UTC under the per-case schema (7d71b7a0, c5974881). - **Triage.** `restate_gpt61sol_adjudications.py` restated the judge fields of 42 entries, every committed decision on a re-opened case. Triage read 10,999 rows in 990 cases and flagged no reference: `reference-flags.csv` and `unresolved-rows.csv` hold only their headers, so triage needed no decision. Triage also applied 1,522 wording amendments in 117 cases, listed in `us_wording-amendments.json`. Round 3 appended 51 more in seven cases, each answering a finding of release review r2: the reasoning, case note and rows of scenarios 005, 064, 120 and 122, and 42 sentence starts that named the package as "Policyengine-us 2.60.0" or "PolicyEngine 2.60.0". The stage's list then held 1,573. Round 4 appended five more in three cases, each answering a finding of release review r3: scenario_122's case note and claude-fable-5's row, scenario_064's case note and reasoning, and scenario_120's reasoning. The stage's list then held 1,578. - **Export and freeze.** The first export and freeze ran on 2026-10-28 at 2677b116, before the wording review (logs: `~/reviews/policybench-gpt-6-1-terra/freeze-run/`). The release ran again on 2026-10-29 with the review's amendments. Export first ran from 08:03 to 08:07 UTC at 4332ff96, and the freeze dry run refused it: export promoted a fixed list of files and left out the adjudication record that triage amends, so the receipt bound bytes the stage did not hold. Commit 4984698a makes export promote every bound file whose bytes triage changed. Export then ran from 08:12 to 08:15 UTC and staged GPT-6.1 Terra at 93.138833%, #3 of 51 (n = 1,935). The freeze dry run (08:15 to 08:18) validated the release (`RELEASE_TAG`): 51 models, payload 214eed92, references on policyengine-us 2.60.0, 49 exclusions (none new), no adjudications beyond the ruled outputs, 1,522 wording amendments, model responses 2026-06-12 to 2026-10-26, snapshot 2026-10-27, nothing written. The freeze (08:18 to 08:22) rebuilt the snapshot and exited 0, its treatment-fingerprint, reference, analysis, adjudication and manifest checks and `verify_frozen` included. Logs: `~/reviews/policybench-gpt-6-1-terra/freeze-run-r2/`. Round 3 ran the release again on 2026-10-29 with the 51 new amendments (logs: `~/reviews/policybench-gpt-6-1-terra/freeze-run-r3/`). Before it, the lead restored the staged adjudication record to d923dd52's bytes, so that export applied every reasoning amendment from the base text. Export ran from 16:30 to 16:34 UTC at 25d779d7 and staged GPT-6.1 Terra at 93.138833%, #3 of 51 (n = 1,935). The freeze dry run refused it at 16:34 UTC: "stage.json's references_installed is not the installed build's record". Commit 25d779d7 had written the CRP ruling into `docs/terra/spec.json`, whose hash install-references binds in `stage.json`. Commit 3654e5d5 restored the bound spec and moved the ruling to this file. The freeze dry run (16:38 to 16:42) then validated the release: 51 models, payload 5de82780, references on policyengine-us 2.60.0, 49 exclusions (none new), no adjudications beyond the ruled outputs, 1,573 wording amendments, model responses 2026-06-12 to 2026-10-26, snapshot 2026-10-27, nothing written. The freeze (16:42 to 16:52) rebuilt the snapshot and exited 0. Round 4 ran the release again on 2026-10-30 with the five new amendments (logs: `~/reviews/policybench-gpt-6-1-terra/freeze-run-r4/`), after the lead again restored the staged adjudication record to d923dd52's bytes. Export ran from 02:13 to 02:16 UTC at 162c1666 and staged GPT-6.1 Terra at 93.138833%, #3 of 51 (n = 1,935). The freeze dry run (02:16 to 02:19) validated the release: 51 models, payload 67cb0d14, references on policyengine-us 2.60.0, 49 exclusions (none new), no adjudications beyond the ruled outputs, 1,578 wording amendments, model responses 2026-06-12 to 2026-10-26, snapshot 2026-10-27, nothing written. The freeze (02:19 to 02:24) rebuilt the snapshot and exited 0. diff --git a/tests/test_notes.py b/tests/test_notes.py index bcdba753..70a74236 100644 --- a/tests/test_notes.py +++ b/tests/test_notes.py @@ -3724,9 +3724,9 @@ def pin(*sentences: str) -> None: # The drift's two causes, as the drift baseline test rebuilds them. test_previous_release_scores_rebuild_from_this_snapshot() pin( - "Together, the new references and the Medicaid exclusion raise the exact " - "rate of every one of the {incumbents} earlier models, by {driftMin} to " - "{driftMax} points.", + "With the new references and the Medicaid exclusion together, every one " + "of the {incumbents} earlier models rises, by {driftMin} to {driftMax} " + "points.", "None of the {incumbents} matched any of the {newExclusions:words} federal " "outputs now excluded, and all {incumbents} had matched the Arizona " "household's old ${azBefore} SNAP reference, which none matches now.", @@ -6068,7 +6068,7 @@ def test_release_20261027_note_claims() -> None: assert (date.fromisoformat(note["boardSnapshot"]) - released).days == 1 assert MODEL_RELEASE_DATES["gpt-6.1-sol"] == "2026-09-29" # "Only GPT-6 Sol and Claude Opus 5.5 score higher", and GPT-5.6 Sol, at - # the same rounded rate, is less than a tenth of a point behind. + # the same rounded rate, trails by less than a tenth of a point. assert names[:4] == ["gpt-6-sol", "claude-opus-5.5", TERRA, "gpt-5.6-sol"] assert facts["sol56Exact"] == facts["terraExact"] assert 0 < exact[TERRA] - exact["gpt-5.6-sol"] < 0.1 @@ -6172,8 +6172,8 @@ def test_release_20261027_note_claims() -> None: SCORE_DEFINITION = ( "PolicyBench counts a dollar amount as right when it falls within $1 of " - "PolicyEngine's reference, and an eligibility answer as right when it " - "matches." + "the reference PolicyEngine computes, and an eligibility answer as right " + "when it matches." ) @@ -6199,17 +6199,20 @@ def test_no_note_gives_the_score_as_answers_within_one_dollar(): pending = {"2026-10-19-america-gov-scores-level-with-gpt-5-5"} for path in sorted(NOTES_DIR.glob("*.json")): note = _load_json(path) + text = " ".join([note["title"], *note["paragraphs"]]) if note["slug"] in pending: + # Fails once #254 lands, so whichever of the two PRs merges + # second drops the exemption. + assert "answers within $1" in text, (path.name, "drop from pending") continue - text = " ".join([note["title"], *note["paragraphs"]]) for phrase in ("answers within $1", "of answers within"): assert phrase not in text, (path.name, phrase) - # A note that gives a score as answers right also says what - # PolicyBench counts as right. + # A note that gives a score as answers right also says, in the same + # words as every other note, what PolicyBench counts as right. if "of answers right" in text: - for clause in ( - "PolicyBench counts a dollar amount as right when it falls within " - "$1 of PolicyEngine's reference", - "an eligibility answer as right when it matches", - ): - assert clause in text, (path.name, clause) + assert SCORE_DEFINITION in text, path.name + # Paragraph 0 doubles as the page description and the list blurb, so + # the news leads and the definition closes it, after every sentence + # that leans on an earlier one. + if SCORE_DEFINITION in note["paragraphs"][0]: + assert note["paragraphs"][0].endswith(SCORE_DEFINITION), path.name From 3c6a06e4f0384d81dde65975db77a22f97c25f15 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 3 Nov 2026 00:00:34 -0500 Subject: [PATCH 06/14] Let the America.gov note state its weighting after the definition #254 gives the America.gov note the same definition sentence. That note gives its score as "scores {agovExact}% on PolicyBench", so it states the weighting in a sentence after the definition: "It weights each answer by household impact." Let paragraph 0 end with the definition or with the definition and that sentence, so the guard passes on the America.gov note once whichever PR merges second drops its exemption. Co-Authored-By: Claude Opus 5.5 --- tests/test_notes.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/test_notes.py b/tests/test_notes.py index 70a74236..c99350bb 100644 --- a/tests/test_notes.py +++ b/tests/test_notes.py @@ -6175,6 +6175,10 @@ def test_release_20261027_note_claims() -> None: "the reference PolicyEngine computes, and an eligibility answer as right " "when it matches." ) +# The America.gov note (#254) gives its score as "scores {agovExact}% on +# PolicyBench", without "weighted by household impact", so it states the +# weighting in a sentence after the definition. +SCORE_WEIGHTING = "It weights each answer by household impact." def _assert_score_definition() -> None: @@ -6213,6 +6217,8 @@ def test_no_note_gives_the_score_as_answers_within_one_dollar(): assert SCORE_DEFINITION in text, path.name # Paragraph 0 doubles as the page description and the list blurb, so # the news leads and the definition closes it, after every sentence - # that leans on an earlier one. - if SCORE_DEFINITION in note["paragraphs"][0]: - assert note["paragraphs"][0].endswith(SCORE_DEFINITION), path.name + # that leans on an earlier one. Only the weighting may follow it. + opening = note["paragraphs"][0] + if SCORE_DEFINITION in opening: + tail = opening.split(SCORE_DEFINITION, 1)[1] + assert tail in ("", f" {SCORE_WEIGHTING}"), path.name From f00bebd23aa668cd17a2c4217c9d3b3cfed3857e Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 3 Nov 2026 00:45:49 -0500 Subject: [PATCH 07/14] Name the leader's metric after a bounded rank, and state the weighting actively (review r2) In the October 13 and 14 openings, "GPT-6 Sol still leads at 95.2%" came right after a bounded-score sentence, so it read as a bounded rate. It now says "95.2% of answers right", and October 31's GPT-6.1 Sol sentence does the same. Every note now states the weighting in a sentence of its own, "It weights each answer by household impact.", after the definition, as #254's note does. It replaces the participle "weighted by household impact". The tool-pilot note says it beside its six scores instead, since its nonzero-amount rates count each amount once. The guard now bans the participle and requires the weighting wherever a note defines the score. An opening that holds the definition must end with it and the weighting. Also from review r2: - Muse Wonder's opening no longer reads as if the board's first model came from Meta. - Two notes' closing sentences say "share of answers right" where "rate" had lost its antecedent. - September 29 gives the drift's causes the verb. - September 5 says "every model" in place of "every row". - September 1 folds its parse count into the opening sentence. - October 14 splits at its semicolon. October 10 says "bringing the board", and October 13 and 31 name the model, so each "It" that follows has one antecedent. app/tests/notes.test.ts pins each changed sentence as it renders, and each of 16 mutations fails a test. Co-Authored-By: Claude Opus 5.5 --- .../2026-09-01-claude-fable-5-1-added.json | 2 +- .../2026-09-05-gpt-6-astra-debuts-second.json | 2 +- .../2026-09-22-gpt-6-sol-debuts-first.json | 2 +- ...-09-29-claude-sonnet-5-5-debuts-fifth.json | 4 +- ...bench-stops-scoring-eight-tax-outputs.json | 2 +- ...0-10-claude-haiku-5-5-joins-the-board.json | 4 +- ...026-10-13-muse-wonder-joins-the-board.json | 4 +- ...-10-14-gemini-4-argon-joins-the-board.json | 2 +- ...gine-tool-three-models-score-above-99.json | 2 +- ...6-10-31-gpt-6-1-terra-joins-the-board.json | 2 +- app/tests/notes.test.ts | 67 ++++++++++++++----- tests/test_notes.py | 45 +++++++------ 12 files changed, 88 insertions(+), 50 deletions(-) diff --git a/app/src/notes/2026-09-01-claude-fable-5-1-added.json b/app/src/notes/2026-09-01-claude-fable-5-1-added.json index 5d2220d2..511d1f32 100644 --- a/app/src/notes/2026-09-01-claude-fable-5-1-added.json +++ b/app/src/notes/2026-09-01-claude-fable-5-1-added.json @@ -3,7 +3,7 @@ "date": "2026-09-01", "title": "Claude Fable 5.1 added", "paragraphs": [ - "Anthropic released Claude Fable 5.1 on 2026-09-01, and it joined the board the same day. It gets {exactRate}% of answers right, weighted by household impact, and ranks #{rank} of {nModels}. PolicyBench parsed {parsed} of the model's {answers} answers. PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches.", + "Anthropic released Claude Fable 5.1 on 2026-09-01, and PolicyBench added it to the board the same day and parsed {parsed} of the model's {answers} answers. Claude Fable 5.1 gets {exactRate}% of answers right and ranks #{rank} of {nModels}. PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches. It weights each answer by household impact.", "The model's API rejects forced tool calls, which is how the board collects most models' answers, so its row answers as a JSON object and reasons at the provider default. The serving-configuration table in the paper records the transport for every row.", "Sensitivity: with the answer tool declared under tool_choice auto, Fable 5.1 scores {autoRate}% (would rank #{autoRank}). Under the same request Claude Fable 5 scores {fable5AutoRate}%. Fable 5's board row ({fable5BoardRate}%) ran chunked under a forced tool call, which switches off its thinking, so the {boardGap}-point gap between the two board rows is mostly request shape.", "The next board version moves every model to tool_choice auto so each provider's default reasoning engages under the recorded request shape (policybench#139)." diff --git a/app/src/notes/2026-09-05-gpt-6-astra-debuts-second.json b/app/src/notes/2026-09-05-gpt-6-astra-debuts-second.json index b028bd42..e8f1e4da 100644 --- a/app/src/notes/2026-09-05-gpt-6-astra-debuts-second.json +++ b/app/src/notes/2026-09-05-gpt-6-astra-debuts-second.json @@ -3,7 +3,7 @@ "date": "2026-09-05", "title": "GPT-6 Astra debuts second: the two rules it invented", "paragraphs": [ - "GPT-6 Astra reached the API on 2026-09-04 and joined the board on 2026-09-05. It gets {astraExact}% of answers right, weighted by household impact, and ranks #{astraRank} of {nModels}, behind GPT-5.6 Sol ({solExact}%) and ahead of Claude Fable 5.1 ({fableExact}%, #{fableRank}). PolicyBench scores every row on {scoredOutputs} of its {totalOutputs} requested outputs. It excludes the other {excludedOutputs} for every model, since their references depend on an engine input the household facts never list. PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches.", + "GPT-6 Astra reached the API on 2026-09-04 and joined the board on 2026-09-05. It gets {astraExact}% of answers right and ranks #{astraRank} of {nModels}, behind GPT-5.6 Sol ({solExact}%) and ahead of Claude Fable 5.1 ({fableExact}%, #{fableRank}). PolicyBench scores every model on {scoredOutputs} of its {totalOutputs} requested outputs and excludes the other {excludedOutputs}, since their references depend on an engine input the household facts never list. PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches. It weights each answer by household impact.", "Row by row, the two OpenAI models agree far more than they differ. Both are right on {bothRight} scored outputs and both wrong on {bothWrong}. The gap is {astraOnlyMisses} outputs Astra misses and Sol gets right, against {solOnlyMisses} the other way.", "Astra's solo misses cluster in two invented rules. In {esiRows} rows it treats a listed employer-sponsored insurance premium as a pre-tax salary reduction and subtracts it from wages before computing payroll tax, federal income tax or state income tax; the household facts state gross wages, and the reference keeps them. In {medicareRows} rows it marks a household member under 65 who is flagged disabled as Medicare-eligible; the reference grants Medicare before 65 only with 24 months of Social Security disability receipt recorded, and none of these households carry that input. The remaining {otherRows} solo misses are one-off rule errors with no shared mechanism. One further Medicare row of the same kind, on scenario_074, sits on an excluded output and is outside every count here.", "The judge's row notes name an employer-sponsored insurance premium on {astraEsiMentions} of Astra's annotated rows and on {solEsiMentions} of Sol's. Sol netted the same premium once, on scenario_045, which cost it {solEsiRows} of its {solOnlyMisses} solo misses; the rest of Sol's solo misses share no mechanism.", diff --git a/app/src/notes/2026-09-22-gpt-6-sol-debuts-first.json b/app/src/notes/2026-09-22-gpt-6-sol-debuts-first.json index eed63db2..0fbbc311 100644 --- a/app/src/notes/2026-09-22-gpt-6-sol-debuts-first.json +++ b/app/src/notes/2026-09-22-gpt-6-sol-debuts-first.json @@ -3,7 +3,7 @@ "date": "2026-09-22", "title": "GPT-6 Sol debuts first, Claude Opus 5.5 second", "paragraphs": [ - "GPT-6 Sol, GPT-6 Luna and Claude Opus 5.5 joined the board on 2026-09-22. GPT-6 Sol gets {solExact}% of answers right, weighted by household impact, {solLead} points above Claude Opus 5.5 ({opusExact}%, #{opusRank}), and ranks #{solRank} of {nModels}. GPT-5.6 Sol, the previous leader, ranks #{sol56Rank} at {sol56Exact}%. PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches.", + "GPT-6 Sol, GPT-6 Luna and Claude Opus 5.5 joined the board on 2026-09-22. GPT-6 Sol gets {solExact}% of answers right, {solLead} points above Claude Opus 5.5 ({opusExact}%, #{opusRank}), and ranks #{solRank} of {nModels}. GPT-5.6 Sol, the previous leader, ranks #{sol56Rank} at {sol56Exact}%. PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches. It weights each answer by household impact.", "GPT-6 Luna scores {lunaExact}% (#{lunaRank}) at ${lunaCost} a household, against ${solCost} for GPT-6 Sol and ${opusCost} for Claude Opus 5.5.", "Claude Opus 5.5's API rejects forced tool calls, as Claude Fable 5.1's does, so its row answers as a JSON object and reasons at the provider default. Claude Opus 5's board row runs the forced tool call, which switches that model's thinking off, and scores {opus5BoardExact}%; its tool_choice auto re-run, where it reasons, scores {opus5AutoExact}%. The two Opus rows differ in serving shape as well as in model, so the {opusGap}-point gap between them is not a measure of the model change alone.", "Every row is scored on {scoredOutputs} of its {totalOutputs} requested outputs. This release regenerated {regenerated} references and excludes {excluded} outputs from scoring for every model; the reference audit note explains both.", diff --git a/app/src/notes/2026-09-29-claude-sonnet-5-5-debuts-fifth.json b/app/src/notes/2026-09-29-claude-sonnet-5-5-debuts-fifth.json index 9f3a715b..145f1568 100644 --- a/app/src/notes/2026-09-29-claude-sonnet-5-5-debuts-fifth.json +++ b/app/src/notes/2026-09-29-claude-sonnet-5-5-debuts-fifth.json @@ -3,13 +3,13 @@ "date": "2026-09-29", "title": "Claude Sonnet 5.5 debuts fifth as the references move to policyengine-us 2.15.17", "paragraphs": [ - "Claude Sonnet 5.5, Grok 4.7 and DeepSeek V4.1 Flash joined the board on 2026-09-29, bringing it to {nModels} models. Claude Sonnet 5.5 gets {sonnetExact}% of answers right, weighted by household impact, and ranks #{sonnetRank} of {nModels}. GPT-6 Luna, #{lunaRank}, also rounds to {lunaExact}% and sits about level with Claude Sonnet 5.5, less than {lunaSonnetGapBelow} points ahead. Grok 4.7 scores {grokExact}% (#{grokRank}) and DeepSeek V4.1 Flash {flashExact}% (#{flashRank}). GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%). PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches.", + "Claude Sonnet 5.5, Grok 4.7 and DeepSeek V4.1 Flash joined the board on 2026-09-29, bringing it to {nModels} models. Claude Sonnet 5.5 gets {sonnetExact}% of answers right and ranks #{sonnetRank} of {nModels}. GPT-6 Luna, #{lunaRank}, also rounds to {lunaExact}% and sits about level with Claude Sonnet 5.5, less than {lunaSonnetGapBelow} points ahead. Grok 4.7 scores {grokExact}% (#{grokRank}) and DeepSeek V4.1 Flash {flashExact}% (#{flashRank}). GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%). PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches. It weights each answer by household impact.", "Claude Sonnet 5.5 costs ${sonnetCost} a household, against ${lunaCost} for GPT-6 Luna. DeepSeek V4.1 Flash costs ${flashCost} at DeepSeek's peak list price, and Grok 4.7 costs ${grokCost}.", "Claude Sonnet 5.5's API rejects forced tool calls, as Claude Opus 5.5's and Claude Fable 5.1's do, so its row answers as a JSON object and reasons with adaptive thinking, the provider default. Grok 4.7 answers through the forced tool call with a {grokTimeoutSeconds}-second request timeout, because its whole-household onboarding probe ran past a {grokOnboardingTimeoutSeconds}-second timeout. DeepSeek serves V4.1 Flash under the alias deepseek-flash, and all {flashAnswers} of the row's answers report that alias and one system fingerprint; none reports a version. PolicyBench labels the row from DeepSeek's September 10 release note, which put V4.1 Flash on that alias.", "The release also moves PolicyBench's scored references from policyengine-us {previousEngine} to {engineVersion}, the newest release when PolicyBench began sweeping the references on 2026-09-29 (uploaded at {engineUploadedUtc} UTC). The newer version includes the {upstreamFixed:words} upstream fixes that PolicyBench applied as sandbox fixes for the September 22 references, and it encodes law the older version lacked. PolicyBench still builds each scored reference from the stated facts and law published before it froze the references on 2026-07-03, so it ported its {conventions:words} publication conventions to the new version. With those conventions and an adapter that keeps Maryland county tax out of state income tax, policyengine-us {checkEngine}, the newest release when PolicyBench checked PyPI on 2026-09-29 at {checkPypiReadUtc} UTC, gives the same value as {engineVersion} for all {totalOutputs} outputs.", "The move changes {scoredChanges:words} scored references. New Jersey's child tax credit schedule for 2026 to 2028, which the state approved on June 30, raises one household's state refundable credits from ${njBefore} to ${njAfter}. Arizona raised the income limit for its broad-based categorical eligibility from {azLimitBefore}% to {azLimitAfter}% of the poverty guideline effective March, and the new limit gives an Arizona household ${azAfter} of SNAP where the reference was ${azBefore}. policyengine-us now counts child support received as school-meal income, which ends a Pennsylvania household's eligibility for reduced-price meals. A fix to the rounding in New York's Empire State child credit phase-out (policyengine-us #9425) raises a New York household's state refundable credits from ${nyBefore} to ${nyAfter}.", "PolicyBench stops scoring {newExclusions:words} federal income tax outputs. policyengine-us {engineVersion} counts the whole of a listed state and local tax refund as income. Federal law counts the refund only to the extent the refunded tax lowered the household's federal tax in the year the household paid it, and the prompts do not say whether it did. PolicyBench also stops scoring one California household head's Medicaid eligibility, which its re-review of the household's excluded SNAP output flagged. On disability, the prompt says only that the head is disabled. The head's income, {medicaidIncomePercent}% of the poverty guideline, is above the {expansionLimitPercent}% limit for the adult expansion group, so only a disability pathway leads to Medi-Cal, California's Medicaid program. Medi-Cal's Working Disabled Program requires SSI's definition of disability, which the prompt does not state, and policyengine-us {engineVersion} tests the general disability flag instead. The same unstated fact already keeps the household's SNAP out of scoring. PolicyBench now scores every model on {scoredOutputs} of its {totalOutputs} requested outputs and excludes {excluded}. Another {withinTolerance:words} references move by less than $1, and PolicyBench re-reviewed the {rechecked} excluded outputs whose values moved; all stay excluded.", - "With the new references and the Medicaid exclusion together, every one of the {incumbents} earlier models rises, by {driftMin} to {driftMax} points. None of the {incumbents} matched any of the {newExclusions:words} federal outputs now excluded, and all {incumbents} had matched the Arizona household's old ${azBefore} SNAP reference, which none matches now. On the Medicaid output, {medicaidMatched} of the {incumbents} had matched the reference and {medicaidMissed} had not. Among the {incumbents}, {riserCount:words} models move up in the order: {riserOne} moves above {riserOnePassed}, {riserTwo} above {riserTwoPassed}, {riserThree} above {riserThreePassed}, {riserFour} above {riserFourPassed}, and {riserFive} above {riserFivePassed}. Every other pair keeps its order.", + "Together, the new references and the Medicaid exclusion lift every one of the {incumbents} earlier models, by {driftMin} to {driftMax} points. None of the {incumbents} matched any of the {newExclusions:words} federal outputs now excluded, and all {incumbents} had matched the Arizona household's old ${azBefore} SNAP reference, which none matches now. On the Medicaid output, {medicaidMatched} of the {incumbents} had matched the reference and {medicaidMissed} had not. Among the {incumbents}, {riserCount:words} models move up in the order: {riserOne} moves above {riserOnePassed}, {riserTwo} above {riserTwoPassed}, {riserThree} above {riserThreePassed}, {riserFour} above {riserFourPassed}, and {riserFive} above {riserFivePassed}. Every other pair keeps its order.", "The Arizona household's income keeps it from qualifying for SNAP under the program's ordinary tests, so it qualifies only through broad-based categorical eligibility, and all {nModels} models answer $0 for it. PolicyBench's October 5 note on such households counts it among five held back by income, with {bbceIncomeHeldCount:words} in {bbceIncomeHeldStates}. That note's data list every model's answer for those five households and for the {bbceAssetHeldCount:words} held back by savings, including the three models this release adds." ], "facts": { diff --git a/app/src/notes/2026-10-06-policybench-stops-scoring-eight-tax-outputs.json b/app/src/notes/2026-10-06-policybench-stops-scoring-eight-tax-outputs.json index 23240da1..0f45d1cb 100644 --- a/app/src/notes/2026-10-06-policybench-stops-scoring-eight-tax-outputs.json +++ b/app/src/notes/2026-10-06-policybench-stops-scoring-eight-tax-outputs.json @@ -8,7 +8,7 @@ "The Virginia household's head is {va114Age}, and the prompt lists no Medicare enrollment or premium. policyengine-us treats everyone eligible for Medicare as enrolled and counts the standard Part B premium, ${partBPremium} for 2026, as a medical expense. The household itemizes, and Virginia's itemized deductions start from the federal ones, so the premium lowers its Virginia income tax: without it, that output is ${va114Alt} rather than the reference's ${va114Ref}. The premium lowers the household's federal income tax too, and that output's exclusion record names both unstated facts.", "The {payrollOutputs:words} payroll outputs count an employee share of a state paid-leave or disability premium: {optionalPrograms}. In each of these states the premium is the employer's, and the employer may deduct up to a set share of it from wages but need not, so the law requires no amount of the employee. The prompts ask for mandatory employee state payroll taxes, and the references count the largest share each state's published 2026 rates let the employer deduct, without the prompts saying whether it does. Without that share, the outputs are ${mnAlt} in Minnesota, ${coAlt} in Colorado, ${maAlt} in Massachusetts, and ${nyAlt} in New York, against references of ${mnRef}, ${coRef}, ${maRef}, and ${nyRef}. PolicyBench still scores the payroll outputs of {mandatoryScored:words} households in {mandatoryStates}, where the law puts the contribution on the worker and requires the employer to withhold it.", "No model's answer is within $1 of the reference on any of the {incomeTaxOutputs:words} income tax outputs. On the {payrollOutputs:words} payroll outputs, {payrollHits} answers match the reference within $1: {mnHits:words} for Minnesota, {coHits} for Colorado, {maHits:words} for Massachusetts, and {nyHits:words} for New York. Another {payrollAltHits} match the amount without the state share.", - "Every model's share of answers right, weighted by household impact, rises by {riseMin} to {riseMax} points. PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches. GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%). Claude Sonnet 5.5 ({sonnetExact}%) passes GPT-6 Luna ({lunaExact}%) for fourth. Further down, {riserTwo} moves above {riserTwoPassed}, {riserThree} above {riserThreePassed}, and {riserFour} above {riserFourPassed}. Every other pair keeps its order.", + "Every model's share of answers right rises by {riseMin} to {riseMax} points. PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches. It weights each answer by household impact. GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%). Claude Sonnet 5.5 ({sonnetExact}%) passes GPT-6 Luna ({lunaExact}%) for fourth. Further down, {riserTwo} moves above {riserTwoPassed}, {riserThree} above {riserThreePassed}, and {riserFour} above {riserFourPassed}. Every other pair keeps its order.", "PolicyBench's next prompt contract remains an unactivated draft (PolicyEngine/policybench#173). It states the state income tax paid for each tax unit or a named convention for estimating withholding; each person's Medicare Part B enrollment and annual net premium; and whether each person's employer withholds the employee share of state paid-leave or disability premiums. Its payroll output definition counts optional employee shares only when the employer withholds them. A later run can score these outputs after its reference builder and a fresh input sweep validate the stated conventions.", "Two corrections in this release move no score. PolicyBench now reports that it collected model responses from {responseStart} to {responseEnd}, 2026, ending on the UTC date of the last answer; release dashboard-data-20260930 gave {previousResponseEnd}, the date of that release. The audit text for a California household head's Medicaid eligibility now explains the head's income test as the engine applies it: California's ${medicaidDisregard} monthly income disregard, which takes the place of SSI's ${ssiExclusion} general income exclusion, brings countable income under the limit. The earlier text said the engine subtracts a Medicare Part B premium from income, which it does not." ], diff --git a/app/src/notes/2026-10-10-claude-haiku-5-5-joins-the-board.json b/app/src/notes/2026-10-10-claude-haiku-5-5-joins-the-board.json index c7dd8206..5c6fb855 100644 --- a/app/src/notes/2026-10-10-claude-haiku-5-5-joins-the-board.json +++ b/app/src/notes/2026-10-10-claude-haiku-5-5-joins-the-board.json @@ -3,12 +3,12 @@ "date": "2026-10-10", "title": "Claude Haiku 5.5 joins the board as the references move to policyengine-us 2.38.6", "paragraphs": [ - "Claude Haiku 5.5 joined the board on {releaseDate}, bringing it to {nModels} models. It gets {haikuExact}% of answers right, weighted by household impact, {haikuGain} points above Claude Haiku 4.5 ({haiku45Exact}%, #{haiku45Rank}), and ranks #{haikuRank} of {nModels}. GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%). PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches.", + "Claude Haiku 5.5 joined the board on {releaseDate}, bringing the board to {nModels} models. It gets {haikuExact}% of answers right, {haikuGain} points above Claude Haiku 4.5 ({haiku45Exact}%, #{haiku45Rank}), and ranks #{haikuRank} of {nModels}. GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%). PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches. It weights each answer by household impact.", "Claude Haiku 5.5 cost ${haikuCost} a household, the least of any row with a cost above zero; Claude Haiku 4.5 cost ${haiku45Cost}. Unlike Claude Opus 5.5 and Claude Sonnet 5.5, its API accepts a forced tool call, so its row takes the board's forced answer tool. A probe with that request returned no thinking block, where a request without the forced tool did, so the row answers without extended thinking, as Claude Opus 5's does. Re-run with tool_choice: auto, it gets {haikuAutoExact}%, which would rank #{haikuAutoRank}.", "The release also moves PolicyBench's references from policyengine-us {previousEngine} to {engineVersion}, a release that fixes the engine defects behind {restored:words} outputs. The previous release excluded {newlyScored:words} of them, and they return to scoring at the new version's values. The other {restoredRuled:words} are the October 6 defects below, which stay scored at the new version's values. Each of the {restored:words} lands within $1 of the corrected value PolicyBench's audits computed for it. Between them they take the upstream fixes for {restoredFixes}. PolicyBench still builds each reference from the stated facts and from law published before it froze the references on 2026-07-03, under the same publication conventions.", "Two rulings of October 6 apply too. PolicyBench's reference adversary flags the scored outputs where many models, or several of the strongest, agree on an answer the reference does not give, and a judge that cannot yet see the engine's derivation works each one from primary law. Its findings led to rulings to exclude {adversaryOutputs:words} outputs. Four were engine defects, in Arizona, Ohio, Colorado and New York; the new version fixes them, so PolicyBench keeps scoring those four instead. The other four are the federal and state income tax of a Pennsylvania and a Missouri household whose dependent earns $45,000 of wages and must file their own return. The output definitions do not say whether the household's income tax includes that return, so PolicyBench stops scoring them. It also stops scoring {louisianaOutputs:words} Louisiana households' state income tax, whose reference rests on a 2026 standard deduction computed from published price indexes; Louisiana published the amount itself on September 28, after the freeze.", "The new version counts Indiana county income tax in local income tax, and the prompt names no county, so PolicyBench stops scoring the local income tax of {indianaOutputs:words} Indiana households. It also counts Idaho's $10 permanent building fund tax in state income tax, which moves one household's reference from ${idahoBefore} to ${idahoAfter}. Every model is now scored on {scoredOutputs} of its {totalOutputs} requested outputs; PolicyBench excludes the other {excluded}.", - "Against the previous release, every one of the {incumbents} earlier models falls, by {fallMin} to {fallMax} points. The cause is the {newlyScored} restored outputs that the previous release did not score, which few models get right: half of them are answered within $1 by at most {newlyScoredHalfAtMost:words} of the {nModels} models, and {newlyScoredUnmatched:words} by none. Had they stayed excluded, every earlier model's rate would have risen, by {riseWithoutMin} to {riseWithoutMax} points." + "Against the previous release, every one of the {incumbents} earlier models falls, by {fallMin} to {fallMax} points. The cause is the {newlyScored} restored outputs that the previous release did not score, which few models get right: half of them are answered within $1 by at most {newlyScoredHalfAtMost:words} of the {nModels} models, and {newlyScoredUnmatched:words} by none. Had they stayed excluded, every earlier model's share of answers right would have risen, by {riseWithoutMin} to {riseWithoutMax} points." ], "facts": { "releaseDate": "2026-10-10", diff --git a/app/src/notes/2026-10-13-muse-wonder-joins-the-board.json b/app/src/notes/2026-10-13-muse-wonder-joins-the-board.json index 24db92f0..c8bcabb3 100644 --- a/app/src/notes/2026-10-13-muse-wonder-joins-the-board.json +++ b/app/src/notes/2026-10-13-muse-wonder-joins-the-board.json @@ -3,10 +3,10 @@ "date": "2026-10-13", "title": "Meta's Muse Wonder joins the board as the references move to policyengine-us 2.41.0", "paragraphs": [ - "Muse Wonder joined the board on {releaseDate}, the day after Meta released it, bringing the board to {nModels} models, the first of them from Meta. It gets {museExact}% of answers right, weighted by household impact, and ranks #{museRank} of {nModels}. On the bounded score, which gives partial credit for amounts close to the reference, Muse Wonder ranks #{museBoundedRank}, behind only GPT-6 Sol and Claude Opus 5.5. GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%). PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches.", + "PolicyBench added Muse Wonder, its first Meta model, to the board on {releaseDate}, the day after Meta released it, bringing the board to {nModels} models. Muse Wonder gets {museExact}% of answers right and ranks #{museRank} of {nModels}. On the bounded score, which gives partial credit for amounts close to the reference, it ranks #{museBoundedRank}, behind only GPT-6 Sol and Claude Opus 5.5. GPT-6 Sol still leads at {solExact}% of answers right, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%). PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches. It weights each answer by household impact.", "Meta released Muse Wonder Full the same day, but only to Muse Code subscribers, with no API. PolicyBench runs every model through its provider's API under one protocol, so Muse Wonder Full is not on the board. PolicyBench reaches Muse Wonder through OpenRouter, whose one endpoint for it is Meta's own API. That API accepts only an automatic tool choice and rejects a forced tool call, so the row answers as a JSON object, as Claude Opus 5.5's and Claude Sonnet 5.5's do, and reasons at Meta's default setting.", "The release also moves PolicyBench's references from policyengine-us {previousEngine} to {engineVersion}, the first release with every fix the previous release was waiting on. Of the outputs the previous release excluded, {restored:words} return to scoring at the new version's values, each within $1 of the corrected value PolicyBench's audits computed for it. Between them they take the upstream fixes for {restoredFixes}. The new version also fixes the defect behind a SNAP output, the heat-and-eat utility allowance, but that output stays excluded because the stated facts do not decide it. No reference PolicyBench already scored changes, and nothing is newly excluded. Every model is now scored on {scoredOutputs} of its {totalOutputs} requested outputs; PolicyBench excludes the other {excluded}.", - "Against the previous release, every one of the {incumbents} earlier models falls, by {fallMin} to {fallMax} points. The cause is the {restored:words} restored outputs, which few models get right: half of them are answered within $1 by at most {restoredHalfAtMost:words} of the {nModels} models, and {restoredUnmatched:words} by none. Muse Wonder answers {museRestored:words} of them within $1. Without them, every earlier model's rate would be the previous release's." + "Against the previous release, every one of the {incumbents} earlier models falls, by {fallMin} to {fallMax} points. The cause is the {restored:words} restored outputs, which few models get right: half of them are answered within $1 by at most {restoredHalfAtMost:words} of the {nModels} models, and {restoredUnmatched:words} by none. Muse Wonder answers {museRestored:words} of them within $1. Without them, every earlier model would keep its share of answers right from the previous release." ], "facts": { "releaseDate": "2026-10-13", diff --git a/app/src/notes/2026-10-14-gemini-4-argon-joins-the-board.json b/app/src/notes/2026-10-14-gemini-4-argon-joins-the-board.json index 0c7ae06f..75742f93 100644 --- a/app/src/notes/2026-10-14-gemini-4-argon-joins-the-board.json +++ b/app/src/notes/2026-10-14-gemini-4-argon-joins-the-board.json @@ -3,7 +3,7 @@ "date": "2026-10-14", "title": "Google's Gemini 4 Argon and Meta's Muse Spark 1.3 join the board", "paragraphs": [ - "Google's Gemini 4 Argon joined the board on {releaseDate}, the day after Google opened it to the public Gemini API, and Meta's Muse Spark 1.3 joined with it, bringing the board to {nModels} models. Gemini 4 Argon gets {argonExact}% of answers right, weighted by household impact, and ranks #{argonRank} of {nModels}, the highest of any Gemini model; Gemini 3.8 Flash, the previous best, gets {flashExact}% (#{flashRank}). Muse Spark 1.3 gets {sparkExact}% (#{sparkRank}), behind Meta's Muse Wonder ({museExact}%, #{museRank}), but on the bounded score, which gives partial credit for amounts close to the reference, it ranks #{sparkBoundedRank}. GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%). PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches.", + "Google's Gemini 4 Argon joined the board on {releaseDate}, the day after Google opened it to the public Gemini API, and Meta's Muse Spark 1.3 joined with it, bringing the board to {nModels} models. Gemini 4 Argon gets {argonExact}% of answers right and ranks #{argonRank} of {nModels}, the highest of any Gemini model. Gemini 3.8 Flash, the previous best, gets {flashExact}% (#{flashRank}). Muse Spark 1.3 gets {sparkExact}% (#{sparkRank}), behind Meta's Muse Wonder ({museExact}%, #{museRank}), but on the bounded score, which gives partial credit for amounts close to the reference, it ranks #{sparkBoundedRank}. GPT-6 Sol still leads at {solExact}% of answers right, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%). PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches. It weights each answer by household impact.", "Google announced Gemini 4 Argon on September 30 for a group of partners and released it on the Gemini API on October 13. PolicyBench runs it there, as it does every Gemini model. The API accepts a forced answer tool, so the row answers through the board's tool and reasons at Google's default setting. A forced tool call suppresses some models' reasoning, as it does Claude Opus 5's, so PolicyBench fixed a test before looking at the results and ran all {households} households both ways. With the forced tool Gemini 4 Argon returned reasoning tokens on {forcedReasoning} of them, and on {boardReasoning} in the board's own run; with an automatic tool choice, on {autoReasoning}. The test found no significant reduction, so the board keeps the forced tool and publishes no second row. At Google's introductory price of $2 per million input tokens and $10 per million output tokens, which doubles on December 1, the {households} households cost ${argonCost}.", "Muse Spark 1.3, which Meta released on September 2, is Meta's second model on the board. PolicyBench reaches it through OpenRouter, whose one endpoint for it is Meta's own API, as it does Muse Wonder. That API rejects a forced tool call, so the row answers as a JSON object and reasons at Meta's default setting.", "Nothing else on the board moves. The newest policyengine-us release, {checkedEngine}, gives the same value as {engineVersion} for every output, so the references stay where they were, and every model is still scored on {scoredOutputs} of its {totalOutputs} requested outputs. Each of the {incumbents} earlier models keeps its score." diff --git a/app/src/notes/2026-10-20-given-a-policyengine-tool-three-models-score-above-99.json b/app/src/notes/2026-10-20-given-a-policyengine-tool-three-models-score-above-99.json index 3e999f6a..bac176c2 100644 --- a/app/src/notes/2026-10-20-given-a-policyengine-tool-three-models-score-above-99.json +++ b/app/src/notes/2026-10-20-given-a-policyengine-tool-three-models-score-above-99.json @@ -3,7 +3,7 @@ "date": "2026-10-20", "title": "Given a tool that runs PolicyEngine, three models score above 99%", "paragraphs": [ - "PolicyBench gave three models a tool that runs PolicyEngine on a household the model writes from the prompt's facts, and ran its {households} US households again on October 14 and 15. GPT-6 Sol, which leads the board at {solBoardExact}% of answers right, weighted by household impact, gets {solToolExact}% with the tool. Claude Opus 5.5 goes from {opusBoardExact}% to {opusToolExact}%, and Gemini 4 Argon from {argonBoardExact}% to {argonToolExact}%. The runs are a pilot, not board rows, since the board scores models answering alone, and the result is circular: the tool runs the engine that computes PolicyBench's references, so the scores measure agreement with that engine, not whether it is right.", + "PolicyBench gave three models a tool that runs PolicyEngine on a household the model writes from the prompt's facts, and ran its {households} US households again on October 14 and 15. GPT-6 Sol, which leads the board at {solBoardExact}% of answers right, gets {solToolExact}% with the tool. Claude Opus 5.5 goes from {opusBoardExact}% to {opusToolExact}%, and Gemini 4 Argon from {argonBoardExact}% to {argonToolExact}%. PolicyBench weights each answer by household impact, on the board and with the tool. The runs are a pilot, not board rows, since the board scores models answering alone, and the result is circular: the tool runs the engine that computes PolicyBench's references, so the scores measure agreement with that engine, not whether it is right.", "With or without the tool, PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches. Most of the answers the tool fixes are dollar amounts that are not zero. Of the {nonzeroAmounts} nonzero amounts, GPT-6 Sol gets {solNonzeroRate}% within $1 on its own and {solNonzeroToolRate}% with the tool; Claude Opus 5.5, {opusNonzeroRate}% and {opusNonzeroToolRate}%; and Gemini 4 Argon, {argonNonzeroRate}% and {argonNonzeroToolRate}%.", "With the tool, {toolMisses} answers are wrong across the three runs: {solMisses:words} of GPT-6 Sol's, {opusMisses:words} of Claude Opus 5.5's and {argonMisses:words} of Gemini 4 Argon's. Each traces to the household the model described to the engine, or to the output it read back, rather than to the calculation: in {enteredElsewhere:words} the model had entered a stated fact under another variable, in {leftOut:words} it had left one out, and in {wrongOutput:words} it read back the wrong output. Prompt labels share the blame for {labelMisses:words}: \"employer sponsored insurance premiums\" never says the employer paid them, and \"disability benefits\" never says they are not Social Security. In {solCapMisses:words} of GPT-6 Sol's, it had used all {turnCap:words} calculation turns the pilot allows. GPT-6 Sol is also the only model that turned right answers wrong, {solFlips:words} of them.", "The tool costs more. For the {households} households, GPT-6 Sol costs ${solToolCost} with it and ${solBoardCost} without, Claude Opus 5.5 ${opusToolCost} and ${opusBoardCost}, and Gemini 4 Argon ${argonToolCost} and ${argonBoardCost}.", diff --git a/app/src/notes/2026-10-31-gpt-6-1-terra-joins-the-board.json b/app/src/notes/2026-10-31-gpt-6-1-terra-joins-the-board.json index fa54ecfd..e87a0458 100644 --- a/app/src/notes/2026-10-31-gpt-6-1-terra-joins-the-board.json +++ b/app/src/notes/2026-10-31-gpt-6-1-terra-joins-the-board.json @@ -3,7 +3,7 @@ "date": "2026-10-31", "title": "OpenAI's GPT-6.1 Terra joins the board at #3 as the references move to policyengine-us 2.60.0", "paragraphs": [ - "PolicyBench added OpenAI's GPT-6.1 Terra to its {snapshotDate} snapshot, the day after OpenAI released it, bringing the board to {nModels} models. It gets {terraExact}% of answers right, weighted by household impact, and ranks #{terraRank} of {nModels}. Only GPT-6 Sol ({solExact}%) and Claude Opus 5.5 ({opusExact}%) score higher, and GPT-5.6 Sol, which also rounds to {sol56Exact}%, trails by less than a tenth of a point. On the bounded score, which gives partial credit for amounts close to the reference, GPT-6.1 Terra ranks #{terraBoundedRank}. GPT-6.1 Sol, which OpenAI released on September 29, gets {sol61Exact}% (#{sol61Rank}). PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches.", + "PolicyBench added OpenAI's GPT-6.1 Terra to its {snapshotDate} snapshot, the day after OpenAI released it, bringing the board to {nModels} models. GPT-6.1 Terra gets {terraExact}% of answers right and ranks #{terraRank} of {nModels}. Only GPT-6 Sol ({solExact}%) and Claude Opus 5.5 ({opusExact}%) score higher, and GPT-5.6 Sol, which also rounds to {sol56Exact}%, trails by less than a tenth of a point. On the bounded score, which gives partial credit for amounts close to the reference, GPT-6.1 Terra ranks #{terraBoundedRank}. GPT-6.1 Sol, which OpenAI released on September 29, gets {sol61Exact}% of answers right (#{sol61Rank}). PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches. It weights each answer by household impact.", "PolicyBench runs GPT-6.1 Terra as it runs GPT-6.1 Sol, on OpenAI's Responses API with the board's forced answer tool at OpenAI's default reasoning effort, and it returned reasoning tokens on all {households} households. At OpenAI's price of $1 per million input tokens and $5 per million output tokens, the {households} households cost ${terraCost}, against ${sol61Cost} for GPT-6.1 Sol and ${solCost} for GPT-6 Sol. GPT-5.6 Terra, the Terra model before it, gets {terra56Exact}% (#{terra56Rank}) and cost ${terra56Cost}.", "The release also moves PolicyBench's references from policyengine-us {previousEngine} to {engineVersion}, the newest release when PolicyBench began recomputing them on October 23. The move changes {changed:words} scored references, each for law in force before the references were frozen. An elderly Californian who lives alone, whose SNAP was $0, now gets the ${snapMonthly} monthly minimum, ${snap031} a year: California's categorical eligibility for CalFresh applies no net income test, even to a household with an elderly member, and the previous version applied one (policyengine-us #10036). Three households in Pennsylvania and Arkansas have too much income for free or reduced-price school lunch, but those states give students at covered schools a free breakfast, so the output asking whether PolicyEngine returns free school meal support is now yes for each (policyengine-us #10238). The new version models both breakfast laws for a student at a covered school and leaves out their conditions: Pennsylvania reimburses schools for them to the extent funds are available, and Arkansas serves them on request.", "Nothing is newly excluded, and no excluded output returns to scoring, so every model is still scored on {scoredOutputs} of its {totalOutputs} requested outputs. When PolicyBench checked PyPI on {checkedOn}, the newest policyengine-us release was {checkedEngine}, and it gives the same value as {engineVersion} for all {scoredOutputs}; it moves {excludedMoved:words} excluded output, which keeps the value it was decided on.", diff --git a/app/tests/notes.test.ts b/app/tests/notes.test.ts index 926f2fc6..60187705 100644 --- a/app/tests/notes.test.ts +++ b/app/tests/notes.test.ts @@ -13,6 +13,10 @@ import artifact from "../src/data.artifact.json"; import paperSnapshot from "../src/paperSnapshot.json"; describe("notes", () => { + // The score definition and the weighting close each release note's opening. + const closing = + "PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches. It weights each answer by household impact."; + test("the index is newest first", () => { expect(notes.map((note) => note.date)).toEqual( [...notes].map((note) => note.date).sort().reverse(), @@ -141,13 +145,13 @@ describe("notes", () => { ); // Board rates keep one decimal. expect(markup).toContain( - "It gets 93.1% of answers right, weighted by household impact, and ranks #3 of 51.", + "GPT-6.1 Terra gets 93.1% of answers right and ranks #3 of 51.", ); + expect(markup).toContain("GPT-6.1 Terra ranks #5."); expect(markup).toContain( - "PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches.", + "GPT-6.1 Sol, which OpenAI released on September 29, gets 90.2% of answers right (#10).", ); - expect(markup).toContain("GPT-6.1 Terra ranks #5."); - expect(markup).toContain("GPT-6.1 Sol, which OpenAI released on September 29, gets 90.2% (#10)."); + expect(markup).toContain(closing); // Costs keep their cents. expect(markup).toContain( "the 100 households cost $1.25, against $2.30 for GPT-6.1 Sol and $2.71 for GPT-6 Sol.", @@ -180,11 +184,14 @@ describe("notes", () => { createElement(NoteArticle, { note, titleLevel: "h1" }), ); expect(markup).toContain( - "GPT-6 Sol, which leads the board at 95.2% of answers right, weighted by household impact, gets 99.2% with the tool.", + "GPT-6 Sol, which leads the board at 95.2% of answers right, gets 99.2% with the tool.", ); expect(markup).toContain( "Claude Opus 5.5 goes from 94.1% to 99.8%, and Gemini 4 Argon from 88.7% to 99.9%.", ); + expect(markup).toContain( + "PolicyBench weights each answer by household impact, on the board and with the tool.", + ); expect(markup).toContain( "Of the 187 nonzero amounts, GPT-6 Sol gets 67.4% within $1 on its own and 96.8% with the tool", ); @@ -251,13 +258,18 @@ describe("notes", () => { "Google's Gemini 4 Argon joined the board on 2026-10-14, the day after Google opened it to the public Gemini API, and Meta's Muse Spark 1.3 joined with it, bringing the board to 50 models.", ); // Board rates keep one decimal. - expect(markup).toContain("Gemini 4 Argon gets 88.7% of answers right"); - expect(markup).toContain("ranks #12 of 50, the highest of any Gemini model"); - expect(markup).toContain("Gemini 3.8 Flash, the previous best, gets 85.6% (#21)"); + expect(markup).toContain( + "Gemini 4 Argon gets 88.7% of answers right and ranks #12 of 50, the highest of any Gemini model. Gemini 3.8 Flash, the previous best, gets 85.6% (#21).", + ); expect(markup).toContain( "Muse Spark 1.3 gets 87.2% (#18), behind Meta's Muse Wonder (90.1%, #10)", ); expect(markup).toContain("it ranks #6."); + // After the bounded rank, the leader's rate names its metric. + expect(markup).toContain( + "GPT-6 Sol still leads at 95.2% of answers right, ahead of Claude Opus 5.5 (94.1%) and GPT-5.6 Sol (93.1%).", + ); + expect(markup).toContain(closing); expect(markup).toContain( "With the forced tool Gemini 4 Argon returned reasoning tokens on 80 of them, and on 85 in the board's own run; with an automatic tool choice, on 82.", ); @@ -282,15 +294,18 @@ describe("notes", () => { createElement(NoteArticle, { note, titleLevel: "h1" }), ); expect(markup).toContain( - "Muse Wonder joined the board on 2026-10-13, the day after Meta released it, bringing the board to 48 models, the first of them from Meta.", + "PolicyBench added Muse Wonder, its first Meta model, to the board on 2026-10-13, the day after Meta released it, bringing the board to 48 models.", ); // Board rates keep one decimal. - expect(markup).toContain("It gets 90.1% of answers right"); - expect(markup).toContain("and ranks #10 of 48."); - expect(markup).toContain("Muse Wonder ranks #3, behind only GPT-6 Sol and Claude Opus 5.5"); + expect(markup).toContain("Muse Wonder gets 90.1% of answers right and ranks #10 of 48."); + expect(markup).toContain( + "On the bounded score, which gives partial credit for amounts close to the reference, it ranks #3, behind only GPT-6 Sol and Claude Opus 5.5.", + ); + // After the bounded rank, the leader's rate names its metric. expect(markup).toContain( - "GPT-6 Sol still leads at 95.2%, ahead of Claude Opus 5.5 (94.1%) and GPT-5.6 Sol (93.1%).", + "GPT-6 Sol still leads at 95.2% of answers right, ahead of Claude Opus 5.5 (94.1%) and GPT-5.6 Sol (93.1%).", ); + expect(markup).toContain(closing); expect(markup).toContain("so Muse Wonder Full is not on the board"); expect(markup).toContain( "Of the outputs the previous release excluded, nine return to scoring", @@ -308,6 +323,9 @@ describe("notes", () => { expect(markup).toContain( "half of them are answered within $1 by at most one of the 48 models, and four by none. Muse Wonder answers one of them within $1.", ); + expect(markup).toContain( + "Without them, every earlier model would keep its share of answers right from the previous release.", + ); expect(markup).toContain('href="/model/muse-wonder"'); expect(markup).toContain( 'href="https://github.com/PolicyEngine/policybench/releases/tag/dashboard-data-20261013"', @@ -318,20 +336,33 @@ describe("notes", () => { test("the rewritten score sentences render", () => { const expected: Record = { "2026-09-01-claude-fable-5-1-added": [ - "Anthropic released Claude Fable 5.1 on 2026-09-01, and it joined the board the same day.", - "PolicyBench parsed 1,984 of the model's 1,984 answers.", + "Anthropic released Claude Fable 5.1 on 2026-09-01, and PolicyBench added it to the board the same day and parsed 1,984 of the model's 1,984 answers.", + "Claude Fable 5.1 gets 86.3% of answers right and ranks #2 of 33.", + closing, ], "2026-09-05-gpt-6-astra-debuts-second": [ - "PolicyBench scores every row on 1,973 of its 1,984 requested outputs. It excludes the other 11 for every model, since their references depend on an engine input the household facts never list.", + "It gets 88.0% of answers right and ranks #2 of 39, behind GPT-5.6 Sol (89.2%) and ahead of Claude Fable 5.1 (86.9%, #3).", + "PolicyBench scores every model on 1,973 of its 1,984 requested outputs and excludes the other 11, since their references depend on an engine input the household facts never list.", + closing, ], "2026-09-22-gpt-6-sol-debuts-first": [ - "GPT-6 Sol gets 94.2% of answers right, weighted by household impact, 1.3 points above Claude Opus 5.5 (92.9%, #2), and ranks #1 of 42. GPT-5.6 Sol, the previous leader, ranks #3 at 92.7%.", + "GPT-6 Sol gets 94.2% of answers right, 1.3 points above Claude Opus 5.5 (92.9%, #2), and ranks #1 of 42. GPT-5.6 Sol, the previous leader, ranks #3 at 92.7%.", + closing, ], "2026-09-29-claude-sonnet-5-5-debuts-fifth": [ - "With the new references and the Medicaid exclusion together, every one of the 42 earlier models rises, by 0.14 to 0.83 points.", + "Claude Sonnet 5.5 gets 92.1% of answers right and ranks #5 of 45.", + closing, + "Together, the new references and the Medicaid exclusion lift every one of the 42 earlier models, by 0.14 to 0.83 points.", + ], + "2026-10-06-policybench-stops-scoring-eight-tax-outputs": [ + "Every model's share of answers right rises by 0.67 to 1.35 points.", + `${closing} GPT-6 Sol still leads at 95.8%`, ], "2026-10-10-claude-haiku-5-5-joins-the-board": [ + "Claude Haiku 5.5 joined the board on 2026-10-10, bringing the board to 47 models. It gets 79.6% of answers right, 4.8 points above Claude Haiku 4.5 (74.8%, #43), and ranks #37 of 47.", + closing, "Against the previous release, every one of the 46 earlier models falls, by 0.24 to 1.37 points.", + "Had they stayed excluded, every earlier model's share of answers right would have risen, by 0.39 to 0.71 points.", ], "2026-10-13-muse-wonder-joins-the-board": [ "Against the previous release, every one of the 47 earlier models falls, by 0.18 to 0.42 points.", diff --git a/tests/test_notes.py b/tests/test_notes.py index c99350bb..fa547f71 100644 --- a/tests/test_notes.py +++ b/tests/test_notes.py @@ -3408,12 +3408,13 @@ def pin(*sentences: str) -> None: pin( "Claude Sonnet 5.5, Grok 4.7 and DeepSeek V4.1 Flash joined the board on " "2026-09-29, bringing it to {nModels} models.", - "Claude Sonnet 5.5 gets {sonnetExact}% of answers right, weighted by " - "household impact, and ranks #{sonnetRank} of {nModels}.", + "Claude Sonnet 5.5 gets {sonnetExact}% of answers right and ranks " + "#{sonnetRank} of {nModels}.", ) _assert_score_definition() pin( SCORE_DEFINITION, + SCORE_WEIGHTING, "GPT-6 Luna, #{lunaRank}, also rounds to {lunaExact}% and sits about level " "with Claude Sonnet 5.5, less than {lunaSonnetGapBelow} points ahead.", "Grok 4.7 scores {grokExact}% (#{grokRank}) and DeepSeek V4.1 Flash " @@ -3724,9 +3725,8 @@ def pin(*sentences: str) -> None: # The drift's two causes, as the drift baseline test rebuilds them. test_previous_release_scores_rebuild_from_this_snapshot() pin( - "With the new references and the Medicaid exclusion together, every one " - "of the {incumbents} earlier models rises, by {driftMin} to {driftMax} " - "points.", + "Together, the new references and the Medicaid exclusion lift every one " + "of the {incumbents} earlier models, by {driftMin} to {driftMax} points.", "None of the {incumbents} matched any of the {newExclusions:words} federal " "outputs now excluded, and all {incumbents} had matched the Arizona " "household's old ${azBefore} SNAP reference, which none matches now.", @@ -4450,12 +4450,12 @@ def rank_then(model: str) -> int: assert rank_now(passed[0]) == rank_then(riser) pin( # The headline the board ranks by, as the September 29 note states it. - "Every model's share of answers right, weighted by household impact, " - "rises by {riseMin} to {riseMax} points.", + "Every model's share of answers right rises by {riseMin} to {riseMax} points.", ) _assert_score_definition() pin( SCORE_DEFINITION, + SCORE_WEIGHTING, "GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 " "({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%).", "Claude Sonnet 5.5 ({sonnetExact}%) passes GPT-6 Luna ({lunaExact}%) for " @@ -5327,8 +5327,9 @@ def test_release_20261013_note_claims() -> None: assert set(scoring["now"]) == {row["model"] for row in board} for row in board: assert scoring["now"][row["model"]] == pytest.approx(row["exact"], abs=1e-9) - # "Without them, every earlier model's rate would be the previous - # release's": leaving the restored outputs unscored gives release 20261010. + # "Without them, every earlier model would keep its share of answers right + # from the previous release": leaving the restored outputs unscored gives + # release 20261010. for model, exact in then.items(): assert scoring["without"][model] == pytest.approx(exact, abs=1e-9), model # The restored outputs are all ones release 20261010 excluded (none was @@ -6175,9 +6176,8 @@ def test_release_20261027_note_claims() -> None: "the reference PolicyEngine computes, and an eligibility answer as right " "when it matches." ) -# The America.gov note (#254) gives its score as "scores {agovExact}% on -# PolicyBench", without "weighted by household impact", so it states the -# weighting in a sentence after the definition. +# "Weighted by household impact" leaves out who weights, so each note says +# it in a sentence of its own, after the definition. SCORE_WEIGHTING = "It weights each answer by household impact." @@ -6197,7 +6197,8 @@ def _assert_score_definition() -> None: def test_no_note_gives_the_score_as_answers_within_one_dollar(): """The board scores eligibility flags by exact match, not within $1, so no note gives an all-outputs score as answers within $1, and a note that - gives a score as answers right defines it.""" + gives a score as answers right defines it and says how PolicyBench + weights the answers.""" _assert_score_definition() # PR #254 rewrites this note's opening; drop it from here once that merges. pending = {"2026-10-19-america-gov-scores-level-with-gpt-5-5"} @@ -6209,16 +6210,22 @@ def test_no_note_gives_the_score_as_answers_within_one_dollar(): # second drops the exemption. assert "answers within $1" in text, (path.name, "drop from pending") continue - for phrase in ("answers within $1", "of answers within"): + for phrase in ( + "answers within $1", + "of answers within", + "weighted by household impact", + ): assert phrase not in text, (path.name, phrase) # A note that gives a score as answers right also says, in the same - # words as every other note, what PolicyBench counts as right. + # words as every other note, what PolicyBench counts as right, and + # says how it weights the answers. if "of answers right" in text: assert SCORE_DEFINITION in text, path.name + if SCORE_DEFINITION in text: + assert "weights each answer by household impact" in text, path.name # Paragraph 0 doubles as the page description and the list blurb, so - # the news leads and the definition closes it, after every sentence - # that leans on an earlier one. Only the weighting may follow it. + # the news leads, and the definition and the weighting close it, after + # every sentence that leans on an earlier one. opening = note["paragraphs"][0] if SCORE_DEFINITION in opening: - tail = opening.split(SCORE_DEFINITION, 1)[1] - assert tail in ("", f" {SCORE_WEIGHTING}"), path.name + assert opening.endswith(f"{SCORE_DEFINITION} {SCORE_WEIGHTING}"), path.name From 248b22814698c20027e04698a680a5a37fd4db25 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 3 Nov 2026 00:47:30 -0500 Subject: [PATCH 08/14] Pin three more rewritten sentences whole September 5's opening sentence, September 29's GPT-6 Luna sentence and October 31's "Only GPT-6 Sol ... score higher" sentence had only partial pins or none. With them, app/tests/notes.test.ts pins every sentence this PR adds or rewrites in the ten notes, as each renders. Co-Authored-By: Claude Opus 5.5 --- app/tests/notes.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/tests/notes.test.ts b/app/tests/notes.test.ts index 60187705..ea2270d7 100644 --- a/app/tests/notes.test.ts +++ b/app/tests/notes.test.ts @@ -341,6 +341,7 @@ describe("notes", () => { closing, ], "2026-09-05-gpt-6-astra-debuts-second": [ + "GPT-6 Astra reached the API on 2026-09-04 and joined the board on 2026-09-05.", "It gets 88.0% of answers right and ranks #2 of 39, behind GPT-5.6 Sol (89.2%) and ahead of Claude Fable 5.1 (86.9%, #3).", "PolicyBench scores every model on 1,973 of its 1,984 requested outputs and excludes the other 11, since their references depend on an engine input the household facts never list.", closing, @@ -351,6 +352,7 @@ describe("notes", () => { ], "2026-09-29-claude-sonnet-5-5-debuts-fifth": [ "Claude Sonnet 5.5 gets 92.1% of answers right and ranks #5 of 45.", + "GPT-6 Luna, #4, also rounds to 92.1% and sits about level with Claude Sonnet 5.5, less than 0.02 points ahead.", closing, "Together, the new references and the Medicaid exclusion lift every one of the 42 earlier models, by 0.14 to 0.83 points.", ], @@ -371,7 +373,7 @@ describe("notes", () => { "With or without the tool, PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches.", ], "2026-10-31-gpt-6-1-terra-joins-the-board": [ - "and GPT-5.6 Sol, which also rounds to 93.1%, trails by less than a tenth of a point.", + "Only GPT-6 Sol (95.2%) and Claude Opus 5.5 (94.1%) score higher, and GPT-5.6 Sol, which also rounds to 93.1%, trails by less than a tenth of a point.", ], }; for (const [slug, sentences] of Object.entries(expected)) { From b74ab2328f9cf7eaf75b9ae2f99202adcf8504cd Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 3 Nov 2026 00:56:37 -0500 Subject: [PATCH 09/14] State the weighting actively in the Terra design record Replace the parenthetical definition and the participle "weighted by household impact" in the result line of docs/terra/design.md with the notes' definition and weighting sentences. No number changes. Co-Authored-By: Claude Opus 5.5 --- docs/terra/design.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/terra/design.md b/docs/terra/design.md index 4eb4de91..fda2c4c9 100644 --- a/docs/terra/design.md +++ b/docs/terra/design.md @@ -141,7 +141,7 @@ Export writes `data-board51.json` and a hash-bound `release-ready.json`. The res ## Lead completion record -- **Result.** GPT-6.1 Terra gets 93.14% of answers right (dollar amounts within $1 of the reference, eligibility answers matching it), weighted by household impact, #3 of 51 on the 1,935 scored outputs, with every output parsed (bounded score 97.19). Its 100 households cost $1.2532 ($0.0125 each) and took a median 22.4 seconds each, all live calls. It answered on 2026-10-26 between 22:11 and 22:28 UTC (board run 2, input pins b0cfa0db). The board's response window is 2026-06-12 to 2026-10-26 and the snapshot is dated 2026-10-27. +- **Result.** GPT-6.1 Terra gets 93.14% of answers right and ranks #3 of 51 on the 1,935 scored outputs, with every output parsed (bounded score 97.19). PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches. It weights each answer by household impact. GPT-6.1 Terra's 100 households cost $1.2532 ($0.0125 each) and took a median 22.4 seconds each, all live calls. It answered on 2026-10-26 between 22:11 and 22:28 UTC (board run 2, input pins b0cfa0db). The board's response window is 2026-06-12 to 2026-10-26 and the snapshot is dated 2026-10-27. - **Judge.** The audit covers 681 cases: 551 verdicts carried over, and 127 cases were re-opened, among them scenario_031's `snap`, one of the four changed references, whose earlier verdict install-references set aside because its prompt re-renders on policyengine-us 2.60.0. The three new cases are the other three changed references, `free_school_meals_eligible` for scenarios 028, 117 and 123. Claude Opus 5.5 judged the 130 on 2026-10-27 under the isolated `setup-token 2` login (`docs/terra/judge_provenance.json`). Scenario 008's `payroll_tax` verdict left out one model in nine judgings in a row; it passed at 14:47 UTC under the per-case schema (7d71b7a0, c5974881). - **Triage.** `restate_gpt61sol_adjudications.py` restated the judge fields of 42 entries, every committed decision on a re-opened case. Triage read 10,999 rows in 990 cases and flagged no reference: `reference-flags.csv` and `unresolved-rows.csv` hold only their headers, so triage needed no decision. Triage also applied 1,522 wording amendments in 117 cases, listed in `us_wording-amendments.json`. Round 3 appended 51 more in seven cases, each answering a finding of release review r2: the reasoning, case note and rows of scenarios 005, 064, 120 and 122, and 42 sentence starts that named the package as "Policyengine-us 2.60.0" or "PolicyEngine 2.60.0". The stage's list then held 1,573. Round 4 appended five more in three cases, each answering a finding of release review r3: scenario_122's case note and claude-fable-5's row, scenario_064's case note and reasoning, and scenario_120's reasoning. The stage's list then held 1,578. - **Export and freeze.** The first export and freeze ran on 2026-10-28 at 2677b116, before the wording review (logs: `~/reviews/policybench-gpt-6-1-terra/freeze-run/`). The release ran again on 2026-10-29 with the review's amendments. Export first ran from 08:03 to 08:07 UTC at 4332ff96, and the freeze dry run refused it: export promoted a fixed list of files and left out the adjudication record that triage amends, so the receipt bound bytes the stage did not hold. Commit 4984698a makes export promote every bound file whose bytes triage changed. Export then ran from 08:12 to 08:15 UTC and staged GPT-6.1 Terra at 93.138833%, #3 of 51 (n = 1,935). The freeze dry run (08:15 to 08:18) validated the release (`RELEASE_TAG`): 51 models, payload 214eed92, references on policyengine-us 2.60.0, 49 exclusions (none new), no adjudications beyond the ruled outputs, 1,522 wording amendments, model responses 2026-06-12 to 2026-10-26, snapshot 2026-10-27, nothing written. The freeze (08:18 to 08:22) rebuilt the snapshot and exited 0, its treatment-fingerprint, reference, analysis, adjudication and manifest checks and `verify_frozen` included. Logs: `~/reviews/policybench-gpt-6-1-terra/freeze-run-r2/`. Round 3 ran the release again on 2026-10-29 with the 51 new amendments (logs: `~/reviews/policybench-gpt-6-1-terra/freeze-run-r3/`). Before it, the lead restored the staged adjudication record to d923dd52's bytes, so that export applied every reasoning amendment from the base text. Export ran from 16:30 to 16:34 UTC at 25d779d7 and staged GPT-6.1 Terra at 93.138833%, #3 of 51 (n = 1,935). The freeze dry run refused it at 16:34 UTC: "stage.json's references_installed is not the installed build's record". Commit 25d779d7 had written the CRP ruling into `docs/terra/spec.json`, whose hash install-references binds in `stage.json`. Commit 3654e5d5 restored the bound spec and moved the ruling to this file. The freeze dry run (16:38 to 16:42) then validated the release: 51 models, payload 5de82780, references on policyengine-us 2.60.0, 49 exclusions (none new), no adjudications beyond the ruled outputs, 1,573 wording amendments, model responses 2026-06-12 to 2026-10-26, snapshot 2026-10-27, nothing written. The freeze (16:42 to 16:52) rebuilt the snapshot and exited 0. Round 4 ran the release again on 2026-10-30 with the five new amendments (logs: `~/reviews/policybench-gpt-6-1-terra/freeze-run-r4/`), after the lead again restored the staged adjudication record to d923dd52's bytes. Export ran from 02:13 to 02:16 UTC at 162c1666 and staged GPT-6.1 Terra at 93.138833%, #3 of 51 (n = 1,935). The freeze dry run (02:16 to 02:19) validated the release: 51 models, payload 67cb0d14, references on policyengine-us 2.60.0, 49 exclusions (none new), no adjudications beyond the ruled outputs, 1,578 wording amendments, model responses 2026-06-12 to 2026-10-26, snapshot 2026-10-27, nothing written. The freeze (02:19 to 02:24) rebuilt the snapshot and exited 0. From 5e9b478229ddc069b9454370748b7e60bfec9d7a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 3 Nov 2026 01:55:23 -0500 Subject: [PATCH 10/14] Limit October 20's weighting sentence to the six scores it weights Review r3 found that "PolicyBench weights each answer by household impact, on the board and with the tool" also covered paragraph 1's six nonzero-amount rates, which `scripts/tool_pilot_note.py` computes as plain counts. We rewrote the sentence as "PolicyBench weights each answer by household impact in these six scores" and added "PolicyBench gives each amount the same weight in these six rates" to paragraph 1. In October 10's paragraph 5, we wrote "Without those outputs" in place of the participial "Had they stayed excluded". In the Terra design record, we wrote "and every one of its outputs parses" in place of "with every output parsed". We also named GPT-6.1 Terra in "GPT-6.1 Terra answered", where "It answered" had followed PolicyBench's "It weights". The guard now checks every sentence that gives a rate within $1 for the word "amount", so it catches "{terraExact}% within $1". We also corrected its comment, which said each note gives the weighting after the definition; the October 20 note gives it first. Co-Authored-By: Claude Opus 5.5 --- ...0-10-claude-haiku-5-5-joins-the-board.json | 2 +- ...gine-tool-three-models-score-above-99.json | 4 ++-- app/tests/notes.test.ts | 8 ++++++-- docs/terra/design.md | 2 +- tests/test_notes.py | 20 +++++++++++++++---- 5 files changed, 26 insertions(+), 10 deletions(-) diff --git a/app/src/notes/2026-10-10-claude-haiku-5-5-joins-the-board.json b/app/src/notes/2026-10-10-claude-haiku-5-5-joins-the-board.json index 5c6fb855..4fc4353f 100644 --- a/app/src/notes/2026-10-10-claude-haiku-5-5-joins-the-board.json +++ b/app/src/notes/2026-10-10-claude-haiku-5-5-joins-the-board.json @@ -8,7 +8,7 @@ "The release also moves PolicyBench's references from policyengine-us {previousEngine} to {engineVersion}, a release that fixes the engine defects behind {restored:words} outputs. The previous release excluded {newlyScored:words} of them, and they return to scoring at the new version's values. The other {restoredRuled:words} are the October 6 defects below, which stay scored at the new version's values. Each of the {restored:words} lands within $1 of the corrected value PolicyBench's audits computed for it. Between them they take the upstream fixes for {restoredFixes}. PolicyBench still builds each reference from the stated facts and from law published before it froze the references on 2026-07-03, under the same publication conventions.", "Two rulings of October 6 apply too. PolicyBench's reference adversary flags the scored outputs where many models, or several of the strongest, agree on an answer the reference does not give, and a judge that cannot yet see the engine's derivation works each one from primary law. Its findings led to rulings to exclude {adversaryOutputs:words} outputs. Four were engine defects, in Arizona, Ohio, Colorado and New York; the new version fixes them, so PolicyBench keeps scoring those four instead. The other four are the federal and state income tax of a Pennsylvania and a Missouri household whose dependent earns $45,000 of wages and must file their own return. The output definitions do not say whether the household's income tax includes that return, so PolicyBench stops scoring them. It also stops scoring {louisianaOutputs:words} Louisiana households' state income tax, whose reference rests on a 2026 standard deduction computed from published price indexes; Louisiana published the amount itself on September 28, after the freeze.", "The new version counts Indiana county income tax in local income tax, and the prompt names no county, so PolicyBench stops scoring the local income tax of {indianaOutputs:words} Indiana households. It also counts Idaho's $10 permanent building fund tax in state income tax, which moves one household's reference from ${idahoBefore} to ${idahoAfter}. Every model is now scored on {scoredOutputs} of its {totalOutputs} requested outputs; PolicyBench excludes the other {excluded}.", - "Against the previous release, every one of the {incumbents} earlier models falls, by {fallMin} to {fallMax} points. The cause is the {newlyScored} restored outputs that the previous release did not score, which few models get right: half of them are answered within $1 by at most {newlyScoredHalfAtMost:words} of the {nModels} models, and {newlyScoredUnmatched:words} by none. Had they stayed excluded, every earlier model's share of answers right would have risen, by {riseWithoutMin} to {riseWithoutMax} points." + "Against the previous release, every one of the {incumbents} earlier models falls, by {fallMin} to {fallMax} points. The cause is the {newlyScored} restored outputs that the previous release did not score, which few models get right: half of them are answered within $1 by at most {newlyScoredHalfAtMost:words} of the {nModels} models, and {newlyScoredUnmatched:words} by none. Without those outputs, every earlier model's share of answers right would have risen, by {riseWithoutMin} to {riseWithoutMax} points." ], "facts": { "releaseDate": "2026-10-10", diff --git a/app/src/notes/2026-10-20-given-a-policyengine-tool-three-models-score-above-99.json b/app/src/notes/2026-10-20-given-a-policyengine-tool-three-models-score-above-99.json index bac176c2..629cacc7 100644 --- a/app/src/notes/2026-10-20-given-a-policyengine-tool-three-models-score-above-99.json +++ b/app/src/notes/2026-10-20-given-a-policyengine-tool-three-models-score-above-99.json @@ -3,8 +3,8 @@ "date": "2026-10-20", "title": "Given a tool that runs PolicyEngine, three models score above 99%", "paragraphs": [ - "PolicyBench gave three models a tool that runs PolicyEngine on a household the model writes from the prompt's facts, and ran its {households} US households again on October 14 and 15. GPT-6 Sol, which leads the board at {solBoardExact}% of answers right, gets {solToolExact}% with the tool. Claude Opus 5.5 goes from {opusBoardExact}% to {opusToolExact}%, and Gemini 4 Argon from {argonBoardExact}% to {argonToolExact}%. PolicyBench weights each answer by household impact, on the board and with the tool. The runs are a pilot, not board rows, since the board scores models answering alone, and the result is circular: the tool runs the engine that computes PolicyBench's references, so the scores measure agreement with that engine, not whether it is right.", - "With or without the tool, PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches. Most of the answers the tool fixes are dollar amounts that are not zero. Of the {nonzeroAmounts} nonzero amounts, GPT-6 Sol gets {solNonzeroRate}% within $1 on its own and {solNonzeroToolRate}% with the tool; Claude Opus 5.5, {opusNonzeroRate}% and {opusNonzeroToolRate}%; and Gemini 4 Argon, {argonNonzeroRate}% and {argonNonzeroToolRate}%.", + "PolicyBench gave three models a tool that runs PolicyEngine on a household the model writes from the prompt's facts, and ran its {households} US households again on October 14 and 15. GPT-6 Sol, which leads the board at {solBoardExact}% of answers right, gets {solToolExact}% with the tool. Claude Opus 5.5 goes from {opusBoardExact}% to {opusToolExact}%, and Gemini 4 Argon from {argonBoardExact}% to {argonToolExact}%. PolicyBench weights each answer by household impact in these six scores. The runs are a pilot, not board rows, since the board scores models answering alone, and the result is circular: the tool runs the engine that computes PolicyBench's references, so the scores measure agreement with that engine, not whether it is right.", + "With or without the tool, PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches. Most of the answers the tool fixes are dollar amounts that are not zero. Of the {nonzeroAmounts} nonzero amounts, GPT-6 Sol gets {solNonzeroRate}% within $1 on its own and {solNonzeroToolRate}% with the tool; Claude Opus 5.5, {opusNonzeroRate}% and {opusNonzeroToolRate}%; and Gemini 4 Argon, {argonNonzeroRate}% and {argonNonzeroToolRate}%. PolicyBench gives each amount the same weight in these six rates.", "With the tool, {toolMisses} answers are wrong across the three runs: {solMisses:words} of GPT-6 Sol's, {opusMisses:words} of Claude Opus 5.5's and {argonMisses:words} of Gemini 4 Argon's. Each traces to the household the model described to the engine, or to the output it read back, rather than to the calculation: in {enteredElsewhere:words} the model had entered a stated fact under another variable, in {leftOut:words} it had left one out, and in {wrongOutput:words} it read back the wrong output. Prompt labels share the blame for {labelMisses:words}: \"employer sponsored insurance premiums\" never says the employer paid them, and \"disability benefits\" never says they are not Social Security. In {solCapMisses:words} of GPT-6 Sol's, it had used all {turnCap:words} calculation turns the pilot allows. GPT-6 Sol is also the only model that turned right answers wrong, {solFlips:words} of them.", "The tool costs more. For the {households} households, GPT-6 Sol costs ${solToolCost} with it and ${solBoardCost} without, Claude Opus 5.5 ${opusToolCost} and ${opusBoardCost}, and Gemini 4 Argon ${argonToolCost} and ${argonBoardCost}.", "The tool applies the conventions PolicyBench uses for its references, so a model that describes a household exactly as the reference does gets the reference's answer. Describing a household is also easier here than it would be from a filer's documents: {labelMatches} of the {statedFacts} facts the prompts state, not counting ages and wages, carry a label that is the engine's own variable name with spaces.", diff --git a/app/tests/notes.test.ts b/app/tests/notes.test.ts index ea2270d7..049eb73e 100644 --- a/app/tests/notes.test.ts +++ b/app/tests/notes.test.ts @@ -190,11 +190,15 @@ describe("notes", () => { "Claude Opus 5.5 goes from 94.1% to 99.8%, and Gemini 4 Argon from 88.7% to 99.9%.", ); expect(markup).toContain( - "PolicyBench weights each answer by household impact, on the board and with the tool.", + "PolicyBench weights each answer by household impact in these six scores.", ); expect(markup).toContain( "Of the 187 nonzero amounts, GPT-6 Sol gets 67.4% within $1 on its own and 96.8% with the tool", ); + // The nonzero-amount rates count each amount once. + expect(markup).toContain( + "and Gemini 4 Argon, 46.5% and 99.5%. PolicyBench gives each amount the same weight in these six rates.", + ); // Whole numbers under ten are spelled out; costs keep their cents. expect(markup).toContain( "With the tool, 12 answers are wrong across the three runs: nine of GPT-6 Sol's, two of Claude Opus 5.5's and one of Gemini 4 Argon's.", @@ -364,7 +368,7 @@ describe("notes", () => { "Claude Haiku 5.5 joined the board on 2026-10-10, bringing the board to 47 models. It gets 79.6% of answers right, 4.8 points above Claude Haiku 4.5 (74.8%, #43), and ranks #37 of 47.", closing, "Against the previous release, every one of the 46 earlier models falls, by 0.24 to 1.37 points.", - "Had they stayed excluded, every earlier model's share of answers right would have risen, by 0.39 to 0.71 points.", + "Without those outputs, every earlier model's share of answers right would have risen, by 0.39 to 0.71 points.", ], "2026-10-13-muse-wonder-joins-the-board": [ "Against the previous release, every one of the 47 earlier models falls, by 0.18 to 0.42 points.", diff --git a/docs/terra/design.md b/docs/terra/design.md index fda2c4c9..5df04c94 100644 --- a/docs/terra/design.md +++ b/docs/terra/design.md @@ -141,7 +141,7 @@ Export writes `data-board51.json` and a hash-bound `release-ready.json`. The res ## Lead completion record -- **Result.** GPT-6.1 Terra gets 93.14% of answers right and ranks #3 of 51 on the 1,935 scored outputs, with every output parsed (bounded score 97.19). PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches. It weights each answer by household impact. GPT-6.1 Terra's 100 households cost $1.2532 ($0.0125 each) and took a median 22.4 seconds each, all live calls. It answered on 2026-10-26 between 22:11 and 22:28 UTC (board run 2, input pins b0cfa0db). The board's response window is 2026-06-12 to 2026-10-26 and the snapshot is dated 2026-10-27. +- **Result.** GPT-6.1 Terra gets 93.14% of answers right and ranks #3 of 51 on the 1,935 scored outputs (bounded score 97.19), and every one of its outputs parses. PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches. It weights each answer by household impact. GPT-6.1 Terra's 100 households cost $1.2532 ($0.0125 each) and took a median 22.4 seconds each, all live calls. GPT-6.1 Terra answered on 2026-10-26 between 22:11 and 22:28 UTC (board run 2, input pins b0cfa0db). The board's response window is 2026-06-12 to 2026-10-26 and the snapshot is dated 2026-10-27. - **Judge.** The audit covers 681 cases: 551 verdicts carried over, and 127 cases were re-opened, among them scenario_031's `snap`, one of the four changed references, whose earlier verdict install-references set aside because its prompt re-renders on policyengine-us 2.60.0. The three new cases are the other three changed references, `free_school_meals_eligible` for scenarios 028, 117 and 123. Claude Opus 5.5 judged the 130 on 2026-10-27 under the isolated `setup-token 2` login (`docs/terra/judge_provenance.json`). Scenario 008's `payroll_tax` verdict left out one model in nine judgings in a row; it passed at 14:47 UTC under the per-case schema (7d71b7a0, c5974881). - **Triage.** `restate_gpt61sol_adjudications.py` restated the judge fields of 42 entries, every committed decision on a re-opened case. Triage read 10,999 rows in 990 cases and flagged no reference: `reference-flags.csv` and `unresolved-rows.csv` hold only their headers, so triage needed no decision. Triage also applied 1,522 wording amendments in 117 cases, listed in `us_wording-amendments.json`. Round 3 appended 51 more in seven cases, each answering a finding of release review r2: the reasoning, case note and rows of scenarios 005, 064, 120 and 122, and 42 sentence starts that named the package as "Policyengine-us 2.60.0" or "PolicyEngine 2.60.0". The stage's list then held 1,573. Round 4 appended five more in three cases, each answering a finding of release review r3: scenario_122's case note and claude-fable-5's row, scenario_064's case note and reasoning, and scenario_120's reasoning. The stage's list then held 1,578. - **Export and freeze.** The first export and freeze ran on 2026-10-28 at 2677b116, before the wording review (logs: `~/reviews/policybench-gpt-6-1-terra/freeze-run/`). The release ran again on 2026-10-29 with the review's amendments. Export first ran from 08:03 to 08:07 UTC at 4332ff96, and the freeze dry run refused it: export promoted a fixed list of files and left out the adjudication record that triage amends, so the receipt bound bytes the stage did not hold. Commit 4984698a makes export promote every bound file whose bytes triage changed. Export then ran from 08:12 to 08:15 UTC and staged GPT-6.1 Terra at 93.138833%, #3 of 51 (n = 1,935). The freeze dry run (08:15 to 08:18) validated the release (`RELEASE_TAG`): 51 models, payload 214eed92, references on policyengine-us 2.60.0, 49 exclusions (none new), no adjudications beyond the ruled outputs, 1,522 wording amendments, model responses 2026-06-12 to 2026-10-26, snapshot 2026-10-27, nothing written. The freeze (08:18 to 08:22) rebuilt the snapshot and exited 0, its treatment-fingerprint, reference, analysis, adjudication and manifest checks and `verify_frozen` included. Logs: `~/reviews/policybench-gpt-6-1-terra/freeze-run-r2/`. Round 3 ran the release again on 2026-10-29 with the 51 new amendments (logs: `~/reviews/policybench-gpt-6-1-terra/freeze-run-r3/`). Before it, the lead restored the staged adjudication record to d923dd52's bytes, so that export applied every reasoning amendment from the base text. Export ran from 16:30 to 16:34 UTC at 25d779d7 and staged GPT-6.1 Terra at 93.138833%, #3 of 51 (n = 1,935). The freeze dry run refused it at 16:34 UTC: "stage.json's references_installed is not the installed build's record". Commit 25d779d7 had written the CRP ruling into `docs/terra/spec.json`, whose hash install-references binds in `stage.json`. Commit 3654e5d5 restored the bound spec and moved the ruling to this file. The freeze dry run (16:38 to 16:42) then validated the release: 51 models, payload 5de82780, references on policyengine-us 2.60.0, 49 exclusions (none new), no adjudications beyond the ruled outputs, 1,573 wording amendments, model responses 2026-06-12 to 2026-10-26, snapshot 2026-10-27, nothing written. The freeze (16:42 to 16:52) rebuilt the snapshot and exited 0. Round 4 ran the release again on 2026-10-30 with the five new amendments (logs: `~/reviews/policybench-gpt-6-1-terra/freeze-run-r4/`), after the lead again restored the staged adjudication record to d923dd52's bytes. Export ran from 02:13 to 02:16 UTC at 162c1666 and staged GPT-6.1 Terra at 93.138833%, #3 of 51 (n = 1,935). The freeze dry run (02:16 to 02:19) validated the release: 51 models, payload 67cb0d14, references on policyengine-us 2.60.0, 49 exclusions (none new), no adjudications beyond the ruled outputs, 1,578 wording amendments, model responses 2026-06-12 to 2026-10-26, snapshot 2026-10-27, nothing written. The freeze (02:19 to 02:24) rebuilt the snapshot and exited 0. diff --git a/tests/test_notes.py b/tests/test_notes.py index fa547f71..f86e84a8 100644 --- a/tests/test_notes.py +++ b/tests/test_notes.py @@ -6177,7 +6177,9 @@ def test_release_20261027_note_claims() -> None: "when it matches." ) # "Weighted by household impact" leaves out who weights, so each note says -# it in a sentence of its own, after the definition. +# it in a sentence of its own, most often this one, after the definition. +# The October 20 note gives its weighted scores before the definition, so +# its sentence names PolicyBench and the six scores it covers. SCORE_WEIGHTING = "It weights each answer by household impact." @@ -6196,9 +6198,10 @@ def _assert_score_definition() -> None: def test_no_note_gives_the_score_as_answers_within_one_dollar(): """The board scores eligibility flags by exact match, not within $1, so no - note gives an all-outputs score as answers within $1, and a note that - gives a score as answers right defines it and says how PolicyBench - weights the answers.""" + note gives an all-outputs score as answers within $1, a note that gives a + rate within $1 says the rate covers amounts, and a note that gives a + score as answers right defines it and says how PolicyBench weights the + answers.""" _assert_score_definition() # PR #254 rewrites this note's opening; drop it from here once that merges. pending = {"2026-10-19-america-gov-scores-level-with-gpt-5-5"} @@ -6216,6 +6219,15 @@ def test_no_note_gives_the_score_as_answers_within_one_dollar(): "weighted by household impact", ): assert phrase not in text, (path.name, phrase) + # A rate within $1 covers dollar amounts only, so its sentence + # names them. + for paragraph in note["paragraphs"]: + for sentence in re.split(r"(?<=[.!?])\s+(?=[A-Z])", paragraph): + if ( + re.search(r"\{\w+(?:Exact|Rate)\}%", sentence) + and "within $1" in sentence + ): + assert "amount" in sentence, (path.name, sentence) # A note that gives a score as answers right also says, in the same # words as every other note, what PolicyBench counts as right, and # says how it weights the answers. From def8324cde7b84d9c9a506d074c452e85b8211a5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 3 Nov 2026 02:48:32 -0500 Subject: [PATCH 11/14] Widen the guard's within-$1 check to titles, literal percentages and shares Review r4 found that the guard's sentence check reads only paragraphs, only placeholder rates and only "within $1", and that the singular "amount" satisfies it. We now check titles too, and treat a literal percentage or the word "share" as a rate. We also catch "within a dollar" and "within one dollar", and require the plural "amounts". We split sentences before a lowercase word, such as policyengine-us, and after a closing quote or parenthesis. We rewrote the test's docstring to state what the check enforces. Co-Authored-By: Claude Opus 5.5 --- tests/test_notes.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/tests/test_notes.py b/tests/test_notes.py index f86e84a8..e11e7d25 100644 --- a/tests/test_notes.py +++ b/tests/test_notes.py @@ -6197,11 +6197,11 @@ def _assert_score_definition() -> None: def test_no_note_gives_the_score_as_answers_within_one_dollar(): - """The board scores eligibility flags by exact match, not within $1, so no - note gives an all-outputs score as answers within $1, a note that gives a - rate within $1 says the rate covers amounts, and a note that gives a - score as answers right defines it and says how PolicyBench weights the - answers.""" + """PolicyBench scores eligibility flags by exact match, not within $1. So no + note gives an all-outputs score as answers within $1. A title or sentence + that gives a percentage or a share within $1, or within a dollar, also + names the amounts that figure covers. A note that gives a score as answers + right defines it and says how PolicyBench weights the answers.""" _assert_score_definition() # PR #254 rewrites this note's opening; drop it from here once that merges. pending = {"2026-10-19-america-gov-scores-level-with-gpt-5-5"} @@ -6219,15 +6219,14 @@ def test_no_note_gives_the_score_as_answers_within_one_dollar(): "weighted by household impact", ): assert phrase not in text, (path.name, phrase) - # A rate within $1 covers dollar amounts only, so its sentence - # names them. - for paragraph in note["paragraphs"]: - for sentence in re.split(r"(?<=[.!?])\s+(?=[A-Z])", paragraph): - if ( - re.search(r"\{\w+(?:Exact|Rate)\}%", sentence) - and "within $1" in sentence - ): - assert "amount" in sentence, (path.name, sentence) + # A percentage or share within $1 covers dollar amounts only, so the + # title or sentence that gives one names them. + for part in (note["title"], *note["paragraphs"]): + for sentence in re.split(r"(?<=[.!?])\s+|(?<=[.!?][\"”’)])\s+", part): + if re.search( + r"(?:\{\w+(?:Exact|Rate)\}|\b\d+(?:\.\d+)?)%|\bshare\b", sentence + ) and re.search(r"within (?:\$1\b|a dollar|one dollar)", sentence): + assert re.search(r"\bamounts\b", sentence), (path.name, sentence) # A note that gives a score as answers right also says, in the same # words as every other note, what PolicyBench counts as right, and # says how it weights the answers. From 5101a8eed3a2c60c8fbb2ce19c4acf571490c93f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 3 Nov 2026 03:31:17 -0500 Subject: [PATCH 12/14] Widen the note guard's percentage and within patterns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review r5 ran four mutations that give a share or a percentage within $1 without naming the amounts, and def8324c's guard passes all four. Two use "share" in the plural or at the start of a sentence, one a {…Share}% placeholder and one "within a $1 tolerance". The guard read only {…Exact}% and {…Rate}% placeholders, a literal percentage and the lowercase word "share", and only "within $1", "within a dollar" and "within one dollar". We take the reviewer's two patterns. The guard now reads any placeholder or literal number before "%", "share" or "shares" in any case, and "within", "within a" or "within one" before "$1" or "dollar". So the docstring and the comment now describe what the guard reads. Co-Authored-By: Claude Opus 5.5 --- tests/test_notes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_notes.py b/tests/test_notes.py index e11e7d25..be4a1a81 100644 --- a/tests/test_notes.py +++ b/tests/test_notes.py @@ -6224,8 +6224,8 @@ def test_no_note_gives_the_score_as_answers_within_one_dollar(): for part in (note["title"], *note["paragraphs"]): for sentence in re.split(r"(?<=[.!?])\s+|(?<=[.!?][\"”’)])\s+", part): if re.search( - r"(?:\{\w+(?:Exact|Rate)\}|\b\d+(?:\.\d+)?)%|\bshare\b", sentence - ) and re.search(r"within (?:\$1\b|a dollar|one dollar)", sentence): + r"(?:\{\w+(?::\w+)?\}|\b\d+(?:\.\d+)?)%|(?i:\bshares?\b)", sentence + ) and re.search(r"within (?:a |one )?(?:\$1\b|dollar\b)", sentence): assert re.search(r"\bamounts\b", sentence), (path.name, sentence) # A note that gives a score as answers right also says, in the same # words as every other note, what PolicyBench counts as right, and From 71b8641e7ecf75e7fb56a422975ac3615c2e4501 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 3 Nov 2026 04:10:02 -0500 Subject: [PATCH 13/14] Make the note guard read "Within" and the banned phrases in any case The r6 reviewer found three sentences that 5101a8ee's guard passes: G1 and G2 open with "Within $1,", and G3 with "Answers within $1". The guard's pattern matched "within" in lowercase, and its phrase check kept the text's capitals. We take the reviewer's two lines, so the guard now matches "within" with a lowercase or capital w and checks the banned phrases against the lowercased text. The guard now fails G1, G2 and G3, the other 30 mutations keep their results, and the guard and schema tests pass (15). Co-Authored-By: Claude Opus 5.5 --- tests/test_notes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_notes.py b/tests/test_notes.py index be4a1a81..25b9e691 100644 --- a/tests/test_notes.py +++ b/tests/test_notes.py @@ -6218,14 +6218,14 @@ def test_no_note_gives_the_score_as_answers_within_one_dollar(): "of answers within", "weighted by household impact", ): - assert phrase not in text, (path.name, phrase) + assert phrase not in text.lower(), (path.name, phrase) # A percentage or share within $1 covers dollar amounts only, so the # title or sentence that gives one names them. for part in (note["title"], *note["paragraphs"]): for sentence in re.split(r"(?<=[.!?])\s+|(?<=[.!?][\"”’)])\s+", part): if re.search( r"(?:\{\w+(?::\w+)?\}|\b\d+(?:\.\d+)?)%|(?i:\bshares?\b)", sentence - ) and re.search(r"within (?:a |one )?(?:\$1\b|dollar\b)", sentence): + ) and re.search(r"[Ww]ithin (?:a |one )?(?:\$1\b|dollar\b)", sentence): assert re.search(r"\bamounts\b", sentence), (path.name, sentence) # A note that gives a score as answers right also says, in the same # words as every other note, what PolicyBench counts as right, and From 7ef8acdb775f4818eb0621efd3e17dd17944f055 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 3 Nov 2026 15:46:02 -0500 Subject: [PATCH 14/14] Drop the America.gov note's exemption from the score-definition guard We rewrote the America.gov note's opening in #254, and the note no longer says "answers within $1". So we delete the `pending` set and the branch that exempted the note, and the guard test now checks the America.gov note as it checks every other note. Co-Authored-By: Claude Opus 5.5 --- tests/test_notes.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/test_notes.py b/tests/test_notes.py index 71b487ef..3baec39c 100644 --- a/tests/test_notes.py +++ b/tests/test_notes.py @@ -6281,16 +6281,9 @@ def test_no_note_gives_the_score_as_answers_within_one_dollar(): names the amounts that figure covers. A note that gives a score as answers right defines it and says how PolicyBench weights the answers.""" _assert_score_definition() - # PR #254 rewrites this note's opening; drop it from here once that merges. - pending = {"2026-10-19-america-gov-scores-level-with-gpt-5-5"} for path in sorted(NOTES_DIR.glob("*.json")): note = _load_json(path) text = " ".join([note["title"], *note["paragraphs"]]) - if note["slug"] in pending: - # Fails once #254 lands, so whichever of the two PRs merges - # second drops the exemption. - assert "answers within $1" in text, (path.name, "drop from pending") - continue for phrase in ( "answers within $1", "of answers within",