diff --git a/app/src/notes/2026-09-01-claude-fable-5-1-added.json b/app/src/notes/2026-09-01-claude-fable-5-1-added.json index fabea8fd..511d1f32 100644 --- a/app/src/notes/2026-09-01-claude-fable-5-1-added.json +++ b/app/src/notes/2026-09-01-claude-fable-5-1-added.json @@ -3,7 +3,7 @@ "date": "2026-09-01", "title": "Claude Fable 5.1 added", "paragraphs": [ - "Claude Fable 5.1 was released on 2026-09-01 and joined the board the same day. Board row: {exactRate}% of answers within $1, #{rank} of {nModels}. {parsed} of {answers} answers parsed.", + "Anthropic released Claude Fable 5.1 on 2026-09-01, and PolicyBench added it to the board the same day and parsed {parsed} of the model's {answers} answers. Claude Fable 5.1 gets {exactRate}% of answers right and ranks #{rank} of {nModels}. PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches. It weights each answer by household impact.", "The model's API rejects forced tool calls, which is how the board collects most models' answers, so its row answers as a JSON object and reasons at the provider default. The serving-configuration table in the paper records the transport for every row.", "Sensitivity: with the answer tool declared under tool_choice auto, Fable 5.1 scores {autoRate}% (would rank #{autoRank}). Under the same request Claude Fable 5 scores {fable5AutoRate}%. Fable 5's board row ({fable5BoardRate}%) ran chunked under a forced tool call, which switches off its thinking, so the {boardGap}-point gap between the two board rows is mostly request shape.", "The next board version moves every model to tool_choice auto so each provider's default reasoning engages under the recorded request shape (policybench#139)." diff --git a/app/src/notes/2026-09-05-gpt-6-astra-debuts-second.json b/app/src/notes/2026-09-05-gpt-6-astra-debuts-second.json index af93d23d..e8f1e4da 100644 --- a/app/src/notes/2026-09-05-gpt-6-astra-debuts-second.json +++ b/app/src/notes/2026-09-05-gpt-6-astra-debuts-second.json @@ -3,7 +3,7 @@ "date": "2026-09-05", "title": "GPT-6 Astra debuts second: the two rules it invented", "paragraphs": [ - "GPT-6 Astra reached the API on 2026-09-04 and joined the board on 2026-09-05 at {astraExact}% of answers within $1, #{astraRank} of {nModels}, behind GPT-5.6 Sol ({solExact}%) and ahead of Claude Fable 5.1 ({fableExact}%, #{fableRank}). Every row on this board is scored on {scoredOutputs} of its {totalOutputs} requested outputs: {excludedOutputs} outputs whose reference depends on an engine input the household facts never list are excluded for every model.", + "GPT-6 Astra reached the API on 2026-09-04 and joined the board on 2026-09-05. It gets {astraExact}% of answers right and ranks #{astraRank} of {nModels}, behind GPT-5.6 Sol ({solExact}%) and ahead of Claude Fable 5.1 ({fableExact}%, #{fableRank}). PolicyBench scores every model on {scoredOutputs} of its {totalOutputs} requested outputs and excludes the other {excludedOutputs}, since their references depend on an engine input the household facts never list. PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches. It weights each answer by household impact.", "Row by row, the two OpenAI models agree far more than they differ. Both are right on {bothRight} scored outputs and both wrong on {bothWrong}. The gap is {astraOnlyMisses} outputs Astra misses and Sol gets right, against {solOnlyMisses} the other way.", "Astra's solo misses cluster in two invented rules. In {esiRows} rows it treats a listed employer-sponsored insurance premium as a pre-tax salary reduction and subtracts it from wages before computing payroll tax, federal income tax or state income tax; the household facts state gross wages, and the reference keeps them. In {medicareRows} rows it marks a household member under 65 who is flagged disabled as Medicare-eligible; the reference grants Medicare before 65 only with 24 months of Social Security disability receipt recorded, and none of these households carry that input. The remaining {otherRows} solo misses are one-off rule errors with no shared mechanism. One further Medicare row of the same kind, on scenario_074, sits on an excluded output and is outside every count here.", "The judge's row notes name an employer-sponsored insurance premium on {astraEsiMentions} of Astra's annotated rows and on {solEsiMentions} of Sol's. Sol netted the same premium once, on scenario_045, which cost it {solEsiRows} of its {solOnlyMisses} solo misses; the rest of Sol's solo misses share no mechanism.", diff --git a/app/src/notes/2026-09-22-gpt-6-sol-debuts-first.json b/app/src/notes/2026-09-22-gpt-6-sol-debuts-first.json index ca8fb480..0fbbc311 100644 --- a/app/src/notes/2026-09-22-gpt-6-sol-debuts-first.json +++ b/app/src/notes/2026-09-22-gpt-6-sol-debuts-first.json @@ -3,7 +3,7 @@ "date": "2026-09-22", "title": "GPT-6 Sol debuts first, Claude Opus 5.5 second", "paragraphs": [ - "GPT-6 Sol, GPT-6 Luna and Claude Opus 5.5 joined the board on 2026-09-22. GPT-6 Sol leads at {solExact}% of answers within $1, weighted by household impact, #{solRank} of {nModels} and {solLead} points above Claude Opus 5.5 ({opusExact}%, #{opusRank}). GPT-5.6 Sol, the previous leader, is #{sol56Rank} at {sol56Exact}%.", + "GPT-6 Sol, GPT-6 Luna and Claude Opus 5.5 joined the board on 2026-09-22. GPT-6 Sol gets {solExact}% of answers right, {solLead} points above Claude Opus 5.5 ({opusExact}%, #{opusRank}), and ranks #{solRank} of {nModels}. GPT-5.6 Sol, the previous leader, ranks #{sol56Rank} at {sol56Exact}%. PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches. It weights each answer by household impact.", "GPT-6 Luna scores {lunaExact}% (#{lunaRank}) at ${lunaCost} a household, against ${solCost} for GPT-6 Sol and ${opusCost} for Claude Opus 5.5.", "Claude Opus 5.5's API rejects forced tool calls, as Claude Fable 5.1's does, so its row answers as a JSON object and reasons at the provider default. Claude Opus 5's board row runs the forced tool call, which switches that model's thinking off, and scores {opus5BoardExact}%; its tool_choice auto re-run, where it reasons, scores {opus5AutoExact}%. The two Opus rows differ in serving shape as well as in model, so the {opusGap}-point gap between them is not a measure of the model change alone.", "Every row is scored on {scoredOutputs} of its {totalOutputs} requested outputs. This release regenerated {regenerated} references and excludes {excluded} outputs from scoring for every model; the reference audit note explains both.", diff --git a/app/src/notes/2026-09-29-claude-sonnet-5-5-debuts-fifth.json b/app/src/notes/2026-09-29-claude-sonnet-5-5-debuts-fifth.json index 58c61e09..145f1568 100644 --- a/app/src/notes/2026-09-29-claude-sonnet-5-5-debuts-fifth.json +++ b/app/src/notes/2026-09-29-claude-sonnet-5-5-debuts-fifth.json @@ -3,13 +3,13 @@ "date": "2026-09-29", "title": "Claude Sonnet 5.5 debuts fifth as the references move to policyengine-us 2.15.17", "paragraphs": [ - "Claude Sonnet 5.5, Grok 4.7 and DeepSeek V4.1 Flash joined the board on 2026-09-29, bringing it to {nModels} models. Claude Sonnet 5.5 scores {sonnetExact}% of answers within $1, weighted by household impact, #{sonnetRank} of {nModels}. GPT-6 Luna, #{lunaRank}, also rounds to {lunaExact}% and sits about level with it, less than {lunaSonnetGapBelow} points ahead. Grok 4.7 scores {grokExact}% (#{grokRank}) and DeepSeek V4.1 Flash {flashExact}% (#{flashRank}). GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%).", + "Claude Sonnet 5.5, Grok 4.7 and DeepSeek V4.1 Flash joined the board on 2026-09-29, bringing it to {nModels} models. Claude Sonnet 5.5 gets {sonnetExact}% of answers right and ranks #{sonnetRank} of {nModels}. GPT-6 Luna, #{lunaRank}, also rounds to {lunaExact}% and sits about level with Claude Sonnet 5.5, less than {lunaSonnetGapBelow} points ahead. Grok 4.7 scores {grokExact}% (#{grokRank}) and DeepSeek V4.1 Flash {flashExact}% (#{flashRank}). GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%). PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches. It weights each answer by household impact.", "Claude Sonnet 5.5 costs ${sonnetCost} a household, against ${lunaCost} for GPT-6 Luna. DeepSeek V4.1 Flash costs ${flashCost} at DeepSeek's peak list price, and Grok 4.7 costs ${grokCost}.", "Claude Sonnet 5.5's API rejects forced tool calls, as Claude Opus 5.5's and Claude Fable 5.1's do, so its row answers as a JSON object and reasons with adaptive thinking, the provider default. Grok 4.7 answers through the forced tool call with a {grokTimeoutSeconds}-second request timeout, because its whole-household onboarding probe ran past a {grokOnboardingTimeoutSeconds}-second timeout. DeepSeek serves V4.1 Flash under the alias deepseek-flash, and all {flashAnswers} of the row's answers report that alias and one system fingerprint; none reports a version. PolicyBench labels the row from DeepSeek's September 10 release note, which put V4.1 Flash on that alias.", "The release also moves PolicyBench's scored references from policyengine-us {previousEngine} to {engineVersion}, the newest release when PolicyBench began sweeping the references on 2026-09-29 (uploaded at {engineUploadedUtc} UTC). The newer version includes the {upstreamFixed:words} upstream fixes that PolicyBench applied as sandbox fixes for the September 22 references, and it encodes law the older version lacked. PolicyBench still builds each scored reference from the stated facts and law published before it froze the references on 2026-07-03, so it ported its {conventions:words} publication conventions to the new version. With those conventions and an adapter that keeps Maryland county tax out of state income tax, policyengine-us {checkEngine}, the newest release when PolicyBench checked PyPI on 2026-09-29 at {checkPypiReadUtc} UTC, gives the same value as {engineVersion} for all {totalOutputs} outputs.", "The move changes {scoredChanges:words} scored references. New Jersey's child tax credit schedule for 2026 to 2028, which the state approved on June 30, raises one household's state refundable credits from ${njBefore} to ${njAfter}. Arizona raised the income limit for its broad-based categorical eligibility from {azLimitBefore}% to {azLimitAfter}% of the poverty guideline effective March, and the new limit gives an Arizona household ${azAfter} of SNAP where the reference was ${azBefore}. policyengine-us now counts child support received as school-meal income, which ends a Pennsylvania household's eligibility for reduced-price meals. A fix to the rounding in New York's Empire State child credit phase-out (policyengine-us #9425) raises a New York household's state refundable credits from ${nyBefore} to ${nyAfter}.", "PolicyBench stops scoring {newExclusions:words} federal income tax outputs. policyengine-us {engineVersion} counts the whole of a listed state and local tax refund as income. Federal law counts the refund only to the extent the refunded tax lowered the household's federal tax in the year the household paid it, and the prompts do not say whether it did. PolicyBench also stops scoring one California household head's Medicaid eligibility, which its re-review of the household's excluded SNAP output flagged. On disability, the prompt says only that the head is disabled. The head's income, {medicaidIncomePercent}% of the poverty guideline, is above the {expansionLimitPercent}% limit for the adult expansion group, so only a disability pathway leads to Medi-Cal, California's Medicaid program. Medi-Cal's Working Disabled Program requires SSI's definition of disability, which the prompt does not state, and policyengine-us {engineVersion} tests the general disability flag instead. The same unstated fact already keeps the household's SNAP out of scoring. PolicyBench now scores every model on {scoredOutputs} of its {totalOutputs} requested outputs and excludes {excluded}. Another {withinTolerance:words} references move by less than $1, and PolicyBench re-reviewed the {rechecked} excluded outputs whose values moved; all stay excluded.", - "Together, the new references and the Medicaid exclusion raise the exact rate of every one of the {incumbents} earlier models, by {driftMin} to {driftMax} points. None of the {incumbents} matched any of the {newExclusions:words} federal outputs now excluded, and all {incumbents} had matched the Arizona household's old ${azBefore} SNAP reference, which none matches now. On the Medicaid output, {medicaidMatched} of the {incumbents} had matched the reference and {medicaidMissed} had not. Among the {incumbents}, {riserCount:words} models move up in the order: {riserOne} moves above {riserOnePassed}, {riserTwo} above {riserTwoPassed}, {riserThree} above {riserThreePassed}, {riserFour} above {riserFourPassed}, and {riserFive} above {riserFivePassed}. Every other pair keeps its order.", + "Together, the new references and the Medicaid exclusion lift every one of the {incumbents} earlier models, by {driftMin} to {driftMax} points. None of the {incumbents} matched any of the {newExclusions:words} federal outputs now excluded, and all {incumbents} had matched the Arizona household's old ${azBefore} SNAP reference, which none matches now. On the Medicaid output, {medicaidMatched} of the {incumbents} had matched the reference and {medicaidMissed} had not. Among the {incumbents}, {riserCount:words} models move up in the order: {riserOne} moves above {riserOnePassed}, {riserTwo} above {riserTwoPassed}, {riserThree} above {riserThreePassed}, {riserFour} above {riserFourPassed}, and {riserFive} above {riserFivePassed}. Every other pair keeps its order.", "The Arizona household's income keeps it from qualifying for SNAP under the program's ordinary tests, so it qualifies only through broad-based categorical eligibility, and all {nModels} models answer $0 for it. PolicyBench's October 5 note on such households counts it among five held back by income, with {bbceIncomeHeldCount:words} in {bbceIncomeHeldStates}. That note's data list every model's answer for those five households and for the {bbceAssetHeldCount:words} held back by savings, including the three models this release adds." ], "facts": { diff --git a/app/src/notes/2026-10-06-policybench-stops-scoring-eight-tax-outputs.json b/app/src/notes/2026-10-06-policybench-stops-scoring-eight-tax-outputs.json index c2e85c32..0f45d1cb 100644 --- a/app/src/notes/2026-10-06-policybench-stops-scoring-eight-tax-outputs.json +++ b/app/src/notes/2026-10-06-policybench-stops-scoring-eight-tax-outputs.json @@ -8,7 +8,7 @@ "The Virginia household's head is {va114Age}, and the prompt lists no Medicare enrollment or premium. policyengine-us treats everyone eligible for Medicare as enrolled and counts the standard Part B premium, ${partBPremium} for 2026, as a medical expense. The household itemizes, and Virginia's itemized deductions start from the federal ones, so the premium lowers its Virginia income tax: without it, that output is ${va114Alt} rather than the reference's ${va114Ref}. The premium lowers the household's federal income tax too, and that output's exclusion record names both unstated facts.", "The {payrollOutputs:words} payroll outputs count an employee share of a state paid-leave or disability premium: {optionalPrograms}. In each of these states the premium is the employer's, and the employer may deduct up to a set share of it from wages but need not, so the law requires no amount of the employee. The prompts ask for mandatory employee state payroll taxes, and the references count the largest share each state's published 2026 rates let the employer deduct, without the prompts saying whether it does. Without that share, the outputs are ${mnAlt} in Minnesota, ${coAlt} in Colorado, ${maAlt} in Massachusetts, and ${nyAlt} in New York, against references of ${mnRef}, ${coRef}, ${maRef}, and ${nyRef}. PolicyBench still scores the payroll outputs of {mandatoryScored:words} households in {mandatoryStates}, where the law puts the contribution on the worker and requires the employer to withhold it.", "No model's answer is within $1 of the reference on any of the {incomeTaxOutputs:words} income tax outputs. On the {payrollOutputs:words} payroll outputs, {payrollHits} answers match the reference within $1: {mnHits:words} for Minnesota, {coHits} for Colorado, {maHits:words} for Massachusetts, and {nyHits:words} for New York. Another {payrollAltHits} match the amount without the state share.", - "Every model's exact rate, its share of answers within $1 weighted by household impact, rises by {riseMin} to {riseMax} points. GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%). Claude Sonnet 5.5 ({sonnetExact}%) passes GPT-6 Luna ({lunaExact}%) for fourth. Further down, {riserTwo} moves above {riserTwoPassed}, {riserThree} above {riserThreePassed}, and {riserFour} above {riserFourPassed}. Every other pair keeps its order.", + "Every model's share of answers right rises by {riseMin} to {riseMax} points. PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches. It weights each answer by household impact. GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%). Claude Sonnet 5.5 ({sonnetExact}%) passes GPT-6 Luna ({lunaExact}%) for fourth. Further down, {riserTwo} moves above {riserTwoPassed}, {riserThree} above {riserThreePassed}, and {riserFour} above {riserFourPassed}. Every other pair keeps its order.", "PolicyBench's next prompt contract remains an unactivated draft (PolicyEngine/policybench#173). It states the state income tax paid for each tax unit or a named convention for estimating withholding; each person's Medicare Part B enrollment and annual net premium; and whether each person's employer withholds the employee share of state paid-leave or disability premiums. Its payroll output definition counts optional employee shares only when the employer withholds them. A later run can score these outputs after its reference builder and a fresh input sweep validate the stated conventions.", "Two corrections in this release move no score. PolicyBench now reports that it collected model responses from {responseStart} to {responseEnd}, 2026, ending on the UTC date of the last answer; release dashboard-data-20260930 gave {previousResponseEnd}, the date of that release. The audit text for a California household head's Medicaid eligibility now explains the head's income test as the engine applies it: California's ${medicaidDisregard} monthly income disregard, which takes the place of SSI's ${ssiExclusion} general income exclusion, brings countable income under the limit. The earlier text said the engine subtracts a Medicare Part B premium from income, which it does not." ], diff --git a/app/src/notes/2026-10-10-claude-haiku-5-5-joins-the-board.json b/app/src/notes/2026-10-10-claude-haiku-5-5-joins-the-board.json index 00d6c8af..4fc4353f 100644 --- a/app/src/notes/2026-10-10-claude-haiku-5-5-joins-the-board.json +++ b/app/src/notes/2026-10-10-claude-haiku-5-5-joins-the-board.json @@ -3,12 +3,12 @@ "date": "2026-10-10", "title": "Claude Haiku 5.5 joins the board as the references move to policyengine-us 2.38.6", "paragraphs": [ - "Claude Haiku 5.5 joined the board on {releaseDate}, bringing it to {nModels} models. It gets {haikuExact}% of answers within $1, weighted by household impact, #{haikuRank} of {nModels} and {haikuGain} points above Claude Haiku 4.5 ({haiku45Exact}%, #{haiku45Rank}). GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%).", + "Claude Haiku 5.5 joined the board on {releaseDate}, bringing the board to {nModels} models. It gets {haikuExact}% of answers right, {haikuGain} points above Claude Haiku 4.5 ({haiku45Exact}%, #{haiku45Rank}), and ranks #{haikuRank} of {nModels}. GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%). PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches. It weights each answer by household impact.", "Claude Haiku 5.5 cost ${haikuCost} a household, the least of any row with a cost above zero; Claude Haiku 4.5 cost ${haiku45Cost}. Unlike Claude Opus 5.5 and Claude Sonnet 5.5, its API accepts a forced tool call, so its row takes the board's forced answer tool. A probe with that request returned no thinking block, where a request without the forced tool did, so the row answers without extended thinking, as Claude Opus 5's does. Re-run with tool_choice: auto, it gets {haikuAutoExact}%, which would rank #{haikuAutoRank}.", "The release also moves PolicyBench's references from policyengine-us {previousEngine} to {engineVersion}, a release that fixes the engine defects behind {restored:words} outputs. The previous release excluded {newlyScored:words} of them, and they return to scoring at the new version's values. The other {restoredRuled:words} are the October 6 defects below, which stay scored at the new version's values. Each of the {restored:words} lands within $1 of the corrected value PolicyBench's audits computed for it. Between them they take the upstream fixes for {restoredFixes}. PolicyBench still builds each reference from the stated facts and from law published before it froze the references on 2026-07-03, under the same publication conventions.", "Two rulings of October 6 apply too. PolicyBench's reference adversary flags the scored outputs where many models, or several of the strongest, agree on an answer the reference does not give, and a judge that cannot yet see the engine's derivation works each one from primary law. Its findings led to rulings to exclude {adversaryOutputs:words} outputs. Four were engine defects, in Arizona, Ohio, Colorado and New York; the new version fixes them, so PolicyBench keeps scoring those four instead. The other four are the federal and state income tax of a Pennsylvania and a Missouri household whose dependent earns $45,000 of wages and must file their own return. The output definitions do not say whether the household's income tax includes that return, so PolicyBench stops scoring them. It also stops scoring {louisianaOutputs:words} Louisiana households' state income tax, whose reference rests on a 2026 standard deduction computed from published price indexes; Louisiana published the amount itself on September 28, after the freeze.", "The new version counts Indiana county income tax in local income tax, and the prompt names no county, so PolicyBench stops scoring the local income tax of {indianaOutputs:words} Indiana households. It also counts Idaho's $10 permanent building fund tax in state income tax, which moves one household's reference from ${idahoBefore} to ${idahoAfter}. Every model is now scored on {scoredOutputs} of its {totalOutputs} requested outputs; PolicyBench excludes the other {excluded}.", - "Against the previous release, the exact rate of every one of the {incumbents} earlier models falls, by {fallMin} to {fallMax} points. The cause is the {newlyScored} restored outputs that the previous release did not score, which few models get right: half of them are answered within $1 by at most {newlyScoredHalfAtMost:words} of the {nModels} models, and {newlyScoredUnmatched:words} by none. Had they stayed excluded, every earlier model's rate would have risen, by {riseWithoutMin} to {riseWithoutMax} points." + "Against the previous release, every one of the {incumbents} earlier models falls, by {fallMin} to {fallMax} points. The cause is the {newlyScored} restored outputs that the previous release did not score, which few models get right: half of them are answered within $1 by at most {newlyScoredHalfAtMost:words} of the {nModels} models, and {newlyScoredUnmatched:words} by none. Without those outputs, every earlier model's share of answers right would have risen, by {riseWithoutMin} to {riseWithoutMax} points." ], "facts": { "releaseDate": "2026-10-10", diff --git a/app/src/notes/2026-10-13-muse-wonder-joins-the-board.json b/app/src/notes/2026-10-13-muse-wonder-joins-the-board.json index 952721eb..c8bcabb3 100644 --- a/app/src/notes/2026-10-13-muse-wonder-joins-the-board.json +++ b/app/src/notes/2026-10-13-muse-wonder-joins-the-board.json @@ -3,10 +3,10 @@ "date": "2026-10-13", "title": "Meta's Muse Wonder joins the board as the references move to policyengine-us 2.41.0", "paragraphs": [ - "Meta's Muse Wonder joined the board on {releaseDate}, the day after Meta released it, bringing the board to {nModels} models. It is the first Meta model on the board. It gets {museExact}% of answers within $1, weighted by household impact, #{museRank} of {nModels}. On the bounded score, which gives partial credit for amounts close to the reference, it ranks #{museBoundedRank}, behind only GPT-6 Sol and Claude Opus 5.5. GPT-6 Sol still leads on answers within $1 at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%).", + "PolicyBench added Muse Wonder, its first Meta model, to the board on {releaseDate}, the day after Meta released it, bringing the board to {nModels} models. Muse Wonder gets {museExact}% of answers right and ranks #{museRank} of {nModels}. On the bounded score, which gives partial credit for amounts close to the reference, it ranks #{museBoundedRank}, behind only GPT-6 Sol and Claude Opus 5.5. GPT-6 Sol still leads at {solExact}% of answers right, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%). PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches. It weights each answer by household impact.", "Meta released Muse Wonder Full the same day, but only to Muse Code subscribers, with no API. PolicyBench runs every model through its provider's API under one protocol, so Muse Wonder Full is not on the board. PolicyBench reaches Muse Wonder through OpenRouter, whose one endpoint for it is Meta's own API. That API accepts only an automatic tool choice and rejects a forced tool call, so the row answers as a JSON object, as Claude Opus 5.5's and Claude Sonnet 5.5's do, and reasons at Meta's default setting.", "The release also moves PolicyBench's references from policyengine-us {previousEngine} to {engineVersion}, the first release with every fix the previous release was waiting on. Of the outputs the previous release excluded, {restored:words} return to scoring at the new version's values, each within $1 of the corrected value PolicyBench's audits computed for it. Between them they take the upstream fixes for {restoredFixes}. The new version also fixes the defect behind a SNAP output, the heat-and-eat utility allowance, but that output stays excluded because the stated facts do not decide it. No reference PolicyBench already scored changes, and nothing is newly excluded. Every model is now scored on {scoredOutputs} of its {totalOutputs} requested outputs; PolicyBench excludes the other {excluded}.", - "Against the previous release, the exact rate of every one of the {incumbents} earlier models falls, by {fallMin} to {fallMax} points. The cause is the {restored:words} restored outputs, which few models get right: half of them are answered within $1 by at most {restoredHalfAtMost:words} of the {nModels} models, and {restoredUnmatched:words} by none. Muse Wonder answers {museRestored:words} of them within $1. Without them, every earlier model's rate would be the previous release's." + "Against the previous release, every one of the {incumbents} earlier models falls, by {fallMin} to {fallMax} points. The cause is the {restored:words} restored outputs, which few models get right: half of them are answered within $1 by at most {restoredHalfAtMost:words} of the {nModels} models, and {restoredUnmatched:words} by none. Muse Wonder answers {museRestored:words} of them within $1. Without them, every earlier model would keep its share of answers right from the previous release." ], "facts": { "releaseDate": "2026-10-13", diff --git a/app/src/notes/2026-10-14-gemini-4-argon-joins-the-board.json b/app/src/notes/2026-10-14-gemini-4-argon-joins-the-board.json index 1a22c33b..75742f93 100644 --- a/app/src/notes/2026-10-14-gemini-4-argon-joins-the-board.json +++ b/app/src/notes/2026-10-14-gemini-4-argon-joins-the-board.json @@ -3,7 +3,7 @@ "date": "2026-10-14", "title": "Google's Gemini 4 Argon and Meta's Muse Spark 1.3 join the board", "paragraphs": [ - "Google's Gemini 4 Argon joined the board on {releaseDate}, the day after Google opened it to the public Gemini API, and Meta's Muse Spark 1.3 joined with it, bringing the board to {nModels} models. Gemini 4 Argon gets {argonExact}% of answers within $1, weighted by household impact, #{argonRank} of {nModels} and the highest of any Gemini model; Gemini 3.8 Flash, the previous best, gets {flashExact}% (#{flashRank}). Muse Spark 1.3 gets {sparkExact}% (#{sparkRank}), behind Meta's Muse Wonder ({museExact}%, #{museRank}), but on the bounded score, which gives partial credit for amounts close to the reference, it ranks #{sparkBoundedRank}. GPT-6 Sol still leads on answers within $1 at {solExact}%, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%).", + "Google's Gemini 4 Argon joined the board on {releaseDate}, the day after Google opened it to the public Gemini API, and Meta's Muse Spark 1.3 joined with it, bringing the board to {nModels} models. Gemini 4 Argon gets {argonExact}% of answers right and ranks #{argonRank} of {nModels}, the highest of any Gemini model. Gemini 3.8 Flash, the previous best, gets {flashExact}% (#{flashRank}). Muse Spark 1.3 gets {sparkExact}% (#{sparkRank}), behind Meta's Muse Wonder ({museExact}%, #{museRank}), but on the bounded score, which gives partial credit for amounts close to the reference, it ranks #{sparkBoundedRank}. GPT-6 Sol still leads at {solExact}% of answers right, ahead of Claude Opus 5.5 ({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%). PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches. It weights each answer by household impact.", "Google announced Gemini 4 Argon on September 30 for a group of partners and released it on the Gemini API on October 13. PolicyBench runs it there, as it does every Gemini model. The API accepts a forced answer tool, so the row answers through the board's tool and reasons at Google's default setting. A forced tool call suppresses some models' reasoning, as it does Claude Opus 5's, so PolicyBench fixed a test before looking at the results and ran all {households} households both ways. With the forced tool Gemini 4 Argon returned reasoning tokens on {forcedReasoning} of them, and on {boardReasoning} in the board's own run; with an automatic tool choice, on {autoReasoning}. The test found no significant reduction, so the board keeps the forced tool and publishes no second row. At Google's introductory price of $2 per million input tokens and $10 per million output tokens, which doubles on December 1, the {households} households cost ${argonCost}.", "Muse Spark 1.3, which Meta released on September 2, is Meta's second model on the board. PolicyBench reaches it through OpenRouter, whose one endpoint for it is Meta's own API, as it does Muse Wonder. That API rejects a forced tool call, so the row answers as a JSON object and reasons at Meta's default setting.", "Nothing else on the board moves. The newest policyengine-us release, {checkedEngine}, gives the same value as {engineVersion} for every output, so the references stay where they were, and every model is still scored on {scoredOutputs} of its {totalOutputs} requested outputs. Each of the {incumbents} earlier models keeps its score." diff --git a/app/src/notes/2026-10-20-given-a-policyengine-tool-three-models-score-above-99.json b/app/src/notes/2026-10-20-given-a-policyengine-tool-three-models-score-above-99.json index 699595d8..629cacc7 100644 --- a/app/src/notes/2026-10-20-given-a-policyengine-tool-three-models-score-above-99.json +++ b/app/src/notes/2026-10-20-given-a-policyengine-tool-three-models-score-above-99.json @@ -3,8 +3,8 @@ "date": "2026-10-20", "title": "Given a tool that runs PolicyEngine, three models score above 99%", "paragraphs": [ - "PolicyBench gave three models a tool that runs PolicyEngine on a household the model writes from the prompt's facts, and ran its {households} US households again on October 14 and 15. GPT-6 Sol, which leads the board at {solBoardExact}% of answers within $1, weighted by household impact, gets {solToolExact}% with the tool. Claude Opus 5.5 goes from {opusBoardExact}% to {opusToolExact}%, and Gemini 4 Argon from {argonBoardExact}% to {argonToolExact}%. The runs are a pilot, not board rows, since the board scores models answering alone, and the result is circular: the tool runs the engine that computes PolicyBench's references, so the scores measure agreement with that engine, not whether it is right.", - "Most of the answers the tool fixes are dollar amounts that are not zero. Of the {nonzeroAmounts} nonzero amounts, GPT-6 Sol gets {solNonzeroRate}% within $1 on its own and {solNonzeroToolRate}% with the tool; Claude Opus 5.5, {opusNonzeroRate}% and {opusNonzeroToolRate}%; and Gemini 4 Argon, {argonNonzeroRate}% and {argonNonzeroToolRate}%.", + "PolicyBench gave three models a tool that runs PolicyEngine on a household the model writes from the prompt's facts, and ran its {households} US households again on October 14 and 15. GPT-6 Sol, which leads the board at {solBoardExact}% of answers right, gets {solToolExact}% with the tool. Claude Opus 5.5 goes from {opusBoardExact}% to {opusToolExact}%, and Gemini 4 Argon from {argonBoardExact}% to {argonToolExact}%. PolicyBench weights each answer by household impact in these six scores. The runs are a pilot, not board rows, since the board scores models answering alone, and the result is circular: the tool runs the engine that computes PolicyBench's references, so the scores measure agreement with that engine, not whether it is right.", + "With or without the tool, PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches. Most of the answers the tool fixes are dollar amounts that are not zero. Of the {nonzeroAmounts} nonzero amounts, GPT-6 Sol gets {solNonzeroRate}% within $1 on its own and {solNonzeroToolRate}% with the tool; Claude Opus 5.5, {opusNonzeroRate}% and {opusNonzeroToolRate}%; and Gemini 4 Argon, {argonNonzeroRate}% and {argonNonzeroToolRate}%. PolicyBench gives each amount the same weight in these six rates.", "With the tool, {toolMisses} answers are wrong across the three runs: {solMisses:words} of GPT-6 Sol's, {opusMisses:words} of Claude Opus 5.5's and {argonMisses:words} of Gemini 4 Argon's. Each traces to the household the model described to the engine, or to the output it read back, rather than to the calculation: in {enteredElsewhere:words} the model had entered a stated fact under another variable, in {leftOut:words} it had left one out, and in {wrongOutput:words} it read back the wrong output. Prompt labels share the blame for {labelMisses:words}: \"employer sponsored insurance premiums\" never says the employer paid them, and \"disability benefits\" never says they are not Social Security. In {solCapMisses:words} of GPT-6 Sol's, it had used all {turnCap:words} calculation turns the pilot allows. GPT-6 Sol is also the only model that turned right answers wrong, {solFlips:words} of them.", "The tool costs more. For the {households} households, GPT-6 Sol costs ${solToolCost} with it and ${solBoardCost} without, Claude Opus 5.5 ${opusToolCost} and ${opusBoardCost}, and Gemini 4 Argon ${argonToolCost} and ${argonBoardCost}.", "The tool applies the conventions PolicyBench uses for its references, so a model that describes a household exactly as the reference does gets the reference's answer. Describing a household is also easier here than it would be from a filer's documents: {labelMatches} of the {statedFacts} facts the prompts state, not counting ages and wages, carry a label that is the engine's own variable name with spaces.", diff --git a/app/src/notes/2026-10-31-gpt-6-1-terra-joins-the-board.json b/app/src/notes/2026-10-31-gpt-6-1-terra-joins-the-board.json index 72f9037a..e87a0458 100644 --- a/app/src/notes/2026-10-31-gpt-6-1-terra-joins-the-board.json +++ b/app/src/notes/2026-10-31-gpt-6-1-terra-joins-the-board.json @@ -3,11 +3,11 @@ "date": "2026-10-31", "title": "OpenAI's GPT-6.1 Terra joins the board at #3 as the references move to policyengine-us 2.60.0", "paragraphs": [ - "PolicyBench added OpenAI's GPT-6.1 Terra to its {snapshotDate} snapshot, the day after OpenAI released it, bringing the board to {nModels} models. It gets {terraExact}% of answers within $1, weighted by household impact, #{terraRank} of {nModels}. Only GPT-6 Sol ({solExact}%) and Claude Opus 5.5 ({opusExact}%) score higher, and GPT-5.6 Sol, which also rounds to {sol56Exact}%, is less than a tenth of a point behind. On the bounded score, which gives partial credit for amounts close to the reference, GPT-6.1 Terra ranks #{terraBoundedRank}. GPT-6.1 Sol, which OpenAI released on September 29, gets {sol61Exact}% (#{sol61Rank}).", + "PolicyBench added OpenAI's GPT-6.1 Terra to its {snapshotDate} snapshot, the day after OpenAI released it, bringing the board to {nModels} models. GPT-6.1 Terra gets {terraExact}% of answers right and ranks #{terraRank} of {nModels}. Only GPT-6 Sol ({solExact}%) and Claude Opus 5.5 ({opusExact}%) score higher, and GPT-5.6 Sol, which also rounds to {sol56Exact}%, trails by less than a tenth of a point. On the bounded score, which gives partial credit for amounts close to the reference, GPT-6.1 Terra ranks #{terraBoundedRank}. GPT-6.1 Sol, which OpenAI released on September 29, gets {sol61Exact}% of answers right (#{sol61Rank}). PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches. It weights each answer by household impact.", "PolicyBench runs GPT-6.1 Terra as it runs GPT-6.1 Sol, on OpenAI's Responses API with the board's forced answer tool at OpenAI's default reasoning effort, and it returned reasoning tokens on all {households} households. At OpenAI's price of $1 per million input tokens and $5 per million output tokens, the {households} households cost ${terraCost}, against ${sol61Cost} for GPT-6.1 Sol and ${solCost} for GPT-6 Sol. GPT-5.6 Terra, the Terra model before it, gets {terra56Exact}% (#{terra56Rank}) and cost ${terra56Cost}.", "The release also moves PolicyBench's references from policyengine-us {previousEngine} to {engineVersion}, the newest release when PolicyBench began recomputing them on October 23. The move changes {changed:words} scored references, each for law in force before the references were frozen. An elderly Californian who lives alone, whose SNAP was $0, now gets the ${snapMonthly} monthly minimum, ${snap031} a year: California's categorical eligibility for CalFresh applies no net income test, even to a household with an elderly member, and the previous version applied one (policyengine-us #10036). Three households in Pennsylvania and Arkansas have too much income for free or reduced-price school lunch, but those states give students at covered schools a free breakfast, so the output asking whether PolicyEngine returns free school meal support is now yes for each (policyengine-us #10238). The new version models both breakfast laws for a student at a covered school and leaves out their conditions: Pennsylvania reimburses schools for them to the extent funds are available, and Arkansas serves them on request.", "Nothing is newly excluded, and no excluded output returns to scoring, so every model is still scored on {scoredOutputs} of its {totalOutputs} requested outputs. When PolicyBench checked PyPI on {checkedOn}, the newest policyengine-us release was {checkedEngine}, and it gives the same value as {engineVersion} for all {scoredOutputs}; it moves {excludedMoved:words} excluded output, which keeps the value it was decided on.", - "Against the previous release, the exact rates of {fallers:words} of the {incumbents} earlier models fall, by {fallMin} to {fallMax} points, and {risers:words} rise by {rise}. No model answers yes for any of the three households' school meals, and only {snapMatch:words} give the Californian's new SNAP: GPT-5.5, GPT-6 Astra and GPT-6.1 Sol, the three that rise. Claude Sonnet 5.5 falls from #{sonnetThen} to #{sonnetNow}, behind GPT-6.1 Terra and GPT-6 Astra." + "Against the previous release, {fallers:words} of the {incumbents} earlier models fall, by {fallMin} to {fallMax} points, and {risers:words} rise by {rise}. No model answers yes for any of the three households' school meals, and only {snapMatch:words} give the Californian's new SNAP: GPT-5.5, GPT-6 Astra and GPT-6.1 Sol, the three that rise. Claude Sonnet 5.5 falls from #{sonnetThen} to #{sonnetNow}, behind GPT-6.1 Terra and GPT-6 Astra." ], "facts": { "snapshotDate": "2026-10-27", diff --git a/app/tests/notes.test.ts b/app/tests/notes.test.ts index 7120b5e5..c1bbe791 100644 --- a/app/tests/notes.test.ts +++ b/app/tests/notes.test.ts @@ -13,6 +13,10 @@ import artifact from "../src/data.artifact.json"; import paperSnapshot from "../src/paperSnapshot.json"; describe("notes", () => { + // The score definition and the weighting close each release note's opening. + const closing = + "PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches. It weights each answer by household impact."; + test("the index is newest first", () => { expect(notes.map((note) => note.date)).toEqual( [...notes].map((note) => note.date).sort().reverse(), @@ -64,8 +68,8 @@ describe("notes", () => { expect(markup).toContain("GPT-6 Astra debuts second: the two rules it invented"); expect(markup).toContain("release dashboard-data-20260905c"); // Whole-number board rates keep one decimal. - expect(markup).toContain("at 88.0% of answers within $1"); - expect(markup).not.toContain("at 88% of answers"); + expect(markup).toContain("It gets 88.0% of answers right"); + expect(markup).not.toContain("gets 88% of answers"); expect(markup).toContain("release dashboard-data-20260901c"); expect(markup).toContain("Six SNAP households the top three models deny"); expect(markup).toContain( @@ -141,10 +145,13 @@ describe("notes", () => { ); // Board rates keep one decimal. expect(markup).toContain( - "It gets 93.1% of answers within $1, weighted by household impact, #3 of 51.", + "GPT-6.1 Terra gets 93.1% of answers right and ranks #3 of 51.", ); expect(markup).toContain("GPT-6.1 Terra ranks #5."); - expect(markup).toContain("GPT-6.1 Sol, which OpenAI released on September 29, gets 90.2% (#10)."); + expect(markup).toContain( + "GPT-6.1 Sol, which OpenAI released on September 29, gets 90.2% of answers right (#10).", + ); + expect(markup).toContain(closing); // Costs keep their cents. expect(markup).toContain( "the 100 households cost $1.25, against $2.30 for GPT-6.1 Sol and $2.71 for GPT-6 Sol.", @@ -158,7 +165,7 @@ describe("notes", () => { "gives the same value as 2.60.0 for all 1,935; it moves one excluded output", ); expect(markup).toContain( - "the exact rates of 47 of the 50 earlier models fall, by 0.03 to 0.07 points, and three rise by 0.02.", + "Against the previous release, 47 of the 50 earlier models fall, by 0.03 to 0.07 points, and three rise by 0.02.", ); expect(markup).toContain("only three give the Californian's new SNAP"); expect(markup).toContain('href="/model/gpt-6.1-terra"'); @@ -177,14 +184,21 @@ describe("notes", () => { createElement(NoteArticle, { note, titleLevel: "h1" }), ); expect(markup).toContain( - "GPT-6 Sol, which leads the board at 95.2% of answers within $1, weighted by household impact, gets 99.2% with the tool.", + "GPT-6 Sol, which leads the board at 95.2% of answers right, gets 99.2% with the tool.", ); expect(markup).toContain( "Claude Opus 5.5 goes from 94.1% to 99.8%, and Gemini 4 Argon from 88.7% to 99.9%.", ); + expect(markup).toContain( + "PolicyBench weights each answer by household impact in these six scores.", + ); expect(markup).toContain( "Of the 187 nonzero amounts, GPT-6 Sol gets 67.4% within $1 on its own and 96.8% with the tool", ); + // The nonzero-amount rates count each amount once. + expect(markup).toContain( + "and Gemini 4 Argon, 46.5% and 99.5%. PolicyBench gives each amount the same weight in these six rates.", + ); // Whole numbers under ten are spelled out; costs keep their cents. expect(markup).toContain( "With the tool, 12 answers are wrong across the three runs: nine of GPT-6 Sol's, two of Claude Opus 5.5's and one of Gemini 4 Argon's.", @@ -300,13 +314,18 @@ describe("notes", () => { "Google's Gemini 4 Argon joined the board on 2026-10-14, the day after Google opened it to the public Gemini API, and Meta's Muse Spark 1.3 joined with it, bringing the board to 50 models.", ); // Board rates keep one decimal. - expect(markup).toContain("Gemini 4 Argon gets 88.7% of answers within $1"); - expect(markup).toContain("#12 of 50 and the highest of any Gemini model"); - expect(markup).toContain("Gemini 3.8 Flash, the previous best, gets 85.6% (#21)"); + expect(markup).toContain( + "Gemini 4 Argon gets 88.7% of answers right and ranks #12 of 50, the highest of any Gemini model. Gemini 3.8 Flash, the previous best, gets 85.6% (#21).", + ); expect(markup).toContain( "Muse Spark 1.3 gets 87.2% (#18), behind Meta's Muse Wonder (90.1%, #10)", ); expect(markup).toContain("it ranks #6."); + // After the bounded rank, the leader's rate names its metric. + expect(markup).toContain( + "GPT-6 Sol still leads at 95.2% of answers right, ahead of Claude Opus 5.5 (94.1%) and GPT-5.6 Sol (93.1%).", + ); + expect(markup).toContain(closing); expect(markup).toContain( "With the forced tool Gemini 4 Argon returned reasoning tokens on 80 of them, and on 85 in the board's own run; with an automatic tool choice, on 82.", ); @@ -331,15 +350,18 @@ describe("notes", () => { createElement(NoteArticle, { note, titleLevel: "h1" }), ); expect(markup).toContain( - "Meta's Muse Wonder joined the board on 2026-10-13, the day after Meta released it, bringing the board to 48 models.", + "PolicyBench added Muse Wonder, its first Meta model, to the board on 2026-10-13, the day after Meta released it, bringing the board to 48 models.", ); // Board rates keep one decimal. - expect(markup).toContain("It gets 90.1% of answers within $1"); - expect(markup).toContain("#10 of 48"); - expect(markup).toContain("it ranks #3, behind only GPT-6 Sol and Claude Opus 5.5"); + expect(markup).toContain("Muse Wonder gets 90.1% of answers right and ranks #10 of 48."); + expect(markup).toContain( + "On the bounded score, which gives partial credit for amounts close to the reference, it ranks #3, behind only GPT-6 Sol and Claude Opus 5.5.", + ); + // After the bounded rank, the leader's rate names its metric. expect(markup).toContain( - "GPT-6 Sol still leads on answers within $1 at 95.2%, ahead of Claude Opus 5.5 (94.1%) and GPT-5.6 Sol (93.1%).", + "GPT-6 Sol still leads at 95.2% of answers right, ahead of Claude Opus 5.5 (94.1%) and GPT-5.6 Sol (93.1%).", ); + expect(markup).toContain(closing); expect(markup).toContain("so Muse Wonder Full is not on the board"); expect(markup).toContain( "Of the outputs the previous release excluded, nine return to scoring", @@ -357,6 +379,9 @@ describe("notes", () => { expect(markup).toContain( "half of them are answered within $1 by at most one of the 48 models, and four by none. Muse Wonder answers one of them within $1.", ); + expect(markup).toContain( + "Without them, every earlier model would keep its share of answers right from the previous release.", + ); expect(markup).toContain('href="/model/muse-wonder"'); expect(markup).toContain( 'href="https://github.com/PolicyEngine/policybench/releases/tag/dashboard-data-20261013"', @@ -364,6 +389,58 @@ describe("notes", () => { expect(markup).not.toMatch(/\{[A-Za-z][A-Za-z0-9]*(?::words)?\}/); }); + test("the rewritten score sentences render", () => { + const expected: Record = { + "2026-09-01-claude-fable-5-1-added": [ + "Anthropic released Claude Fable 5.1 on 2026-09-01, and PolicyBench added it to the board the same day and parsed 1,984 of the model's 1,984 answers.", + "Claude Fable 5.1 gets 86.3% of answers right and ranks #2 of 33.", + closing, + ], + "2026-09-05-gpt-6-astra-debuts-second": [ + "GPT-6 Astra reached the API on 2026-09-04 and joined the board on 2026-09-05.", + "It gets 88.0% of answers right and ranks #2 of 39, behind GPT-5.6 Sol (89.2%) and ahead of Claude Fable 5.1 (86.9%, #3).", + "PolicyBench scores every model on 1,973 of its 1,984 requested outputs and excludes the other 11, since their references depend on an engine input the household facts never list.", + closing, + ], + "2026-09-22-gpt-6-sol-debuts-first": [ + "GPT-6 Sol gets 94.2% of answers right, 1.3 points above Claude Opus 5.5 (92.9%, #2), and ranks #1 of 42. GPT-5.6 Sol, the previous leader, ranks #3 at 92.7%.", + closing, + ], + "2026-09-29-claude-sonnet-5-5-debuts-fifth": [ + "Claude Sonnet 5.5 gets 92.1% of answers right and ranks #5 of 45.", + "GPT-6 Luna, #4, also rounds to 92.1% and sits about level with Claude Sonnet 5.5, less than 0.02 points ahead.", + closing, + "Together, the new references and the Medicaid exclusion lift every one of the 42 earlier models, by 0.14 to 0.83 points.", + ], + "2026-10-06-policybench-stops-scoring-eight-tax-outputs": [ + "Every model's share of answers right rises by 0.67 to 1.35 points.", + `${closing} GPT-6 Sol still leads at 95.8%`, + ], + "2026-10-10-claude-haiku-5-5-joins-the-board": [ + "Claude Haiku 5.5 joined the board on 2026-10-10, bringing the board to 47 models. It gets 79.6% of answers right, 4.8 points above Claude Haiku 4.5 (74.8%, #43), and ranks #37 of 47.", + closing, + "Against the previous release, every one of the 46 earlier models falls, by 0.24 to 1.37 points.", + "Without those outputs, every earlier model's share of answers right would have risen, by 0.39 to 0.71 points.", + ], + "2026-10-13-muse-wonder-joins-the-board": [ + "Against the previous release, every one of the 47 earlier models falls, by 0.18 to 0.42 points.", + ], + "2026-10-20-given-a-policyengine-tool-three-models-score-above-99": [ + "With or without the tool, PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches.", + ], + "2026-10-31-gpt-6-1-terra-joins-the-board": [ + "Only GPT-6 Sol (95.2%) and Claude Opus 5.5 (94.1%) score higher, and GPT-5.6 Sol, which also rounds to 93.1%, trails by less than a tenth of a point.", + ], + }; + for (const [slug, sentences] of Object.entries(expected)) { + const note = notes.find((n) => n.slug === slug)!; + const markup = renderToStaticMarkup( + createElement(NoteArticle, { note, titleLevel: "h1" }), + ); + for (const sentence of sentences) expect(markup).toContain(sentence); + } + }); + test("a words placeholder spells out whole numbers under ten", () => { // Any note serves as the template; its facts and paragraphs are replaced. const note = { diff --git a/docs/terra/design.md b/docs/terra/design.md index cd0be468..5df04c94 100644 --- a/docs/terra/design.md +++ b/docs/terra/design.md @@ -141,7 +141,7 @@ Export writes `data-board51.json` and a hash-bound `release-ready.json`. The res ## Lead completion record -- **Result.** GPT-6.1 Terra gets 93.14% of answers within $1, weighted by household impact, #3 of 51 on the 1,935 scored outputs, with every output parsed (bounded score 97.19). Its 100 households cost $1.2532 ($0.0125 each) and took a median 22.4 seconds each, all live calls. It answered on 2026-10-26 between 22:11 and 22:28 UTC (board run 2, input pins b0cfa0db). The board's response window is 2026-06-12 to 2026-10-26 and the snapshot is dated 2026-10-27. +- **Result.** GPT-6.1 Terra gets 93.14% of answers right and ranks #3 of 51 on the 1,935 scored outputs (bounded score 97.19), and every one of its outputs parses. PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches. It weights each answer by household impact. GPT-6.1 Terra's 100 households cost $1.2532 ($0.0125 each) and took a median 22.4 seconds each, all live calls. GPT-6.1 Terra answered on 2026-10-26 between 22:11 and 22:28 UTC (board run 2, input pins b0cfa0db). The board's response window is 2026-06-12 to 2026-10-26 and the snapshot is dated 2026-10-27. - **Judge.** The audit covers 681 cases: 551 verdicts carried over, and 127 cases were re-opened, among them scenario_031's `snap`, one of the four changed references, whose earlier verdict install-references set aside because its prompt re-renders on policyengine-us 2.60.0. The three new cases are the other three changed references, `free_school_meals_eligible` for scenarios 028, 117 and 123. Claude Opus 5.5 judged the 130 on 2026-10-27 under the isolated `setup-token 2` login (`docs/terra/judge_provenance.json`). Scenario 008's `payroll_tax` verdict left out one model in nine judgings in a row; it passed at 14:47 UTC under the per-case schema (7d71b7a0, c5974881). - **Triage.** `restate_gpt61sol_adjudications.py` restated the judge fields of 42 entries, every committed decision on a re-opened case. Triage read 10,999 rows in 990 cases and flagged no reference: `reference-flags.csv` and `unresolved-rows.csv` hold only their headers, so triage needed no decision. Triage also applied 1,522 wording amendments in 117 cases, listed in `us_wording-amendments.json`. Round 3 appended 51 more in seven cases, each answering a finding of release review r2: the reasoning, case note and rows of scenarios 005, 064, 120 and 122, and 42 sentence starts that named the package as "Policyengine-us 2.60.0" or "PolicyEngine 2.60.0". The stage's list then held 1,573. Round 4 appended five more in three cases, each answering a finding of release review r3: scenario_122's case note and claude-fable-5's row, scenario_064's case note and reasoning, and scenario_120's reasoning. The stage's list then held 1,578. - **Export and freeze.** The first export and freeze ran on 2026-10-28 at 2677b116, before the wording review (logs: `~/reviews/policybench-gpt-6-1-terra/freeze-run/`). The release ran again on 2026-10-29 with the review's amendments. Export first ran from 08:03 to 08:07 UTC at 4332ff96, and the freeze dry run refused it: export promoted a fixed list of files and left out the adjudication record that triage amends, so the receipt bound bytes the stage did not hold. Commit 4984698a makes export promote every bound file whose bytes triage changed. Export then ran from 08:12 to 08:15 UTC and staged GPT-6.1 Terra at 93.138833%, #3 of 51 (n = 1,935). The freeze dry run (08:15 to 08:18) validated the release (`RELEASE_TAG`): 51 models, payload 214eed92, references on policyengine-us 2.60.0, 49 exclusions (none new), no adjudications beyond the ruled outputs, 1,522 wording amendments, model responses 2026-06-12 to 2026-10-26, snapshot 2026-10-27, nothing written. The freeze (08:18 to 08:22) rebuilt the snapshot and exited 0, its treatment-fingerprint, reference, analysis, adjudication and manifest checks and `verify_frozen` included. Logs: `~/reviews/policybench-gpt-6-1-terra/freeze-run-r2/`. Round 3 ran the release again on 2026-10-29 with the 51 new amendments (logs: `~/reviews/policybench-gpt-6-1-terra/freeze-run-r3/`). Before it, the lead restored the staged adjudication record to d923dd52's bytes, so that export applied every reasoning amendment from the base text. Export ran from 16:30 to 16:34 UTC at 25d779d7 and staged GPT-6.1 Terra at 93.138833%, #3 of 51 (n = 1,935). The freeze dry run refused it at 16:34 UTC: "stage.json's references_installed is not the installed build's record". Commit 25d779d7 had written the CRP ruling into `docs/terra/spec.json`, whose hash install-references binds in `stage.json`. Commit 3654e5d5 restored the bound spec and moved the ruling to this file. The freeze dry run (16:38 to 16:42) then validated the release: 51 models, payload 5de82780, references on policyengine-us 2.60.0, 49 exclusions (none new), no adjudications beyond the ruled outputs, 1,573 wording amendments, model responses 2026-06-12 to 2026-10-26, snapshot 2026-10-27, nothing written. The freeze (16:42 to 16:52) rebuilt the snapshot and exited 0. Round 4 ran the release again on 2026-10-30 with the five new amendments (logs: `~/reviews/policybench-gpt-6-1-terra/freeze-run-r4/`), after the lead again restored the staged adjudication record to d923dd52's bytes. Export ran from 02:13 to 02:16 UTC at 162c1666 and staged GPT-6.1 Terra at 93.138833%, #3 of 51 (n = 1,935). The freeze dry run (02:16 to 02:19) validated the release: 51 models, payload 67cb0d14, references on policyengine-us 2.60.0, 49 exclusions (none new), no adjudications beyond the ruled outputs, 1,578 wording amendments, model responses 2026-06-12 to 2026-10-26, snapshot 2026-10-27, nothing written. The freeze (02:19 to 02:24) rebuilt the snapshot and exited 0. diff --git a/tests/test_notes.py b/tests/test_notes.py index f1b58cf2..3baec39c 100644 --- a/tests/test_notes.py +++ b/tests/test_notes.py @@ -3408,10 +3408,15 @@ def pin(*sentences: str) -> None: pin( "Claude Sonnet 5.5, Grok 4.7 and DeepSeek V4.1 Flash joined the board on " "2026-09-29, bringing it to {nModels} models.", - "Claude Sonnet 5.5 scores {sonnetExact}% of answers within $1, weighted by " - "household impact, #{sonnetRank} of {nModels}.", + "Claude Sonnet 5.5 gets {sonnetExact}% of answers right and ranks " + "#{sonnetRank} of {nModels}.", + ) + _assert_score_definition() + pin( + SCORE_DEFINITION, + SCORE_WEIGHTING, "GPT-6 Luna, #{lunaRank}, also rounds to {lunaExact}% and sits about level " - "with it, less than {lunaSonnetGapBelow} points ahead.", + "with Claude Sonnet 5.5, less than {lunaSonnetGapBelow} points ahead.", "Grok 4.7 scores {grokExact}% (#{grokRank}) and DeepSeek V4.1 Flash " "{flashExact}% (#{flashRank}).", # The previous release's leader, recomputed on its references. @@ -3720,9 +3725,8 @@ def pin(*sentences: str) -> None: # The drift's two causes, as the drift baseline test rebuilds them. test_previous_release_scores_rebuild_from_this_snapshot() pin( - "Together, the new references and the Medicaid exclusion raise the exact " - "rate of every one of the {incumbents} earlier models, by {driftMin} to " - "{driftMax} points.", + "Together, the new references and the Medicaid exclusion lift every one " + "of the {incumbents} earlier models, by {driftMin} to {driftMax} points.", "None of the {incumbents} matched any of the {newExclusions:words} federal " "outputs now excluded, and all {incumbents} had matched the Arizona " "household's old ${azBefore} SNAP reference, which none matches now.", @@ -4446,8 +4450,12 @@ def rank_then(model: str) -> int: assert rank_now(passed[0]) == rank_then(riser) pin( # The headline the board ranks by, as the September 29 note states it. - "Every model's exact rate, its share of answers within $1 weighted by " - "household impact, rises by {riseMin} to {riseMax} points.", + "Every model's share of answers right rises by {riseMin} to {riseMax} points.", + ) + _assert_score_definition() + pin( + SCORE_DEFINITION, + SCORE_WEIGHTING, "GPT-6 Sol still leads at {solExact}%, ahead of Claude Opus 5.5 " "({opusExact}%) and GPT-5.6 Sol ({sol56Exact}%).", "Claude Sonnet 5.5 ({sonnetExact}%) passes GPT-6 Luna ({lunaExact}%) for " @@ -5319,8 +5327,9 @@ def test_release_20261013_note_claims() -> None: assert set(scoring["now"]) == {row["model"] for row in board} for row in board: assert scoring["now"][row["model"]] == pytest.approx(row["exact"], abs=1e-9) - # "Without them, every earlier model's rate would be the previous - # release's": leaving the restored outputs unscored gives release 20261010. + # "Without them, every earlier model would keep its share of answers right + # from the previous release": leaving the restored outputs unscored gives + # release 20261010. for model, exact in then.items(): assert scoring["without"][model] == pytest.approx(exact, abs=1e-9), model # The restored outputs are all ones release 20261010 excluded (none was @@ -6138,7 +6147,7 @@ def test_release_20261027_note_claims() -> None: assert (date.fromisoformat(note["boardSnapshot"]) - released).days == 1 assert MODEL_RELEASE_DATES["gpt-6.1-sol"] == "2026-09-29" # "Only GPT-6 Sol and Claude Opus 5.5 score higher", and GPT-5.6 Sol, at - # the same rounded rate, is less than a tenth of a point behind. + # the same rounded rate, trails by less than a tenth of a point. assert names[:4] == ["gpt-6-sol", "claude-opus-5.5", TERRA, "gpt-5.6-sol"] assert facts["sol56Exact"] == facts["terraExact"] assert 0 < exact[TERRA] - exact["gpt-5.6-sol"] < 0.1 @@ -6238,3 +6247,67 @@ def test_release_20261027_note_claims() -> None: ): if href.startswith(prefix): assert (ROOT / href.removeprefix(prefix)).exists(), href + + +SCORE_DEFINITION = ( + "PolicyBench counts a dollar amount as right when it falls within $1 of " + "the reference PolicyEngine computes, and an eligibility answer as right " + "when it matches." +) +# "Weighted by household impact" leaves out who weights, so each note says +# it in a sentence of its own, most often this one, after the definition. +# The October 20 note gives its weighted scores before the definition, so +# its sentence names PolicyBench and the six scores it covers. +SCORE_WEIGHTING = "It weights each answer by household impact." + + +def _assert_score_definition() -> None: + """The headline exact score, as ``row_hit_scores`` computes it: a dollar + amount hits within $1 of the reference, and an eligibility flag hits only + when it matches.""" + from policybench.analysis import BINARY_PROGRAMS, row_hit_scores + + assert row_hit_scores("snap", 100.0, 101.0)["exact"] == 1.0 + assert row_hit_scores("snap", 100.0, 101.5)["exact"] == 0.0 + assert "person_medicaid_eligible" in BINARY_PROGRAMS + assert row_hit_scores("person_medicaid_eligible", 1.0, 1.0)["exact"] == 1.0 + assert row_hit_scores("person_medicaid_eligible", 1.0, 0.0)["exact"] == 0.0 + + +def test_no_note_gives_the_score_as_answers_within_one_dollar(): + """PolicyBench scores eligibility flags by exact match, not within $1. So no + note gives an all-outputs score as answers within $1. A title or sentence + that gives a percentage or a share within $1, or within a dollar, also + names the amounts that figure covers. A note that gives a score as answers + right defines it and says how PolicyBench weights the answers.""" + _assert_score_definition() + for path in sorted(NOTES_DIR.glob("*.json")): + note = _load_json(path) + text = " ".join([note["title"], *note["paragraphs"]]) + for phrase in ( + "answers within $1", + "of answers within", + "weighted by household impact", + ): + assert phrase not in text.lower(), (path.name, phrase) + # A percentage or share within $1 covers dollar amounts only, so the + # title or sentence that gives one names them. + for part in (note["title"], *note["paragraphs"]): + for sentence in re.split(r"(?<=[.!?])\s+|(?<=[.!?][\"”’)])\s+", part): + if re.search( + r"(?:\{\w+(?::\w+)?\}|\b\d+(?:\.\d+)?)%|(?i:\bshares?\b)", sentence + ) and re.search(r"[Ww]ithin (?:a |one )?(?:\$1\b|dollar\b)", sentence): + assert re.search(r"\bamounts\b", sentence), (path.name, sentence) + # A note that gives a score as answers right also says, in the same + # words as every other note, what PolicyBench counts as right, and + # says how it weights the answers. + if "of answers right" in text: + assert SCORE_DEFINITION in text, path.name + if SCORE_DEFINITION in text: + assert "weights each answer by household impact" in text, path.name + # Paragraph 0 doubles as the page description and the list blurb, so + # the news leads, and the definition and the weighting close it, after + # every sentence that leans on an earlier one. + opening = note["paragraphs"][0] + if SCORE_DEFINITION in opening: + assert opening.endswith(f"{SCORE_DEFINITION} {SCORE_WEIGHTING}"), path.name