diff --git a/app/public/paper/policybench.pdf b/app/public/paper/policybench.pdf index b6c1483d..415928a6 100644 Binary files a/app/public/paper/policybench.pdf and b/app/public/paper/policybench.pdf differ diff --git a/app/public/paper/web/index.html b/app/public/paper/web/index.html index 5710129c..09c69634 100644 --- a/app/public/paper/web/index.html +++ b/app/public/paper/web/index.html @@ -299,7 +299,7 @@

Frozen snapshot an -
+
@@ -854,7 +854,7 @@

Reference-output credibility

PolicyBench computes each scored reference with policyengine_us.Simulation. The snapshot also records policyengine.py 6.1.2 for provenance; its certified US bundle carries an older policyengine-us, and policyengine.py does not load beside 2.60.0. On 2026-09-29 PolicyBench rebuilt the references with policyengine-us 2.15.17 in place of 1.755.4 and ported the publication conventions to the new version. The SNAP convention holds each FY2026 figure directly for October to December 2026, because the new version carries USDA’s FY2027 figures as published values. policyengine-us 2.15.17 counts Maryland county income tax in its state income tax. PolicyBench’s state income tax output leaves local tax out, so an adapter restores that scope, and the county tax stays in the federal state and local tax deduction. policyengine-us #9261 changed from 40 hours to 0 the default of the weekly-hours input that SNAP’s work rules read, so the scenario builder also passes each prompt’s stated usual weekly hours under that input’s name. After the port, PolicyBench grouped every output that still moved by root cause. An investigator settled each root cause from the upstream commits, the engine code of both versions and primary legal sources with their publication dates, and an independent reviewer tried to refute each settlement. The move changes 4 scored references (Table 4). Two take law published before the freeze that policyengine-us 1.755.4 did not encode: New Jersey’s child tax credit schedule for 2026 to 2028 (P.L.2026, c.26, approved June 30, 2026), and Arizona’s broad-based categorical eligibility limit for SNAP, which Arizona raised from 185 to 200 percent of the poverty guideline starting with benefit month March 2026. Two follow engine corrections: policyengine-us added child support received to school-meal income (7 CFR 245.6(a)(5)(ii)), and policyengine-us #9425 corrected the rounding in New York’s Empire State child credit phase-out. The investigation found, and PolicyBench excluded, 3 federal income tax outputs that depend on an unlisted input: the new version counts a listed state and local tax refund as income (policyengine-us #9422), and whether any of it is income depends on whether the refunded tax reduced federal tax in the year the household paid it (26 U.S.C. 111(a)), which the prompt does not say. The re-review of a California household’s excluded SNAP output also flagged a scored output the move did not change: the head’s Medicaid eligibility. The head’s income is above the 138 percent limit for the adult expansion group, so only a disability pathway leads to Medi-Cal, and California’s Working Disabled Program requires SSI’s definition of disability, the unlisted input that already excludes the household’s SNAP. policyengine-us 2.15.17 tests the general disability flag there and gives 1 under either reading; with the program’s test reading SSI’s definition, the head qualifies only by meeting it, and PolicyBench excluded the output. Another 2 references move by less than the dollar tolerance. PolicyBench re-reviewed the 19 excluded outputs whose values move on 2.15.17; all stay excluded, each with the value it was decided on. The reference sidecar lists every change, and reference_audit/2026-09-28 holds the recomputation of every output, the investigations and the reviews.

A review of release dashboard-data-20260930 on 2026-10-05 found three more inputs that no prompt states behind scored references; no judge had flagged the scored outputs they move (reference_audit/2026-10-05, reference_audit/2026-10-05-medicare-part-b and reference_audit/2026-10-05-payroll). The first is the state income tax in the federal state and local tax deduction, which takes the state income tax a household paid during the year, or general sales tax instead (26 U.S.C. 164(a)(3) and (b)(5)). No prompt states state income tax withheld or paid. policyengine-us fills that input with state_withheld_income_tax, a formula estimate of withholding on the person’s whole federal adjusted gross income, which can differ from the household’s state liability, and the references take the estimate. Read as the household paying its 2026 state income tax during the year, or as paying none, the federal income tax of five households moves, three of them scored. The second is Medicare enrollment and a Part B premium. policyengine-us treats every Medicare-eligible person as enrolled and adds the modeled Part B premium, $2,434.80 for 2026 where the Medicare Savings Program does not pay it, to medical expenses; no prompt states either. The prompt’s instruction to assume program take-up supports the reference, and its rules that unlisted numbers are zero and that unlisted expenses and health coverage are not to be inferred support the other reading. Without the premium, the outputs of one household move: it itemizes, so its federal income tax moves, and Virginia’s itemized deductions start from the federal ones, so its Virginia income tax moves too. The third is whether the employer deducts the employee share of a state paid-leave or disability premium. The payroll output counts mandatory employee state payroll taxes. In Minnesota, Colorado, Massachusetts and New York the law lets the employer deduct an employee share of such a premium from wages, up to a cap, but does not require it, so the employee pays from $0 to the cap at the employer’s choice. The four payroll references from those states count the cap, and the prompt does not state the employer’s choice; read as the amount the law requires of the employee, each is employee federal payroll tax alone. California’s, Connecticut’s, New Jersey’s and Pennsylvania’s employee contributions are owed by the worker and withheld by the employer, and the review left those references as they were. Each audit recomputed every output on policyengine-us 2.15.17 with the publication conventions, under each reading, and each sweep’s baseline reproduces all 1,928 references that release scored. Beyond the one-dollar tolerance, the sweeps move eight scored outputs, which PolicyBench excluded, and two federal income tax outputs already excluded for other reasons; they move no output that is still scored by any amount. The eight records are computed on policyengine-us 2.15.17, whose values are the references.

On 2026-10-06 PolicyBench ruled to exclude ten more outputs. Eight came from a reference adversary (reference_audit/2026-10-05-reference-adversary). It flags the scored outputs where many models, or several of the strongest, agree on an answer the reference does not give. A judge that cannot yet see how the engine derived the reference works each flagged output from primary law, and only then reconciles its answer with the derivation. The adversary flagged 61 outputs and ran 52, the other nine belonging to other audits, and it held 45 references. An independent check of its seven other verdicts confirmed four as engine defects: Arizona’s 2026 standard deduction left unindexed (A.R.S. 43-1041(H)), Ohio’s medical deduction counting health insurance premiums only for a Medicare-eligible filer with no employer-paid plan (R.C. 5747.01(A)(10)), Colorado’s 2026 sales tax refund paid without the excess state revenues that C.R.S. 39-22-2003(2) requires, and New York’s 2026 child and dependent care credit, which Tax Law 606(c-2) replaced. The check refuted two, the Medicaid eligibility of two children in a North Carolina household, whose references stand. The seventh is a question of definition. A Pennsylvania household’s dependent earns $45,000 of wages and must file their own return, and the output definitions name no return, so the household’s federal and state income tax can be read with or without that return; a scan for the same question found a Missouri household, and PolicyBench excluded the four outputs. The other two outputs are two Louisiana households’ 2026 state income tax (reference_audit/2026-10-05-louisiana). Their references rest on a standard deduction that policyengine-us computed from published price indexes under Louisiana’s formula; Louisiana published the return amount itself on 2026-09-28, after the freeze. All ten records are computed on policyengine-us 2.15.17.

-

On 2026-10-10 PolicyBench moved the references from policyengine-us 2.15.17 to 2.38.6, a release that fixes engine defects behind excluded outputs, and recomputed every output under the same publication conventions and adapter (reference_audit/2026-10-09-engine-upgrade). The move fixes the engine defects behind 18 outputs. Earlier releases excluded 14 of them, which return to scoring at the new version’s values; the other four, ruled on 2026-10-06 while still scored, stay scored at those values. Of the 18, 16 land within $1 of the corrected values their exclusion records carry. The other two are held instead to the values the audit’s fix modules give on policyengine-us 2.37.2, an engine that still has the defects; each lands within $1 of its own. The restored outputs take the upstream fixes for the IRA deduction’s compensation limit and a dependent’s contributions (policyengine-us #10032, #9801; three outputs), the IRA deduction’s active-participant phase-out and California’s itemized deduction conformity (policyengine-us #10032, #10034), New Jersey’s worker unemployment and workforce contributions (policyengine-us #10031), Arizona’s standard deduction indexing (policyengine-us #9928), California’s itemized deduction conformity (policyengine-us #10034), Ohio’s medical deduction for health insurance premiums (policyengine-us #10020), estate income in gross income (policyengine-us #10027, #9633; two outputs), Colorado’s 2026 sales tax refund (policyengine-us #9946), the IRA deduction’s active-participant phase-out (policyengine-us #10032; four outputs), New York’s 2026 child and dependent care credit (policyengine-us #9948) and the IRA deduction’s active-participant phase-out and estate income in gross income (policyengine-us #10032, #10027; two outputs). The move changes one scored reference beyond the dollar tolerance and two within it, and removes two outputs from scoring. The changed scored reference follows Idaho’s $10 permanent building fund tax (Idaho Code 63-3082), which policyengine-us now counts in state income tax (policyengine-us #9810). The two outputs that leave scoring depend on an input no prompt states: county of residence (Indiana county income tax).

+

On 2026-10-10 PolicyBench moved the references from policyengine-us 2.15.17 to 2.38.6, a release that fixes engine defects behind excluded outputs, and recomputed every output under the same publication conventions and adapter (reference_audit/2026-10-09-engine-upgrade). The move fixes the engine defects behind 18 outputs. Earlier releases excluded 14 of them, which return to scoring at the new version’s values; the other four, ruled on 2026-10-06 while still scored, stay scored at those values. Of the 18, 16 land within $1 of the corrected values their exclusion records carry. The other two are held instead to the values the audit’s fix modules give on policyengine-us 2.37.2, an engine that still has the defects; each lands within $1 of its own. The fixed outputs take the upstream fixes for the IRA deduction’s compensation limit and a dependent’s contributions (policyengine-us #10032, #9801; three outputs), the IRA deduction’s active-participant phase-out and California’s itemized deduction conformity (policyengine-us #10032, #10034), New Jersey’s worker unemployment and workforce contributions (policyengine-us #10031), Arizona’s standard deduction indexing (policyengine-us #9928), California’s itemized deduction conformity (policyengine-us #10034), Ohio’s medical deduction for health insurance premiums (policyengine-us #10020), estate income in gross income (policyengine-us #10027, #9633; two outputs), Colorado’s 2026 sales tax refund (policyengine-us #9946), the IRA deduction’s active-participant phase-out (policyengine-us #10032; four outputs), New York’s 2026 child and dependent care credit (policyengine-us #9948) and the IRA deduction’s active-participant phase-out and estate income in gross income (policyengine-us #10032, #10027; two outputs). The move changes one scored reference beyond the dollar tolerance and two within it, and removes two outputs from scoring. The changed scored reference follows Idaho’s $10 permanent building fund tax (Idaho Code 63-3082), which policyengine-us now counts in state income tax (policyengine-us #9810). The two outputs that leave scoring depend on an input no prompt states: county of residence (Indiana county income tax).

On 2026-10-13 PolicyBench moved the references again, from policyengine-us 2.38.6 to 2.41.0, the first release with every fix the previous move was waiting on. With the same publication conventions and adapter, it recomputed every output (reference_audit/2026-10-12-engine-upgrade). The move fixes the engine defects behind nine outputs. Earlier releases excluded them, and they return to scoring at the new version’s values. Of the nine, six land within $1 of the corrected values their exclusion records carry. The other three are held instead to the values the audit’s fix modules give on policyengine-us 2.38.6, an engine that still has the defects; each lands within $1 of its own. The reference sidecar records each target with the upstream fix it names (regenerated_exclusions). The restored outputs take the upstream fixes for Idaho’s subtraction for health insurance premiums (policyengine-us #10023; two outputs), Wisconsin’s retirement income exclusion before refundable credits and Wisconsin’s capital gain subtraction for distributions (policyengine-us #10026; two outputs), the IRA deduction’s active-participant phase-out, California’s itemized deduction conformity and California’s educator expense addback (policyengine-us #10032, #10034, #10052) and elective deferrals in the earned income behind the EITC and the refundable child tax credit (policyengine-us #10030; four outputs). The move changes no scored reference, by any amount, and removes no output from scoring. PolicyBench re-reviewed the 12 excluded outputs whose values move on 2.41.0; all stay excluded, each with the value it was decided on.

On 2026-10-23 PolicyBench moved the references once more, from policyengine-us 2.41.0 to 2.60.0, and recomputed every output under the same publication conventions and adapter (reference_audit/2026-10-22-engine-upgrade). policyengine-us 2.60.0 was the newest release when PolicyBench began sweeping the references on 2026-10-23 (uploaded 2026-10-22 at 22:47 UTC). policyengine-us 2.78.0, the newest release when PolicyBench checked PyPI on 2026-10-29 at 16:52 UTC, gives the same value as 2.60.0 for all 1,935 scored outputs under the same conventions and adapter, and moves one of the excluded outputs, which keep the values they were decided on. No excluded output returns to scoring. The move changes four scored references beyond the dollar tolerance and none within it, and removes no output from scoring. The changed scored references follow the free school breakfast that Pennsylvania (Act 33 of 2023) and Arkansas (Act 123 of 2025) give students at covered schools (policyengine-us #10238) and California’s modified categorical eligibility for CalFresh, which has no net income test even for a household with an elderly or disabled member (policyengine-us #10036). policyengine-us models both breakfast laws for a student at a covered school without their conditions: Pennsylvania reimburses schools to the extent funds are available, and Arkansas serves the breakfast on request. PolicyBench re-reviewed the 16 excluded outputs whose values on 2.60.0 differ from the values they were decided on; all stay excluded. The 49 excluded outputs keep the values they were decided on (29 computed with policyengine-us 1.755.4, 18 with 2.15.17, 2 with 2.38.6).

Federal income tax references apply the section 1 rate schedule to taxable income, to the cent, at every income. The Form 1040 instructions send filers with taxable income under $100,000 to the IRS Tax Table, whose looked-up amounts can differ from the schedule by a few dollars. PolicyBench keeps the schedule: an answer that follows the Tax Table can miss the $1 tolerance on those outputs (reference_audit/2026-10-10-tax-table).

@@ -930,7 +930,7 @@

Reference-output credibility US references on policyengine-us 2.38.6 (October 10 upgrade) -The move from policyengine-us 2.15.17 fixes the engine defects behind 18 outputs (14 that earlier releases excluded return to scoring at the new version’s values; the 4 ruled on 2026-10-06 while still scored stay scored). The move changes one scored reference beyond the dollar tolerance and two within it, and removes two outputs from scoring. PolicyBench re-reviewed the 17 excluded outputs whose values move; all stay excluded. The restored outputs take the upstream fixes for the IRA deduction’s compensation limit and a dependent’s contributions (policyengine-us #10032, #9801; three outputs), the IRA deduction’s active-participant phase-out and California’s itemized deduction conformity (policyengine-us #10032, #10034), New Jersey’s worker unemployment and workforce contributions (policyengine-us #10031), Arizona’s standard deduction indexing (policyengine-us #9928), California’s itemized deduction conformity (policyengine-us #10034), Ohio’s medical deduction for health insurance premiums (policyengine-us #10020), estate income in gross income (policyengine-us #10027, #9633; two outputs), Colorado’s 2026 sales tax refund (policyengine-us #9946), the IRA deduction’s active-participant phase-out (policyengine-us #10032; four outputs), New York’s 2026 child and dependent care credit (policyengine-us #9948) and the IRA deduction’s active-participant phase-out and estate income in gross income (policyengine-us #10032, #10027; two outputs). The changed scored reference follows Idaho’s $10 permanent building fund tax (Idaho Code 63-3082), which policyengine-us now counts in state income tax (policyengine-us #9810). The two outputs that leave scoring depend on an input no prompt states: county of residence (Indiana county income tax). +The move from policyengine-us 2.15.17 fixes the engine defects behind 18 outputs (14 that earlier releases excluded return to scoring at the new version’s values; the 4 ruled on 2026-10-06 while still scored stay scored). The move changes one scored reference beyond the dollar tolerance and two within it, and removes two outputs from scoring. PolicyBench re-reviewed the 17 excluded outputs whose values move; all stay excluded. The fixed outputs take the upstream fixes for the IRA deduction’s compensation limit and a dependent’s contributions (policyengine-us #10032, #9801; three outputs), the IRA deduction’s active-participant phase-out and California’s itemized deduction conformity (policyengine-us #10032, #10034), New Jersey’s worker unemployment and workforce contributions (policyengine-us #10031), Arizona’s standard deduction indexing (policyengine-us #9928), California’s itemized deduction conformity (policyengine-us #10034), Ohio’s medical deduction for health insurance premiums (policyengine-us #10020), estate income in gross income (policyengine-us #10027, #9633; two outputs), Colorado’s 2026 sales tax refund (policyengine-us #9946), the IRA deduction’s active-participant phase-out (policyengine-us #10032; four outputs), New York’s 2026 child and dependent care credit (policyengine-us #9948) and the IRA deduction’s active-participant phase-out and estate income in gross income (policyengine-us #10032, #10027; two outputs). The changed scored reference follows Idaho’s $10 permanent building fund tax (Idaho Code 63-3082), which policyengine-us now counts in state income tax (policyengine-us #9810). The two outputs that leave scoring depend on an input no prompt states: county of residence (Indiana county income tax). US references on policyengine-us 2.41.0 (October 13 upgrade) @@ -1110,7 +1110,7 @@

Uncertainty

Sampling uncertainty is substantial at this sample size. In the frozen snapshot, 7 of the other 50 models have an exact-match 95% interval that overlaps the leader’s. Small differences in the leaderboard ordering should therefore be read as ties, and the rank-range column makes the set of plausible positions explicit. These intervals are a manuscript artifact: the public leaderboard and app report point estimates only, because the benchmark sample is the dominant source of uncertainty here, ahead of run-to-run model variation.

Cost and latency

Accuracy is one axis; the price and speed of a no-tools answer are another. Table 7 reports each model’s estimated cost to compute one household’s full output vector and its median per-household latency, next to the headline exact rate. Per-household cost spans $0.001 (Claude Haiku 5.5) to $0.541 (Claude Fable 5), and median per-household latency 3 s (Gemini 3.5 Flash-Lite) to 1023 s (Kimi K3). Neither dimension cleanly tracks accuracy: the top-scoring model, GPT-6 Sol at 95.2% exact, sits mid-pack at $0.027 per household and 23 s.

-
+
diff --git a/app/src/paperSnapshot.json b/app/src/paperSnapshot.json index 1e536615..913ddc46 100644 --- a/app/src/paperSnapshot.json +++ b/app/src/paperSnapshot.json @@ -1,6 +1,6 @@ { - "pdfVersion": "501fb9038a2f", + "pdfVersion": "23a6a3009d67", "responseWindow": "June 12 and October 26, 2026", "snapshotDate": "2026-10-27", - "webVersion": "f9d806df25d9" + "webVersion": "af0ecbe96fac" } diff --git a/paper/index.qmd b/paper/index.qmd index d5c0d31d..7be9a9a8 100644 --- a/paper/index.qmd +++ b/paper/index.qmd @@ -906,22 +906,9 @@ def october_upgrade_row(view) -> list[str]: """A discrepancy-table row for one October engine move (``view``'s last upgrade): what it fixes and restores, what it changes, and its rechecks.""" move = view.last_engine_upgrade - restored = view.engine_upgrade_restored_count - ruled = view.engine_upgrade_restored_ruled_count - split = ( - f"{restored - ruled} that earlier releases excluded return to scoring at " - f"the new version's values; the {ruled} ruled on {r.ruling_date} while " - "still scored stay scored" - if ruled - else "earlier releases excluded them, and they return to scoring at the " - "new version's values" - ) fixes = ( - f"The move from policyengine-us {move.previous_engine_version} fixes the " - f"engine defects behind {restored} outputs ({split}). " - if restored - else f"The move from policyengine-us {move.previous_engine_version} " - "returns no excluded output to scoring. " + f"The move from policyengine-us {move.previous_engine_version} " + f"{view.engine_upgrade_restored_split_clause}. " ) return [ f"US references on policyengine-us {move.engine_version} " diff --git a/paper/snapshot/20260501/manifest.json b/paper/snapshot/20260501/manifest.json index a16b7375..77238b59 100644 --- a/paper/snapshot/20260501/manifest.json +++ b/paper/snapshot/20260501/manifest.json @@ -165,12 +165,12 @@ "rendered_paper_artifacts": { "pdf": { "path": "app/public/paper/policybench.pdf", - "sha256": "501fb9038a2f85068bf82512935b56ee703e37617807e3df703008f98ab5aa06" + "sha256": "23a6a3009d677dba02e332b054be15f19416c42512d85930588d94eff5d61113" }, "web": { "files": { "figures/positive_zero_scatter.png": "85851be224bcceddc3ee15776db2b1917babcfc58e8dc7fbf0f8c26f57b2b9a6", - "index.html": "f9d806df25d92f2546250e5fb39e71302ca7f58a074d514267c82f7f04bde274", + "index.html": "af0ecbe96fac2eb5c1f80fead5e122575e5f7024efcac80d0342e640019e3b5c", "pe-tokens.css": "8f24d8da26f583c8ffddffcdcd172b6d52cbecfec20eda55bd39d7aa829f41d8", "policybench-theme.css": "0e12c5fd615558259e5bce0167a38424e54f9ceb280666c4afd660d759cd1cb9", "site_libs/clipboard/clipboard.min.js": "e17a1d816e13c0826e0ed7febfabc3277f45571234bde0bf9120829a7169edc9", diff --git a/policybench/paper_results.py b/policybench/paper_results.py index fd751dc0..3b9bec14 100644 --- a/policybench/paper_results.py +++ b/policybench/paper_results.py @@ -329,8 +329,10 @@ def partition_engine_upgrade_changes( """Split an engine upgrade's changed outputs into four disjoint groups. ``excluded`` is the exclusion record once the upgrade was decided, and - ``restored`` the excluded outputs the upgrade returned to scoring at the - new engine's values. A change to an excluded output is a new exclusion; a + ``restored`` the outputs whose engine defects the upgrade fixes, scored at + the new engine's values (those a published board excluded return to + scoring; any a ruling decided while still scored stay scored). A change to + an excluded output is a new exclusion; a change to a restored output is a restoration. Any other change is scored, and moves beyond the exact-match tolerance ($1 for an amount, any change for a 0/1 flag) or within it. Every change lands in exactly one group, so @@ -394,7 +396,7 @@ def restored_split_sentence(restored: int, ruled: int, ruling_date: str) -> str: values, and ``ruled`` that rulings on ``ruling_date`` decided while they were still scored stay scored at them.""" if not 0 <= ruled <= restored: - raise ValueError(f"{ruled} ruled of {restored} restored outputs") + raise ValueError(f"{ruled} ruled of {restored} fixed outputs") if not restored: return "" returned = restored - ruled @@ -430,6 +432,58 @@ def restored_split_sentence(restored: int, ruled: int, ruling_date: str) -> str: ) +def restored_split_clause(restored: int, ruled: int, ruling_date: str) -> str: + """The upgrades table's form of ``restored_split_sentence``: a verb phrase + finishing "The move from policyengine-us X ...", with numerals as the rest + of that cell writes counts. It names the ``restored`` outputs whose engine + defects the upgrade fixes and, in parentheses, splits them as the sentence + does: ``restored - ruled`` that earlier releases excluded return to + scoring, and ``ruled`` that rulings on ``ruling_date`` decided while they + were still scored stay scored. A lone output in either group is "one", not + "1". With nothing fixed, it says the move returns no excluded output.""" + if not 0 <= ruled <= restored: + raise ValueError(f"{ruled} ruled of {restored} fixed outputs") + if not restored: + return "returns no excluded output to scoring" + returned = restored - ruled + defects = "defect" if restored == 1 else "defects" + noun = "output" if restored == 1 else "outputs" + lead = f"fixes the engine {defects} behind {restored} {noun}" + if not ruled: + excluded = ( + "an earlier release excluded it, and it returns" + if restored == 1 + else "earlier releases excluded them, and they return" + ) + return f"{lead} ({excluded} to scoring at the new version's values)" + if not returned: + decided = ( + "it while it was still scored, and it stays" + if ruled == 1 + else "them while they were still scored, and they stay" + ) + return ( + f"{lead} (the {ruling_date} rulings decided {decided} scored at the new " + "version's values)" + ) + back = "one that earlier releases excluded returns" + if returned > 1: + back = f"{returned} that earlier releases excluded return" + kept = f"the one ruled on {ruling_date} while still scored stays" + if ruled > 1: + kept = f"the {ruled} ruled on {ruling_date} while still scored stay" + return f"{lead} ({back} to scoring at the new version's values; {kept} scored)" + + +def fixed_or_restored(ruled: int) -> str: + """What the sentence naming an upgrade's fixes calls its outputs: "fixed" + once rulings decided any of them while still scored, since those never + left scoring, and "restored" when every one returns to scoring.""" + if ruled < 0: + raise ValueError(f"{ruled} ruled outputs") + return "fixed" if ruled else "restored" + + def _key(entry: dict) -> tuple[str, str]: return entry["scenario_id"], entry.get("variable", "snap") @@ -471,8 +525,9 @@ class EngineUpgrade: holds those excluded once it was decided: records decided by the end of its date and not removed by then. A record decided after the upgrade never counts here, and a record a later revision removed still does. - ``restored`` holds the outputs the upgrade returned to scoring (its - ``regenerated_exclusions``). + ``restored`` holds the outputs whose engine defects the upgrade fixes (its + ``regenerated_exclusions``): those a published board excluded return to + scoring, and any a ruling decided while still scored stay scored. """ def __init__( @@ -2082,7 +2137,10 @@ def engine_upgrade_new_exclusion_count(self) -> int: @property def engine_upgrade_restored_count(self) -> int: - """Excluded outputs the last upgrade returned to scoring.""" + """Outputs whose engine defects the last upgrade fixes (its + ``regenerated_exclusions``): those a published board excluded return + to scoring, and those the 2026-10-06 rulings decided while still + scored (``engine_upgrade_restored_ruled_count``) stay scored.""" last = self.last_engine_upgrade return 0 if last is None else last.restored_count @@ -2092,7 +2150,7 @@ def engine_upgrade_restored_count_word(self) -> str: @property def engine_upgrade_restored_ruled_count(self) -> int: - """Restored outputs the 2026-10-06 rulings had decided to exclude. No + """Fixed outputs the 2026-10-06 rulings had decided to exclude. No published board excluded them: they were still scored when the rulings came, so they stay scored rather than return to scoring.""" last = self.last_engine_upgrade @@ -2114,6 +2172,16 @@ def engine_upgrade_restored_split_sentence(self) -> str: RULING_DATE, ) + @property + def engine_upgrade_restored_split_clause(self) -> str: + """The same split as a clause for the upgrades table, after "The move + from policyengine-us X".""" + return restored_split_clause( + self.engine_upgrade_restored_count, + self.engine_upgrade_restored_ruled_count, + RULING_DATE, + ) + @cached_property def engine_upgrade_timing(self) -> dict | None: """The last upgrade's sweep timing record, when that upgrade is one @@ -2192,10 +2260,11 @@ def engine_upgrade_timing_sentence(self) -> str: @property def engine_upgrade_restored_fixes(self) -> list[tuple[str, int, list[str]]]: - """The defects behind the outputs the last upgrade returned to scoring: - each one's name, how many outputs it restores and the upstream pull - requests its regenerated_exclusions entries name (not the related - ones), in the sidecar's order. A record with several root causes names + """The defects the last upgrade fixes behind its fixed outputs (those + that return to scoring and any still scored alike): each one's name, + how many outputs it fixes and the upstream pull requests its + regenerated_exclusions entries name (not the related ones), in the + sidecar's order. A record with several root causes names each, and so does a fix-modules target whose modules correct a cause the record does not name (scenario_099's educator expense addback, policyengine-us#10052). Outputs whose causes are the same or one @@ -2267,7 +2336,7 @@ def root(i: int) -> int: @property def engine_upgrade_restored_fix_phrase(self) -> str: - """The fixes behind the restored outputs as one series, each with the + """The fixes behind the fixed outputs as one series, each with the upstream pull requests it names and, past one, how many outputs.""" parts = [] for label, count, prs in self.engine_upgrade_restored_fixes: @@ -2278,11 +2347,16 @@ def engine_upgrade_restored_fix_phrase(self) -> str: @property def engine_upgrade_restored_sentence(self) -> str: - """One sentence naming the fixes behind the restored outputs.""" + """One sentence naming the fixes behind all the outputs the last + upgrade fixes. When the 2026-10-06 rulings decided some of them while + still scored, those never left scoring, so the sentence calls them all + "fixed"; when every one of them returns to scoring, "restored" holds + and it says that.""" phrase = self.engine_upgrade_restored_fix_phrase if not phrase: return "" - return f"The restored outputs take the upstream fixes for {phrase}." + kind = fixed_or_restored(self.engine_upgrade_restored_ruled_count) + return f"The {kind} outputs take the upstream fixes for {phrase}." @property def engine_upgrade_audit_dir(self) -> str: @@ -2341,7 +2415,7 @@ def upgrade_view(self, version: str) -> "PaperResults": @property def engine_upgrade_restored_targets(self) -> dict[str, list[dict]]: - """The last upgrade's restored outputs by how their corrected value is + """The last upgrade's fixed outputs by how their corrected value is known (the sidecar's ``target.kind``): ``record`` for the value the exclusion record carries, ``fix_modules`` for the value the audit's fix modules give on a later engine that still has the defect. Refuses, @@ -2358,14 +2432,14 @@ def engine_upgrade_restored_targets(self) -> dict[str, list[dict]]: raise ValueError(f"{key} has an unknown target kind {target['kind']!r}") tolerance = entry["tolerance"] if not 0 <= tolerance <= 1: - raise ValueError(f"{key} is restored on a tolerance of {tolerance}") + raise ValueError(f"{key} is fixed on a tolerance of {tolerance}") variable = entry["variable"] if abs( entry["regenerated"] - target["value"] ) > tolerance or moves_beyond_tolerance( variable, target["value"], entry["regenerated"] ): - raise ValueError(f"{key} is restored off its target") + raise ValueError(f"{key} is fixed off its target") if target["kind"] == "record": if target["value"] != entry["record"]["alternative_value"]: raise ValueError(f"{key}'s target is not its record's value") @@ -2378,7 +2452,7 @@ def engine_upgrade_restored_targets(self) -> dict[str, list[dict]]: @property def engine_upgrade_restored_target_sentence(self) -> str: - """How the restored outputs' corrected values are known: by the + """How the fixed outputs' corrected values are known: by the exclusion record, or by the audit's fix modules on a later engine.""" targets = self.engine_upgrade_restored_targets by_record, by_modules = targets["record"], targets["fix_modules"] diff --git a/tests/test_disclosures.py b/tests/test_disclosures.py index ec178890..d1a5c02e 100644 --- a/tests/test_disclosures.py +++ b/tests/test_disclosures.py @@ -741,6 +741,35 @@ def test_rendered_pdf_takes_its_engine_times_from_the_timing_record(): assert _across_page_breaks(sentence).search(text), sentence +def test_rendered_html_calls_the_upgrade_fixes_fixed_not_restored(): + """Round-2 final review nits: each October move's upgrades-table row + renders its split from restored_split_clause, and the sentence naming the + upstream fixes, in the table and in the move's paragraph, calls the outputs + "fixed" when some were ruled on 2026-10-06 while still scored and never + left scoring (the October 10 move), and "restored" when every one returns + to scoring (the October 13 move).""" + text = _html_text() + for version, kind in (("2.38.6", "fixed"), ("2.41.0", "restored")): + view = r.upgrade_view(version) + assert (view.engine_upgrade_restored_ruled_count > 0) == (kind == "fixed") + assert view.engine_upgrade_restored_split_clause in text + sentence = view.engine_upgrade_restored_sentence + assert sentence.startswith(f"The {kind} outputs take the upstream fixes for ") + assert text.count(sentence) == 2, version + assert r.engine_upgrade_restored_split_clause == ( + "returns no excluded output to scoring" + ) + assert r.engine_upgrade_restored_split_clause in text + assert text.count("fixed outputs take the upstream fixes for") == 2 + assert text.count("restored outputs take the upstream fixes for") == 2 + + +def test_rendered_pdf_calls_the_upgrade_fixes_fixed_not_restored(): + text = _straight_quotes(re.sub(r"\s+", " ", _pdf_running_text())) + assert text.count("fixed outputs take the upstream fixes for") == 2 + assert text.count("restored outputs take the upstream fixes for") == 2 + + def _engine_groups(exclusions) -> list[tuple[str, int]]: """Excluded outputs by the engine version of their value, oldest first.""" by_engine = Counter( @@ -1014,13 +1043,13 @@ def test_card_counts_follow_the_frozen_records(): r = PaperResults() card = re.sub(r"\s+", " ", BENCHMARK_CARD.read_text()) - # The October 10 move to 2.38.6: what it restores, from where, and why. + # The October 10 move to 2.38.6: what it fixes, from where, and why. v10 = r.upgrade_view("2.38.6") - restored, ruled = v10.engine_upgrade_restored_count, r.ruled_regenerated_count + fixed, ruled = v10.engine_upgrade_restored_count, r.ruled_regenerated_count assert ruled == v10.engine_upgrade_restored_ruled_count assert ( - f"That version fixes the defects behind {count_word(restored)} outputs. " - f"Release 20261006 excluded {count_word(restored - ruled)} of them, which " + f"That version fixes the defects behind {count_word(fixed)} outputs. " + f"Release 20261006 excluded {count_word(fixed - ruled)} of them, which " f"return to scoring; the other {count_word(ruled)} are the 2026-10-06 " "defects below, which stay scored." ) in card diff --git a/tests/test_paper_results.py b/tests/test_paper_results.py index 85115b57..634337ae 100644 --- a/tests/test_paper_results.py +++ b/tests/test_paper_results.py @@ -1663,6 +1663,14 @@ def test_the_upgrade_sentences_name_what_the_sidecar_records(): assert sum(count for _, count, _ in fixes) == len(entries) assert len(entries) == view.engine_upgrade_restored_count sentence = view.engine_upgrade_restored_sentence + # The sentence covers every output the move fixes. It calls them + # "fixed" when some were ruled on 2026-10-06 while still scored and + # never left scoring, and "restored" only when every one returns + # (round-2 final review nit 2). + kind = "fixed" if view.engine_upgrade_restored_ruled_count else "restored" + assert sentence.startswith( + f"The {kind} outputs take the upstream fixes for " if entries else "" + ) for entry in entries: for cause in entry["record"]["root_cause"].split("+"): assert ROOT_CAUSE_LABELS[cause] in sentence @@ -1700,6 +1708,10 @@ def test_the_upgrade_sentences_name_what_the_sidecar_records(): assert [view.engine_upgrade_new_exclusion_count for view in views] == [2, 0, 0] assert [view.engine_upgrade_restored_count for view in views] == [18, 9, 0] assert [view.engine_upgrade_scored_change_count for view in views] == [1, 0, 4] + assert [view.engine_upgrade_restored_ruled_count for view in views] == [4, 0, 0] + assert views[0].engine_upgrade_restored_sentence.startswith("The fixed outputs ") + assert views[1].engine_upgrade_restored_sentence.startswith("The restored ") + assert views[2].engine_upgrade_restored_sentence == "" paper = (ROOT / "paper/index.qmd").read_text() assert "`{python} r12.engine_upgrade_restored_sentence`" in paper # The change sentences render inside one non-empty accessor, so an empty @@ -1722,10 +1734,11 @@ def test_the_upgrade_sentences_name_what_the_sidecar_records(): assert f"{{view.{accessor}}}" in paper assert "`{python} r12.engine_upgrade_restored_target_sentence`" in paper assert "RELEASE AUTHOR" not in paper + assert "restored outputs take" not in paper def _MOCK_fix_modules_target(entry: dict, engine: str = "policyengine-us 9.9.9"): - """MOCK: turn a restored output's target into the fix-module kind, with a + """MOCK: turn a fixed output's target into the fix-module kind, with a record value and a defective engine value both beyond the tolerance.""" value = entry["regenerated"] entry["record"]["alternative_value"] = value + 50.0 @@ -1743,8 +1756,31 @@ def _MOCK_fix_modules_target(entry: dict, engine: str = "policyengine-us 9.9.9") RESTORING_MOVE = "2.41.0" +def test_one_ruled_output_makes_the_fix_sentence_say_fixed(monkeypatch): + """The fix sentence says "fixed" from the first output the rulings decided + while still scored. No frozen move has exactly one, so this gives the + October 13 move (nine fixed, none ruled) a MOCK ruled count of one (#209 + review r2, finding 1).""" + from policybench.paper_results import PaperResults + + view = r.upgrade_view(RESTORING_MOVE) + assert ( + view.engine_upgrade_restored_count, + view.engine_upgrade_restored_ruled_count, + ) == (9, 0) + assert view.engine_upgrade_restored_sentence.startswith( + "The restored outputs take " + ) + monkeypatch.setattr( + PaperResults, + "engine_upgrade_restored_ruled_count", + property(lambda self: 1), # MOCK + ) + assert view.engine_upgrade_restored_sentence.startswith("The fixed outputs take ") + + def _native_fix_modules() -> tuple[set[int], str]: - """The frozen sidecar's restored entries already held to fix modules, and + """The frozen sidecar's fixed-output entries already held to fix modules, and the engine a MOCK target must name to sit beside them (the sentence names one engine).""" restoring = r.upgrade_view(RESTORING_MOVE).last_engine_upgrade @@ -1813,10 +1849,34 @@ def test_the_fixed_outputs_split_into_returned_and_still_scored(): "values; the other four, ruled on 2026-10-06 while still scored, stay " "scored at those values." ) + # The upgrades table's clause gives the same splits in the cell's numerals + # (round-2 final review nit 1). + assert october_10.engine_upgrade_restored_split_clause == ( + "fixes the engine defects behind 18 outputs (14 that earlier releases " + "excluded return to scoring at the new version's values; the 4 ruled on " + "2026-10-06 while still scored stay scored)" + ) + assert restoring.engine_upgrade_restored_split_clause == ( + "fixes the engine defects behind 9 outputs (earlier releases excluded " + "them, and they return to scoring at the new version's values)" + ) + assert r.engine_upgrade_restored_split_clause == ( + "returns no excluded output to scoring" + ) paper = (ROOT / "paper/index.qmd").read_text() assert "`{python} r12.engine_upgrade_restored_split_sentence`" in paper assert "excluded outputs to scoring" not in paper assert "PolicyBench excluded `{python} r.ruled_exclusion_count_word`" not in paper + # Each move's table row builds its split from the helper, not inline + # arithmetic (october_upgrade_row). + assert ( + 'f"The move from policyengine-us {move.previous_engine_version} "\n' + ' f"{view.engine_upgrade_restored_split_clause}. "' in paper + ) + assert "restored - ruled" not in paper + assert "engine_upgrade_restored_count -" not in paper + assert "{ruled} ruled on" not in paper + assert "{r.engine_upgrade_restored_ruled_count}" not in paper @given(st.integers(0, 40), st.data()) @@ -1844,8 +1904,71 @@ def test_the_split_sentence_counts_both_kinds(restored, data): restored_split_sentence(restored, restored + 1, "2026-10-06") +@given(st.integers(0, 40), st.data()) +def test_the_split_clause_counts_both_kinds(restored, data): + """For any split, the upgrades table's clause names the total in numerals + with a matching noun, says "return" only when some outputs return and + "stay" only when some stay (as the split sentence does), agrees each verb + with its count, and never prints "the 0", "the 1" or a lone "1" in the + split (round-2 final review nit 1). The fix sentence calls the outputs + "fixed" from the first ruled one (#209 review r2).""" + from policybench.paper_results import ( + fixed_or_restored, + restored_split_clause, + restored_split_sentence, + ) + + ruled = data.draw(st.integers(0, restored)) + date = data.draw(st.sampled_from(["2026-10-06", "2027-01-02"])) + clause = restored_split_clause(restored, ruled, date) + sentence = restored_split_sentence(restored, ruled, date) + with pytest.raises(ValueError): + restored_split_clause(restored, restored + 1, date) + with pytest.raises(ValueError): + restored_split_clause(restored, -1, date) + assert fixed_or_restored(ruled) == ("fixed" if ruled > 0 else "restored") + with pytest.raises(ValueError): + fixed_or_restored(-1) + if not restored: + # As the table row for a move that fixes nothing has always said it. + assert clause == "returns no excluded output to scoring" + assert sentence == "" + return + returned = restored - ruled + noun = "output" if restored == 1 else "outputs" + defects = "defect" if restored == 1 else "defects" + assert clause.startswith(f"fixes the engine {defects} behind {restored} {noun} (") + assert clause.endswith(")") and clause.count("(") == clause.count(")") == 1 + split = clause[clause.index("(") + 1 : -1] + for kind in ("return", "stay"): + assert (kind in clause) == (kind in sentence) + assert ("return" in split) == (returned > 0) + assert ("stay" in split) == (ruled > 0) + assert (date in split) == (ruled > 0) + if returned and ruled: + back = ( + "one that earlier releases excluded returns to scoring" + if returned == 1 + else f"{returned} that earlier releases excluded return to scoring" + ) + kept = ( + f"the one ruled on {date} while still scored stays scored" + if ruled == 1 + else f"the {ruled} ruled on {date} while still scored stay scored" + ) + assert split.startswith(back) and split.endswith(kept) + elif returned: + verb = "it returns" if restored == 1 else "they return" + assert f"{verb} to scoring" in split + else: + verb = "it stays" if ruled == 1 else "they stay" + assert f"{verb} scored" in split + assert re.search(r"\b(the [01]|[01] that)\b", split) is None, split + assert re.search(r"(?