Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file modified app/public/paper/policybench.pdf
Binary file not shown.
8 changes: 4 additions & 4 deletions app/public/paper/web/index.html

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions app/src/paperSnapshot.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"pdfVersion": "501fb9038a2f",
"pdfVersion": "23a6a3009d67",
"responseWindow": "June 12 and October 26, 2026",
"snapshotDate": "2026-10-27",
"webVersion": "f9d806df25d9"
"webVersion": "af0ecbe96fac"
}
17 changes: 2 additions & 15 deletions paper/index.qmd
Original file line number Diff line number Diff line change
Expand Up @@ -906,22 +906,9 @@ def october_upgrade_row(view) -> list[str]:
"""A discrepancy-table row for one October engine move (``view``'s last
upgrade): what it fixes and restores, what it changes, and its rechecks."""
move = view.last_engine_upgrade
restored = view.engine_upgrade_restored_count
ruled = view.engine_upgrade_restored_ruled_count
split = (
f"{restored - ruled} that earlier releases excluded return to scoring at "
f"the new version's values; the {ruled} ruled on {r.ruling_date} while "
"still scored stay scored"
if ruled
else "earlier releases excluded them, and they return to scoring at the "
"new version's values"
)
fixes = (
f"The move from policyengine-us {move.previous_engine_version} fixes the "
f"engine defects behind {restored} outputs ({split}). "
if restored
else f"The move from policyengine-us {move.previous_engine_version} "
"returns no excluded output to scoring. "
f"The move from policyengine-us {move.previous_engine_version} "
f"{view.engine_upgrade_restored_split_clause}. "
)
return [
f"US references on policyengine-us {move.engine_version} "
Expand Down
4 changes: 2 additions & 2 deletions paper/snapshot/20260501/manifest.json
Original file line number Diff line number Diff line change
Expand Up @@ -165,12 +165,12 @@
"rendered_paper_artifacts": {
"pdf": {
"path": "app/public/paper/policybench.pdf",
"sha256": "501fb9038a2f85068bf82512935b56ee703e37617807e3df703008f98ab5aa06"
"sha256": "23a6a3009d677dba02e332b054be15f19416c42512d85930588d94eff5d61113"
},
"web": {
"files": {
"figures/positive_zero_scatter.png": "85851be224bcceddc3ee15776db2b1917babcfc58e8dc7fbf0f8c26f57b2b9a6",
"index.html": "f9d806df25d92f2546250e5fb39e71302ca7f58a074d514267c82f7f04bde274",
"index.html": "af0ecbe96fac2eb5c1f80fead5e122575e5f7024efcac80d0342e640019e3b5c",
"pe-tokens.css": "8f24d8da26f583c8ffddffcdcd172b6d52cbecfec20eda55bd39d7aa829f41d8",
"policybench-theme.css": "0e12c5fd615558259e5bce0167a38424e54f9ceb280666c4afd660d759cd1cb9",
"site_libs/clipboard/clipboard.min.js": "e17a1d816e13c0826e0ed7febfabc3277f45571234bde0bf9120829a7169edc9",
Expand Down
110 changes: 92 additions & 18 deletions policybench/paper_results.py
Original file line number Diff line number Diff line change
Expand Up @@ -329,8 +329,10 @@ def partition_engine_upgrade_changes(
"""Split an engine upgrade's changed outputs into four disjoint groups.

``excluded`` is the exclusion record once the upgrade was decided, and
``restored`` the excluded outputs the upgrade returned to scoring at the
new engine's values. A change to an excluded output is a new exclusion; a
``restored`` the outputs whose engine defects the upgrade fixes, scored at
the new engine's values (those a published board excluded return to
scoring; any a ruling decided while still scored stay scored). A change to
an excluded output is a new exclusion; a
change to a restored output is a restoration. Any other change is scored,
and moves beyond the exact-match tolerance ($1 for an amount, any change
for a 0/1 flag) or within it. Every change lands in exactly one group, so
Expand Down Expand Up @@ -394,7 +396,7 @@ def restored_split_sentence(restored: int, ruled: int, ruling_date: str) -> str:
values, and ``ruled`` that rulings on ``ruling_date`` decided while they
were still scored stay scored at them."""
if not 0 <= ruled <= restored:
raise ValueError(f"{ruled} ruled of {restored} restored outputs")
raise ValueError(f"{ruled} ruled of {restored} fixed outputs")
if not restored:
return ""
returned = restored - ruled
Expand Down Expand Up @@ -430,6 +432,58 @@ def restored_split_sentence(restored: int, ruled: int, ruling_date: str) -> str:
)


def restored_split_clause(restored: int, ruled: int, ruling_date: str) -> str:
"""The upgrades table's form of ``restored_split_sentence``: a verb phrase
finishing "The move from policyengine-us X ...", with numerals as the rest
of that cell writes counts. It names the ``restored`` outputs whose engine
defects the upgrade fixes and, in parentheses, splits them as the sentence
does: ``restored - ruled`` that earlier releases excluded return to
scoring, and ``ruled`` that rulings on ``ruling_date`` decided while they
were still scored stay scored. A lone output in either group is "one", not
"1". With nothing fixed, it says the move returns no excluded output."""
if not 0 <= ruled <= restored:
raise ValueError(f"{ruled} ruled of {restored} fixed outputs")
if not restored:
return "returns no excluded output to scoring"
returned = restored - ruled
defects = "defect" if restored == 1 else "defects"
noun = "output" if restored == 1 else "outputs"
lead = f"fixes the engine {defects} behind {restored} {noun}"
if not ruled:
excluded = (
"an earlier release excluded it, and it returns"
if restored == 1
else "earlier releases excluded them, and they return"
)
return f"{lead} ({excluded} to scoring at the new version's values)"
if not returned:
decided = (
"it while it was still scored, and it stays"
if ruled == 1
else "them while they were still scored, and they stay"
)
return (
f"{lead} (the {ruling_date} rulings decided {decided} scored at the new "
"version's values)"
)
back = "one that earlier releases excluded returns"
if returned > 1:
back = f"{returned} that earlier releases excluded return"
kept = f"the one ruled on {ruling_date} while still scored stays"
if ruled > 1:
kept = f"the {ruled} ruled on {ruling_date} while still scored stay"
return f"{lead} ({back} to scoring at the new version's values; {kept} scored)"


def fixed_or_restored(ruled: int) -> str:
"""What the sentence naming an upgrade's fixes calls its outputs: "fixed"
once rulings decided any of them while still scored, since those never
left scoring, and "restored" when every one returns to scoring."""
if ruled < 0:
raise ValueError(f"{ruled} ruled outputs")
return "fixed" if ruled else "restored"


def _key(entry: dict) -> tuple[str, str]:
return entry["scenario_id"], entry.get("variable", "snap")

Expand Down Expand Up @@ -471,8 +525,9 @@ class EngineUpgrade:
holds those excluded once it was decided: records decided by the end of
its date and not removed by then. A record decided after the upgrade never
counts here, and a record a later revision removed still does.
``restored`` holds the outputs the upgrade returned to scoring (its
``regenerated_exclusions``).
``restored`` holds the outputs whose engine defects the upgrade fixes (its
``regenerated_exclusions``): those a published board excluded return to
scoring, and any a ruling decided while still scored stay scored.
"""

def __init__(
Expand Down Expand Up @@ -2082,7 +2137,10 @@ def engine_upgrade_new_exclusion_count(self) -> int:

@property
def engine_upgrade_restored_count(self) -> int:
"""Excluded outputs the last upgrade returned to scoring."""
"""Outputs whose engine defects the last upgrade fixes (its
``regenerated_exclusions``): those a published board excluded return
to scoring, and those the 2026-10-06 rulings decided while still
scored (``engine_upgrade_restored_ruled_count``) stay scored."""
last = self.last_engine_upgrade
return 0 if last is None else last.restored_count

Expand All @@ -2092,7 +2150,7 @@ def engine_upgrade_restored_count_word(self) -> str:

@property
def engine_upgrade_restored_ruled_count(self) -> int:
"""Restored outputs the 2026-10-06 rulings had decided to exclude. No
"""Fixed outputs the 2026-10-06 rulings had decided to exclude. No
published board excluded them: they were still scored when the rulings
came, so they stay scored rather than return to scoring."""
last = self.last_engine_upgrade
Expand All @@ -2114,6 +2172,16 @@ def engine_upgrade_restored_split_sentence(self) -> str:
RULING_DATE,
)

@property
def engine_upgrade_restored_split_clause(self) -> str:
"""The same split as a clause for the upgrades table, after "The move
from policyengine-us X"."""
return restored_split_clause(
self.engine_upgrade_restored_count,
self.engine_upgrade_restored_ruled_count,
RULING_DATE,
)

@cached_property
def engine_upgrade_timing(self) -> dict | None:
"""The last upgrade's sweep timing record, when that upgrade is one
Expand Down Expand Up @@ -2192,10 +2260,11 @@ def engine_upgrade_timing_sentence(self) -> str:

@property
def engine_upgrade_restored_fixes(self) -> list[tuple[str, int, list[str]]]:
"""The defects behind the outputs the last upgrade returned to scoring:
each one's name, how many outputs it restores and the upstream pull
requests its regenerated_exclusions entries name (not the related
ones), in the sidecar's order. A record with several root causes names
"""The defects the last upgrade fixes behind its fixed outputs (those
that return to scoring and any still scored alike): each one's name,
how many outputs it fixes and the upstream pull requests its
regenerated_exclusions entries name (not the related ones), in the
sidecar's order. A record with several root causes names
each, and so does a fix-modules target whose modules correct a cause
the record does not name (scenario_099's educator expense addback,
policyengine-us#10052). Outputs whose causes are the same or one
Expand Down Expand Up @@ -2267,7 +2336,7 @@ def root(i: int) -> int:

@property
def engine_upgrade_restored_fix_phrase(self) -> str:
"""The fixes behind the restored outputs as one series, each with the
"""The fixes behind the fixed outputs as one series, each with the
upstream pull requests it names and, past one, how many outputs."""
parts = []
for label, count, prs in self.engine_upgrade_restored_fixes:
Expand All @@ -2278,11 +2347,16 @@ def engine_upgrade_restored_fix_phrase(self) -> str:

@property
def engine_upgrade_restored_sentence(self) -> str:
"""One sentence naming the fixes behind the restored outputs."""
"""One sentence naming the fixes behind all the outputs the last
upgrade fixes. When the 2026-10-06 rulings decided some of them while
still scored, those never left scoring, so the sentence calls them all
"fixed"; when every one of them returns to scoring, "restored" holds
and it says that."""
phrase = self.engine_upgrade_restored_fix_phrase
if not phrase:
return ""
return f"The restored outputs take the upstream fixes for {phrase}."
kind = fixed_or_restored(self.engine_upgrade_restored_ruled_count)
return f"The {kind} outputs take the upstream fixes for {phrase}."

@property
def engine_upgrade_audit_dir(self) -> str:
Expand Down Expand Up @@ -2341,7 +2415,7 @@ def upgrade_view(self, version: str) -> "PaperResults":

@property
def engine_upgrade_restored_targets(self) -> dict[str, list[dict]]:
"""The last upgrade's restored outputs by how their corrected value is
"""The last upgrade's fixed outputs by how their corrected value is
known (the sidecar's ``target.kind``): ``record`` for the value the
exclusion record carries, ``fix_modules`` for the value the audit's
fix modules give on a later engine that still has the defect. Refuses,
Expand All @@ -2358,14 +2432,14 @@ def engine_upgrade_restored_targets(self) -> dict[str, list[dict]]:
raise ValueError(f"{key} has an unknown target kind {target['kind']!r}")
tolerance = entry["tolerance"]
if not 0 <= tolerance <= 1:
raise ValueError(f"{key} is restored on a tolerance of {tolerance}")
raise ValueError(f"{key} is fixed on a tolerance of {tolerance}")
variable = entry["variable"]
if abs(
entry["regenerated"] - target["value"]
) > tolerance or moves_beyond_tolerance(
variable, target["value"], entry["regenerated"]
):
raise ValueError(f"{key} is restored off its target")
raise ValueError(f"{key} is fixed off its target")
if target["kind"] == "record":
if target["value"] != entry["record"]["alternative_value"]:
raise ValueError(f"{key}'s target is not its record's value")
Expand All @@ -2378,7 +2452,7 @@ def engine_upgrade_restored_targets(self) -> dict[str, list[dict]]:

@property
def engine_upgrade_restored_target_sentence(self) -> str:
"""How the restored outputs' corrected values are known: by the
"""How the fixed outputs' corrected values are known: by the
exclusion record, or by the audit's fix modules on a later engine."""
targets = self.engine_upgrade_restored_targets
by_record, by_modules = targets["record"], targets["fix_modules"]
Expand Down
37 changes: 33 additions & 4 deletions tests/test_disclosures.py
Original file line number Diff line number Diff line change
Expand Up @@ -741,6 +741,35 @@ def test_rendered_pdf_takes_its_engine_times_from_the_timing_record():
assert _across_page_breaks(sentence).search(text), sentence


def test_rendered_html_calls_the_upgrade_fixes_fixed_not_restored():
"""Round-2 final review nits: each October move's upgrades-table row
renders its split from restored_split_clause, and the sentence naming the
upstream fixes, in the table and in the move's paragraph, calls the outputs
"fixed" when some were ruled on 2026-10-06 while still scored and never
left scoring (the October 10 move), and "restored" when every one returns
to scoring (the October 13 move)."""
text = _html_text()
for version, kind in (("2.38.6", "fixed"), ("2.41.0", "restored")):
view = r.upgrade_view(version)
assert (view.engine_upgrade_restored_ruled_count > 0) == (kind == "fixed")
assert view.engine_upgrade_restored_split_clause in text
sentence = view.engine_upgrade_restored_sentence
assert sentence.startswith(f"The {kind} outputs take the upstream fixes for ")
assert text.count(sentence) == 2, version
assert r.engine_upgrade_restored_split_clause == (
"returns no excluded output to scoring"
)
assert r.engine_upgrade_restored_split_clause in text
assert text.count("fixed outputs take the upstream fixes for") == 2
assert text.count("restored outputs take the upstream fixes for") == 2


def test_rendered_pdf_calls_the_upgrade_fixes_fixed_not_restored():
text = _straight_quotes(re.sub(r"\s+", " ", _pdf_running_text()))
assert text.count("fixed outputs take the upstream fixes for") == 2
assert text.count("restored outputs take the upstream fixes for") == 2


def _engine_groups(exclusions) -> list[tuple[str, int]]:
"""Excluded outputs by the engine version of their value, oldest first."""
by_engine = Counter(
Expand Down Expand Up @@ -1014,13 +1043,13 @@ def test_card_counts_follow_the_frozen_records():

r = PaperResults()
card = re.sub(r"\s+", " ", BENCHMARK_CARD.read_text())
# The October 10 move to 2.38.6: what it restores, from where, and why.
# The October 10 move to 2.38.6: what it fixes, from where, and why.
v10 = r.upgrade_view("2.38.6")
restored, ruled = v10.engine_upgrade_restored_count, r.ruled_regenerated_count
fixed, ruled = v10.engine_upgrade_restored_count, r.ruled_regenerated_count
assert ruled == v10.engine_upgrade_restored_ruled_count
assert (
f"That version fixes the defects behind {count_word(restored)} outputs. "
f"Release 20261006 excluded {count_word(restored - ruled)} of them, which "
f"That version fixes the defects behind {count_word(fixed)} outputs. "
f"Release 20261006 excluded {count_word(fixed - ruled)} of them, which "
f"return to scoring; the other {count_word(ruled)} are the 2026-10-06 "
"defects below, which stay scored."
) in card
Expand Down
Loading
Loading