diff --git a/app/src/notes/2026-10-19-america-gov-scores-level-with-gpt-5-5.json b/app/src/notes/2026-10-19-america-gov-scores-level-with-gpt-5-5.json index c8c8cb4c..63c117b5 100644 --- a/app/src/notes/2026-10-19-america-gov-scores-level-with-gpt-5-5.json +++ b/app/src/notes/2026-10-19-america-gov-scores-level-with-gpt-5-5.json @@ -3,45 +3,50 @@ "date": "2026-10-19", "title": "America.gov, the government's new assistant, scores level with GPT-5.5", "paragraphs": [ - "America.gov, the General Services Administration's assistant, live since September 29, gets {agovExact}% of PolicyBench answers within $1, weighted by household impact. On the board it would rank #{agovRank} of {nRanked}, level with GPT-5.5 ({gpt55Exact}%); GPT-6 Sol leads at {solExact}%. It does not join the board, which scores models answering alone, with no tools and no web access. America.gov says it uses \"Super Intelligence (SI) to find and explain information from official government sources.\"", - "Between October 14 and 18, PolicyBench sent each of its {households} households to America.gov in a new chat, with the board's own prompt in the JSON answer format, and scored every reply as it scores the board. Six prompts run past the {charLimit} characters America.gov asks a message to stay within, so each went in as two chats asking for half the outputs apiece. Before freezing a run, PolicyBench asks a board model again when its response leaves answers out, and America.gov got the same kind of retry: each refusal went back word for word in a fresh chat, up to twice. On first attempts alone it gets {agovFirstExact}% (#{agovFirstRank}).", - "America.gov's first replies included {firstRefusals:words} refusals. \"I cannot estimate or calculate federal or state tax, credit, payroll, or benefit amounts,\" it replied to the prompt for a Virginia single parent of two. Asked again, {answeredOnRetry:words} of the refusals turned into full answers and {refusedThrice:words} came back word for word the same, which leaves {unansweredRate}% of its scored answers blank; {fullModels} of the {nModels} models on the board answer every one.", - "America.gov is strongest on taxes. It gets {fedTaxRate}% of federal income tax amounts within $1, against a median of {fedTaxMedianRate}% across the board's models, and {stateTaxRate}% of state income tax amounts, against {stateTaxMedianRate}%. Benefits are weaker. On SNAP it gets {snapRate}%, below the median of {snapMedianRate}%, and it put a Michigan couple's SNAP at ${snap072} a year where the reference and {snap072Models} of the {nModels} models say $0. On Medicaid eligibility, the output that carries the most weight, it gets {medicaidRate}% against a median of {medicaidMedianRate}%, and it called both adults in a New York couple eligible where the reference and {medicaid004Models} models say neither is.", - "The current prompts define {peOutputs:words} outputs by PolicyEngine: Medicaid and CHIP eligibility \"under PolicyEngine rules,\" and the two school-meal outputs as whether \"PolicyEngine returns\" support. PolicyBench is revising that wording. At least {peRefusals:words} of America.gov's {firstRefusals:words} refusals cited PolicyEngine; no model on the board declined a household on that ground, though some models' recorded reasoning notes that they cannot run PolicyEngine. Counting only the answers each gave, America.gov is never more than {peMaxGap} points below the median model on those {peOutputs:words} outputs, and without them it would rank #{noPeRank} of {nRanked}.", - "This is one run, from one week in October, and America.gov's terms say answers generated with SI \"can be incomplete or wrong.\" The board's last repair step, asking again for each answer still missing, one at a time, would take at least {rowRepairChats} more chats and could lift America.gov to {rowRepairMaxRate}% at most. The replies, the scores and the method, including how the long prompts were split, are in the data below." + "America.gov, which says it answers questions with AI and information from government websites, scores {agovExact}% on PolicyBench. GPT-5.5, which OpenAI released in April, matches that score with AI alone and no web access, while GPT-6 Sol leads at {solExact}%, also with AI alone. America.gov would rank #{agovRank} of {nRanked}, but PolicyBench keeps its board to AI alone. PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches. It weights each answer by household impact.", + "The General Services Administration has run America.gov since September 29. Between October 14 and 18, PolicyBench sent each of its {households} households to America.gov in a new chat, with the board's own prompt in the JSON answer format. It scored every reply as it scores the board. Six prompts run past the {charLimit} characters America.gov asks a message to stay within, so PolicyBench split each across two chats asking for half the outputs apiece. Before publishing a run, PolicyBench asks again whenever a board model leaves answers out. America.gov got the same kind of retry: PolicyBench sent each refused prompt again, word for word, in a fresh chat, up to twice. On first attempts alone, America.gov gets {agovFirstExact}% (#{agovFirstRank}).", + "America.gov refused prompts for {firstRefusalHouseholds:words} households on the first attempt. \"I cannot estimate or calculate federal or state tax, credit, payroll, or benefit amounts,\" it replied to the prompt for a Virginia single parent of two. When PolicyBench asked again, America.gov answered {answeredOnRetry:words} of those prompts in full and repeated its refusal word for word on the other {refusedThrice:words}, which leaves {unansweredRate}% of its scored answers blank. Of the {nModels} models on the board, {fullModels} answer every one.", + "On income taxes, America.gov falls between GPT-5.5 and GPT-6 Sol: it gets {fedTaxRate}% of federal amounts within $1, against their {gpt55FedTaxRate}% and {solFedTaxRate}%, and {stateTaxRate}% of state amounts, against {gpt55StateTaxRate}% and {solStateTaxRate}%. It trails both on SNAP amounts and Medicaid eligibility. On SNAP it gets {snapRate}%, against {gpt55SnapRate}% and {solSnapRate}%, and it put a Michigan couple's SNAP at ${snap072} a year, where PolicyEngine and {snap072Models} of the {nModels} models give $0. On Medicaid eligibility, which PolicyBench weights most heavily, it gets {medicaidRate}%, against {gpt55MedicaidRate}% and {solMedicaidRate}%. It called both adults in a New York couple eligible, where PolicyEngine and {medicaid004Models} models find neither eligible.", + "The board's prompts name PolicyEngine, the open-source tax and benefit model behind the references, in {peOutputs:words} output definitions. They ask about Medicaid and CHIP eligibility \"under PolicyEngine rules,\" and ask \"whether PolicyEngine returns\" support for free and for reduced-price school meals. PolicyBench is revising that wording. At least {peRefusals:words} of America.gov's {firstRefusals:words} refusals cited PolicyEngine, and no model on the board declined a household on that ground. On the answers each gave, America.gov leads GPT-5.5 on {peLeads:words} of those {peOutputs:words} outputs and trails it by at most {peMaxGap} points on the others. If PolicyBench left all {peOutputs:words} out for every model, America.gov would rank #{noPeRank} of {nRanked}.", + "America.gov's terms say the answers it generates with AI \"can be incomplete or wrong.\" PolicyBench tested America.gov once and skipped the last retry it gives board models, in which it asks again, one at a time, for each answer still missing. If PolicyBench ran that retry, it would need at least {rowRepairChats} more chats, and America.gov could rise to {rowRepairMaxRate}% at most." ], "facts": { "agovExact": 87.9, "agovRank": 14, "nRanked": 51, - "gpt55Exact": 87.9, "solExact": 95.2, "households": 100, "charLimit": 8000, "agovFirstExact": 86.6, "agovFirstRank": 19, "firstRefusals": 5, + "firstRefusalHouseholds": 5, "answeredOnRetry": 2, "refusedThrice": 3, "unansweredRate": 3.7, "fullModels": 46, "nModels": 50, "fedTaxRate": 72.6, - "fedTaxMedianRate": 54.8, + "gpt55FedTaxRate": 70.2, + "solFedTaxRate": 90.5, "stateTaxRate": 66.7, - "stateTaxMedianRate": 60.2, + "gpt55StateTaxRate": 65.6, + "solStateTaxRate": 72.0, "snapRate": 81.7, - "snapMedianRate": 84.9, + "gpt55SnapRate": 84.9, + "solSnapRate": 89.2, "snap072": 6432, "snap072Models": 49, "medicaidRate": 92.5, - "medicaidMedianRate": 95.4, + "gpt55MedicaidRate": 96.5, + "solMedicaidRate": 97.7, "medicaid004Models": 49, "rowRepairChats": 73, "rowRepairMaxRate": 89.8, "peRefusals": 2, "peOutputs": 4, - "peMaxGap": "0.8", + "peLeads": 2, + "peMaxGap": "1.3", "noPeRank": 13 }, "data": [ diff --git a/app/tests/notes.test.ts b/app/tests/notes.test.ts index 9c3b8642..7120b5e5 100644 --- a/app/tests/notes.test.ts +++ b/app/tests/notes.test.ts @@ -213,24 +213,76 @@ describe("notes", () => { createElement(NoteArticle, { note, titleLevel: "h1" }), ); expect(markup).toContain( - "gets 87.9% of PolicyBench answers within $1, weighted by household impact.", + "America.gov, which says it answers questions with AI and information from government websites, scores 87.9% on PolicyBench.", ); expect(markup).toContain( - "On the board it would rank #14 of 51, level with GPT-5.5 (87.9%); GPT-6 Sol leads at 95.2%.", + "PolicyBench counts a dollar amount as right when it falls within $1 of the reference PolicyEngine computes, and an eligibility answer as right when it matches. It weights each answer by household impact.", ); + expect(markup).toContain( + "GPT-5.5, which OpenAI released in April, matches that score with AI alone and no web access, while GPT-6 Sol leads at 95.2%, also with AI alone.", + ); + expect(markup).toContain( + "America.gov would rank #14 of 51, but PolicyBench keeps its board to AI alone.", + ); + // Rates keep one decimal, so GPT-6 Sol's state income tax rate reads 72.0%. + expect(markup).toContain( + "On income taxes, America.gov falls between GPT-5.5 and GPT-6 Sol: it gets 72.6% of federal amounts within $1, against their 70.2% and 90.5%, and 66.7% of state amounts, against 65.6% and 72.0%.", + ); + expect(markup).toContain( + "It trails both on SNAP amounts and Medicaid eligibility. On SNAP it gets 81.7%, against 84.9% and 89.2%,", + ); + expect(markup).toContain( + "On Medicaid eligibility, which PolicyBench weights most heavily, it gets 92.5%, against 96.5% and 97.7%. It called both adults in a New York couple eligible, where PolicyEngine and 49 models find neither eligible.", + ); + expect(interpolateNoteText(note, note.paragraphs[0]).length).toBeLessThan(600); // Whole numbers under ten are spelled out; thousands get separators. - expect(markup).toContain("America.gov's first replies included five refusals."); - expect(markup).toContain("Six prompts run past the 8,000 characters"); - expect(markup).toContain("put a Michigan couple's SNAP at $6,432 a year"); expect(markup).toContain( - "At least two of America.gov's five refusals cited PolicyEngine; no model on the board declined a household on that ground, though some models' recorded reasoning notes that they cannot run PolicyEngine.", + "America.gov refused prompts for five households on the first attempt.", + ); + expect(markup).toContain( + "Six prompts run past the 8,000 characters America.gov asks a message to stay within, so PolicyBench split each across two chats asking for half the outputs apiece.", + ); + expect(markup).toContain( + "When PolicyBench asked again, America.gov answered two of those prompts in full and repeated its refusal word for word on the other three, which leaves 3.7% of its scored answers blank. Of the 50 models on the board, 46 answer every one.", + ); + expect(markup).toContain( + "and it put a Michigan couple's SNAP at $6,432 a year, where PolicyEngine and 49 of the 50 models give $0.", + ); + expect(markup).toContain( + "The board's prompts name PolicyEngine, the open-source tax and benefit model behind the references, in four output definitions.", + ); + // React writes the note's straight double quotes as ". + expect(markup).toContain( + "They ask about Medicaid and CHIP eligibility "under PolicyEngine rules," and ask "whether PolicyEngine returns" support for free and for reduced-price school meals.", + ); + expect(markup).toContain( + "PolicyBench is revising that wording. At least two of America.gov's five refusals cited PolicyEngine, and no model on the board declined a household on that ground.", ); expect(markup).toContain("it replied to the prompt for a Virginia single parent of two."); - expect(markup).toContain("Between October 14 and 18, PolicyBench sent"); expect(markup).toContain( - "never more than 0.8 points below the median model on those four outputs, and without them it would rank #13 of 51.", + "The General Services Administration has run America.gov since September 29.", + ); + expect(markup).toContain( + "Between October 14 and 18, PolicyBench sent each of its 100 households to America.gov in a new chat, with the board's own prompt in the JSON answer format. It scored every reply as it scores the board.", + ); + expect(markup).toContain( + "Before publishing a run, PolicyBench asks again whenever a board model leaves answers out. America.gov got the same kind of retry: PolicyBench sent each refused prompt again, word for word, in a fresh chat, up to twice. On first attempts alone, America.gov gets 86.6% (#19).", + ); + expect(markup).toContain( + "On the answers each gave, America.gov leads GPT-5.5 on two of those four outputs and trails it by at most 1.3 points on the others.", + ); + expect(markup).toContain( + "If PolicyBench left all four out for every model, America.gov would rank #13 of 51.", + ); + expect(markup).toContain( + "America.gov's terms say the answers it generates with AI "can be incomplete or wrong."", + ); + expect(markup).toContain( + "PolicyBench tested America.gov once and skipped the last retry it gives board models, in which it asks again, one at a time, for each answer still missing.", + ); + expect(markup).toContain( + "If PolicyBench ran that retry, it would need at least 73 more chats, and America.gov could rise to 89.8% at most.", ); - expect(markup).toContain("would take at least 73 more chats and could lift America.gov to 89.8% at most."); expect(markup).toContain( 'href="https://github.com/PolicyEngine/policybench/blob/main/notes/data/america_gov_20261019.csv"', ); diff --git a/scripts/america_gov_note.py b/scripts/america_gov_note.py index c6448858..83744dd3 100644 --- a/scripts/america_gov_note.py +++ b/scripts/america_gov_note.py @@ -39,7 +39,6 @@ import hashlib import json import math -import statistics import sys from collections import defaultdict from pathlib import Path @@ -79,6 +78,9 @@ "reduced_price_school_meals_eligible", ) ROUNDS = (1, 2) +FEDERAL_TAX = "federal_income_tax_before_refundable_credits" +STATE_TAX = "state_income_tax_before_refundable_credits" +MEDICAID = "person_medicaid_eligible" # America.gov's own words, as its pages read on 2026-10-19. SITE_QUOTES = { "fetched_utc": "2026-10-19", @@ -364,6 +366,7 @@ def facts(payload: dict, rows: list[dict], meta: dict) -> dict: gw = payload["globalWeights"]["household"] w = weights(ref.keys(), gw) board = board_predictions(payload) + gpt55, sol = board["gpt-5.5"], board["gpt-6-sol"] first = view(rows, payload, meta, retried=False) retried = view(rows, payload, meta, retried=True) board_exact = {m: weighted_exact(p, ref, w) for m, p in board.items()} @@ -372,9 +375,6 @@ def facts(payload: dict, rows: list[dict], meta: dict) -> dict: r["model"]: r for r in payload["modelStats"] if r["condition"] == "no_tools" } - def median_group(group: str) -> float: - return statistics.median(group_rate(p, ref, group) for p in board.values()) - def models_right(key) -> int: return sum( row_hit_scores(key[1], ref[key], p.get(key))["exact"] @@ -400,14 +400,12 @@ def models_right(key) -> int: } w_no_pe = weights(no_pe.keys(), gw) no_pe_others = [weighted_exact(p, no_pe, w_no_pe) for p in board.values()] - # Answered cells on both sides: America.gov's, and each board model's own. - pe_gap = max( - statistics.median( - group_rate(p, ref, g, answered_only=True) for p in board.values() - ) + # Answered cells on both sides: America.gov's, and GPT-5.5's own. + pe_gaps = [ + group_rate(gpt55, ref, g, answered_only=True) - group_rate(retried, ref, g, answered_only=True) for g in POLICYENGINE_GROUPS - ) + ] best = {**retried, **{k: ref[k] for k in unanswered}} def one(x: float) -> float: @@ -417,13 +415,15 @@ def one(x: float) -> float: "agovExact": one(weighted_exact(retried, ref, w)), "agovRank": rank(weighted_exact(retried, ref, w), others), "nRanked": len(board) + 1, - "gpt55Exact": one(by_model["gpt-5.5"]["exact"]), "solExact": one(by_model["gpt-6-sol"]["exact"]), "households": len({k[0] for k in ref}), "charLimit": meta["message_limit_characters"], "agovFirstExact": one(weighted_exact(first, ref, w)), "agovFirstRank": rank(weighted_exact(first, ref, w), others), "firstRefusals": len(first_refused), + "firstRefusalHouseholds": len( + {by_unit[u][1][0]["scenario_id"] for u in first_refused} + ), "answeredOnRetry": len(retry_answered), "refusedThrice": sum( all(not answered(by_unit[u][a]) for a in (1, 2, 3) if a in by_unit[u]) @@ -433,24 +433,20 @@ def one(x: float) -> float: "unansweredRate": one(100 * len(unanswered) / len(ref)), "fullModels": sum(r["nParsed"] == r["n"] for r in by_model.values()), "nModels": len(by_model), - "fedTaxRate": one( - group_rate(retried, ref, "federal_income_tax_before_refundable_credits") - ), - "fedTaxMedianRate": one( - median_group("federal_income_tax_before_refundable_credits") - ), - "stateTaxRate": one( - group_rate(retried, ref, "state_income_tax_before_refundable_credits") - ), - "stateTaxMedianRate": one( - median_group("state_income_tax_before_refundable_credits") - ), + "fedTaxRate": one(group_rate(retried, ref, FEDERAL_TAX)), + "gpt55FedTaxRate": one(group_rate(gpt55, ref, FEDERAL_TAX)), + "solFedTaxRate": one(group_rate(sol, ref, FEDERAL_TAX)), + "stateTaxRate": one(group_rate(retried, ref, STATE_TAX)), + "gpt55StateTaxRate": one(group_rate(gpt55, ref, STATE_TAX)), + "solStateTaxRate": one(group_rate(sol, ref, STATE_TAX)), "snapRate": one(group_rate(retried, ref, "snap")), - "snapMedianRate": one(median_group("snap")), + "gpt55SnapRate": one(group_rate(gpt55, ref, "snap")), + "solSnapRate": one(group_rate(sol, ref, "snap")), "snap072": int(retried[("scenario_072", "snap")]), "snap072Models": models_right(("scenario_072", "snap")), - "medicaidRate": one(group_rate(retried, ref, "person_medicaid_eligible")), - "medicaidMedianRate": one(median_group("person_medicaid_eligible")), + "medicaidRate": one(group_rate(retried, ref, MEDICAID)), + "gpt55MedicaidRate": one(group_rate(gpt55, ref, MEDICAID)), + "solMedicaidRate": one(group_rate(sol, ref, MEDICAID)), # Models right on both adults. "medicaid004Models": sum( all( @@ -465,8 +461,10 @@ def one(x: float) -> float: "peRefusals": sum( "PolicyEngine" in by_unit[u][1][0]["refusal_text"] for u in first_refused ), - # A bound ("never more than X points below"), so it rounds up. - "peMaxGap": f"{math.ceil(max(pe_gap, 0) * 10) / 10:.1f}", + # "leads GPT-5.5 on two of those four outputs". + "peLeads": sum(gap < 0 for gap in pe_gaps), + # "trails it by at most X points on the others": a bound, so it rounds up. + "peMaxGap": f"{math.ceil(max(max(pe_gaps), 0) * 10) / 10:.1f}", "noPeRank": rank(weighted_exact(retried, no_pe, w_no_pe), no_pe_others), "rowRepairChats": sum( len(unit_requested(u, payload, meta)) for u in still_refused @@ -518,46 +516,6 @@ def board_declined_households(predictions_path: Path) -> tuple[list, list]: ) -def board_cannot_run_policyengine(predictions_path: Path) -> list[tuple]: - """(model, household) pairs whose recorded response or explanations say the model - cannot run, use or access PolicyEngine. A floor: many models record no reasoning, - and other wordings exist.""" - import re - - pe = r"policy\s*engine" - patterns = [ - re.compile( - r"(?:can't|cannot|can not|unable to)\s+(?:\w+\s+){0,2}?" - r"(?:run|access|invoke|use|call|execute)\s+(?:the\s+)?(?:\w+\s+){0,2}?" - + pe, - re.IGNORECASE, - ), - re.compile( - r"(?:don't|do not)\s+have\s+(?:access\s+to\s+)?(?:the\s+)?" + pe, - re.IGNORECASE, - ), - re.compile( - r"without\s+(?:access\s+to\s+|running\s+)(?:the\s+)?(?:\w+\s+){0,2}?" + pe, - re.IGNORECASE, - ), - re.compile( - pe + r"[^.\n]{0,40}\.\s*But\s+(?:I\s+)?(?:can't|cannot|can not)\s+run", - re.IGNORECASE, - ), - ] - found = set() - for row in _board_rows(predictions_path): - key = (row["model"], row["scenario_id"]) - if key in found: - continue - text = (row["raw_response"] or "") + " " + (row["explanation"] or "") - # Stored responses can carry escaped apostrophes and newlines. - text = text.replace("\\u2019", "'").replace("\u2019", "'").replace("\\n", "\n") - if any(pattern.search(text) for pattern in patterns): - found.add(key) - return sorted(found) - - if __name__ == "__main__": if sys.argv[1:2] == ["build"]: build(Path(sys.argv[2]).expanduser()) diff --git a/tests/test_notes.py b/tests/test_notes.py index 4baf5507..f1b58cf2 100644 --- a/tests/test_notes.py +++ b/tests/test_notes.py @@ -5651,9 +5651,9 @@ def test_america_gov_split_units_partition_the_prompt() -> None: def test_america_gov_note_quotes_are_verbatim() -> None: """The note's quote of America.gov's refusal to the Virginia household - (scenario_119) is its first reply, verbatim. (Its quotes of America.gov's - pages are checked against the data's metadata in - test_america_gov_note_predicates.)""" + (scenario_119) is its first reply, verbatim. (test_america_gov_note_predicates + checks its terms quote, and the how-it-works page behind its opening, + against the data's metadata.)""" agov = _america_gov() note = _note(AGOV_NOTE) text = " ".join(note["paragraphs"]) @@ -5673,9 +5673,10 @@ def test_america_gov_note_quotes_are_verbatim() -> None: def test_america_gov_note_predicates(monkeypatch) -> None: """The America.gov note's claims beyond its numeric facts: who leads, the - households it describes, the heaviest output, the over-limit prompts, the - repeated refusals, and its quotes of America.gov's own pages as the data's - metadata records them (fetched 2026-10-19).""" + households it describes and America.gov's answers for them, the heaviest + output, the over-limit prompts, the repeated refusals, and the America.gov + pages behind its opening and its terms quote, as the data's metadata records + them (fetched 2026-10-19).""" agov = _america_gov() note = _note(AGOV_NOTE) text = " ".join(note["paragraphs"]) @@ -5684,9 +5685,60 @@ def test_america_gov_note_predicates(monkeypatch) -> None: ref, _ = agov.references() board = [r for r in payload["modelStats"] if r["condition"] == "no_tools"] + # The note's ranks set America.gov against each board model's score as + # the script computes it, which must match the score the release publishes. + w = agov.weights(ref.keys(), payload["globalWeights"]["household"]) + computed = { + model: agov.weighted_exact(predictions, ref, w) + for model, predictions in agov.board_predictions(payload).items() + } + published = {r["model"]: r["exact"] for r in board} + assert computed.keys() == published.keys() + for model, score in computed.items(): + assert abs(score - published[model]) < 1e-9, model + # "GPT-6 Sol leads". assert max(board, key=lambda r: r["exact"])["model"] == "gpt-6-sol" - # "the output that carries the most weight". + # "GPT-5.5, which OpenAI released in April, matches that score". + from policybench.paper_results import MODEL_RELEASE_DATES + + assert MODEL_RELEASE_DATES["gpt-5.5"].startswith("2026-04-") + facts = note["facts"] + gpt55 = next(r for r in board if r["model"] == "gpt-5.5") + assert round(gpt55["exact"], 1) == facts["agovExact"] + # "falls between GPT-5.5 and GPT-6 Sol". + assert facts["gpt55FedTaxRate"] < facts["fedTaxRate"] < facts["solFedTaxRate"] + assert facts["gpt55StateTaxRate"] < facts["stateTaxRate"] < facts["solStateTaxRate"] + # "It trails both on SNAP amounts and Medicaid eligibility". + for name in ("Snap", "Medicaid"): + rate = facts[f"{name.lower()}Rate"] + assert rate < min(facts[f"gpt55{name}Rate"], facts[f"sol{name}Rate"]) + # "leads GPT-5.5 on two of those four outputs and trails it by at most 1.3 + # points on the others": each output's gap on the answers each gave. + assert ( + "leads GPT-5.5 on {peLeads:words} of those {peOutputs:words} outputs and " + "trails it by at most {peMaxGap} points on the others" in text + ) + retried = agov.view(agov.load_rows(), payload, meta, retried=True) + gpt55_predictions = agov.board_predictions(payload)["gpt-5.5"] + gaps = [ + agov.group_rate(gpt55_predictions, ref, g, answered_only=True) + - agov.group_rate(retried, ref, g, answered_only=True) + for g in agov.POLICYENGINE_GROUPS + ] + assert len(gaps) == facts["peOutputs"] + assert all(gap != 0 for gap in gaps) + trailing = [gap for gap in gaps if gap > 0] + assert len(gaps) - len(trailing) == facts["peLeads"] + assert 0 < facts["peLeads"] < facts["peOutputs"] + assert max(trailing) <= float(facts["peMaxGap"]) < max(trailing) + 0.1 + # The note quotes the board's prompts, and every prompt carries both phrases. + for phrase in ("under PolicyEngine rules", "whether PolicyEngine returns"): + assert f'"{phrase}' in text + assert all(phrase in s["prompt"]["json"] for s in payload["scenarios"].values()) + # The note compares America.gov with models it names, each at its own rate. + assert "median" not in text + # "which PolicyBench weights most heavily". weights = payload["globalWeights"]["household"] assert max(weights, key=weights.get) == "person_medicaid_eligible" @@ -5701,8 +5753,11 @@ def household(sid: str) -> tuple: assert ref[("scenario_072", "snap")] == 0 for adult in ("head", "spouse"): assert ref[("scenario_004", f"{adult}_medicaid_eligible")] == 0 + # "It called both adults in a New York couple eligible". + assert retried[("scenario_004", f"{adult}_medicaid_eligible")] == 1 - # "the reference and 49 of the 50 models say $0" / "say neither is": exactly 0. + # "PolicyEngine and 49 of the 50 models give $0" / "find neither eligible": + # exactly 0. def exact_zeros(sid: str, variable: str) -> int: rows = payload["scenarioPredictions"][sid][variable].values() return sum(r.get("parsed") and r.get("prediction") == 0 for r in rows) @@ -5713,6 +5768,10 @@ def exact_zeros(sid: str, variable: str) -> int: exact_zeros("scenario_004", f"{adult}_medicaid_eligible") == note["facts"]["medicaid004Models"] ) + # Both counts render as 49 on this run, so only the raw text shows which + # sentence takes which. + assert "PolicyEngine and {snap072Models} of the {nModels} models give $0" in text + assert "PolicyEngine and {medicaid004Models} models find neither eligible" in text # "Six prompts run past the 8,000 characters": exactly these six. limit = meta["message_limit_characters"] @@ -5723,7 +5782,8 @@ def exact_zeros(sid: str, variable: str) -> int: ) assert over == sorted(meta["split_units"]) - # "came back word for word the same": the three units refused at every attempt. + # "repeated its refusal word for word on the other three": the three units + # refused at every attempt. by_unit = agov.units(agov.load_rows()) refused_every_time = [ u @@ -5734,6 +5794,23 @@ def exact_zeros(sid: str, variable: str) -> int: for unit in refused_every_time: texts = {by_unit[unit][a][0]["refusal_text"] for a in (1, 2, 3)} assert len(texts) == 1 + # "answered two of those prompts in full ... on the other three": the units + # answered on a retry and the units refused every time make up the first + # refusals. + answered_on_retry = [ + u + for u, attempts in by_unit.items() + if not agov.answered(attempts[1]) + and any(agov.answered(attempts[a]) for a in attempts if a > 1) + ] + assert len(answered_on_retry) == facts["answeredOnRetry"] + assert len(answered_on_retry) + len(refused_every_time) == facts["firstRefusals"] + # answeredOnRetry, peRefusals and peLeads all render as "two" on this run, so + # only the raw text shows which fact paragraph 2 counts. + assert ( + "answered {answeredOnRetry:words} of those prompts in full and repeated " + "its refusal word for word on the other {refusedThrice:words}" in text + ) # "At least two ... cited PolicyEngine": two complete refusals do; the one # refusal cut off in capture does not in the text that was kept. @@ -5743,13 +5820,25 @@ def exact_zeros(sid: str, variable: str) -> int: u for u in first_refusals if "PolicyEngine" in by_unit[u][1][0]["refusal_text"] ] assert len(citing) == note["facts"]["peRefusals"] + # "At least two of America.gov's five refusals": P4 counts refused chats, so + # it takes firstRefusals, not firstRefusalHouseholds, and peRefusals, not + # answeredOnRetry or peLeads. + assert ( + "At least {peRefusals:words} of America.gov's {firstRefusals:words} refusals" + in text + ) + # "refused prompts for five households": a split prompt's two halves + # count once. + assert "refused prompts for {firstRefusalHouseholds:words} households" in text + assert len({u[:12] for u in first_refusals}) == facts["firstRefusalHouseholds"] assert truncated <= set(first_refusals) and not truncated & set(citing) - # America.gov's own words, as quoted. + # America.gov's own words. The opening attributes how it answers to its + # how-it-works page, and the last paragraph quotes its terms. quotes = meta["site_quotes"] assert ( - "Super Intelligence (SI) to find and explain information from official " - "government sources" in text + "which says it answers questions with AI and information from " + "government websites" in text ) assert quotes["how_it_works"]["quote"] == ( "Super Intelligence (SI) to find and explain information from official " @@ -5872,8 +5961,7 @@ def test_tool_pilot_note_predicates(monkeypatch) -> None: def test_no_board_model_declined_a_household_over_policyengine() -> None: """The America.gov note says no board model declined a household on PolicyEngine - grounds, though some models' recorded reasoning notes they cannot run - PolicyEngine. No board model left a household fully unanswered with text in its + grounds. No board model left a household fully unanswered with text in its response, so none declined one on any ground; the households some models left fully unanswered hold no response text at all.""" agov = _america_gov() @@ -5885,16 +5973,6 @@ def test_no_board_model_declined_a_household_over_policyengine() -> None: assert no_text # the check reaches the fully unanswered households text = " ".join(note["paragraphs"]) assert "no model on the board declined a household on that ground" in text - # A floor: the pairs whose recorded reasoning says so, from more than one model. - cannot = set(agov.board_cannot_run_policyengine(predictions)) - assert { - ("grok-4.6", "scenario_007"), - ("kimi-k2.6", "scenario_053"), - ("kimi-k2.6", "scenario_119"), - ("qwen3.8-max", "scenario_121"), - } <= cannot - assert len({model for model, _ in cannot}) >= 2 - assert "some models' recorded reasoning notes that they cannot run" in text def test_methodology_quotes_the_policyengine_named_definitions() -> None: