From fa93a0ce674455759d6701d518ce49f81acdb32f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 13 Oct 2026 19:40:07 -0400 Subject: [PATCH 1/3] Leave recorded ceiling-exhaustion rows alone when applying an adjudication apply_adjudications rewrote every row of an adjudicated case except parse_contract_failure rows, so it overwrote a recorded budget_exhausted_at_ceiling row with the adjudicated class and subtype. collect_audit and case_annotations force that recorded class onto its row, and the dashboard export restores it, but the annotation CSVs kept the overwrite (and the export kept the overwritten subtype). NON_SUBSTANTIVE_SOURCES names both classes an adjudication leaves alone. verify_adjudications_applied uses the same mask for its substantive rows and for its stray-verdict check, so the release drivers' apply-then-verify does not demand the adjudicated class on a recorded ceiling row, or call it a stray verdict when the judge also chose that class for the case. No committed annotation moves: no committed row records ceiling exhaustion, and rebuilding releases 20261013 and 20261013b from their frozen audit trees on this code gives the committed CSVs byte for byte. Co-Authored-By: Claude Opus 5.5 --- policybench/adjudications.py | 33 +++--- tests/test_adjudications.py | 188 +++++++++++++++++++++++++++++++++++ 2 files changed, 209 insertions(+), 12 deletions(-) diff --git a/policybench/adjudications.py b/policybench/adjudications.py index 28871282..a044b8c4 100644 --- a/policybench/adjudications.py +++ b/policybench/adjudications.py @@ -37,6 +37,12 @@ class and the reasoning, so the override stays auditable, and this module "reasoning", ) FINAL_SOURCES = frozenset({"llm_error", "parse_contract_failure"}) +# Row classes an adjudication leaves alone: a parse failure stays a parse +# failure, and a ceiling exhaustion is the run data's own record (collect_audit +# and the dashboard export force it onto its row), not a judge's verdict. +NON_SUBSTANTIVE_SOURCES = frozenset( + {"parse_contract_failure", "budget_exhausted_at_ceiling"} +) # A developer may affirm a judge's (or override a judge's) verdict as prompt # ambiguity, or record a reference engine defect, only when the output is also # removed from scoring for every model (see policybench.reference_exclusions); @@ -207,8 +213,8 @@ def apply_adjudications( """Return adjudicated copies of the row annotations and case notes. An adjudication is a case-level decision: every substantive row of the - case (anything but a ``parse_contract_failure`` row, which stays a parse - failure) takes the adjudicated source and subtype, so re-applying a + case (anything but a :data:`NON_SUBSTANTIVE_SOURCES` row, which keeps its + class) takes the adjudicated source and subtype, so re-applying a revised record updates rows an earlier application already rewrote. The case note's ``case_failure_sources`` and ``case_failure_subtypes`` take the adjudicated class, and the case @@ -219,8 +225,8 @@ def apply_adjudications( cases = cases.copy() report: list[dict] = [] for entry in adjudications: - row_mask = _case_mask(rows, entry) & ( - rows["failure_source"].astype(str) != "parse_contract_failure" + row_mask = _case_mask(rows, entry) & ~rows["failure_source"].astype(str).isin( + NON_SUBSTANTIVE_SOURCES ) case_mask = _case_mask(cases, entry) if not case_mask.any(): @@ -271,9 +277,10 @@ def verify_adjudications_applied( ) -> None: """Raise unless the annotations agree with the complete adjudication record. - For every entry: no row still carries the judge's non-final verdict; every - substantive row of the case (anything but a parse failure) carries the - adjudicated source and subtype; the case note carries the adjudicated + For every entry: no substantive row of the case (anything but a + :data:`NON_SUBSTANTIVE_SOURCES` row, which apply_adjudications leaves + alone) still carries the judge's non-final verdict, and every one carries + the adjudicated source and subtype; the case note carries the adjudicated source and subtype and the exact adjudication sentence, reasoning included. A revised record therefore fails verification until it is re-applied, and a record edited to disagree with the rows fails closed. @@ -285,8 +292,13 @@ def verify_adjudications_applied( if not case_mask.any() or not row_mask.any(): raise AdjudicationError(f"adjudicated case missing from annotations: {key}") case_rows = rows.loc[row_mask] - stray = case_rows[ - case_rows["failure_source"].astype(str) == entry["judge_failure_source"] + substantive = case_rows[ + ~case_rows["failure_source"].astype(str).isin(NON_SUBSTANTIVE_SOURCES) + ] + # A recorded ceiling exhaustion is run data even when the judge also + # chose that class for the case. + stray = substantive[ + substantive["failure_source"].astype(str) == entry["judge_failure_source"] ] affirmed = entry["judge_failure_source"] == entry["adjudicated_failure_source"] if ( @@ -298,9 +310,6 @@ def verify_adjudications_applied( f"{len(stray)} rows of {key} still carry the judge's " f"{entry['judge_failure_source']} verdict" ) - substantive = case_rows[ - case_rows["failure_source"].astype(str) != "parse_contract_failure" - ] off_source = substantive[ substantive["failure_source"].astype(str) != entry["adjudicated_failure_source"] diff --git a/tests/test_adjudications.py b/tests/test_adjudications.py index 40bbda42..760d759e 100644 --- a/tests/test_adjudications.py +++ b/tests/test_adjudications.py @@ -5,17 +5,24 @@ from collections import Counter from functools import cache from pathlib import Path +from unittest import mock import pandas as pd import pytest +from policybench import adjudications from policybench.adjudications import ( AdjudicationError, apply_adjudications, load_adjudications, + parse_adjudications, unresolved_suspect_cases, verify_adjudications_applied, ) +from policybench.annotation_taxonomy import ( + FAILURE_SOURCE_VALUES, + FAILURE_SUBTYPE_VALUES, +) ROOT = Path(__file__).resolve().parents[1] RUN_LABEL = "us_full_run_20260612_policyengine_4_16_1_populace" @@ -430,6 +437,72 @@ def test_verify_requires_agreement_with_the_complete_record(): verify_adjudications_applied(bad_rows, out_cases, [_entry()]) +def _ceiling_frames() -> tuple[pd.DataFrame, pd.DataFrame]: + """_frames, with m3's missing answer on scenario_001 recorded by the run as + ceiling exhaustion rather than a parse failure.""" + rows, cases = _frames() + rows.loc[2, ["failure_source", "failure_subtype"]] = [ + "budget_exhausted_at_ceiling", + "missing_output", + ] + return rows, cases + + +def test_apply_leaves_a_recorded_ceiling_exhaustion_alone(): + """Like a parse failure, a recorded ceiling exhaustion keeps its class and + subtype through an adjudication of its case; collect_audit and the + dashboard export force it onto the row, so the CSVs must keep it too.""" + rows, cases = _ceiling_frames() + out_rows, out_cases, report = apply_adjudications(rows, cases, [_entry()]) + + case_rows = out_rows[out_rows["scenario_id"] == "scenario_001"] + assert case_rows[["failure_source", "failure_subtype"]].values.tolist() == [ + ["llm_error", "age_disability"], + ["llm_error", "age_disability"], + ["budget_exhausted_at_ceiling", "missing_output"], + ] + assert report[0]["rows_rewritten"] == 2 + assert out_cases.loc[0, "case_failure_sources"] == "llm_error" + verify_adjudications_applied(out_rows, out_cases, [_entry()]) + + +def test_verify_does_not_hold_a_ceiling_row_to_the_adjudicated_class(): + """The verifier's substantive rows are apply's: a recorded ceiling row is + neither off the adjudicated class nor a stray judge verdict, even when the + judge chose ceiling exhaustion for the case. The rows apply does rewrite + are still held to the record.""" + rows, cases = _ceiling_frames() + entry = _entry(judge_failure_source="budget_exhausted_at_ceiling") + out_rows, out_cases, _ = apply_adjudications(rows, cases, [entry]) + verify_adjudications_applied(out_rows, out_cases, [entry]) + + with pytest.raises(AdjudicationError, match="still carry"): + verify_adjudications_applied(rows, cases, [_entry()]) + bad_rows = out_rows.copy() + bad_rows.loc[0, "failure_source"] = "reference_model_issue_fixed" + with pytest.raises(AdjudicationError, match="source other than"): + verify_adjudications_applied(bad_rows, out_cases, [entry]) + + +def test_skipping_ceiling_rows_moves_no_committed_annotation(): + """No committed row records ceiling exhaustion, so the committed record + gives the committed annotations whether or not apply skips those rows.""" + entries = load_adjudications(ANNOTATIONS / "us_adjudications.json") + rows = pd.read_csv(ANNOTATIONS / "us_audit_row_annotations.csv") + cases = pd.read_csv(ANNOTATIONS / "us_case_notes.csv") + with mock.patch.object( + adjudications, "NON_SUBSTANTIVE_SOURCES", frozenset({"parse_contract_failure"}) + ): + before = apply_adjudications(rows, cases, entries) + after = apply_adjudications(rows, cases, entries) + for old, new in zip(before, after): + if isinstance(old, pd.DataFrame): + pd.testing.assert_frame_equal(new, old) + else: + assert new == old + verify_adjudications_applied(*after[:2], entries) + + def _suspect_frames() -> tuple[pd.DataFrame, pd.DataFrame]: rows, cases = _frames() rows.loc[rows["scenario_id"] == "scenario_001", "reference_suspect"] = True @@ -1545,3 +1618,118 @@ def test_release_20261013_restates_only_the_cases_muse_wonder_reopened(tmp_path) if history[:-1] == before.get("judge_previous", []) and history: replaced += 1 assert replaced == len(verdicts) == 50 + + +# The rows an adjudication skips, spelled out so the oracle below does not +# share the module's constant. They are drawn more often than their share of +# the taxonomy, so most examples hold a row an adjudication must skip. +_SKIPPED = ("budget_exhausted_at_ceiling", "parse_contract_failure") +_ROW_SOURCES = st.one_of( + st.sampled_from(_SKIPPED), + st.sampled_from(FAILURE_SOURCE_VALUES), +) +_SUBTYPES = st.sampled_from(FAILURE_SUBTYPE_VALUES) + + +@st.composite +def _adjudicated_annotations(draw): + """Row annotations and case notes for one to three cases of one to four + rows, each row of any class and subtype, and a valid adjudication of any + judge verdict for some of the cases.""" + rows, cases, entries = [], [], [] + for n in range(draw(st.integers(1, 3))): + scenario = f"scenario_{n:03d}" + count = draw(st.integers(1, 4)) + for m in range(count): + rows.append( + ("us", scenario, "ssi", f"m{m}", draw(_ROW_SOURCES), draw(_SUBTYPES)) + ) + cases.append( + ( + "us", + scenario, + "ssi", + count, + draw(_ROW_SOURCES), + draw(_SUBTYPES), + f"note {n}.", + ) + ) + if draw(st.booleans()): + adjudicated = draw( + st.sampled_from( + ["llm_error", "parse_contract_failure", "prompt_ambiguity"] + ) + ) + entry = _entry( + scenario_id=scenario, + judge_failure_source=draw(st.sampled_from(FAILURE_SOURCE_VALUES)), + judge_failure_subtype=draw(_SUBTYPES), + adjudicated_failure_source=adjudicated, + adjudicated_failure_subtype=draw(_SUBTYPES), + ) + if adjudicated == "prompt_ambiguity": + entry["excluded_from_scoring"] = True + entries.append(entry) + row_frame = pd.DataFrame( + rows, + columns=[ + "country", + "scenario_id", + "variable", + "model", + "failure_source", + "failure_subtype", + ], + ) + row_frame["reference_suspect"] = False + row_frame["annotation"] = "diagnosis" + case_frame = pd.DataFrame( + cases, + columns=[ + "country", + "scenario_id", + "variable", + "wrong_model_count", + "case_failure_sources", + "case_failure_subtypes", + "case_annotation", + ], + ) + return row_frame, case_frame, parse_adjudications({"adjudications": entries}, "") + + +@settings(max_examples=300, deadline=None) +@given(data=_adjudicated_annotations()) +def test_adjudication_skips_non_substantive_rows_properties(data): + """For any annotations and valid record: + - a parse-failure or ceiling-exhaustion row keeps its class and subtype, + and a row of a case no entry decides is untouched; + - every other row of a decided case takes the adjudicated class; + - the report counts exactly the rows whose class or subtype changed; + - the verifier accepts what apply builds, and a second apply changes + nothing. + """ + rows, cases, entries = data + out_rows, out_cases, report = apply_adjudications(rows, cases, entries) + + decided = {e["scenario_id"]: e for e in entries} + classes = ["failure_source", "failure_subtype"] + expected = rows.copy() + for index, row in rows.iterrows(): + entry = decided.get(row["scenario_id"]) + if entry and row["failure_source"] not in _SKIPPED: + expected.loc[index, classes] = [ + entry["adjudicated_failure_source"], + entry["adjudicated_failure_subtype"], + ] + pd.testing.assert_frame_equal(out_rows, expected) + changed = (rows[classes] != out_rows[classes]).any(axis=1) + assert [item["rows_rewritten"] for item in report] == [ + int(changed[rows["scenario_id"] == e["scenario_id"]].sum()) for e in entries + ] + + verify_adjudications_applied(out_rows, out_cases, entries) + again_rows, again_cases, _ = apply_adjudications(out_rows, out_cases, entries) + pd.testing.assert_frame_equal(again_rows, out_rows) + pd.testing.assert_frame_equal(again_cases, out_cases) From 93256a3e4872514d427207283c89d705ded82fe0 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 23:37:03 +0000 Subject: [PATCH 2/3] Generate every adjudicated outcome in the non-substantive-row property The property drew only llm_error, parse_contract_failure and prompt_ambiguity. It now draws reference_engine_defect and reference_later_law too, each with the exclusion and reference verdict a valid record requires, and any reference verdict the other classes admit. Rows and case notes carry a drawn reference_suspect flag, and the oracle expects a reference verdict to clear it on every row of its case, skipped rows included. Co-Authored-By: Claude Opus 5.5 --- tests/test_adjudications.py | 50 +++++++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/tests/test_adjudications.py b/tests/test_adjudications.py index dabf072b..f9b28710 100644 --- a/tests/test_adjudications.py +++ b/tests/test_adjudications.py @@ -1756,20 +1756,40 @@ def test_release_20261013_restates_only_the_cases_muse_wonder_reopened(tmp_path) st.sampled_from(FAILURE_SOURCE_VALUES), ) _SUBTYPES = st.sampled_from(FAILURE_SUBTYPE_VALUES) +# Every adjudicated class a valid record admits, each with the reference +# verdicts parse_adjudications accepts beside it (None: no verdict) and whether +# it removes the output from scoring. Spelled out, like _SKIPPED, so the +# generator does not share the module's constants. +_OUTCOMES = { + "llm_error": ((None, "affirmed", "regenerated"), False), + "parse_contract_failure": ((None, "affirmed", "regenerated"), False), + "prompt_ambiguity": ((None, "unlisted_input"), True), + "reference_engine_defect": (("engine_defect",), True), + "reference_later_law": (("later_law",), True), +} @st.composite def _adjudicated_annotations(draw): """Row annotations and case notes for one to three cases of one to four - rows, each row of any class and subtype, and a valid adjudication of any - judge verdict for some of the cases.""" + rows, each row of any class, subtype and reference-suspect flag, and a + valid adjudication of any judge verdict, to any adjudicated class with + any reference verdict that class admits, for some of the cases.""" rows, cases, entries = [], [], [] for n in range(draw(st.integers(1, 3))): scenario = f"scenario_{n:03d}" count = draw(st.integers(1, 4)) for m in range(count): rows.append( - ("us", scenario, "ssi", f"m{m}", draw(_ROW_SOURCES), draw(_SUBTYPES)) + ( + "us", + scenario, + "ssi", + f"m{m}", + draw(_ROW_SOURCES), + draw(_SUBTYPES), + draw(st.booleans()), + ) ) cases.append( ( @@ -1780,14 +1800,12 @@ def _adjudicated_annotations(draw): draw(_ROW_SOURCES), draw(_SUBTYPES), f"note {n}.", + draw(st.booleans()), ) ) if draw(st.booleans()): - adjudicated = draw( - st.sampled_from( - ["llm_error", "parse_contract_failure", "prompt_ambiguity"] - ) - ) + adjudicated = draw(st.sampled_from(sorted(_OUTCOMES))) + verdicts, excluded = _OUTCOMES[adjudicated] entry = _entry( scenario_id=scenario, judge_failure_source=draw(st.sampled_from(FAILURE_SOURCE_VALUES)), @@ -1795,8 +1813,13 @@ def _adjudicated_annotations(draw): adjudicated_failure_source=adjudicated, adjudicated_failure_subtype=draw(_SUBTYPES), ) - if adjudicated == "prompt_ambiguity": + if excluded: entry["excluded_from_scoring"] = True + verdict = draw(st.sampled_from(verdicts)) + if verdict is not None: + entry["reference_verdict"] = verdict + entry["reference_basis"] = "recorded basis" + entry["judge_reference_suspect"] = draw(st.booleans()) entries.append(entry) row_frame = pd.DataFrame( rows, @@ -1807,9 +1830,9 @@ def _adjudicated_annotations(draw): "model", "failure_source", "failure_subtype", + "reference_suspect", ], ) - row_frame["reference_suspect"] = False row_frame["annotation"] = "diagnosis" case_frame = pd.DataFrame( cases, @@ -1821,6 +1844,7 @@ def _adjudicated_annotations(draw): "case_failure_sources", "case_failure_subtypes", "case_annotation", + "reference_suspect", ], ) return row_frame, case_frame, parse_adjudications({"adjudications": entries}, "") @@ -1829,10 +1853,12 @@ def _adjudicated_annotations(draw): @settings(max_examples=300, deadline=None) @given(data=_adjudicated_annotations()) def test_adjudication_skips_non_substantive_rows_properties(data): - """For any annotations and valid record: + """For any annotations and valid record, whatever class it adjudicates: - a parse-failure or ceiling-exhaustion row keeps its class and subtype, and a row of a case no entry decides is untouched; - every other row of a decided case takes the adjudicated class; + - a reference verdict clears the flag on every row of its case, skipped + rows included, and touches no other row's flag; - the report counts exactly the rows whose class or subtype changed; - the verifier accepts what apply builds, and a second apply changes nothing. @@ -1850,6 +1876,8 @@ def test_adjudication_skips_non_substantive_rows_properties(data): entry["adjudicated_failure_source"], entry["adjudicated_failure_subtype"], ] + if entry and entry.get("reference_verdict"): + expected.loc[index, "reference_suspect"] = False pd.testing.assert_frame_equal(out_rows, expected) changed = (rows[classes] != out_rows[classes]).any(axis=1) assert [item["rows_rewritten"] for item in report] == [ From 52b15e61b96b563e49bf1ee779dc9b4a6572cc8c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 03:00:30 +0000 Subject: [PATCH 3/3] Test case-note flags and every outcome-verdict pair in the adjudication property - The differential test asserts its premise: no committed row of an adjudicated case records budget_exhausted_at_ceiling, and says to delete the test once one does. - The property's oracle now checks each case note's reference_suspect flag: cleared only on decided cases whose entry has a reference verdict. - A parametrized test runs each class-and-verdict pair in _OUTCOMES on a decided case with ceiling, parse and substantive rows beside an undecided case, through the same oracle as the property (_check_adjudication). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LwQq6KUzqoALNASH6Yqjbk --- tests/test_adjudications.py | 114 ++++++++++++++++++++++++++++++++++-- 1 file changed, 108 insertions(+), 6 deletions(-) diff --git a/tests/test_adjudications.py b/tests/test_adjudications.py index f9b28710..a8aaf449 100644 --- a/tests/test_adjudications.py +++ b/tests/test_adjudications.py @@ -600,6 +600,16 @@ def test_skipping_ceiling_rows_moves_no_committed_annotation(): entries = load_adjudications(ANNOTATIONS / "us_adjudications.json") rows = pd.read_csv(ANNOTATIONS / "us_audit_row_annotations.csv") cases = pd.read_csv(ANNOTATIONS / "us_case_notes.csv") + decided = {(str(e["scenario_id"]), str(e["variable"])) for e in entries} + in_decided = ( + rows[["scenario_id", "variable"]].astype(str).apply(tuple, axis=1).isin(decided) + ) + assert not ( + in_decided & (rows["failure_source"] == "budget_exhausted_at_ceiling") + ).any(), ( + "a committed adjudicated case now records ceiling exhaustion; this " + "differential no longer applies (see PR #224), delete it" + ) with mock.patch.object( adjudications, "NON_SUBSTANTIVE_SOURCES", frozenset({"parse_contract_failure"}) ): @@ -1850,20 +1860,20 @@ def _adjudicated_annotations(draw): return row_frame, case_frame, parse_adjudications({"adjudications": entries}, "") -@settings(max_examples=300, deadline=None) -@given(data=_adjudicated_annotations()) -def test_adjudication_skips_non_substantive_rows_properties(data): - """For any annotations and valid record, whatever class it adjudicates: +def _check_adjudication( + rows: pd.DataFrame, cases: pd.DataFrame, entries: list[dict] +) -> None: + """The oracle the property and the outcome-by-outcome test share: - a parse-failure or ceiling-exhaustion row keeps its class and subtype, and a row of a case no entry decides is untouched; - every other row of a decided case takes the adjudicated class; - a reference verdict clears the flag on every row of its case, skipped - rows included, and touches no other row's flag; + rows included, and on its case note, and touches no other row's or + note's flag; - the report counts exactly the rows whose class or subtype changed; - the verifier accepts what apply builds, and a second apply changes nothing. """ - rows, cases, entries = data out_rows, out_cases, report = apply_adjudications(rows, cases, entries) decided = {e["scenario_id"]: e for e in entries} @@ -1879,6 +1889,12 @@ def test_adjudication_skips_non_substantive_rows_properties(data): if entry and entry.get("reference_verdict"): expected.loc[index, "reference_suspect"] = False pd.testing.assert_frame_equal(out_rows, expected) + expected_flags = cases["reference_suspect"].copy() + for index, case in cases.iterrows(): + entry = decided.get(case["scenario_id"]) + if entry and entry.get("reference_verdict"): + expected_flags.loc[index] = False + pd.testing.assert_series_equal(out_cases["reference_suspect"], expected_flags) changed = (rows[classes] != out_rows[classes]).any(axis=1) assert [item["rows_rewritten"] for item in report] == [ int(changed[rows["scenario_id"] == e["scenario_id"]].sum()) for e in entries @@ -1890,6 +1906,92 @@ def test_adjudication_skips_non_substantive_rows_properties(data): pd.testing.assert_frame_equal(again_cases, out_cases) +@settings(max_examples=300, deadline=None) +@given(data=_adjudicated_annotations()) +def test_adjudication_skips_non_substantive_rows_properties(data): + """For any annotations and valid record, whatever class it adjudicates, + apply meets the oracle in _check_adjudication.""" + _check_adjudication(*data) + + +@pytest.mark.parametrize( + ("adjudicated", "verdict"), + [ + (adjudicated, verdict) + for adjudicated, (verdicts, _) in sorted(_OUTCOMES.items()) + for verdict in verdicts + ], +) +def test_every_outcome_and_verdict_leaves_skipped_rows_alone(adjudicated, verdict): + """Each adjudicated class with each reference verdict it admits, on a + decided case holding a ceiling row, a parse row and a substantive row, + all flagged, beside an undecided case with the same rows. The judge chose + ceiling exhaustion, so the verifier must not take the recorded ceiling + row for its stray verdict.""" + rows, cases = [], [] + for scenario in ("scenario_000", "scenario_001"): + for model, source, subtype in ( + ("m0", "budget_exhausted_at_ceiling", "missing_output"), + ("m1", "parse_contract_failure", "missing_output"), + ("m2", "needs_review", "other"), + ): + rows.append(("us", scenario, "ssi", model, source, subtype, True)) + cases.append( + ( + "us", + scenario, + "ssi", + 3, + "needs_review", + "other", + f"note {scenario}.", + True, + ) + ) + rows = pd.DataFrame( + rows, + columns=[ + "country", + "scenario_id", + "variable", + "model", + "failure_source", + "failure_subtype", + "reference_suspect", + ], + ) + rows["annotation"] = "diagnosis" + cases = pd.DataFrame( + cases, + columns=[ + "country", + "scenario_id", + "variable", + "wrong_model_count", + "case_failure_sources", + "case_failure_subtypes", + "case_annotation", + "reference_suspect", + ], + ) + _, excluded = _OUTCOMES[adjudicated] + entry = _entry( + scenario_id="scenario_000", + judge_failure_source="budget_exhausted_at_ceiling", + judge_failure_subtype="missing_output", + adjudicated_failure_source=adjudicated, + adjudicated_failure_subtype="age_disability", + ) + if excluded: + entry["excluded_from_scoring"] = True + if verdict is not None: + entry["reference_verdict"] = verdict + entry["reference_basis"] = "recorded basis" + entry["judge_reference_suspect"] = True + entries = parse_adjudications({"adjudications": [entry]}, "") + _check_adjudication(rows, cases, entries) + + def test_release_20261014_restates_only_the_reopened_cases_and_adds_its_wave( tmp_path, ):